From 177fcb9154fed58d582f41fb9aa14a16930d4202 Mon Sep 17 00:00:00 2001 From: Mathieu Baudier Date: Thu, 21 Oct 2021 13:39:03 +0200 Subject: [PATCH] Dynamic secondary instances based on queries, and supporting CSV in addition to XML. --- .../src/org/argeo/entity/EntityConstants.java | 2 +- .../src/org/argeo/entity/EntityMimeType.java | 60 ++++++++ .../src/org/argeo/entity/entity.cnd | 6 + .../entity/core/JcrEntityDefinition.java | 6 +- .../src/org/argeo/support/odk/OdkUtils.java | 84 +++++++++--- .../odk/servlet/OdkManifestServlet.java | 129 +++++++++++++----- 6 files changed, 229 insertions(+), 58 deletions(-) create mode 100644 core/org.argeo.entity.api/src/org/argeo/entity/EntityMimeType.java diff --git a/core/org.argeo.entity.api/src/org/argeo/entity/EntityConstants.java b/core/org.argeo.entity.api/src/org/argeo/entity/EntityConstants.java index f7a2de8..93021e0 100644 --- a/core/org.argeo.entity.api/src/org/argeo/entity/EntityConstants.java +++ b/core/org.argeo.entity.api/src/org/argeo/entity/EntityConstants.java @@ -3,6 +3,6 @@ package org.argeo.entity; /** Constant related to entities, typically used in an OSGi context. */ public interface EntityConstants { final static String TYPE = "entity.type"; - final static String DEFAULT_EDITORY_ID = "entity.defaultEditorId"; + final static String DEFAULT_EDITOR_ID = "entity.defaultEditorId"; } diff --git a/core/org.argeo.entity.api/src/org/argeo/entity/EntityMimeType.java b/core/org.argeo.entity.api/src/org/argeo/entity/EntityMimeType.java new file mode 100644 index 0000000..997f1be --- /dev/null +++ b/core/org.argeo.entity.api/src/org/argeo/entity/EntityMimeType.java @@ -0,0 +1,60 @@ +package org.argeo.entity; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** Supported mime types. */ +public enum EntityMimeType { + XML("text/xml", "xml"), CSV("text/csv", "csv"); + + private final String mimeType; + private final String[] extensions; + + EntityMimeType(String mimeType, String... extensions) { + this.mimeType = mimeType; + this.extensions = extensions; + } + + public String getMimeType() { + return mimeType; + } + + public String[] getExtensions() { + return extensions; + } + + public String getDefaultExtension() { + if (extensions.length > 0) + return extensions[0]; + else + return null; + } + + public String toHttpContentType(Charset charset) { + if (charset == null) + return mimeType; + return mimeType + "; charset=" + charset.name(); + } + + public String toHttpContentType() { + if (mimeType.startsWith("text/")) { + return toHttpContentType(StandardCharsets.UTF_8); + } else { + return mimeType; + } + } + + public static EntityMimeType find(String mimeType) { + for (EntityMimeType entityMimeType : values()) { + if (entityMimeType.mimeType.equals(mimeType)) + return entityMimeType; + } + return null; + } + + @Override + public String toString() { + return mimeType; + } + +} diff --git a/core/org.argeo.entity.api/src/org/argeo/entity/entity.cnd b/core/org.argeo.entity.api/src/org/argeo/entity/entity.cnd index 151df4e..2ea89f9 100644 --- a/core/org.argeo.entity.api/src/org/argeo/entity/entity.cnd +++ b/core/org.argeo.entity.api/src/org/argeo/entity/entity.cnd @@ -35,6 +35,11 @@ mixin //+ * (entity:reference) //+ * (entity:composite) +[entity:query] > nt:query, mix:referenceable + +[entity:querySet] ++ * (entity:query) = entity:query * + // // STRUCTURE // @@ -65,6 +70,7 @@ orderable // [entity:form] mixin ++ queries (entity:querySet) = entity:querySet [entity:formSubmission] mixin diff --git a/core/org.argeo.entity.core/src/org/argeo/entity/core/JcrEntityDefinition.java b/core/org.argeo.entity.core/src/org/argeo/entity/core/JcrEntityDefinition.java index 7fd26d1..6eb086d 100644 --- a/core/org.argeo.entity.core/src/org/argeo/entity/core/JcrEntityDefinition.java +++ b/core/org.argeo.entity.core/src/org/argeo/entity/core/JcrEntityDefinition.java @@ -18,7 +18,7 @@ public class JcrEntityDefinition implements EntityDefinition { private Repository repository; private String type; - private String defaultEditoryId; + private String defaultEditorId; public void init(BundleContext bundleContext, Map properties) throws RepositoryException { Session adminSession = NodeUtils.openDataAdminSession(repository, null); @@ -26,7 +26,7 @@ public class JcrEntityDefinition implements EntityDefinition { type = properties.get(EntityConstants.TYPE); if (type == null) throw new IllegalArgumentException("Entity type property " + EntityConstants.TYPE + " must be set."); - defaultEditoryId = properties.get(EntityConstants.DEFAULT_EDITORY_ID); + defaultEditorId = properties.get(EntityConstants.DEFAULT_EDITOR_ID); // String definitionPath = EntityNames.ENTITY_DEFINITIONS_PATH + '/' + type; // if (!adminSession.itemExists(definitionPath)) { // Node entityDefinition = JcrUtils.mkdirs(adminSession, definitionPath, EntityTypes.ENTITY_DEFINITION); @@ -50,7 +50,7 @@ public class JcrEntityDefinition implements EntityDefinition { @Override public String getEditorId(Node entity) { - return defaultEditoryId; + return defaultEditorId; } @Override diff --git a/knowledge/org.argeo.support.odk/src/org/argeo/support/odk/OdkUtils.java b/knowledge/org.argeo.support.odk/src/org/argeo/support/odk/OdkUtils.java index 585b897..f4bb936 100644 --- a/knowledge/org.argeo.support.odk/src/org/argeo/support/odk/OdkUtils.java +++ b/knowledge/org.argeo.support.odk/src/org/argeo/support/odk/OdkUtils.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import javax.jcr.ImportUUIDBehavior; import javax.jcr.Node; @@ -12,10 +13,16 @@ import javax.jcr.NodeIterator; import javax.jcr.Property; import javax.jcr.RepositoryException; import javax.jcr.Session; +import javax.jcr.Value; import javax.jcr.nodetype.NodeType; +import javax.jcr.query.Query; +import javax.jcr.query.QueryResult; +import javax.jcr.query.Row; +import javax.jcr.query.RowIterator; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.argeo.entity.EntityMimeType; import org.argeo.entity.EntityType; import org.argeo.jcr.Jcr; import org.argeo.jcr.JcrUtils; @@ -26,7 +33,8 @@ import org.argeo.util.DigestUtils; public class OdkUtils { private final static Log log = LogFactory.getLog(OdkUtils.class); - public static void loadOdkForm(Node formBase, String name, InputStream in) throws RepositoryException, IOException { + public static Node loadOdkForm(Node formBase, String name, InputStream in, InputStream... additionalNodes) + throws RepositoryException, IOException { if (!formBase.isNodeType(EntityType.formSet.get())) throw new IllegalArgumentException( "Parent path " + formBase + " must be of type " + EntityType.formSet.get()); @@ -37,10 +45,12 @@ public class OdkUtils { String previousFormVersion = Jcr.get(form, OrxListName.version.get()); Session s = formBase.getSession(); -// String res = "/odk/apafSession.odk.xml"; -// try (InputStream in = getClass().getClassLoader().getResourceAsStream(res)) { s.importXML(form.getPath(), in, ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING); -// } + + for (InputStream additionalIn : additionalNodes) { + s.importXML(form.getPath(), additionalIn, ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING); + } + s.save(); // manage instances // NodeIterator instances = @@ -62,29 +72,62 @@ public class OdkUtils { } if (instanceUri != null) { if ("jr".equals(instanceUri.getScheme())) { + String uuid; + String mimeType; + String encoding = StandardCharsets.UTF_8.name(); String type = instanceUri.getHost(); + String path = instanceUri.getPath(); if ("file".equals(type)) { - Node manifest = JcrUtils.getOrAdd(form, OrxManifestName.manifest.name(), - OrxManifestName.manifest.get()); - Node file = JcrUtils.getOrAdd(manifest, instanceId); - String path = instanceUri.getPath(); if (!path.endsWith(".xml")) throw new IllegalArgumentException("File uri " + instanceUri + " must end with .xml"); // Work around bug in ODK Collect not supporting paths // path = path.substring(0, path.length() - ".xml".length()); // Node target = file.getSession().getNode(path); - String uuid = path.substring(1, path.length() - ".xml".length()); - Node target = file.getSession().getNodeByIdentifier(uuid); - // FIXME hard code terms path in order to test ODK Collect bug - if (target.isNodeType(NodeType.MIX_REFERENCEABLE)) { - file.setProperty(Property.JCR_ID, target); - if (file.hasProperty(Property.JCR_PATH)) - file.getProperty(Property.JCR_PATH).remove(); - } else { - file.setProperty(Property.JCR_PATH, target.getPath()); - if (file.hasProperty(Property.JCR_ID)) - file.getProperty(Property.JCR_ID).remove(); + uuid = path.substring(1, path.length() - ".xml".length()); + mimeType = EntityMimeType.XML.getMimeType(); + } else if ("file-csv".equals(type)) { + if (!path.endsWith(".csv")) + throw new IllegalArgumentException("File uri " + instanceUri + " must end with .csv"); + // Work around bug in ODK Collect not supporting paths + // path = path.substring(0, path.length() - ".csv".length()); + // Node target = file.getSession().getNode(path); + uuid = path.substring(1, path.length() - ".csv".length()); + mimeType = EntityMimeType.CSV.getMimeType(); + } else { + throw new IllegalArgumentException("Unsupported instance type " + type); + } + Node manifest = JcrUtils.getOrAdd(form, OrxManifestName.manifest.name(), + OrxManifestName.manifest.get()); + Node file = JcrUtils.getOrAdd(manifest, instanceId); + file.addMixin(NodeType.MIX_MIMETYPE); + file.setProperty(Property.JCR_MIMETYPE, mimeType); + file.setProperty(Property.JCR_ENCODING, encoding); + Node target = file.getSession().getNodeByIdentifier(uuid); + + if (target.isNodeType(NodeType.NT_QUERY)) { + Query query = target.getSession().getWorkspace().getQueryManager().getQuery(target); + query.setLimit(10); + QueryResult queryResult = query.execute(); + RowIterator rit = queryResult.getRows(); + while (rit.hasNext()) { + Row row = rit.nextRow(); + for (Value value : row.getValues()) { + System.out.print(value.getString()); + System.out.print(','); + } + System.out.print('\n'); } + + } + + if (target.isNodeType(NodeType.MIX_REFERENCEABLE)) { + file.setProperty(Property.JCR_ID, target); + if (file.hasProperty(Property.JCR_PATH)) + file.getProperty(Property.JCR_PATH).remove(); + } else { + file.setProperty(Property.JCR_PATH, target.getPath()); + if (file.hasProperty(Property.JCR_ID)) + file.getProperty(Property.JCR_ID).remove(); } } } @@ -126,7 +169,7 @@ public class OdkUtils { s.refresh(false); if (log.isDebugEnabled()) log.debug("Unmodified form " + form); - return; + return form; } else { if (formVersion.equals(previousFormVersion)) { s.refresh(false); @@ -145,6 +188,7 @@ public class OdkUtils { } } } + return form; } /** Singleton. */ diff --git a/knowledge/org.argeo.support.odk/src/org/argeo/support/odk/servlet/OdkManifestServlet.java b/knowledge/org.argeo.support.odk/src/org/argeo/support/odk/servlet/OdkManifestServlet.java index 091f4da..3510e06 100644 --- a/knowledge/org.argeo.support.odk/src/org/argeo/support/odk/servlet/OdkManifestServlet.java +++ b/knowledge/org.argeo.support.odk/src/org/argeo/support/odk/servlet/OdkManifestServlet.java @@ -1,26 +1,40 @@ package org.argeo.support.odk.servlet; -import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.Writer; -import java.nio.charset.StandardCharsets; - +import java.nio.charset.Charset; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; + +import javax.jcr.ItemNotFoundException; import javax.jcr.Node; import javax.jcr.NodeIterator; import javax.jcr.Property; import javax.jcr.Repository; import javax.jcr.RepositoryException; import javax.jcr.Session; +import javax.jcr.Value; import javax.jcr.nodetype.NodeType; +import javax.jcr.query.Query; +import javax.jcr.query.QueryResult; +import javax.jcr.query.Row; +import javax.jcr.query.RowIterator; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.output.NullOutputStream; import org.argeo.cms.servlet.ServletAuthUtils; +import org.argeo.entity.EntityMimeType; import org.argeo.jcr.Jcr; import org.argeo.jcr.JcrException; import org.argeo.support.odk.OrxManifestName; +import org.argeo.util.CsvWriter; import org.argeo.util.DigestUtils; /** Describe additional files. */ @@ -31,7 +45,6 @@ public class OdkManifestServlet extends HttpServlet { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - resp.setContentType("text/xml; charset=utf-8"); resp.setHeader("X-OpenRosa-Version", "1.0"); resp.setDateHeader("Date", System.currentTimeMillis()); @@ -48,40 +61,50 @@ public class OdkManifestServlet extends HttpServlet { try { Node node = session.getNode(pathInfo); if (node.isNodeType(OrxManifestName.manifest.get())) { + resp.setContentType(EntityMimeType.XML.toHttpContentType()); Writer writer = resp.getWriter(); writer.append(""); writer.append(""); NodeIterator nit = node.getNodes(); - while (nit.hasNext()) { + children: while (nit.hasNext()) { Node file = nit.nextNode(); if (file.isNodeType(OrxManifestName.mediaFile.get())) { - writer.append(""); + EntityMimeType mimeType = EntityMimeType + .find(file.getProperty(Property.JCR_MIMETYPE).getString()); + Charset charset = Charset.forName(file.getProperty(Property.JCR_ENCODING).getString()); if (file.isNodeType(NodeType.NT_ADDRESS)) { - Node target = file.getProperty(Property.JCR_ID).getNode(); + Node target; + try { + target = file.getProperty(Property.JCR_ID).getNode(); + } catch (ItemNotFoundException e) { + // TODO remove old manifests + continue children; + } + writer.append(""); writer.append(""); // Work around bug in ODK Collect not supporting paths // writer.append(target.getPath().substring(1) + ".xml"); - writer.append(target.getIdentifier() + ".xml"); + writer.append(target.getIdentifier() + "." + mimeType.getDefaultExtension()); writer.append(""); -// StringBuilder xml = new StringBuilder(); -// xml.append(""); -// JcrUtils.toSimpleXml(target, xml); -// String fileCsum = DigestUtils.digest(DigestUtils.MD5, -// xml.toString().getBytes(StandardCharsets.UTF_8)); -// writer.append(""); -// writer.append("md5sum:" + fileCsum); -// writer.append(""); - - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - session.exportDocumentView(target.getPath(), out, true, false); - String fileCsum = DigestUtils.digest(DigestUtils.MD5, out.toByteArray()); -// JcrxApi.addChecksum(file, fileCsum); + MessageDigest messageDigest = MessageDigest.getInstance(DigestUtils.MD5); + // TODO cache a temp file ? + try (DigestOutputStream out = new DigestOutputStream(new NullOutputStream(), + messageDigest)) { + writeMediaFile(out, target, mimeType, charset); writer.append(""); - writer.append("md5sum:" + fileCsum); + writer.append("md5sum:" + DigestUtils.encodeHexString(out.getMessageDigest().digest())); writer.append(""); } + +// try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { +// session.exportDocumentView(target.getPath(), out, true, false); +// String fileCsum = DigestUtils.digest(DigestUtils.MD5, out.toByteArray()); +// writer.append(""); +// writer.append("md5sum:" + fileCsum); +// writer.append(""); +// } writer.append("" + protocol + "://" + serverName + (serverPort == 80 || serverPort == 443 ? "" : ":" + serverPort) + "/api/odk/formManifest" + file.getPath() + ""); @@ -92,21 +115,18 @@ public class OdkManifestServlet extends HttpServlet { writer.append(""); } else if (node.isNodeType(OrxManifestName.mediaFile.get())) { + EntityMimeType mimeType = EntityMimeType.find(node.getProperty(Property.JCR_MIMETYPE).getString()); + Charset charset = Charset.forName(node.getProperty(Property.JCR_ENCODING).getString()); + resp.setContentType(mimeType.toHttpContentType(charset)); if (node.isNodeType(NodeType.NT_ADDRESS)) { Node target = node.getProperty(Property.JCR_ID).getNode(); -// StringBuilder xml = new StringBuilder(); -// xml.append(""); -// JcrUtils.toSimpleXml(target, xml); -// System.out.println(xml); -// resp.getOutputStream().write(xml.toString().getBytes(StandardCharsets.UTF_8)); -// resp.flushBuffer(); - - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - session.exportDocumentView(target.getPath(), out, true, false); - System.out.println(new String(out.toByteArray(), StandardCharsets.UTF_8)); - resp.getOutputStream().write(out.toByteArray()); - } + writeMediaFile(resp.getOutputStream(), target, mimeType, charset); +// try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { +// session.exportDocumentView(target.getPath(), out, true, false); +// System.out.println(new String(out.toByteArray(), StandardCharsets.UTF_8)); +// resp.getOutputStream().write(out.toByteArray()); +// } } else { throw new IllegalArgumentException("Unsupported node " + node); } @@ -115,12 +135,53 @@ public class OdkManifestServlet extends HttpServlet { } } catch (RepositoryException e) { throw new JcrException(e); + } catch (NoSuchAlgorithmException e) { + throw new ServletException(e); } finally { Jcr.logout(session); } } + protected void writeMediaFile(OutputStream out, Node target, EntityMimeType mimeType, Charset charset) + throws RepositoryException, IOException { + if (target.isNodeType(NodeType.NT_QUERY)) { + Query query = target.getSession().getWorkspace().getQueryManager().getQuery(target); + QueryResult queryResult = query.execute(); + String[] columnNames = queryResult.getColumnNames(); + if (EntityMimeType.XML.equals(mimeType)) { + } else if (EntityMimeType.CSV.equals(mimeType)) { + CsvWriter csvWriter = new CsvWriter(out, charset); + csvWriter.writeLine(columnNames); + RowIterator rit = queryResult.getRows(); + while (rit.hasNext()) { + Row row = rit.nextRow(); + Value[] values = row.getValues(); + List lst = new ArrayList<>(); + for (Value value : values) { + lst.add(value.getString()); + } + csvWriter.writeLine(lst); + } + } + } else { + if (EntityMimeType.XML.equals(mimeType)) { + target.getSession().exportDocumentView(target.getPath(), out, true, false); + } else if (EntityMimeType.CSV.equals(mimeType)) { + CsvWriter csvWriter = new CsvWriter(out, charset); + csvWriter.writeLine(new String[] { "name", "label" }); + NodeIterator children = target.getNodes(); + while (children.hasNext()) { + Node child = children.nextNode(); + String label = Jcr.getTitle(child); + csvWriter.writeLine(new String[] { child.getIdentifier(), label }); + } + } + + } + + } + public void setRepository(Repository repository) { this.repository = repository; } -- 2.30.2