Make indexing more robust
authorMathieu Baudier <mbaudier@argeo.org>
Tue, 17 Jul 2018 18:21:18 +0000 (20:21 +0200)
committerMathieu Baudier <mbaudier@argeo.org>
Tue, 17 Jul 2018 18:21:18 +0000 (20:21 +0200)
org.argeo.slc.client.ui.dist/src/org/argeo/slc/client/ui/dist/commands/NormalizeWorkspace.java
org.argeo.slc.repo/src/org/argeo/slc/repo/ModularDistributionFactory.java
org.argeo.slc.repo/src/org/argeo/slc/repo/ModularDistributionIndexer.java
org.argeo.slc.repo/src/org/argeo/slc/repo/PdeSourcesIndexer.java
org.argeo.slc.repo/src/org/argeo/slc/repo/RepoConstants.java
org.argeo.slc.repo/src/org/argeo/slc/repo/RepoUtils.java

index de82a3e2fddc7b28b686666c5d0cc8458a112327..b6db2ad16d3b50128123bb06c47289a5efd7c452 100644 (file)
@@ -173,6 +173,9 @@ public class NormalizeWorkspace extends AbstractHandler implements SlcNames {
                                        monitor.worked(1);
                                        if (log.isDebugEnabled())
                                                log.debug("Processed source artifact " + node.getPath());
+                               } else if (node.getName().endsWith("-javadoc.jar")) {
+                                       if (log.isDebugEnabled())
+                                               log.debug("Skip indexing of Javadoc jar " + node.getPath());
                                } else if (node.getName().endsWith(".jar")) {
                                        if (jarFileIndexer.support(node.getPath()))
                                                if (artifactIndexer.support(node.getPath())) {
index 25b6c48c4f9f4d1d5408c4c0464e6d8586f0b5dc..9c13ce048260348baf1fd8fb020eb1489e7c343d 100644 (file)
@@ -149,7 +149,7 @@ public class ModularDistributionFactory implements Runnable {
                addManifestAttribute(manifest, Constants.BUNDLE_NAME, osgiDistribution.getName());
 
                // Categorized name version
-               addManifestAttribute(manifest, RepoConstants.SLC_GROUP_ID, osgiDistribution.getCategory());
+               addManifestAttribute(manifest, RepoConstants.SLC_CATEGORY_ID, osgiDistribution.getCategory());
                addManifestAttribute(manifest, Constants.BUNDLE_SYMBOLICNAME, osgiDistribution.getName());
                String version = osgiDistribution.getVersion();
                if (version.endsWith("-SNAPSHOT")) {
index 65238df5583d6e5913aa0d6de34b41ad03a94846..d89077322a73af49d2ef074b9ac7306dea5e4203 100644 (file)
@@ -20,6 +20,7 @@ import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.argeo.jcr.JcrUtils;
 import org.argeo.slc.CategorizedNameVersion;
 import org.argeo.slc.DefaultNameVersion;
 import org.argeo.slc.NameVersion;
@@ -35,46 +36,29 @@ import org.osgi.framework.Constants;
 /**
  * Create or update JCR meta-data for an SLC Modular Distribution
  * 
- * Currently, following types are managed: <list><li>* .jar: dependency
- * artifacts with csv index</li> <li>@Deprecated : .pom: artifact (binaries)
- * that indexes a group, the .pom file contains a tag "dependencyManagement"
- * that list all modules</li> </list>
+ * Currently, following types are managed: <list>
+ * <li>* .jar: dependency artifacts with csv index</li>
+ * <li>@Deprecated : .pom: artifact (binaries) that indexes a group, the .pom
+ * file contains a tag "dependencyManagement" that list all modules</li> </list>
  */
 public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
-       private final static Log log = LogFactory
-                       .getLog(ModularDistributionIndexer.class);
+       private final static Log log = LogFactory.getLog(ModularDistributionIndexer.class);
 
        // Constants for csv indexing
        private final static String INDEX_FILE_NAME = "modularDistribution.csv";
        private String separator = ",";
 
-       // TODO remove this: binaries have been replaced by modular distribution
-       // // Artifact indexing
-       // private final static List<String> BINARIES_ARTIFACTS_NAME;
-       // static {
-       // List<String> tmpList = new ArrayList<String>();
-       // tmpList.add(RepoConstants.BINARIES_ARTIFACT_ID);
-       // // tmpList.add(RepoConstants.SOURCES_ARTIFACT_ID);
-       // // tmpList.add(RepoConstants.SDK_ARTIFACT_ID);
-       // BINARIES_ARTIFACTS_NAME = Collections.unmodifiableList(tmpList);
-       // }
-
        private Manifest manifest;
 
        public Boolean support(String path) {
                if (FilenameUtils.getExtension(path).equals("jar"))
                        return true;
-               // if (FilenameUtils.getExtension(path).equals("pom")
-               // && BINARIES_ARTIFACTS_NAME.contains(FilenameUtils.getName(path)
-               // .split("-")[0]))
-               // return true;
                return false;
        }
 
        public void index(Node fileNode) {
                Binary fileBinary = null;
                try {
-
                        String fileNodePath = fileNode.getPath();
                        if (!support(fileNodePath))
                                return;
@@ -88,12 +72,9 @@ public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
                        MyModularDistribution currDist = null;
                        if (FilenameUtils.getExtension(fileNode.getPath()).equals("jar"))
                                currDist = listModulesFromCsvIndex(fileNode, fileBinary);
-                       // else if (FilenameUtils.getExtension(fileNode.getPath()).equals(
-                       // "pom"))
-                       // currDist = listModulesFromPomIndex(fileNode, fileBinary);
 
-                       if (fileNode.isNodeType(SlcTypes.SLC_MODULAR_DISTRIBUTION)
-                                       || currDist == null || !currDist.nameVersions().hasNext())
+                       if (fileNode.isNodeType(SlcTypes.SLC_MODULAR_DISTRIBUTION) || currDist == null
+                                       || !currDist.nameVersions().hasNext())
                                return; // already indexed or no modules found
                        else {
                                fileNode.addMixin(SlcTypes.SLC_MODULAR_DISTRIBUTION);
@@ -108,13 +89,13 @@ public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
                        if (log.isTraceEnabled())
                                log.trace("Indexed " + fileNode + " as modular distribution");
                } catch (Exception e) {
-                       throw new SlcException("Cannot list dependencies from " + fileNode,
-                                       e);
+                       throw new SlcException("Cannot list dependencies from " + fileNode, e);
+               } finally {
+                       JcrUtils.closeQuietly(fileBinary);
                }
        }
 
-       private void indexDistribution(ArgeoOsgiDistribution osgiDist, Node distNode)
-                       throws RepositoryException {
+       private void indexDistribution(ArgeoOsgiDistribution osgiDist, Node distNode) throws RepositoryException {
                distNode.addMixin(SlcTypes.SLC_MODULAR_DISTRIBUTION);
                distNode.addMixin(SlcTypes.SLC_CATEGORIZED_NAME_VERSION);
                distNode.setProperty(SlcNames.SLC_CATEGORY, osgiDist.getCategory());
@@ -124,26 +105,22 @@ public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
                        distNode.getNode(SLC_MODULES).remove();
                Node modules = distNode.addNode(SLC_MODULES, NodeType.NT_UNSTRUCTURED);
 
-               for (Iterator<? extends NameVersion> it = osgiDist.nameVersions(); it
-                               .hasNext();)
+               for (Iterator<? extends NameVersion> it = osgiDist.nameVersions(); it.hasNext();)
                        addModule(modules, it.next());
        }
 
        // Helpers
-       private Node addModule(Node modules, NameVersion nameVersion)
-                       throws RepositoryException {
+       private Node addModule(Node modules, NameVersion nameVersion) throws RepositoryException {
                CategorizedNameVersion cnv = (CategorizedNameVersion) nameVersion;
                Node moduleCoord = null;
-               moduleCoord = modules.addNode(cnv.getName(),
-                               SlcTypes.SLC_MODULE_COORDINATES);
+               moduleCoord = modules.addNode(cnv.getName(), SlcTypes.SLC_MODULE_COORDINATES);
                moduleCoord.setProperty(SlcNames.SLC_CATEGORY, cnv.getCategory());
                moduleCoord.setProperty(SlcNames.SLC_NAME, cnv.getName());
                moduleCoord.setProperty(SlcNames.SLC_VERSION, cnv.getVersion());
                return moduleCoord;
        }
 
-       private MyModularDistribution listModulesFromCsvIndex(Node fileNode,
-                       Binary fileBinary) {
+       private MyModularDistribution listModulesFromCsvIndex(Node fileNode, Binary fileBinary) {
                JarInputStream jarIn = null;
                BufferedReader reader = null;
                try {
@@ -157,15 +134,11 @@ public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
                                log.error(fileNode + " has no MANIFEST");
                                return null;
                        }
-                       String category = manifest.getMainAttributes().getValue(
-                                       RepoConstants.SLC_GROUP_ID);
-                       String name = manifest.getMainAttributes().getValue(
-                                       Constants.BUNDLE_SYMBOLICNAME);
-                       String version = manifest.getMainAttributes().getValue(
-                                       Constants.BUNDLE_VERSION);
-
-                       Artifact distribution = new DefaultArtifact(category, name, "jar",
-                                       version);
+                       String category = manifest.getMainAttributes().getValue(RepoConstants.SLC_CATEGORY_ID);
+                       String name = manifest.getMainAttributes().getValue(Constants.BUNDLE_SYMBOLICNAME);
+                       String version = manifest.getMainAttributes().getValue(Constants.BUNDLE_VERSION);
+
+                       Artifact distribution = new DefaultArtifact(category, name, "jar", version);
                        // Retrieve the index file
                        JarEntry indexEntry;
                        while ((indexEntry = jarIn.getNextJarEntry()) != null) {
@@ -176,14 +149,17 @@ public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
                                try {
                                        jarIn.closeEntry();
                                } catch (SecurityException se) {
-                                       log.error("Invalid signature file digest "
-                                                       + "for Manifest main attributes: " + entryName
+                                       log.error("Invalid signature file digest " + "for Manifest main attributes: " + entryName
                                                        + " while looking for an index in bundle " + name);
                                }
                        }
                        if (indexEntry == null)
                                return null; // Not a modular definition
 
+                       if (category == null) {
+                               log.warn("Modular definition found but no " + RepoConstants.SLC_CATEGORY_ID + " in " + fileNode);
+                       }
+
                        // Process the index
                        reader = new BufferedReader(new InputStreamReader(jarIn));
                        String line = null;
@@ -192,11 +168,9 @@ public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
                                st.nextToken(); // moduleName
                                st.nextToken(); // moduleVersion
                                String relativeUrl = st.nextToken();
-                               Artifact currModule = AetherUtils.convertPathToArtifact(
-                                               relativeUrl, null);
-                               modules.add(new MyCategorizedNameVersion(currModule
-                                               .getGroupId(), currModule.getArtifactId(), currModule
-                                               .getVersion()));
+                               Artifact currModule = AetherUtils.convertPathToArtifact(relativeUrl, null);
+                               modules.add(new MyCategorizedNameVersion(currModule.getGroupId(), currModule.getArtifactId(),
+                                               currModule.getVersion()));
                        }
                        return new MyModularDistribution(distribution, modules);
                } catch (Exception e) {
@@ -207,75 +181,11 @@ public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
                }
        }
 
-       // private MyModularDistribution listModulesFromPomIndex(Node fileNode,
-       // Binary fileBinary) {
-       // InputStream input = null;
-       // List<CategorizedNameVersion> modules = new
-       // ArrayList<CategorizedNameVersion>();
-       // try {
-       // input = fileBinary.getStream();
-       //
-       // DocumentBuilder documentBuilder = DocumentBuilderFactory
-       // .newInstance().newDocumentBuilder();
-       // Document doc = documentBuilder.parse(input);
-       // // properties
-       // Properties props = new Properties();
-       // // props.setProperty("project.version",
-       // // pomArtifact.getBaseVersion());
-       // NodeList properties = doc.getElementsByTagName("properties");
-       // if (properties.getLength() > 0) {
-       // NodeList propertiesElems = properties.item(0).getChildNodes();
-       // for (int i = 0; i < propertiesElems.getLength(); i++) {
-       // if (propertiesElems.item(i) instanceof Element) {
-       // Element property = (Element) propertiesElems.item(i);
-       // props.put(property.getNodeName(),
-       // property.getTextContent());
-       // }
-       // }
-       // }
-       //
-       // // full coordinates are under <dependencyManagement><dependencies>
-       // NodeList dependencies = ((Element) doc.getElementsByTagName(
-       // "dependencyManagement").item(0))
-       // .getElementsByTagName("dependency");
-       // for (int i = 0; i < dependencies.getLength(); i++) {
-       // Element dependency = (Element) dependencies.item(i);
-       // String groupId = dependency.getElementsByTagName("groupId")
-       // .item(0).getTextContent().trim();
-       // String artifactId = dependency
-       // .getElementsByTagName("artifactId").item(0)
-       // .getTextContent().trim();
-       // String version = dependency.getElementsByTagName("version")
-       // .item(0).getTextContent().trim();
-       // modules.add(new MyCategorizedNameVersion(groupId, artifactId,
-       // version));
-       // }
-       //
-       // String groupId = doc.getElementsByTagName("groupId").item(0)
-       // .getTextContent().trim();
-       // String artifactId = doc.getElementsByTagName("artifactId").item(0)
-       // .getTextContent().trim();
-       // String version = doc.getElementsByTagName("version").item(0)
-       // .getTextContent().trim();
-       //
-       // Artifact currDist = new DefaultArtifact(groupId, artifactId, "pom",
-       // version);
-       //
-       // return new MyModularDistribution(currDist, modules);
-       // } catch (Exception e) {
-       // throw new SlcException("Cannot process pom " + fileNode, e);
-       // } finally {
-       // IOUtils.closeQuietly(input);
-       // }
-       // }
-
        /** The created modular distribution */
-       private static class MyCategorizedNameVersion extends DefaultNameVersion
-                       implements CategorizedNameVersion {
+       private static class MyCategorizedNameVersion extends DefaultNameVersion implements CategorizedNameVersion {
                private final String category;
 
-               public MyCategorizedNameVersion(String category, String name,
-                               String version) {
+               public MyCategorizedNameVersion(String category, String name, String version) {
                        super(name, version);
                        this.category = category;
                }
@@ -286,16 +196,13 @@ public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
        }
 
        /**
-        * A consistent and versioned OSGi distribution, which can be built and
-        * tested.
+        * A consistent and versioned OSGi distribution, which can be built and tested.
         */
-       private class MyModularDistribution extends ArtifactDistribution implements
-                       ArgeoOsgiDistribution {
+       private class MyModularDistribution extends ArtifactDistribution implements ArgeoOsgiDistribution {
 
                private List<CategorizedNameVersion> modules;
 
-               public MyModularDistribution(Artifact artifact,
-                               List<CategorizedNameVersion> modules) {
+               public MyModularDistribution(Artifact artifact, List<CategorizedNameVersion> modules) {
                        super(artifact);
                        this.modules = modules;
                }
@@ -305,8 +212,7 @@ public class ModularDistributionIndexer implements NodeIndexer, SlcNames {
                }
 
                // Modular distribution interface methods. Not yet used.
-               public Distribution getModuleDistribution(String moduleName,
-                               String moduleVersion) {
+               public Distribution getModuleDistribution(String moduleName, String moduleVersion) {
                        return null;
                }
 
index 793f9bd9e82bb7efa099d88b30ee15adc728f70c..b6f87e0e24901c2acc3ae1ef84a805d20d6f9b08 100644 (file)
@@ -57,8 +57,7 @@ public class PdeSourcesIndexer implements NodeIndexer {
                // int lastInd = name.lastIndexOf("[");
                // if (lastInd != -1)
                // name = name.substring(0, lastInd);
-               return name.endsWith("-sources")
-                               && FilenameUtils.getExtension(path).equals("jar");
+               return name.endsWith("-sources") && FilenameUtils.getExtension(path).equals("jar");
        }
 
        public void index(Node sourcesNode) {
@@ -68,8 +67,7 @@ public class PdeSourcesIndexer implements NodeIndexer {
 
                        packageSourcesAsPdeSource(sourcesNode);
                } catch (Exception e) {
-                       throw new SlcException("Cannot generate pde sources for node "
-                                       + sourcesNode, e);
+                       throw new SlcException("Cannot generate pde sources for node " + sourcesNode, e);
                }
        }
 
@@ -78,44 +76,34 @@ public class PdeSourcesIndexer implements NodeIndexer {
                Binary osgiBinary = null;
                try {
                        Session session = sourcesNode.getSession();
-                       Artifact sourcesArtifact = AetherUtils.convertPathToArtifact(
-                                       sourcesNode.getPath(), null);
+                       Artifact sourcesArtifact = AetherUtils.convertPathToArtifact(sourcesNode.getPath(), null);
 
                        // read name version from manifest
-                       Artifact osgiArtifact = new DefaultArtifact(
-                                       sourcesArtifact.getGroupId(),
-                                       sourcesArtifact.getArtifactId(),
-                                       sourcesArtifact.getExtension(),
-                                       sourcesArtifact.getVersion());
-                       String osgiPath = MavenConventionsUtils.artifactPath(
-                                       artifactBasePath, osgiArtifact);
-                       osgiBinary = session.getNode(osgiPath).getNode(Node.JCR_CONTENT)
-                                       .getProperty(Property.JCR_DATA).getBinary();
-
-                       NameVersion nameVersion = RepoUtils.readNameVersion(osgiBinary
-                                       .getStream());
+                       Artifact osgiArtifact = new DefaultArtifact(sourcesArtifact.getGroupId(), sourcesArtifact.getArtifactId(),
+                                       sourcesArtifact.getExtension(), sourcesArtifact.getVersion());
+                       String osgiPath = MavenConventionsUtils.artifactPath(artifactBasePath, osgiArtifact);
+                       osgiBinary = session.getNode(osgiPath).getNode(Node.JCR_CONTENT).getProperty(Property.JCR_DATA).getBinary();
+
+                       NameVersion nameVersion = RepoUtils.readNameVersion(osgiBinary.getStream());
+                       if (nameVersion == null) {
+                               log.warn("Cannot package PDE sources for " + osgiPath + " as it is probably not an OSGi bundle");
+                               return;
+                       }
 
                        // create PDe sources artifact
-                       Artifact pdeSourceArtifact = new DefaultArtifact(
-                                       sourcesArtifact.getGroupId(),
-                                       sourcesArtifact.getArtifactId() + ".source",
-                                       sourcesArtifact.getExtension(),
+                       Artifact pdeSourceArtifact = new DefaultArtifact(sourcesArtifact.getGroupId(),
+                                       sourcesArtifact.getArtifactId() + ".source", sourcesArtifact.getExtension(),
                                        sourcesArtifact.getVersion());
-                       String targetSourceParentPath = MavenConventionsUtils
-                                       .artifactParentPath(artifactBasePath, pdeSourceArtifact);
-                       String targetSourceFileName = MavenConventionsUtils
-                                       .artifactFileName(pdeSourceArtifact);
+                       String targetSourceParentPath = MavenConventionsUtils.artifactParentPath(artifactBasePath,
+                                       pdeSourceArtifact);
+                       String targetSourceFileName = MavenConventionsUtils.artifactFileName(pdeSourceArtifact);
                        // String targetSourceJarPath = targetSourceParentPath + '/'
                        // + targetSourceFileName;
 
-                       Node targetSourceParentNode = JcrUtils.mkfolders(session,
-                                       targetSourceParentPath);
-                       origBinary = sourcesNode.getNode(Node.JCR_CONTENT)
-                                       .getProperty(Property.JCR_DATA).getBinary();
-                       byte[] targetJarBytes = RepoUtils.packageAsPdeSource(
-                                       origBinary.getStream(), nameVersion);
-                       JcrUtils.copyBytesAsFile(targetSourceParentNode,
-                                       targetSourceFileName, targetJarBytes);
+                       Node targetSourceParentNode = JcrUtils.mkfolders(session, targetSourceParentPath);
+                       origBinary = sourcesNode.getNode(Node.JCR_CONTENT).getProperty(Property.JCR_DATA).getBinary();
+                       byte[] targetJarBytes = RepoUtils.packageAsPdeSource(origBinary.getStream(), nameVersion);
+                       JcrUtils.copyBytesAsFile(targetSourceParentNode, targetSourceFileName, targetJarBytes);
 
                        // reindex
                        // Automagically done via the various listeners or manually
@@ -124,12 +112,10 @@ public class PdeSourcesIndexer implements NodeIndexer {
                        // artifactIndexer.index(targetSourceJarNode);
                        // jarFileIndexer.index(targetSourceJarNode);
                        if (log.isTraceEnabled())
-                               log.trace("Created pde source artifact " + pdeSourceArtifact
-                                               + " from " + sourcesNode);
+                               log.trace("Created pde source artifact " + pdeSourceArtifact + " from " + sourcesNode);
 
                } catch (RepositoryException e) {
-                       throw new SlcException("Cannot add PDE sources for " + sourcesNode,
-                                       e);
+                       throw new SlcException("Cannot add PDE sources for " + sourcesNode, e);
                } finally {
                        JcrUtils.closeQuietly(origBinary);
                        JcrUtils.closeQuietly(osgiBinary);
index 229defcf5ae2a262e9f60acc07f083996cffa980..f2d26eea675c3e93d5974d7a363d2feec787ab9d 100644 (file)
@@ -33,7 +33,7 @@ public interface RepoConstants {
        String SDK_ARTIFACT_ID = "sdk";
 
        // TODO might exists somewhere else
-       String SLC_GROUP_ID = "SLC-GroupId";
+       String SLC_CATEGORY_ID = "SLC-Category";
 
        // TODO find a more generic way
        String DEFAULT_DEFAULT_WORKSPACE = "main";
index 13ad7583dc0f14f40c3c0cbe63a741a96e0b2c97..7bc7854cf80f92e1bfd1ee5f24c64bc611cfffa5 100644 (file)
@@ -269,7 +269,6 @@ public class RepoUtils implements ArgeoNames, SlcNames {
                        // probably not a jar, skipping
                        if (log.isDebugEnabled()) {
                                log.debug("Skipping because of " + e);
-                               e.printStackTrace();
                        }
                } finally {
                        IOUtils.closeQuietly(jarInputStream);