From 8bc47c1403b47e1e35413ead26a606195c21549e Mon Sep 17 00:00:00 2001 From: Mathieu Baudier Date: Sat, 8 Dec 2012 16:10:18 +0000 Subject: [PATCH] Improve workspace normalization git-svn-id: https://svn.argeo.org/slc/trunk@5929 4cfe0d0a-d680-48aa-b62c-e0a02a3f76cc --- .../dist/commands/NormalizeDistribution.java | 157 ++++++++++++++---- pom.xml | 12 +- .../org/argeo/slc/repo/ArtifactIndexer.java | 8 +- .../java/org/argeo/slc/repo/RepoUtils.java | 19 ++- .../slc/repo/maven/MavenConventionsUtils.java | 8 +- .../argeo/slc/repo/osgi/NormalizeGroup.java | 136 ++++++++------- 6 files changed, 235 insertions(+), 105 deletions(-) diff --git a/plugins/org.argeo.slc.client.ui.dist/src/main/java/org/argeo/slc/client/ui/dist/commands/NormalizeDistribution.java b/plugins/org.argeo.slc.client.ui.dist/src/main/java/org/argeo/slc/client/ui/dist/commands/NormalizeDistribution.java index 005ff2c4f..967ea770b 100644 --- a/plugins/org.argeo.slc.client.ui.dist/src/main/java/org/argeo/slc/client/ui/dist/commands/NormalizeDistribution.java +++ b/plugins/org.argeo.slc.client.ui.dist/src/main/java/org/argeo/slc/client/ui/dist/commands/NormalizeDistribution.java @@ -15,7 +15,9 @@ */ package org.argeo.slc.client.ui.dist.commands; +import javax.jcr.Binary; import javax.jcr.Node; +import javax.jcr.NodeIterator; import javax.jcr.Property; import javax.jcr.Repository; import javax.jcr.RepositoryException; @@ -27,12 +29,21 @@ import javax.jcr.util.TraversingItemVisitor; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.argeo.ArgeoMonitor; +import org.argeo.eclipse.ui.EclipseArgeoMonitor; +import org.argeo.eclipse.ui.dialogs.SingleValue; import org.argeo.jcr.JcrUtils; +import org.argeo.slc.NameVersion; import org.argeo.slc.SlcException; +import org.argeo.slc.aether.AetherUtils; import org.argeo.slc.client.ui.dist.DistPlugin; import org.argeo.slc.jcr.SlcNames; +import org.argeo.slc.jcr.SlcTypes; import org.argeo.slc.repo.ArtifactIndexer; import org.argeo.slc.repo.JarFileIndexer; +import org.argeo.slc.repo.RepoUtils; +import org.argeo.slc.repo.maven.MavenConventionsUtils; +import org.argeo.slc.repo.osgi.NormalizeGroup; import org.eclipse.core.commands.AbstractHandler; import org.eclipse.core.commands.ExecutionEvent; import org.eclipse.core.commands.ExecutionException; @@ -40,6 +51,8 @@ import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.core.runtime.IStatus; import org.eclipse.core.runtime.Status; import org.eclipse.core.runtime.jobs.Job; +import org.sonatype.aether.artifact.Artifact; +import org.sonatype.aether.util.artifact.DefaultArtifact; /** Make sure than Maven and OSGi metadata are consistent */ public class NormalizeDistribution extends AbstractHandler implements SlcNames { @@ -59,9 +72,14 @@ public class NormalizeDistribution extends AbstractHandler implements SlcNames { public Object execute(ExecutionEvent event) throws ExecutionException { String workspace = event.getParameter(PARAM_WORKSPACE); + String version = SingleValue.ask("Version", + "Enter Distribution Version"); + if (version == null) + return null; + NormalizeJob job; try { - job = new NormalizeJob(repository.login(workspace)); + job = new NormalizeJob(repository.login(workspace), version); } catch (RepositoryException e) { throw new SlcException("Cannot normalize " + workspace, e); } @@ -70,34 +88,84 @@ public class NormalizeDistribution extends AbstractHandler implements SlcNames { return null; } + protected void packageSourcesAsPdeSource(Node sourcesNode) { + Binary origBinary = null; + Binary osgiBinary = null; + try { + Session session = sourcesNode.getSession(); + Artifact sourcesArtifact = AetherUtils.convertPathToArtifact( + sourcesNode.getPath(), null); + + // read name version from manifest + Artifact osgiArtifact = new DefaultArtifact( + sourcesArtifact.getGroupId(), + sourcesArtifact.getArtifactId(), + sourcesArtifact.getExtension(), + sourcesArtifact.getVersion()); + String osgiPath = MavenConventionsUtils.artifactPath( + artifactBasePath, osgiArtifact); + osgiBinary = session.getNode(osgiPath).getNode(Node.JCR_CONTENT) + .getProperty(Property.JCR_DATA).getBinary(); + + NameVersion nameVersion = RepoUtils.readNameVersion(osgiBinary + .getStream()); + + // create PDe sources artifact + Artifact pdeSourceArtifact = new DefaultArtifact( + sourcesArtifact.getGroupId(), + sourcesArtifact.getArtifactId() + ".source", + sourcesArtifact.getExtension(), + sourcesArtifact.getVersion()); + String targetSourceParentPath = MavenConventionsUtils + .artifactParentPath(artifactBasePath, pdeSourceArtifact); + String targetSourceFileName = MavenConventionsUtils + .artifactFileName(pdeSourceArtifact); + String targetSourceJarPath = targetSourceParentPath + '/' + + targetSourceFileName; + + Node targetSourceParentNode = JcrUtils.mkfolders(session, + targetSourceParentPath); + origBinary = sourcesNode.getNode(Node.JCR_CONTENT) + .getProperty(Property.JCR_DATA).getBinary(); + byte[] targetJarBytes = RepoUtils.packageAsPdeSource( + origBinary.getStream(), nameVersion); + JcrUtils.copyBytesAsFile(targetSourceParentNode, + targetSourceFileName, targetJarBytes); + + // reindex + Node targetSourceJarNode = session.getNode(targetSourceJarPath); + artifactIndexer.index(targetSourceJarNode); + jarFileIndexer.index(targetSourceJarNode); + } catch (RepositoryException e) { + throw new SlcException("Cannot add PDE sources for " + sourcesNode, + e); + } finally { + JcrUtils.closeQuietly(origBinary); + JcrUtils.closeQuietly(osgiBinary); + } + + } + public void setRepository(Repository repository) { this.repository = repository; } private class NormalizeJob extends Job { private Session session; + private String version; - public NormalizeJob(Session session) { + public NormalizeJob(Session session, String version) { super("Normalize Distribution"); this.session = session; + this.version = version; } @Override - protected IStatus run(IProgressMonitor monitor) { - // Session session = null; + protected IStatus run(IProgressMonitor progressMonitor) { + try { - // session = repository.login(workspace); - // QueryManager qm = session.getWorkspace().getQueryManager(); - // Query query = qm - // .createQuery( - // "select * from [nt:file] where NAME([nt:file]) like '%.jar'", - // Query.JCR_SQL2); - // // Query query = qm.createQuery("//*jar", Query.XPATH); - // long count = query.execute().getRows().getSize(); - // if (log.isDebugEnabled()) - // log.debug("Count: " + count); - // long count = query.execute().getRows().nextRow() - // .getValue("count").getLong(); + ArgeoMonitor monitor = new EclipseArgeoMonitor(progressMonitor); + // normalize artifacts Query countQuery = session .getWorkspace() .getQueryManager() @@ -105,12 +173,29 @@ public class NormalizeDistribution extends AbstractHandler implements SlcNames { Query.JCR_SQL2); QueryResult result = countQuery.execute(); Long expectedCount = result.getNodes().getSize(); - - monitor.beginTask("Normalize " + monitor.beginTask("Normalize artifacts of " + session.getWorkspace().getName(), expectedCount.intValue()); NormalizingTraverser tiv = new NormalizingTraverser(monitor); session.getNode(artifactBasePath).accept(tiv); + + // normalize groups + Query groupQuery = session + .getWorkspace() + .getQueryManager() + .createQuery( + "select group from [" + SlcTypes.SLC_GROUP_BASE + + "] as group", Query.JCR_SQL2); + NodeIterator groups = groupQuery.execute().getNodes(); + monitor.beginTask("Normalize groups of " + + session.getWorkspace().getName(), + (int) groups.getSize()); + while (groups.hasNext()) { + NormalizeGroup normalizeGroup = new NormalizeGroup(); + normalizeGroup.setArtifactBasePath(artifactBasePath); + normalizeGroup.processGroupNode(groups.nextNode(), version, + monitor); + } } catch (Exception e) { return new Status(IStatus.ERROR, DistPlugin.ID, "Cannot normalize distribution " @@ -124,9 +209,9 @@ public class NormalizeDistribution extends AbstractHandler implements SlcNames { } private class NormalizingTraverser extends TraversingItemVisitor { - IProgressMonitor monitor; + ArgeoMonitor monitor; - public NormalizingTraverser(IProgressMonitor monitor) { + public NormalizingTraverser(ArgeoMonitor monitor) { super(); this.monitor = monitor; } @@ -140,16 +225,28 @@ public class NormalizeDistribution extends AbstractHandler implements SlcNames { protected void entering(Node node, int level) throws RepositoryException { if (node.isNodeType(NodeType.NT_FILE)) { - if (jarFileIndexer.support(node.getPath())) - if (artifactIndexer.support(node.getPath())) { - monitor.subTask(node.getName()); - artifactIndexer.index(node); - jarFileIndexer.index(node); - node.getSession().save(); - monitor.worked(1); - if (log.isDebugEnabled()) - log.debug("Processed " + node); - } + if (node.getName().endsWith("-sources.jar")) { + monitor.subTask(node.getName()); + packageSourcesAsPdeSource(node); + node.getSession().save(); + monitor.worked(1); + if (log.isDebugEnabled()) + log.debug("Processed source artifact " + node.getPath()); + } else if (node.getName().endsWith(".jar")) { + if (jarFileIndexer.support(node.getPath())) + if (artifactIndexer.support(node.getPath())) { + monitor.subTask(node.getName()); + artifactIndexer.index(node); + jarFileIndexer.index(node); + node.getSession().save(); + monitor.worked(1); + if (log.isDebugEnabled()) + log.debug("Processed artifact " + + node.getPath()); + } + } else { + monitor.worked(1); + } } } diff --git a/pom.xml b/pom.xml index 46b9bde70..44e290b83 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.argeo.commons argeo-commons - 1.1.8-SNAPSHOT + 1.1.8 org.argeo.slc argeo-slc @@ -13,9 +13,7 @@ 1.1 2012-06-27 - file:///srv/projects/www/slc/site - http://projects.argeo.org/slc/site - 1.1.6 + 1.1.7-SNAPSHOT runtime @@ -27,7 +25,7 @@ dist demo - ${site.urlBase}/${developmentCycle.slc} + http://projects.argeo.org/slc/ scm:svn:https://svn.argeo.org/slc/trunk scm:svn:https://svn.argeo.org/slc/trunk @@ -118,13 +116,13 @@ limitations under the License. staging - dav:https://repo.argeo.org/data/files/java/org.argeo.slc-1.1.x + dav:https://repo.argeo.org/data/files/java/org.argeo.slc-${developmentCycle.slc}.x false site SLC Site - file:///srv/projects/www/slc/site/${project.version} + dav:https://repo.argeo.org/data/files/docs/org.argeo.slc-${developmentCycle.slc}.x diff --git a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/ArtifactIndexer.java b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/ArtifactIndexer.java index 183a70ad3..acb05b483 100644 --- a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/ArtifactIndexer.java +++ b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/ArtifactIndexer.java @@ -73,13 +73,19 @@ public class ArtifactIndexer implements NodeIndexer { artifact.getClassifier()); JcrUtils.updateLastModified(fileNode); - // make sure there is a checksum + // make sure there are checksums String shaNodeName = fileNode.getName() + ".sha1"; if (!fileNode.getParent().hasNode(shaNodeName)) { String sha = JcrUtils.checksumFile(fileNode, "SHA-1"); JcrUtils.copyBytesAsFile(fileNode.getParent(), shaNodeName, sha.getBytes()); } + String md5NodeName = fileNode.getName() + ".md5"; + if (!fileNode.getParent().hasNode(md5NodeName)) { + String md5 = JcrUtils.checksumFile(fileNode, "MD5"); + JcrUtils.copyBytesAsFile(fileNode.getParent(), md5NodeName, + md5.getBytes()); + } // set higher levels Node artifactVersionBase = fileNode.getParent(); diff --git a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoUtils.java b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoUtils.java index fe833bc07..3d7d5920d 100644 --- a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoUtils.java +++ b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoUtils.java @@ -237,15 +237,28 @@ public class RepoUtils implements ArgeoNames, SlcNames { /** Read the OSGi {@link NameVersion} */ public static NameVersion readNameVersion(File artifactFile) { + try { + return readNameVersion(new FileInputStream(artifactFile)); + } catch (Exception e) { + // probably not a jar, skipping + if (log.isDebugEnabled()) { + log.debug("Skipping " + artifactFile + " because of " + e); + // e.printStackTrace(); + } + } + return null; + } + + /** Read the OSGi {@link NameVersion} */ + public static NameVersion readNameVersion(InputStream in) { JarInputStream jarInputStream = null; try { - jarInputStream = new JarInputStream(new FileInputStream( - artifactFile)); + jarInputStream = new JarInputStream(in); return readNameVersion(jarInputStream.getManifest()); } catch (Exception e) { // probably not a jar, skipping if (log.isDebugEnabled()) { - log.debug("Skipping " + artifactFile + " because of " + e); + log.debug("Skipping because of " + e); // e.printStackTrace(); } } finally { diff --git a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/MavenConventionsUtils.java b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/MavenConventionsUtils.java index 9b3bfe7e0..4e8d3fb83 100644 --- a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/MavenConventionsUtils.java +++ b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/MavenConventionsUtils.java @@ -100,7 +100,7 @@ public class MavenConventionsUtils { /** Relative path to the directories where the files will be stored */ public static String artifactParentPath(Artifact artifact) { return artifact.getGroupId().replace('.', '/') + '/' - + artifact.getArtifactId() + '/' + artifact.getVersion(); + + artifact.getArtifactId() + '/' + artifact.getBaseVersion(); } public static String artifactsAsDependencyPom(Artifact pomArtifact, @@ -163,9 +163,9 @@ public class MavenConventionsUtils { p.append("\n"); // Repositories -// p.append("\n"); -// p.append("argeohttp://maven.argeo.org/argeo\n"); -// p.append("\n"); + // p.append("\n"); + // p.append("argeohttp://maven.argeo.org/argeo\n"); + // p.append("\n"); p.append("\n"); return p.toString(); diff --git a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/osgi/NormalizeGroup.java b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/osgi/NormalizeGroup.java index 965eb207c..0d734335b 100644 --- a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/osgi/NormalizeGroup.java +++ b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/osgi/NormalizeGroup.java @@ -31,6 +31,7 @@ import javax.jcr.Session; import org.apache.commons.io.FilenameUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.argeo.ArgeoMonitor; import org.argeo.jcr.JcrUtils; import org.argeo.slc.SlcException; import org.argeo.slc.aether.ArtifactIdComparator; @@ -83,68 +84,9 @@ public class NormalizeGroup implements Runnable, SlcNames { Session session = null; try { session = repository.login(workspace); - Node groupNode = session.getNode(MavenConventionsUtils.groupPath( artifactBasePath, groupId)); - // TODO factorize with a traverser pattern? - for (NodeIterator artifactBases = groupNode.getNodes(); artifactBases - .hasNext();) { - Node artifactBase = artifactBases.nextNode(); - if (artifactBase.isNodeType(SlcTypes.SLC_ARTIFACT_BASE)) { - for (NodeIterator artifactVersions = artifactBase - .getNodes(); artifactVersions.hasNext();) { - Node artifactVersion = artifactVersions.nextNode(); - if (artifactVersion - .isNodeType(SlcTypes.SLC_ARTIFACT_VERSION_BASE)) - for (NodeIterator files = artifactVersion - .getNodes(); files.hasNext();) { - Node file = files.nextNode(); - if (file.isNodeType(SlcTypes.SLC_BUNDLE_ARTIFACT)) { - preProcessBundleArtifact(file); - file.getSession().save(); - if (log.isDebugEnabled()) - log.debug("Pre-processed " - + file.getName()); - } - - } - } - } - } - // NodeIterator bundlesIt = listBundleArtifacts(session); - // - // while (bundlesIt.hasNext()) { - // Node bundleNode = bundlesIt.nextNode(); - // preProcessBundleArtifact(bundleNode); - // bundleNode.getSession().save(); - // if (log.isDebugEnabled()) - // log.debug("Pre-processed " + bundleNode.getName()); - // } - - int bundleCount = symbolicNamesToNodes.size(); - if (log.isDebugEnabled()) - log.debug("Indexed " + bundleCount + " bundles"); - - int count = 1; - for (Node bundleNode : symbolicNamesToNodes.values()) { - processBundleArtifact(bundleNode); - bundleNode.getSession().save(); - if (log.isDebugEnabled()) - log.debug(count + "/" + bundleCount + " Processed " - + bundleNode.getName()); - count++; - } - - // indexes - Set indexes = new TreeSet( - new ArtifactIdComparator()); - Artifact indexArtifact = writeIndex(session, BINARIES_ARTIFACT_ID, - binaries); - indexes.add(indexArtifact); - indexArtifact = writeIndex(session, SOURCES_ARTIFACT_ID, sources); - indexes.add(indexArtifact); - // sdk - writeIndex(session, SDK_ARTIFACT_ID, indexes); + processGroupNode(groupNode, null); } catch (Exception e) { throw new SlcException("Cannot normalize group " + groupId + " in " + workspace, e); @@ -153,6 +95,80 @@ public class NormalizeGroup implements Runnable, SlcNames { } } + public synchronized void processGroupNode(Node groupNode, String version, + ArgeoMonitor monitor) throws RepositoryException { + // FIXME better encapsulate + groupId = groupNode.getProperty(SlcNames.SLC_GROUP_BASE_ID).getString(); + this.version = version; + processGroupNode(groupNode, monitor); + } + + protected void processGroupNode(Node groupNode, ArgeoMonitor monitor) + throws RepositoryException { + if (monitor != null) + monitor.subTask("Group " + groupId); + Session session = groupNode.getSession(); + for (NodeIterator artifactBases = groupNode.getNodes(); artifactBases + .hasNext();) { + Node artifactBase = artifactBases.nextNode(); + if (artifactBase.isNodeType(SlcTypes.SLC_ARTIFACT_BASE)) { + for (NodeIterator artifactVersions = artifactBase.getNodes(); artifactVersions + .hasNext();) { + Node artifactVersion = artifactVersions.nextNode(); + if (artifactVersion + .isNodeType(SlcTypes.SLC_ARTIFACT_VERSION_BASE)) + for (NodeIterator files = artifactVersion.getNodes(); files + .hasNext();) { + Node file = files.nextNode(); + if (file.isNodeType(SlcTypes.SLC_BUNDLE_ARTIFACT)) { + preProcessBundleArtifact(file); + file.getSession().save(); + if (log.isDebugEnabled()) + log.debug("Pre-processed " + file.getName()); + } + + } + } + } + } + // NodeIterator bundlesIt = listBundleArtifacts(session); + // + // while (bundlesIt.hasNext()) { + // Node bundleNode = bundlesIt.nextNode(); + // preProcessBundleArtifact(bundleNode); + // bundleNode.getSession().save(); + // if (log.isDebugEnabled()) + // log.debug("Pre-processed " + bundleNode.getName()); + // } + + int bundleCount = symbolicNamesToNodes.size(); + if (log.isDebugEnabled()) + log.debug("Indexed " + bundleCount + " bundles"); + + int count = 1; + for (Node bundleNode : symbolicNamesToNodes.values()) { + processBundleArtifact(bundleNode); + bundleNode.getSession().save(); + if (log.isDebugEnabled()) + log.debug(count + "/" + bundleCount + " Processed " + + bundleNode.getName()); + count++; + } + + // indexes + Set indexes = new TreeSet( + new ArtifactIdComparator()); + Artifact indexArtifact = writeIndex(session, BINARIES_ARTIFACT_ID, + binaries); + indexes.add(indexArtifact); + indexArtifact = writeIndex(session, SOURCES_ARTIFACT_ID, sources); + indexes.add(indexArtifact); + // sdk + writeIndex(session, SDK_ARTIFACT_ID, indexes); + if (monitor != null) + monitor.worked(1); + } + private Artifact writeIndex(Session session, String artifactId, Set artifacts) throws RepositoryException { Artifact artifact = new DefaultArtifact(groupId, artifactId, "pom", -- 2.39.2