runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/ImportMavenDependencies.java

   1 /*
   2  * Copyright (C) 2007-2012 Mathieu Baudier
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.repo.maven;
  17
  18 import java.io.ByteArrayOutputStream;
  19 import java.io.File;
  20 import java.util.Comparator;
  21 import java.util.HashSet;
  22 import java.util.Properties;
  23 import java.util.Set;
  24 import java.util.TreeSet;
  25
  26 import javax.jcr.Node;
  27 import javax.jcr.NodeIterator;
  28 import javax.jcr.Repository;
  29 import javax.jcr.Session;
  30 import javax.jcr.nodetype.NodeType;
  31
  32 import org.apache.commons.io.IOUtils;
  33 import org.apache.commons.logging.Log;
  34 import org.apache.commons.logging.LogFactory;
  35 import org.argeo.jcr.JcrUtils;
  36 import org.argeo.slc.NameVersion;
  37 import org.argeo.slc.SlcException;
  38 import org.argeo.slc.aether.AetherTemplate;
  39 import org.argeo.slc.aether.ArtifactIdComparator;
  40 import org.argeo.slc.repo.ArtifactIndexer;
  41 import org.argeo.slc.repo.JarFileIndexer;
  42 import org.argeo.slc.repo.RepoConstants;
  43 import org.argeo.slc.repo.RepoUtils;
  44 import org.sonatype.aether.artifact.Artifact;
  45 import org.sonatype.aether.graph.DependencyNode;
  46 import org.sonatype.aether.util.artifact.DefaultArtifact;
  47
  48 /**
  49  * Import all the dependencies listed in a POM and their dependency graphs to a
  50  * workspace.
  51  */
  52 public class ImportMavenDependencies implements Runnable {
  53         private final static Log log = LogFactory
  54                         .getLog(ImportMavenDependencies.class);
  55
  56         private AetherTemplate aetherTemplate;
  57         private String rootCoordinates = "org.argeo.dep:versions-all:pom:1.2.0";
  58         private String distCoordinates = "org.argeo.tp:dist:pom:1.3.0";
  59         private Set<String> excludedArtifacts = new HashSet<String>();
  60
  61         private Repository repository;
  62         private String workspace;
  63
  64         private String artifactBasePath = RepoConstants.ARTIFACTS_BASE_PATH;
  65
  66         private ArtifactIndexer artifactIndexer = new ArtifactIndexer();
  67         private JarFileIndexer jarFileIndexer = new JarFileIndexer();
  68         private Comparator<Artifact> artifactComparator = new ArtifactIdComparator();
  69
  70         public void run() {
  71                 // resolve
  72                 Set<Artifact> artifacts = resolveDistribution();
  73
  74                 // sync
  75                 sync(artifacts);
  76         }
  77
  78         void sync(Set<Artifact> artifacts) {
  79                 Session session = null;
  80                 try {
  81                         session = JcrUtils.loginOrCreateWorkspace(repository, workspace);
  82                         // clear
  83                         NodeIterator nit = session.getNode(artifactBasePath).getNodes();
  84                         while (nit.hasNext()) {
  85                                 Node node = nit.nextNode();
  86                                 if (node.isNodeType(NodeType.NT_FOLDER)
  87                                                 || node.isNodeType(NodeType.NT_UNSTRUCTURED))
  88                                         node.remove();
  89                         }
  90                         session.save();
  91
  92                         // sync
  93                         syncDistribution(session, artifacts);
  94                 } catch (Exception e) {
  95                         throw new SlcException("Cannot import distribution", e);
  96                 } finally {
  97                         JcrUtils.logoutQuietly(session);
  98                 }
  99         }
 100
 101         /**
 102          * Generate a POM with all the artifacts declared in root coordinates as
 103          * dependencies AND in dependency management.
 104          */
 105         void createDistPom() {
 106                 try {
 107                         Artifact pomArtifact = new DefaultArtifact(rootCoordinates);
 108
 109                         Set<Artifact> registeredArtifacts = new TreeSet<Artifact>(
 110                                         artifactComparator);
 111                         MavenConventionsUtils.gatherPomDependencies(aetherTemplate,
 112                                         registeredArtifacts, pomArtifact);
 113                         Artifact sdkArtifact = new DefaultArtifact(distCoordinates);
 114                         String sdkPom = MavenConventionsUtils.artifactsAsDependencyPom(
 115                                         sdkArtifact, registeredArtifacts);
 116                         if (log.isDebugEnabled())
 117                                 log.debug("Gathered " + registeredArtifacts.size()
 118                                                 + " artifacts:\n" + sdkPom);
 119                 } catch (Exception e) {
 120                         throw new SlcException("Cannot resolve distribution", e);
 121                 }
 122         }
 123
 124         /** Returns all transitive dependencies of dist POM */
 125         private Set<Artifact> resolveDistribution() {
 126                 try {
 127                         Artifact distArtifact = new DefaultArtifact(distCoordinates);
 128                         Set<Artifact> artifacts = new TreeSet<Artifact>(artifactComparator);
 129
 130                         DependencyNode node = aetherTemplate
 131                                         .resolveDependencies(distArtifact);
 132                         addDependencies(artifacts, node, null);
 133
 134                         if (log.isDebugEnabled()) {
 135                                 log.debug("Resolved " + artifacts.size() + " artifacts");
 136
 137                                 // Properties distributionDescriptor =
 138                                 // generateDistributionDescriptor(artifacts);
 139                                 // ByteArrayOutputStream out = new ByteArrayOutputStream();
 140                                 // distributionDescriptor.store(out, "");
 141                                 // log.debug(new String(out.toByteArray()));
 142                                 // out.close();
 143                         }
 144
 145                         /*
 146                          * for (Artifact artifact : registeredArtifacts) { try { Boolean
 147                          * wasAdded = addArtifact(artifacts, artifact); if (wasAdded) {
 148                          * DependencyNode node = aetherTemplate
 149                          * .resolveDependencies(artifact); addDependencies(artifacts, node,
 150                          * null); } } catch (Exception e) {
 151                          * log.error("Could not resolve dependencies of " + artifact + ": "
 152                          * + e.getCause().getMessage()); }
 153                          *
 154                          * }
 155                          *
 156                          * if (log.isDebugEnabled()) log.debug("Resolved " +
 157                          * artifacts.size() + " artifacts");
 158                          *
 159                          * // distribution descriptor // Properties distributionDescriptor =
 160                          * // generateDistributionDescriptor(artifacts); //
 161                          * ByteArrayOutputStream out = new ByteArrayOutputStream(); //
 162                          * distributionDescriptor.store(out, ""); // log.debug(new
 163                          * String(out.toByteArray())); // out.close();
 164                          */
 165                         return artifacts;
 166                 } catch (Exception e) {
 167                         throw new SlcException("Cannot resolve distribution", e);
 168                 }
 169         }
 170
 171         protected Properties generateDistributionDescriptor(Set<Artifact> artifacts) {
 172                 Properties distributionDescriptor = new Properties();
 173                 for (Artifact artifact : artifacts) {
 174                         log.debug(artifact.getArtifactId() + " [" + artifact.getVersion()
 175                                         + "]\t(" + artifact + ")");
 176                         distributionDescriptor.setProperty(artifact.getArtifactId() + ":"
 177                                         + artifact.getVersion(), artifact.toString());
 178                 }
 179                 return distributionDescriptor;
 180         }
 181
 182         /** Write artifacts to the target workspace, skipping excluded ones */
 183         protected void syncDistribution(Session jcrSession, Set<Artifact> artifacts) {
 184                 Set<Artifact> artifactsWithoutSources = new TreeSet<Artifact>(
 185                                 artifactComparator);
 186                 Long begin = System.currentTimeMillis();
 187                 try {
 188                         JcrUtils.mkfolders(jcrSession, artifactBasePath);
 189                         artifacts: for (Artifact artifact : artifacts) {
 190                                 // skip excluded
 191                                 if (excludedArtifacts.contains(artifact.getGroupId() + ":"
 192                                                 + artifact.getArtifactId())) {
 193                                         if (log.isDebugEnabled())
 194                                                 log.debug("Exclude " + artifact);
 195                                         continue artifacts;
 196                                 }
 197
 198                                 File jarFile = MavenConventionsUtils.artifactToFile(artifact);
 199                                 if (!jarFile.exists()) {
 200                                         log.warn("Generated file " + jarFile + " for " + artifact
 201                                                         + " does not exist");
 202                                         continue artifacts;
 203                                 }
 204                                 artifact.setFile(jarFile);
 205
 206                                 try {
 207                                         String parentPath = MavenConventionsUtils
 208                                                         .artifactParentPath(artifactBasePath, artifact);
 209                                         Node parentNode;
 210                                         if (!jcrSession.itemExists(parentPath))
 211                                                 parentNode = JcrUtils.mkfolders(jcrSession, parentPath);
 212                                         else
 213                                                 parentNode = jcrSession.getNode(parentPath);
 214
 215                                         Node fileNode;
 216                                         if (!parentNode.hasNode(jarFile.getName())) {
 217                                                 fileNode = createFileNode(parentNode, jarFile);
 218                                         } else {
 219                                                 fileNode = parentNode.getNode(jarFile.getName());
 220                                         }
 221
 222                                         if (artifactIndexer.support(fileNode.getPath()))
 223                                                 artifactIndexer.index(fileNode);
 224                                         if (jarFileIndexer.support(fileNode.getPath()))
 225                                                 jarFileIndexer.index(fileNode);
 226                                         jcrSession.save();
 227
 228                                         addPdeSource(jcrSession, artifact, jarFile,
 229                                                         artifactsWithoutSources);
 230                                         jcrSession.save();
 231
 232                                         if (log.isDebugEnabled())
 233                                                 log.debug("Synchronized " + fileNode);
 234                                 } catch (Exception e) {
 235                                         log.error("Could not synchronize " + artifact, e);
 236                                         jcrSession.refresh(false);
 237                                         throw e;
 238                                 }
 239                         }
 240
 241                         Long duration = (System.currentTimeMillis() - begin) / 1000;
 242                         if (log.isDebugEnabled()) {
 243                                 log.debug("Synchronized distribution in " + duration + "s");
 244                                 log.debug("The following artifacts have no sources:");
 245                                 for (Artifact artifact : artifactsWithoutSources) {
 246                                         log.debug(artifact);
 247                                 }
 248                         }
 249                 } catch (Exception e) {
 250                         throw new SlcException("Cannot synchronize distribution", e);
 251                 }
 252         }
 253
 254         /** Try to add PDE sources */
 255         private void addPdeSource(Session session, Artifact artifact,
 256                         File artifactFile, Set<Artifact> artifactsWithoutSources) {
 257                 ByteArrayOutputStream out = new ByteArrayOutputStream();
 258                 try {
 259                         File origSourceFile = null;
 260                         Artifact origSourceArtifact = new DefaultArtifact(
 261                                         artifact.getGroupId(), artifact.getArtifactId(), "sources",
 262                                         artifact.getExtension(), artifact.getVersion());
 263                         Artifact newSourceArtifact = new DefaultArtifact(
 264                                         artifact.getGroupId(),
 265                                         artifact.getArtifactId() + ".source",
 266                                         artifact.getExtension(), artifact.getVersion());
 267                         try {
 268                                 origSourceFile = aetherTemplate
 269                                                 .getResolvedFile(origSourceArtifact);
 270                         } catch (Exception e) {
 271                                 // also try artifact following the conventions
 272                                 origSourceArtifact = newSourceArtifact;
 273                                 origSourceFile = aetherTemplate
 274                                                 .getResolvedFile(origSourceArtifact);
 275                         }
 276
 277                         String newSourceParentPath = MavenConventionsUtils
 278                                         .artifactParentPath(artifactBasePath, newSourceArtifact);
 279                         Node newSourceParentNode = JcrUtils.mkfolders(session,
 280                                         newSourceParentPath);
 281                         NameVersion bundleNameVersion = RepoUtils
 282                                         .readNameVersion(artifactFile);
 283                         RepoUtils.packagesAsPdeSource(origSourceFile, bundleNameVersion,
 284                                         out);
 285                         String newSourceFileName = MavenConventionsUtils
 286                                         .artifactFileName(newSourceArtifact);
 287                         JcrUtils.copyBytesAsFile(newSourceParentNode, newSourceFileName,
 288                                         out.toByteArray());
 289                 } catch (Exception e) {
 290                         log.error("Cannot add PDE source for " + artifact + ": " + e);
 291                         artifactsWithoutSources.add(artifact);
 292                 } finally {
 293                         IOUtils.closeQuietly(out);
 294                 }
 295         }
 296
 297         private Node createFileNode(Node parentNode, File file) {
 298                 return JcrUtils.copyFile(parentNode, file);
 299         }
 300
 301         /** Recursively adds non optional dependencies */
 302         private void addDependencies(Set<Artifact> artifacts, DependencyNode node,
 303                         String ancestors) {
 304                 // if (artifacts.contains(node.getDependency().getArtifact()))
 305                 // return;
 306                 String currentArtifactId = node.getDependency().getArtifact()
 307                                 .getArtifactId();
 308                 if (log.isDebugEnabled()) {
 309                         log.debug("# Add dependency for " + currentArtifactId);
 310                         if (ancestors != null)
 311                                 log.debug(ancestors);
 312                 }
 313                 for (DependencyNode child : node.getChildren()) {
 314                         if (!child.getDependency().isOptional()) {
 315                                 if (willAdd(child.getDependency().getArtifact())) {
 316                                         addArtifact(artifacts, child.getDependency().getArtifact());
 317                                         addDependencies(artifacts, child, currentArtifactId + "\n"
 318                                                         + (ancestors != null ? ancestors : ""));
 319                                 }
 320                         }
 321                 }
 322         }
 323
 324         /** @return whether it was added */
 325         private Boolean addArtifact(Set<Artifact> artifacts, Artifact artifact) {
 326                 Boolean willAdd = willAdd(artifact);
 327                 if (willAdd)
 328                         artifacts.add(artifact);
 329                 else
 330                         log.info("Skip " + artifact);
 331                 return willAdd;
 332         }
 333
 334         private Boolean willAdd(Artifact artifact) {
 335                 Boolean willAdd = true;
 336                 if (excludedArtifacts.contains(artifact.getGroupId() + ":"
 337                                 + artifact.getArtifactId()))
 338                         willAdd = false;
 339                 else if (excludedArtifacts.contains(artifact.getGroupId() + ":*"))
 340                         willAdd = false;
 341                 return willAdd;
 342         }
 343
 344         public void setAetherTemplate(AetherTemplate aetherTemplate) {
 345                 this.aetherTemplate = aetherTemplate;
 346         }
 347
 348         public void setExcludedArtifacts(Set<String> excludedArtifactIds) {
 349                 this.excludedArtifacts = excludedArtifactIds;
 350         }
 351
 352         public void setRootCoordinates(String rootCoordinates) {
 353                 this.rootCoordinates = rootCoordinates;
 354         }
 355
 356         public void setRepository(Repository repository) {
 357                 this.repository = repository;
 358         }
 359
 360         public void setWorkspace(String workspace) {
 361                 this.workspace = workspace;
 362         }
 363
 364         public void setDistCoordinates(String distCoordinates) {
 365                 this.distCoordinates = distCoordinates;
 366         }
 367
 368         public void setArtifactBasePath(String artifactBasePath) {
 369                 this.artifactBasePath = artifactBasePath;
 370         }
 371
 372 }