runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/ImportMavenDependencies.java

   1 /*
   2  * Copyright (C) 2007-2012 Mathieu Baudier
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.repo.maven;
  17
  18 import java.io.ByteArrayOutputStream;
  19 import java.io.File;
  20 import java.util.Comparator;
  21 import java.util.HashSet;
  22 import java.util.Properties;
  23 import java.util.Set;
  24 import java.util.TreeSet;
  25
  26 import javax.jcr.Node;
  27 import javax.jcr.NodeIterator;
  28 import javax.jcr.Repository;
  29 import javax.jcr.Session;
  30 import javax.jcr.nodetype.NodeType;
  31
  32 import org.apache.commons.io.IOUtils;
  33 import org.apache.commons.logging.Log;
  34 import org.apache.commons.logging.LogFactory;
  35 import org.argeo.jcr.JcrUtils;
  36 import org.argeo.slc.NameVersion;
  37 import org.argeo.slc.SlcException;
  38 import org.argeo.slc.aether.AetherTemplate;
  39 import org.argeo.slc.repo.ArtifactIndexer;
  40 import org.argeo.slc.repo.JarFileIndexer;
  41 import org.argeo.slc.repo.RepoConstants;
  42 import org.argeo.slc.repo.RepoUtils;
  43 import org.sonatype.aether.artifact.Artifact;
  44 import org.sonatype.aether.graph.DependencyNode;
  45 import org.sonatype.aether.util.artifact.DefaultArtifact;
  46
  47 /**
  48  * Import all the dependencies listed in a POM and their dependency graphs to a
  49  * workspace.
  50  */
  51 public class ImportMavenDependencies implements Runnable {
  52         private final static Log log = LogFactory
  53                         .getLog(ImportMavenDependencies.class);
  54
  55         private AetherTemplate aetherTemplate;
  56         private String rootCoordinates = "org.argeo.dep:versions-all:pom:1.2.0";
  57         private String distCoordinates = "org.argeo.tp:dist:pom:1.3.0";
  58         private Set<String> excludedArtifacts = new HashSet<String>();
  59
  60         private Repository repository;
  61         private String workspace;
  62
  63         private String artifactBasePath = RepoConstants.ARTIFACTS_BASE_PATH;
  64
  65         private ArtifactIndexer artifactIndexer = new ArtifactIndexer();
  66         private JarFileIndexer jarFileIndexer = new JarFileIndexer();
  67         private Comparator<Artifact> artifactComparator = new Comparator<Artifact>() {
  68                 public int compare(Artifact o1, Artifact o2) {
  69                         return o1.getArtifactId().compareTo(o2.getArtifactId());
  70                 }
  71         };
  72
  73         public void run() {
  74                 // resolve
  75                 Set<Artifact> artifacts = resolveDistribution();
  76
  77                 // sync
  78                 sync(artifacts);
  79         }
  80
  81         void sync(Set<Artifact> artifacts) {
  82                 Session session = null;
  83                 try {
  84                         session = JcrUtils.loginOrCreateWorkspace(repository, workspace);
  85                         // clear
  86                         NodeIterator nit = session.getNode(artifactBasePath).getNodes();
  87                         while (nit.hasNext()) {
  88                                 Node node = nit.nextNode();
  89                                 if (node.isNodeType(NodeType.NT_FOLDER)
  90                                                 || node.isNodeType(NodeType.NT_UNSTRUCTURED))
  91                                         node.remove();
  92                         }
  93                         session.save();
  94                         syncDistribution(session, artifacts);
  95                 } catch (Exception e) {
  96                         throw new SlcException("Cannot import distribution", e);
  97                 } finally {
  98                         JcrUtils.logoutQuietly(session);
  99                 }
 100         }
 101
 102         /**
 103          * Generate a POM with all the artifacts declared in root coordinates as
 104          * dependencies AND in dependency management.
 105          */
 106         void createDistPom() {
 107                 try {
 108                         Artifact pomArtifact = new DefaultArtifact(rootCoordinates);
 109
 110                         Set<Artifact> registeredArtifacts = new TreeSet<Artifact>(
 111                                         artifactComparator);
 112                         MavenConventionsUtils.gatherPomDependencies(aetherTemplate,
 113                                         registeredArtifacts, pomArtifact);
 114                         Artifact sdkArtifact = new DefaultArtifact(distCoordinates);
 115                         String sdkPom = MavenConventionsUtils.artifactsAsDependencyPom(
 116                                         sdkArtifact, registeredArtifacts);
 117                         if (log.isDebugEnabled())
 118                                 log.debug("Gathered " + registeredArtifacts.size()
 119                                                 + " artifacts:\n" + sdkPom);
 120                 } catch (Exception e) {
 121                         throw new SlcException("Cannot resolve distribution", e);
 122                 }
 123         }
 124
 125         /** Returns all transitive dependencies of dist POM */
 126         private Set<Artifact> resolveDistribution() {
 127                 try {
 128                         Artifact distArtifact = new DefaultArtifact(distCoordinates);
 129                         Set<Artifact> artifacts = new TreeSet<Artifact>(artifactComparator);
 130
 131                         DependencyNode node = aetherTemplate
 132                                         .resolveDependencies(distArtifact);
 133                         addDependencies(artifacts, node, null);
 134
 135                         if (log.isDebugEnabled()) {
 136                                 log.debug("Resolved " + artifacts.size() + " artifacts");
 137
 138                                 // Properties distributionDescriptor =
 139                                 // generateDistributionDescriptor(artifacts);
 140                                 // ByteArrayOutputStream out = new ByteArrayOutputStream();
 141                                 // distributionDescriptor.store(out, "");
 142                                 // log.debug(new String(out.toByteArray()));
 143                                 // out.close();
 144                         }
 145
 146                         /*
 147                          * for (Artifact artifact : registeredArtifacts) { try { Boolean
 148                          * wasAdded = addArtifact(artifacts, artifact); if (wasAdded) {
 149                          * DependencyNode node = aetherTemplate
 150                          * .resolveDependencies(artifact); addDependencies(artifacts, node,
 151                          * null); } } catch (Exception e) {
 152                          * log.error("Could not resolve dependencies of " + artifact + ": "
 153                          * + e.getCause().getMessage()); }
 154                          *
 155                          * }
 156                          *
 157                          * if (log.isDebugEnabled()) log.debug("Resolved " +
 158                          * artifacts.size() + " artifacts");
 159                          *
 160                          * // distribution descriptor // Properties distributionDescriptor =
 161                          * // generateDistributionDescriptor(artifacts); //
 162                          * ByteArrayOutputStream out = new ByteArrayOutputStream(); //
 163                          * distributionDescriptor.store(out, ""); // log.debug(new
 164                          * String(out.toByteArray())); // out.close();
 165                          */
 166                         return artifacts;
 167                 } catch (Exception e) {
 168                         throw new SlcException("Cannot resolve distribution", e);
 169                 }
 170         }
 171
 172         protected Properties generateDistributionDescriptor(Set<Artifact> artifacts) {
 173                 Properties distributionDescriptor = new Properties();
 174                 for (Artifact artifact : artifacts) {
 175                         log.debug(artifact.getArtifactId() + " [" + artifact.getVersion()
 176                                         + "]\t(" + artifact + ")");
 177                         distributionDescriptor.setProperty(artifact.getArtifactId() + ":"
 178                                         + artifact.getVersion(), artifact.toString());
 179                 }
 180                 return distributionDescriptor;
 181         }
 182
 183         /** Write artifacts to the target workspace, skipping excluded ones */
 184         protected void syncDistribution(Session jcrSession, Set<Artifact> artifacts) {
 185                 Set<Artifact> artifactsWithoutSources = new TreeSet<Artifact>(
 186                                 artifactComparator);
 187                 Long begin = System.currentTimeMillis();
 188                 try {
 189                         JcrUtils.mkdirs(jcrSession, artifactBasePath, NodeType.NT_FOLDER,
 190                                         NodeType.NT_FOLDER, false);
 191                         artifacts: for (Artifact artifact : artifacts) {
 192                                 // skip excluded
 193                                 if (excludedArtifacts.contains(artifact.getGroupId() + ":"
 194                                                 + artifact.getArtifactId())) {
 195                                         if (log.isDebugEnabled())
 196                                                 log.debug("Exclude " + artifact);
 197                                         continue artifacts;
 198                                 }
 199
 200                                 File jarFile = MavenConventionsUtils.artifactToFile(artifact);
 201                                 if (!jarFile.exists()) {
 202                                         log.warn("Generated file " + jarFile + " for " + artifact
 203                                                         + " does not exist");
 204                                         continue artifacts;
 205                                 }
 206                                 artifact.setFile(jarFile);
 207
 208                                 try {
 209                                         String parentPath = MavenConventionsUtils
 210                                                         .artifactParentPath(artifactBasePath, artifact);
 211                                         Node parentNode;
 212                                         if (!jcrSession.itemExists(parentPath))
 213                                                 parentNode = JcrUtils.mkdirs(jcrSession, parentPath,
 214                                                                 NodeType.NT_FOLDER, NodeType.NT_FOLDER, false);
 215                                         else
 216                                                 parentNode = jcrSession.getNode(parentPath);
 217
 218                                         Node fileNode;
 219                                         if (!parentNode.hasNode(jarFile.getName())) {
 220                                                 fileNode = createFileNode(parentNode, jarFile);
 221                                         } else {
 222                                                 fileNode = parentNode.getNode(jarFile.getName());
 223                                         }
 224
 225                                         if (artifactIndexer.support(fileNode.getPath()))
 226                                                 artifactIndexer.index(fileNode);
 227                                         if (jarFileIndexer.support(fileNode.getPath()))
 228                                                 jarFileIndexer.index(fileNode);
 229                                         jcrSession.save();
 230
 231                                         addPdeSource(jcrSession, artifact, jarFile,
 232                                                         artifactsWithoutSources);
 233                                         jcrSession.save();
 234
 235                                         if (log.isDebugEnabled())
 236                                                 log.debug("Synchronized " + fileNode);
 237                                 } catch (Exception e) {
 238                                         log.error("Could not synchronize " + artifact, e);
 239                                         jcrSession.refresh(false);
 240                                         throw e;
 241                                 }
 242                         }
 243
 244                         Long duration = (System.currentTimeMillis() - begin) / 1000;
 245                         if (log.isDebugEnabled()) {
 246                                 log.debug("Synchronized distribution in " + duration + "s");
 247                                 log.debug("The following artifacts have no sources:");
 248                                 for (Artifact artifact : artifactsWithoutSources) {
 249                                         log.debug(artifact);
 250                                 }
 251                         }
 252                 } catch (Exception e) {
 253                         throw new SlcException("Cannot synchronize distribution", e);
 254                 }
 255         }
 256
 257         /** Try to add PDE sources */
 258         private void addPdeSource(Session session, Artifact artifact,
 259                         File artifactFile, Set<Artifact> artifactsWithoutSources) {
 260                 ByteArrayOutputStream out = new ByteArrayOutputStream();
 261                 try {
 262                         File origSourceFile = null;
 263                         Artifact origSourceArtifact = new DefaultArtifact(
 264                                         artifact.getGroupId(), artifact.getArtifactId(), "sources",
 265                                         artifact.getExtension(), artifact.getVersion());
 266                         Artifact targetSourceArtifact = new DefaultArtifact(
 267                                         artifact.getGroupId(),
 268                                         artifact.getArtifactId() + ".source",
 269                                         artifact.getExtension(), artifact.getVersion());
 270                         try {
 271                                 origSourceFile = aetherTemplate
 272                                                 .getResolvedFile(origSourceArtifact);
 273                         } catch (Exception e) {
 274                                 // also try artifact following the conventions
 275                                 origSourceArtifact = targetSourceArtifact;
 276                                 origSourceFile = aetherTemplate
 277                                                 .getResolvedFile(origSourceArtifact);
 278                         }
 279
 280                         String sourceParentPath = MavenConventionsUtils.artifactParentPath(
 281                                         artifactBasePath, targetSourceArtifact);
 282                         Node sourceParentNode = JcrUtils.mkdirs(session, sourceParentPath,
 283                                         NodeType.NT_FOLDER);
 284                         NameVersion bundleNameVersion = RepoUtils
 285                                         .readNameVersion(artifactFile);
 286                         RepoUtils.packagesAsPdeSource(origSourceFile, bundleNameVersion,
 287                                         out);
 288                         String targetSourceFileName = MavenConventionsUtils
 289                                         .artifactFileName(targetSourceArtifact);
 290                         JcrUtils.copyBytesAsFile(sourceParentNode, targetSourceFileName,
 291                                         out.toByteArray());
 292                 } catch (Exception e) {
 293                         log.error("Cannot add PDE source for " + artifact + ": " + e);
 294                         artifactsWithoutSources.add(artifact);
 295                 } finally {
 296                         IOUtils.closeQuietly(out);
 297                 }
 298         }
 299
 300         private Node createFileNode(Node parentNode, File file) {
 301                 return JcrUtils.copyFile(parentNode, file);
 302         }
 303
 304         /** Recursively adds non optional dependencies */
 305         private void addDependencies(Set<Artifact> artifacts, DependencyNode node,
 306                         String ancestors) {
 307                 // if (artifacts.contains(node.getDependency().getArtifact()))
 308                 // return;
 309                 String currentArtifactId = node.getDependency().getArtifact()
 310                                 .getArtifactId();
 311                 if (log.isDebugEnabled()) {
 312                         log.debug("# Add dependency for " + currentArtifactId);
 313                         if (ancestors != null)
 314                                 log.debug(ancestors);
 315                 }
 316                 for (DependencyNode child : node.getChildren()) {
 317                         if (!child.getDependency().isOptional()) {
 318                                 if (willAdd(child.getDependency().getArtifact())) {
 319                                         addArtifact(artifacts, child.getDependency().getArtifact());
 320                                         addDependencies(artifacts, child, currentArtifactId + "\n"
 321                                                         + (ancestors != null ? ancestors : ""));
 322                                 }
 323                         }
 324                 }
 325         }
 326
 327         /** @return whether it was added */
 328         private Boolean addArtifact(Set<Artifact> artifacts, Artifact artifact) {
 329                 Boolean willAdd = willAdd(artifact);
 330                 if (willAdd)
 331                         artifacts.add(artifact);
 332                 else
 333                         log.info("Skip " + artifact);
 334                 return willAdd;
 335         }
 336
 337         private Boolean willAdd(Artifact artifact) {
 338                 Boolean willAdd = true;
 339                 if (excludedArtifacts.contains(artifact.getGroupId() + ":"
 340                                 + artifact.getArtifactId()))
 341                         willAdd = false;
 342                 else if (excludedArtifacts.contains(artifact.getGroupId() + ":*"))
 343                         willAdd = false;
 344                 return willAdd;
 345         }
 346
 347         public void setAetherTemplate(AetherTemplate aetherTemplate) {
 348                 this.aetherTemplate = aetherTemplate;
 349         }
 350
 351         public void setExcludedArtifacts(Set<String> excludedArtifactIds) {
 352                 this.excludedArtifacts = excludedArtifactIds;
 353         }
 354
 355         public void setRootCoordinates(String rootCoordinates) {
 356                 this.rootCoordinates = rootCoordinates;
 357         }
 358
 359         public void setRepository(Repository repository) {
 360                 this.repository = repository;
 361         }
 362
 363         public void setWorkspace(String workspace) {
 364                 this.workspace = workspace;
 365         }
 366
 367         public void setDistCoordinates(String distCoordinates) {
 368                 this.distCoordinates = distCoordinates;
 369         }
 370
 371         public void setArtifactBasePath(String artifactBasePath) {
 372                 this.artifactBasePath = artifactBasePath;
 373         }
 374
 375 }