org.argeo.slc.repo/src/org/argeo/slc/repo/maven/ImportMavenDependencies.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.repo.maven;
  17
  18 import java.io.ByteArrayOutputStream;
  19 import java.io.File;
  20 import java.util.Comparator;
  21 import java.util.HashSet;
  22 import java.util.Properties;
  23 import java.util.Set;
  24 import java.util.TreeSet;
  25
  26 import javax.jcr.Node;
  27 import javax.jcr.NodeIterator;
  28 import javax.jcr.Repository;
  29 import javax.jcr.Session;
  30 import javax.jcr.nodetype.NodeType;
  31
  32 import org.apache.commons.io.IOUtils;
  33 import org.apache.commons.logging.Log;
  34 import org.apache.commons.logging.LogFactory;
  35 import org.argeo.jcr.JcrUtils;
  36 import org.argeo.slc.NameVersion;
  37 import org.argeo.slc.SlcException;
  38 import org.argeo.slc.aether.AetherTemplate;
  39 import org.argeo.slc.aether.ArtifactIdComparator;
  40 import org.argeo.slc.repo.ArtifactIndexer;
  41 import org.argeo.slc.repo.JarFileIndexer;
  42 import org.argeo.slc.repo.RepoConstants;
  43 import org.argeo.slc.repo.RepoUtils;
  44 import org.eclipse.aether.artifact.Artifact;
  45 import org.eclipse.aether.graph.DependencyNode;
  46 import org.eclipse.aether.artifact.DefaultArtifact;
  47
  48 /**
  49  * Import all the dependencies listed in a POM and their dependency graphs to a
  50  * workspace.
  51  */
  52 public class ImportMavenDependencies implements Runnable {
  53         private final static Log log = LogFactory
  54                         .getLog(ImportMavenDependencies.class);
  55
  56         private AetherTemplate aetherTemplate;
  57         private String rootCoordinates = "org.argeo.dep:versions-all:pom:1.2.0";
  58         private String distCoordinates = "org.argeo.tp:dist:pom:1.3.0";
  59         private String parentPomCoordinates = "org.argeo:parent:1.2.0";
  60         private Set<String> excludedArtifacts = new HashSet<String>();
  61
  62         private Repository repository;
  63         private String workspace;
  64
  65         private String artifactBasePath = RepoConstants.DEFAULT_ARTIFACTS_BASE_PATH;
  66
  67         private ArtifactIndexer artifactIndexer = new ArtifactIndexer();
  68         private JarFileIndexer jarFileIndexer = new JarFileIndexer();
  69         private Comparator<Artifact> artifactComparator = new ArtifactIdComparator();
  70
  71         public void run() {
  72                 // resolve
  73                 Set<Artifact> artifacts = resolveDistribution();
  74
  75                 // sync
  76                 sync(artifacts);
  77         }
  78
  79         void sync(Set<Artifact> artifacts) {
  80                 Session session = null;
  81                 try {
  82                         session = JcrUtils.loginOrCreateWorkspace(repository, workspace);
  83                         // clear
  84                         NodeIterator nit = session.getNode(artifactBasePath).getNodes();
  85                         while (nit.hasNext()) {
  86                                 Node node = nit.nextNode();
  87                                 if (node.isNodeType(NodeType.NT_FOLDER)
  88                                                 || node.isNodeType(NodeType.NT_UNSTRUCTURED))
  89                                         node.remove();
  90                         }
  91                         session.save();
  92
  93                         // sync
  94                         syncDistribution(session, artifacts);
  95                 } catch (Exception e) {
  96                         throw new SlcException("Cannot import distribution", e);
  97                 } finally {
  98                         JcrUtils.logoutQuietly(session);
  99                 }
 100         }
 101
 102         /**
 103          * Generate a POM with all the artifacts declared in root coordinates as
 104          * dependencies AND in dependency management.
 105          */
 106         void createDistPom() {
 107                 try {
 108                         Artifact pomArtifact = new DefaultArtifact(rootCoordinates);
 109
 110                         Set<Artifact> registeredArtifacts = new TreeSet<Artifact>(
 111                                         artifactComparator);
 112                         MavenConventionsUtils.gatherPomDependencies(aetherTemplate,
 113                                         registeredArtifacts, pomArtifact);
 114                         Artifact sdkArtifact = new DefaultArtifact(distCoordinates);
 115                         String sdkPom = MavenConventionsUtils.artifactsAsDependencyPom(
 116                                         sdkArtifact, registeredArtifacts, new DefaultArtifact(
 117                                                         parentPomCoordinates));
 118                         if (log.isDebugEnabled())
 119                                 log.debug("Gathered " + registeredArtifacts.size()
 120                                                 + " artifacts:\n" + sdkPom);
 121                 } catch (Exception e) {
 122                         throw new SlcException("Cannot resolve distribution", e);
 123                 }
 124         }
 125
 126         /** Returns all transitive dependencies of dist POM */
 127         private Set<Artifact> resolveDistribution() {
 128                 try {
 129                         Artifact distArtifact = new DefaultArtifact(distCoordinates);
 130                         Set<Artifact> artifacts = new TreeSet<Artifact>(artifactComparator);
 131
 132                         DependencyNode node = aetherTemplate
 133                                         .resolveDependencies(distArtifact);
 134                         addDependencies(artifacts, node, null);
 135
 136                         if (log.isDebugEnabled()) {
 137                                 log.debug("Resolved " + artifacts.size() + " artifacts");
 138
 139                                 // Properties distributionDescriptor =
 140                                 // generateDistributionDescriptor(artifacts);
 141                                 // ByteArrayOutputStream out = new ByteArrayOutputStream();
 142                                 // distributionDescriptor.store(out, "");
 143                                 // log.debug(new String(out.toByteArray()));
 144                                 // out.close();
 145                         }
 146
 147                         /*
 148                          * for (Artifact artifact : registeredArtifacts) { try { Boolean
 149                          * wasAdded = addArtifact(artifacts, artifact); if (wasAdded) {
 150                          * DependencyNode node = aetherTemplate
 151                          * .resolveDependencies(artifact); addDependencies(artifacts, node,
 152                          * null); } } catch (Exception e) {
 153                          * log.error("Could not resolve dependencies of " + artifact + ": "
 154                          * + e.getCause().getMessage()); }
 155                          *
 156                          * }
 157                          *
 158                          * if (log.isDebugEnabled()) log.debug("Resolved " +
 159                          * artifacts.size() + " artifacts");
 160                          *
 161                          * // distribution descriptor // Properties distributionDescriptor =
 162                          * // generateDistributionDescriptor(artifacts); //
 163                          * ByteArrayOutputStream out = new ByteArrayOutputStream(); //
 164                          * distributionDescriptor.store(out, ""); // log.debug(new
 165                          * String(out.toByteArray())); // out.close();
 166                          */
 167                         return artifacts;
 168                 } catch (Exception e) {
 169                         throw new SlcException("Cannot resolve distribution", e);
 170                 }
 171         }
 172
 173         protected Properties generateDistributionDescriptor(Set<Artifact> artifacts) {
 174                 Properties distributionDescriptor = new Properties();
 175                 for (Artifact artifact : artifacts) {
 176                         log.debug(artifact.getArtifactId() + " [" + artifact.getVersion()
 177                                         + "]\t(" + artifact + ")");
 178                         distributionDescriptor.setProperty(artifact.getArtifactId() + ":"
 179                                         + artifact.getVersion(), artifact.toString());
 180                 }
 181                 return distributionDescriptor;
 182         }
 183
 184         /** Write artifacts to the target workspace, skipping excluded ones */
 185         protected void syncDistribution(Session jcrSession, Set<Artifact> artifacts) {
 186                 Set<Artifact> artifactsWithoutSources = new TreeSet<Artifact>(
 187                                 artifactComparator);
 188                 Long begin = System.currentTimeMillis();
 189                 try {
 190                         JcrUtils.mkfolders(jcrSession, artifactBasePath);
 191                         artifacts: for (Artifact artifact : artifacts) {
 192                                 // skip excluded
 193                                 if (excludedArtifacts.contains(artifact.getGroupId() + ":"
 194                                                 + artifact.getArtifactId())) {
 195                                         if (log.isDebugEnabled())
 196                                                 log.debug("Exclude " + artifact);
 197                                         continue artifacts;
 198                                 }
 199
 200                                 File jarFile = MavenConventionsUtils.artifactToFile(artifact);
 201                                 if (!jarFile.exists()) {
 202                                         log.warn("Generated file " + jarFile + " for " + artifact
 203                                                         + " does not exist");
 204                                         continue artifacts;
 205                                 }
 206                                 artifact.setFile(jarFile);
 207
 208                                 try {
 209                                         String parentPath = MavenConventionsUtils
 210                                                         .artifactParentPath(artifactBasePath, artifact);
 211                                         Node parentNode;
 212                                         if (!jcrSession.itemExists(parentPath))
 213                                                 parentNode = JcrUtils.mkfolders(jcrSession, parentPath);
 214                                         else
 215                                                 parentNode = jcrSession.getNode(parentPath);
 216
 217                                         Node fileNode;
 218                                         if (!parentNode.hasNode(jarFile.getName())) {
 219                                                 fileNode = createFileNode(parentNode, jarFile);
 220                                         } else {
 221                                                 fileNode = parentNode.getNode(jarFile.getName());
 222                                         }
 223
 224                                         if (artifactIndexer.support(fileNode.getPath()))
 225                                                 artifactIndexer.index(fileNode);
 226                                         if (jarFileIndexer.support(fileNode.getPath()))
 227                                                 jarFileIndexer.index(fileNode);
 228                                         jcrSession.save();
 229
 230                                         addPdeSource(jcrSession, artifact, jarFile,
 231                                                         artifactsWithoutSources);
 232                                         jcrSession.save();
 233
 234                                         if (log.isDebugEnabled())
 235                                                 log.debug("Synchronized " + fileNode);
 236                                 } catch (Exception e) {
 237                                         log.error("Could not synchronize " + artifact, e);
 238                                         jcrSession.refresh(false);
 239                                         throw e;
 240                                 }
 241                         }
 242
 243                         Long duration = (System.currentTimeMillis() - begin) / 1000;
 244                         if (log.isDebugEnabled()) {
 245                                 log.debug("Synchronized distribution in " + duration + "s");
 246                                 log.debug("The following artifacts have no sources:");
 247                                 for (Artifact artifact : artifactsWithoutSources) {
 248                                         log.debug(artifact);
 249                                 }
 250                         }
 251                 } catch (Exception e) {
 252                         throw new SlcException("Cannot synchronize distribution", e);
 253                 }
 254         }
 255
 256         /** Try to add PDE sources */
 257         private void addPdeSource(Session session, Artifact artifact,
 258                         File artifactFile, Set<Artifact> artifactsWithoutSources) {
 259                 ByteArrayOutputStream out = new ByteArrayOutputStream();
 260                 try {
 261                         File origSourceFile = null;
 262                         Artifact origSourceArtifact = new DefaultArtifact(
 263                                         artifact.getGroupId(), artifact.getArtifactId(), "sources",
 264                                         artifact.getExtension(), artifact.getVersion());
 265                         Artifact newSourceArtifact = new DefaultArtifact(
 266                                         artifact.getGroupId(),
 267                                         artifact.getArtifactId() + ".source",
 268                                         artifact.getExtension(), artifact.getVersion());
 269                         try {
 270                                 origSourceFile = aetherTemplate
 271                                                 .getResolvedFile(origSourceArtifact);
 272                         } catch (Exception e) {
 273                                 // also try artifact following the conventions
 274                                 origSourceArtifact = newSourceArtifact;
 275                                 origSourceFile = aetherTemplate
 276                                                 .getResolvedFile(origSourceArtifact);
 277                         }
 278
 279                         String newSourceParentPath = MavenConventionsUtils
 280                                         .artifactParentPath(artifactBasePath, newSourceArtifact);
 281                         Node newSourceParentNode = JcrUtils.mkfolders(session,
 282                                         newSourceParentPath);
 283                         NameVersion bundleNameVersion = RepoUtils
 284                                         .readNameVersion(artifactFile);
 285                         RepoUtils.packagesAsPdeSource(origSourceFile, bundleNameVersion,
 286                                         out);
 287                         String newSourceFileName = MavenConventionsUtils
 288                                         .artifactFileName(newSourceArtifact);
 289                         JcrUtils.copyBytesAsFile(newSourceParentNode, newSourceFileName,
 290                                         out.toByteArray());
 291                 } catch (Exception e) {
 292                         log.error("Cannot add PDE source for " + artifact + ": " + e);
 293                         artifactsWithoutSources.add(artifact);
 294                 } finally {
 295                         IOUtils.closeQuietly(out);
 296                 }
 297         }
 298
 299         private Node createFileNode(Node parentNode, File file) {
 300                 return JcrUtils.copyFile(parentNode, file);
 301         }
 302
 303         /** Recursively adds non optional dependencies */
 304         private void addDependencies(Set<Artifact> artifacts, DependencyNode node,
 305                         String ancestors) {
 306                 // if (artifacts.contains(node.getDependency().getArtifact()))
 307                 // return;
 308                 String currentArtifactId = node.getDependency().getArtifact()
 309                                 .getArtifactId();
 310                 if (log.isDebugEnabled()) {
 311                         log.debug("# Add dependency for " + currentArtifactId);
 312                         if (ancestors != null)
 313                                 log.debug(ancestors);
 314                 }
 315                 for (DependencyNode child : node.getChildren()) {
 316                         if (!child.getDependency().isOptional()) {
 317                                 if (willAdd(child.getDependency().getArtifact())) {
 318                                         addArtifact(artifacts, child.getDependency().getArtifact());
 319                                         addDependencies(artifacts, child, currentArtifactId + "\n"
 320                                                         + (ancestors != null ? ancestors : ""));
 321                                 }
 322                         }
 323                 }
 324         }
 325
 326         /** @return whether it was added */
 327         private Boolean addArtifact(Set<Artifact> artifacts, Artifact artifact) {
 328                 Boolean willAdd = willAdd(artifact);
 329                 if (willAdd)
 330                         artifacts.add(artifact);
 331                 else
 332                         log.info("Skip " + artifact);
 333                 return willAdd;
 334         }
 335
 336         private Boolean willAdd(Artifact artifact) {
 337                 Boolean willAdd = true;
 338                 if (excludedArtifacts.contains(artifact.getGroupId() + ":"
 339                                 + artifact.getArtifactId()))
 340                         willAdd = false;
 341                 else if (excludedArtifacts.contains(artifact.getGroupId() + ":*"))
 342                         willAdd = false;
 343                 return willAdd;
 344         }
 345
 346         public void setAetherTemplate(AetherTemplate aetherTemplate) {
 347                 this.aetherTemplate = aetherTemplate;
 348         }
 349
 350         public void setExcludedArtifacts(Set<String> excludedArtifactIds) {
 351                 this.excludedArtifacts = excludedArtifactIds;
 352         }
 353
 354         public void setRootCoordinates(String rootCoordinates) {
 355                 this.rootCoordinates = rootCoordinates;
 356         }
 357
 358         public void setRepository(Repository repository) {
 359                 this.repository = repository;
 360         }
 361
 362         public void setWorkspace(String workspace) {
 363                 this.workspace = workspace;
 364         }
 365
 366         public void setDistCoordinates(String distCoordinates) {
 367                 this.distCoordinates = distCoordinates;
 368         }
 369
 370         public void setArtifactBasePath(String artifactBasePath) {
 371                 this.artifactBasePath = artifactBasePath;
 372         }
 373
 374 }