org.argeo.slc.repo/src/org/argeo/slc/repo/maven/GenerateBinaries.java

   1 package org.argeo.slc.repo.maven;
   2
   3 import java.util.ArrayList;
   4 import java.util.List;
   5 import java.util.Set;
   6 import java.util.StringTokenizer;
   7 import java.util.TreeSet;
   8
   9 import javax.jcr.Credentials;
  10 import javax.jcr.Node;
  11 import javax.jcr.NodeIterator;
  12 import javax.jcr.Repository;
  13 import javax.jcr.RepositoryException;
  14 import javax.jcr.Session;
  15
  16 import org.argeo.api.cms.CmsLog;
  17 import org.argeo.jcr.JcrMonitor;
  18 import org.argeo.jcr.JcrUtils;
  19 import org.argeo.slc.SlcException;
  20 import org.argeo.slc.SlcNames;
  21 import org.argeo.slc.SlcTypes;
  22 import org.argeo.slc.repo.ArtifactIndexer;
  23 import org.argeo.slc.repo.RepoConstants;
  24 import org.argeo.slc.repo.RepoUtils;
  25 import org.eclipse.aether.artifact.Artifact;
  26 import org.eclipse.aether.artifact.DefaultArtifact;
  27 import org.osgi.framework.Version;
  28
  29 /**
  30  * Generates binaries-, sources- and sdk-version.pom artifacts for a given
  31  * group.
  32  */
  33 public class GenerateBinaries implements Runnable, SlcNames {
  34         private final static CmsLog log = CmsLog.getLog(GenerateBinaries.class);
  35
  36         // Connection info
  37         private Repository repository;
  38         private Credentials credentials;
  39         private String workspace;
  40
  41         // Business info
  42         private String groupId;
  43         private String parentPomCoordinates;
  44         private String version = null;
  45
  46         // Constants
  47         private String artifactBasePath = RepoConstants.DEFAULT_ARTIFACTS_BASE_PATH;
  48         private List<String> excludedSuffixes = new ArrayList<String>();
  49
  50         // Indexes
  51         private Set<Artifact> binaries = new TreeSet<Artifact>(new ArtifactIdComparator());
  52         private Set<Artifact> sources = new TreeSet<Artifact>(new ArtifactIdComparator());
  53
  54         // local cache
  55         private ArtifactIndexer artifactIndexer = new ArtifactIndexer();
  56         private Node allArtifactsHighestVersion;
  57
  58         public void run() {
  59                 Session session = null;
  60                 try {
  61                         session = repository.login(credentials, workspace);
  62                         Node groupNode = session.getNode(MavenConventionsUtils.groupPath(artifactBasePath, groupId));
  63                         internalPreProcessing(groupNode, null);
  64                         internalProcessing(groupNode, null);
  65                 } catch (Exception e) {
  66                         throw new SlcException("Cannot normalize group " + groupId + " in " + workspace, e);
  67                 } finally {
  68                         JcrUtils.logoutQuietly(session);
  69                 }
  70         }
  71
  72         /**
  73          * Generates binaries-, sources- and sdk-version.pom artifacts for the given
  74          * version (or the highest of all children version if none is precised).
  75          *
  76          * By default, it includes each latest version of all artifact of this group.
  77          *
  78          * The 3 generated artifacts are then marked as modular distributions and
  79          * indexed.
  80          */
  81         public static void processGroupNode(Node groupNode, String version, JcrMonitor monitor) throws RepositoryException {
  82                 // TODO set artifactsBase based on group node
  83                 GenerateBinaries gb = new GenerateBinaries();
  84                 String groupId = groupNode.getProperty(SlcNames.SLC_GROUP_BASE_ID).getString();
  85                 gb.setGroupId(groupId);
  86                 gb.setVersion(version);
  87                 // TODO use already done pre-processing
  88                 gb.internalPreProcessing(groupNode, monitor);
  89                 gb.internalProcessing(groupNode, monitor);
  90         }
  91
  92         /** Only builds local indexes. Does not change anything in the local Session */
  93         public static GenerateBinaries preProcessGroupNode(Node groupNode, JcrMonitor monitor) throws RepositoryException {
  94                 // TODO set artifactsBase based on group node
  95                 GenerateBinaries gb = new GenerateBinaries();
  96                 String groupId = groupNode.getProperty(SlcNames.SLC_GROUP_BASE_ID).getString();
  97                 gb.setGroupId(groupId);
  98                 // gb.setVersion(version);
  99                 // gb.setOverridePoms(overridePoms);
 100                 gb.internalPreProcessing(groupNode, monitor);
 101                 return gb;
 102         }
 103
 104         // exposes indexes. to display results of the pre-processing phase.
 105         public Set<Artifact> getBinaries() {
 106                 return binaries;
 107         }
 108
 109         public Artifact getHighestArtifactVersion() throws RepositoryException {
 110                 return allArtifactsHighestVersion == null ? null : RepoUtils.asArtifact(allArtifactsHighestVersion);
 111         }
 112
 113         // //////////////////////////////////////
 114         // INTERNAL METHODS
 115
 116         /**
 117          * Browse all children of a Node considered as a folder that follows Aether
 118          * conventions i.e that has Aether's artifact base as children.
 119          *
 120          * Each of such child contains a set of Aether artifact versions. This methods
 121          * build the binaries {@code Set<Artifact>} and other indexes. It does not
 122          * impact the
 123          */
 124         protected void internalPreProcessing(Node groupNode, JcrMonitor monitor) throws RepositoryException {
 125                 if (monitor != null)
 126                         monitor.subTask("Pre processing group " + groupId);
 127
 128                 // Process all direct children nodes,
 129                 // gathering latest versions of each artifact
 130                 allArtifactsHighestVersion = null;
 131
 132                 aBases: for (NodeIterator aBases = groupNode.getNodes(); aBases.hasNext();) {
 133                         Node aBase = aBases.nextNode();
 134                         if (aBase.isNodeType(SlcTypes.SLC_ARTIFACT_BASE)) {
 135                                 Node highestAVersion = getArtifactLatestVersion(aBase);
 136                                 if (highestAVersion == null)
 137                                         continue aBases;
 138                                 else {
 139                                         // retrieve relevant child node
 140                                         // Information is stored on the NT_FILE child node.
 141                                         for (NodeIterator files = highestAVersion.getNodes(); files.hasNext();) {
 142                                                 Node file = files.nextNode();
 143                                                 if (file.isNodeType(SlcTypes.SLC_BUNDLE_ARTIFACT)) {
 144                                                         if (log.isDebugEnabled())
 145                                                                 log.debug("Pre-Processing " + file.getName());
 146                                                         preProcessBundleArtifact(file);
 147                                                 }
 148                                         }
 149                                 }
 150                         }
 151                 }
 152                 // if (log.isDebugEnabled()) {
 153                 // int bundleCount = symbolicNamesToNodes.size();
 154                 // log.debug("" + bundleCount + " bundles have been indexed for "
 155                 // + groupId);
 156                 // }
 157         }
 158
 159         /** Does the real job : writes JCR META-DATA and generates binaries */
 160         protected void internalProcessing(Node groupNode, JcrMonitor monitor) throws RepositoryException {
 161                 if (monitor != null)
 162                         monitor.subTask("Processing group " + groupId);
 163
 164                 Session session = groupNode.getSession();
 165
 166                 // if version not set or empty, use the highest version
 167                 // useful when indexing a product maven repository where
 168                 // all artifacts have the same version for a given release
 169                 // => the version can then be left empty
 170                 if (version == null || version.trim().equals(""))
 171                         if (allArtifactsHighestVersion != null)
 172                                 version = allArtifactsHighestVersion.getProperty(SLC_ARTIFACT_VERSION).getString();
 173                         else
 174                                 throw new SlcException("Group version " + version + " is empty.");
 175
 176                 // int bundleCount = symbolicNamesToNodes.size();
 177                 // int count = 1;
 178                 // for (Node bundleNode : symbolicNamesToNodes.values()) {
 179                 // if (log.isDebugEnabled())
 180                 // log.debug("Processing " + bundleNode.getName() + " ( " + count
 181                 // + "/" + bundleCount + " )");
 182                 //
 183                 // // processBundleArtifact(bundleNode);
 184                 // // bundleNode.getSession().save();
 185                 // count++;
 186                 // }
 187
 188                 // indexes
 189                 Set<Artifact> indexes = new TreeSet<Artifact>(new ArtifactIdComparator());
 190
 191                 Artifact indexArtifact;
 192                 indexArtifact = writeIndex(session, RepoConstants.BINARIES_ARTIFACT_ID, binaries);
 193                 indexes.add(indexArtifact);
 194
 195                 indexArtifact = writeIndex(session, RepoConstants.SOURCES_ARTIFACT_ID, sources);
 196                 indexes.add(indexArtifact);
 197
 198                 // sdk
 199                 writeIndex(session, RepoConstants.SDK_ARTIFACT_ID, indexes);
 200
 201                 if (monitor != null)
 202                         monitor.worked(1);
 203         }
 204
 205         protected void preProcessBundleArtifact(Node bundleNode) throws RepositoryException {
 206
 207                 String symbolicName = JcrUtils.get(bundleNode, SLC_SYMBOLIC_NAME);
 208                 // Sanity check.
 209                 if (symbolicName == null)
 210                         log.warn("Symbolic name is null for bundle " + bundleNode);
 211
 212                 // Manage source bundles
 213                 if (symbolicName.endsWith(".source")) {
 214                         // TODO make a shared node with classifier 'sources'?
 215                         String bundleName = RepoUtils.extractBundleNameFromSourceName(symbolicName);
 216                         for (String excludedSuffix : excludedSuffixes) {
 217                                 if (bundleName.endsWith(excludedSuffix))
 218                                         return;// skip adding to sources
 219                         }
 220                         sources.add(RepoUtils.asArtifact(bundleNode));
 221                         return;
 222                 }
 223
 224                 // // Build indexes
 225                 // NodeIterator exportPackages = bundleNode.getNodes(SLC_
 226                 // + Constants.EXPORT_PACKAGE);
 227                 // while (exportPackages.hasNext()) {
 228                 // Node exportPackage = exportPackages.nextNode();
 229                 // String pkg = JcrUtils.get(exportPackage, SLC_NAME);
 230                 // packagesToSymbolicNames.put(pkg, symbolicName);
 231                 // }
 232                 //
 233                 // symbolicNamesToNodes.put(symbolicName, bundleNode);
 234                 // for (String excludedSuffix : excludedSuffixes) {
 235                 // if (symbolicName.endsWith(excludedSuffix))
 236                 // return;// skip adding to binaries
 237                 // }
 238
 239                 binaries.add(RepoUtils.asArtifact(bundleNode));
 240
 241                 // Extra check. to remove
 242                 if (bundleNode.getSession().hasPendingChanges())
 243                         throw new SlcException("Pending changes in the session, " + "this should not be true here.");
 244         }
 245
 246         // protected void processBundleArtifact(Node bundleNode)
 247         // throws RepositoryException {
 248         // Node artifactFolder = bundleNode.getParent();
 249         // String baseName = FilenameUtils.getBaseName(bundleNode.getName());
 250         //
 251         // // pom
 252         // String pomName = baseName + ".pom";
 253         // if (artifactFolder.hasNode(pomName) && !overridePoms)
 254         // return;// skip
 255         //
 256         // String pom = generatePomForBundle(bundleNode);
 257         // Node pomNode = JcrUtils.copyBytesAsFile(artifactFolder, pomName,
 258         // pom.getBytes());
 259         // // checksum
 260         // String bundleSha = JcrUtils.checksumFile(bundleNode, "SHA-1");
 261         // JcrUtils.copyBytesAsFile(artifactFolder,
 262         // bundleNode.getName() + ".sha1", bundleSha.getBytes());
 263         // String pomSha = JcrUtils.checksumFile(pomNode, "SHA-1");
 264         // JcrUtils.copyBytesAsFile(artifactFolder, pomNode.getName() + ".sha1",
 265         // pomSha.getBytes());
 266         // }
 267
 268         // ////////////////////
 269         // LOCAL WRITERS
 270         //
 271
 272         private Artifact writeIndex(Session session, String artifactId, Set<Artifact> artifacts)
 273                         throws RepositoryException {
 274                 Artifact artifact = new DefaultArtifact(groupId, artifactId, "pom", version);
 275                 Artifact parentArtifact = parentPomCoordinates != null ? new DefaultArtifact(parentPomCoordinates) : null;
 276                 String pom = MavenConventionsUtils.artifactsAsDependencyPom(artifact, artifacts, parentArtifact);
 277                 Node node = RepoUtils.copyBytesAsArtifact(session.getNode(artifactBasePath), artifact, pom.getBytes());
 278                 artifactIndexer.index(node);
 279
 280                 // TODO factorize
 281                 String pomSha = JcrUtils.checksumFile(node, "SHA-1");
 282                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".sha1", pomSha.getBytes());
 283                 String pomMd5 = JcrUtils.checksumFile(node, "MD5");
 284                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".md5", pomMd5.getBytes());
 285                 session.save();
 286                 return artifact;
 287         }
 288
 289         // Helpers
 290         private Node getArtifactLatestVersion(Node artifactBase) {
 291                 try {
 292                         Node highestAVersion = null;
 293                         for (NodeIterator aVersions = artifactBase.getNodes(); aVersions.hasNext();) {
 294                                 Node aVersion = aVersions.nextNode();
 295                                 if (aVersion.isNodeType(SlcTypes.SLC_ARTIFACT_VERSION_BASE)) {
 296                                         if (highestAVersion == null) {
 297                                                 highestAVersion = aVersion;
 298                                                 if (allArtifactsHighestVersion == null)
 299                                                         allArtifactsHighestVersion = aVersion;
 300                                                 // Correctly handle following arrival order:
 301                                                 // Name1 - V1, name2 - V3
 302                                                 else {
 303                                                         Version cachedHighestVersion = extractOsgiVersion(allArtifactsHighestVersion);
 304                                                         Version currVersion = extractOsgiVersion(aVersion);
 305                                                         if (currVersion.compareTo(cachedHighestVersion) > 0)
 306                                                                 allArtifactsHighestVersion = aVersion;
 307                                                 }
 308                                         } else {
 309                                                 Version currVersion = extractOsgiVersion(aVersion);
 310                                                 Version currentHighestVersion = extractOsgiVersion(highestAVersion);
 311                                                 if (currVersion.compareTo(currentHighestVersion) > 0) {
 312                                                         highestAVersion = aVersion;
 313                                                 }
 314                                                 if (currVersion.compareTo(extractOsgiVersion(allArtifactsHighestVersion)) > 0) {
 315                                                         allArtifactsHighestVersion = aVersion;
 316                                                 }
 317                                         }
 318
 319                                 }
 320                         }
 321                         return highestAVersion;
 322                 } catch (RepositoryException re) {
 323                         throw new SlcException("Unable to get latest version for node " + artifactBase, re);
 324                 }
 325         }
 326
 327         private Version extractOsgiVersion(Node artifactVersion) throws RepositoryException {
 328                 String rawVersion = artifactVersion.getProperty(SLC_ARTIFACT_VERSION).getString();
 329                 String cleanVersion = rawVersion.replace("-SNAPSHOT", ".SNAPSHOT");
 330                 Version osgiVersion = null;
 331                 // log invalid version value to enable tracking them
 332                 try {
 333                         osgiVersion = new Version(cleanVersion);
 334                 } catch (IllegalArgumentException e) {
 335                         log.error("Version string " + cleanVersion + " is invalid ");
 336                         String twickedVersion = twickInvalidVersion(cleanVersion);
 337                         osgiVersion = new Version(twickedVersion);
 338                         log.error("Using " + twickedVersion + " instead");
 339                         // throw e;
 340                 }
 341                 return osgiVersion;
 342         }
 343
 344         private String twickInvalidVersion(String tmpVersion) {
 345                 String[] tokens = tmpVersion.split("\\.");
 346                 if (tokens.length == 3 && tokens[2].lastIndexOf("-") > 0) {
 347                         String newSuffix = tokens[2].replaceFirst("-", ".");
 348                         tmpVersion = tmpVersion.replaceFirst(tokens[2], newSuffix);
 349                 } else if (tokens.length > 4) {
 350                         // FIXME manually remove other "."
 351                         StringTokenizer st = new StringTokenizer(tmpVersion, ".", true);
 352                         StringBuilder builder = new StringBuilder();
 353                         // Major
 354                         builder.append(st.nextToken()).append(st.nextToken());
 355                         // Minor
 356                         builder.append(st.nextToken()).append(st.nextToken());
 357                         // Micro
 358                         builder.append(st.nextToken()).append(st.nextToken());
 359                         // Qualifier
 360                         builder.append(st.nextToken());
 361                         while (st.hasMoreTokens()) {
 362                                 // consume delimiter
 363                                 st.nextToken();
 364                                 if (st.hasMoreTokens())
 365                                         builder.append("-").append(st.nextToken());
 366                         }
 367                         tmpVersion = builder.toString();
 368                 }
 369                 return tmpVersion;
 370         }
 371
 372         // private String generatePomForBundle(Node n) throws RepositoryException {
 373         // String ownSymbolicName = JcrUtils.get(n, SLC_SYMBOLIC_NAME);
 374         //
 375         // StringBuffer p = new StringBuffer();
 376         //
 377         // // XML header
 378         // p.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
 379         // p.append("<project xmlns=\"http://maven.apache.org/POM/4.0.0\"
 380         // xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"
 381         // xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0
 382         // http://maven.apache.org/maven-v4_0_0.xsd\">\n");
 383         // p.append("<modelVersion>4.0.0</modelVersion>");
 384         //
 385         // // Artifact
 386         // p.append("<groupId>").append(JcrUtils.get(n, SLC_GROUP_ID))
 387         // .append("</groupId>\n");
 388         // p.append("<artifactId>").append(JcrUtils.get(n, SLC_ARTIFACT_ID))
 389         // .append("</artifactId>\n");
 390         // p.append("<version>").append(JcrUtils.get(n, SLC_ARTIFACT_VERSION))
 391         // .append("</version>\n");
 392         // p.append("<packaging>pom</packaging>\n");
 393         // if (n.hasProperty(SLC_ + Constants.BUNDLE_NAME))
 394         // p.append("<name>")
 395         // .append(JcrUtils.get(n, SLC_ + Constants.BUNDLE_NAME))
 396         // .append("</name>\n");
 397         // if (n.hasProperty(SLC_ + Constants.BUNDLE_DESCRIPTION))
 398         // p.append("<description>")
 399         // .append(JcrUtils
 400         // .get(n, SLC_ + Constants.BUNDLE_DESCRIPTION))
 401         // .append("</description>\n");
 402         //
 403         // // Dependencies
 404         // Set<String> dependenciesSymbolicNames = new TreeSet<String>();
 405         // Set<String> optionalSymbolicNames = new TreeSet<String>();
 406         // NodeIterator importPackages = n.getNodes(SLC_
 407         // + Constants.IMPORT_PACKAGE);
 408         // while (importPackages.hasNext()) {
 409         // Node importPackage = importPackages.nextNode();
 410         // String pkg = JcrUtils.get(importPackage, SLC_NAME);
 411         // if (packagesToSymbolicNames.containsKey(pkg)) {
 412         // String dependencySymbolicName = packagesToSymbolicNames
 413         // .get(pkg);
 414         // if (JcrUtils.check(importPackage, SLC_OPTIONAL))
 415         // optionalSymbolicNames.add(dependencySymbolicName);
 416         // else
 417         // dependenciesSymbolicNames.add(dependencySymbolicName);
 418         // } else {
 419         // if (!JcrUtils.check(importPackage, SLC_OPTIONAL)
 420         // && !systemPackages.contains(pkg))
 421         // log.warn("No bundle found for pkg " + pkg);
 422         // }
 423         // }
 424         //
 425         // if (n.hasNode(SLC_ + Constants.FRAGMENT_HOST)) {
 426         // String fragmentHost = JcrUtils.get(
 427         // n.getNode(SLC_ + Constants.FRAGMENT_HOST),
 428         // SLC_SYMBOLIC_NAME);
 429         // dependenciesSymbolicNames.add(fragmentHost);
 430         // }
 431         //
 432         // // TODO require bundles
 433         //
 434         // List<Node> dependencyNodes = new ArrayList<Node>();
 435         // for (String depSymbName : dependenciesSymbolicNames) {
 436         // if (depSymbName.equals(ownSymbolicName))
 437         // continue;// skip self
 438         //
 439         // if (symbolicNamesToNodes.containsKey(depSymbName))
 440         // dependencyNodes.add(symbolicNamesToNodes.get(depSymbName));
 441         // else
 442         // log.warn("Could not find node for " + depSymbName);
 443         // }
 444         // List<Node> optionalDependencyNodes = new ArrayList<Node>();
 445         // for (String depSymbName : optionalSymbolicNames) {
 446         // if (symbolicNamesToNodes.containsKey(depSymbName))
 447         // optionalDependencyNodes.add(symbolicNamesToNodes
 448         // .get(depSymbName));
 449         // else
 450         // log.warn("Could not find node for " + depSymbName);
 451         // }
 452         //
 453         // p.append("<dependencies>\n");
 454         // for (Node dependencyNode : dependencyNodes) {
 455         // p.append("<dependency>\n");
 456         // p.append("\t<groupId>")
 457         // .append(JcrUtils.get(dependencyNode, SLC_GROUP_ID))
 458         // .append("</groupId>\n");
 459         // p.append("\t<artifactId>")
 460         // .append(JcrUtils.get(dependencyNode, SLC_ARTIFACT_ID))
 461         // .append("</artifactId>\n");
 462         // p.append("</dependency>\n");
 463         // }
 464         //
 465         // if (optionalDependencyNodes.size() > 0)
 466         // p.append("<!-- OPTIONAL -->\n");
 467         // for (Node dependencyNode : optionalDependencyNodes) {
 468         // p.append("<dependency>\n");
 469         // p.append("\t<groupId>")
 470         // .append(JcrUtils.get(dependencyNode, SLC_GROUP_ID))
 471         // .append("</groupId>\n");
 472         // p.append("\t<artifactId>")
 473         // .append(JcrUtils.get(dependencyNode, SLC_ARTIFACT_ID))
 474         // .append("</artifactId>\n");
 475         // p.append("\t<optional>true</optional>\n");
 476         // p.append("</dependency>\n");
 477         // }
 478         // p.append("</dependencies>\n");
 479         //
 480         // // Dependency management
 481         // p.append("<dependencyManagement>\n");
 482         // p.append("<dependencies>\n");
 483         // p.append("<dependency>\n");
 484         // p.append("\t<groupId>").append(groupId).append("</groupId>\n");
 485         // p.append("\t<artifactId>")
 486         // .append(ownSymbolicName.endsWith(".source") ?
 487         // RepoConstants.SOURCES_ARTIFACT_ID
 488         // : RepoConstants.BINARIES_ARTIFACT_ID)
 489         // .append("</artifactId>\n");
 490         // p.append("\t<version>").append(version).append("</version>\n");
 491         // p.append("\t<type>pom</type>\n");
 492         // p.append("\t<scope>import</scope>\n");
 493         // p.append("</dependency>\n");
 494         // p.append("</dependencies>\n");
 495         // p.append("</dependencyManagement>\n");
 496         //
 497         // p.append("</project>\n");
 498         // return p.toString();
 499         // }
 500
 501         /* SETTERS */
 502         public void setRepository(Repository repository) {
 503                 this.repository = repository;
 504         }
 505
 506         public void setCredentials(Credentials credentials) {
 507                 this.credentials = credentials;
 508         }
 509
 510         public void setWorkspace(String workspace) {
 511                 this.workspace = workspace;
 512         }
 513
 514         public void setGroupId(String groupId) {
 515                 this.groupId = groupId;
 516         }
 517
 518         public void setParentPomCoordinates(String parentPomCoordinates) {
 519                 this.parentPomCoordinates = parentPomCoordinates;
 520         }
 521
 522         public void setArtifactBasePath(String artifactBasePath) {
 523                 this.artifactBasePath = artifactBasePath;
 524         }
 525
 526         public void setVersion(String version) {
 527                 this.version = version;
 528         }
 529
 530         public void setExcludedSuffixes(List<String> excludedSuffixes) {
 531                 this.excludedSuffixes = excludedSuffixes;
 532         }
 533
 534         public void setArtifactIndexer(ArtifactIndexer artifactIndexer) {
 535                 this.artifactIndexer = artifactIndexer;
 536         }
 537 }