runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/GenerateBinaries.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.repo.maven;
  17
  18 import java.util.ArrayList;
  19 import java.util.HashMap;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.TreeSet;
  24
  25 import javax.jcr.Credentials;
  26 import javax.jcr.Node;
  27 import javax.jcr.NodeIterator;
  28 import javax.jcr.Repository;
  29 import javax.jcr.RepositoryException;
  30 import javax.jcr.Session;
  31
  32 import org.apache.commons.io.FilenameUtils;
  33 import org.apache.commons.logging.Log;
  34 import org.apache.commons.logging.LogFactory;
  35 import org.argeo.ArgeoMonitor;
  36 import org.argeo.jcr.JcrUtils;
  37 import org.argeo.slc.SlcException;
  38 import org.argeo.slc.aether.ArtifactIdComparator;
  39 import org.argeo.slc.jcr.SlcNames;
  40 import org.argeo.slc.jcr.SlcTypes;
  41 import org.argeo.slc.repo.ArtifactIndexer;
  42 import org.argeo.slc.repo.RepoConstants;
  43 import org.argeo.slc.repo.RepoUtils;
  44 import org.argeo.slc.repo.osgi.OsgiProfile;
  45 import org.osgi.framework.Constants;
  46 import org.osgi.framework.Version;
  47 import org.sonatype.aether.artifact.Artifact;
  48 import org.sonatype.aether.util.artifact.DefaultArtifact;
  49
  50 /**
  51  * Make sure that all JCR metadata and Maven metadata are consistent for this
  52  * group of OSGi bundles.
  53  */
  54 public class GenerateBinaries implements Runnable, SlcNames {
  55         private final static Log log = LogFactory.getLog(GenerateBinaries.class);
  56
  57         // Connection info
  58         private Repository repository;
  59         private Credentials credentials;
  60         private String workspace;
  61
  62         // Business info
  63         private String groupId;
  64         private String parentPomCoordinates;
  65         private Boolean overridePoms = false;
  66         private String version = null;
  67
  68         // Constants
  69         private String artifactBasePath = RepoConstants.DEFAULT_ARTIFACTS_BASE_PATH;
  70         private List<String> excludedSuffixes = new ArrayList<String>();
  71         /** TODO make it more generic */
  72         private List<String> systemPackages = OsgiProfile.PROFILE_JAVA_SE_1_6
  73                         .getSystemPackages();
  74
  75         private ArtifactIndexer artifactIndexer = new ArtifactIndexer();
  76
  77         // Local indexes
  78         private Map<String, String> packagesToSymbolicNames = new HashMap<String, String>();
  79         private Map<String, Node> symbolicNamesToNodes = new HashMap<String, Node>();
  80         private Set<Artifact> binaries = new TreeSet<Artifact>(
  81                         new ArtifactIdComparator());
  82         private Set<Artifact> sources = new TreeSet<Artifact>(
  83                         new ArtifactIdComparator());
  84         private Node allArtifactsHighestVersion;
  85
  86         public void run() {
  87                 Session session = null;
  88                 try {
  89                         session = repository.login(workspace);
  90                         Node groupNode = session.getNode(MavenConventionsUtils.groupPath(
  91                                         artifactBasePath, groupId));
  92                         internalPreProcessing(groupNode, null);
  93                         processGroupNode(groupNode, null);
  94                 } catch (Exception e) {
  95                         throw new SlcException("Cannot normalize group " + groupId + " in "
  96                                         + workspace, e);
  97                 } finally {
  98                         JcrUtils.logoutQuietly(session);
  99                 }
 100         }
 101
 102         public static void processGroupNode(Node groupNode, String version,
 103                         Boolean overridePoms, ArgeoMonitor monitor)
 104                         throws RepositoryException {
 105                 // TODO set artifactsBase based on group node
 106                 GenerateBinaries gb = new GenerateBinaries();
 107                 String groupId = groupNode.getProperty(SlcNames.SLC_GROUP_BASE_ID)
 108                                 .getString();
 109                 gb.setGroupId(groupId);
 110                 gb.setVersion(version);
 111                 gb.setOverridePoms(overridePoms);
 112                 // TODO use already done pre-processing
 113                 gb.internalPreProcessing(groupNode, monitor);
 114                 gb.processGroupNode(groupNode, monitor);
 115         }
 116
 117         /** Only builds local indexes. Does not change anything in the local Session */
 118         public static GenerateBinaries preProcessGroupNode(Node groupNode,
 119                         ArgeoMonitor monitor) throws RepositoryException {
 120                 // TODO set artifactsBase based on group node
 121                 GenerateBinaries gb = new GenerateBinaries();
 122                 String groupId = groupNode.getProperty(SlcNames.SLC_GROUP_BASE_ID)
 123                                 .getString();
 124                 gb.setGroupId(groupId);
 125                 // gb.setVersion(version);
 126                 // gb.setOverridePoms(overridePoms);
 127                 gb.internalPreProcessing(groupNode, monitor);
 128                 return gb;
 129         }
 130
 131         // exposes indexes. to display results of the pre-processing phase.
 132         public Set<Artifact> getBinaries() {
 133                 return binaries;
 134         }
 135
 136         public Artifact getHighestArtifactVersion() throws RepositoryException {
 137                 return allArtifactsHighestVersion == null ? null : RepoUtils
 138                                 .asArtifact(allArtifactsHighestVersion);
 139         }
 140
 141         protected void internalPreProcessing(Node groupNode, ArgeoMonitor monitor)
 142                         throws RepositoryException {
 143                 if (monitor != null)
 144                         monitor.subTask("Pre processing group " + groupId);
 145
 146                 // Process all direct children nodes,
 147                 // gathering latest versions of all artifact base
 148                 allArtifactsHighestVersion = null;
 149                 // Session session = groupNode.getSession();
 150                 aBases: for (NodeIterator aBases = groupNode.getNodes(); aBases
 151                                 .hasNext();) {
 152                         Node aBase = aBases.nextNode();
 153                         if (aBase.isNodeType(SlcTypes.SLC_ARTIFACT_BASE)) {
 154                                 Node highestAVersion = getArtifactLatestVersion(aBase);
 155                                 if (highestAVersion == null)
 156                                         continue aBases;
 157                                 else {
 158                                         // retrieve relevant child node
 159                                         for (NodeIterator files = highestAVersion.getNodes(); files
 160                                                         .hasNext();) {
 161                                                 Node file = files.nextNode();
 162                                                 if (file.isNodeType(SlcTypes.SLC_BUNDLE_ARTIFACT)) {
 163                                                         preProcessBundleArtifact(file);
 164                                                         if (log.isDebugEnabled())
 165                                                                 log.debug("Pre-processed " + file.getName());
 166                                                 }
 167                                         }
 168                                 }
 169                         }
 170                 }
 171                 if (log.isDebugEnabled()) {
 172                         int bundleCount = symbolicNamesToNodes.size();
 173                         log.debug("" + bundleCount + " bundles have been indexed for "
 174                                         + groupId);
 175                 }
 176         }
 177
 178         /** Does the real job : writes JCR META-DATA and generates binaries */
 179         protected void processGroupNode(Node groupNode, ArgeoMonitor monitor)
 180                         throws RepositoryException {
 181                 if (monitor != null)
 182                         monitor.subTask("Processing group " + groupId);
 183
 184                 Session session = groupNode.getSession();
 185
 186                 // if version not set or empty, use the highest version
 187                 // useful when indexing a product maven repository where
 188                 // all artifacts have the same version for a given release
 189                 // => the version can then be left empty
 190                 if (version == null || version.trim().equals(""))
 191                         if (allArtifactsHighestVersion != null)
 192                                 version = allArtifactsHighestVersion.getProperty(
 193                                                 SLC_ARTIFACT_VERSION).getString();
 194                         else
 195                                 throw new SlcException("Group version " + version
 196                                                 + " is empty.");
 197
 198                 int bundleCount = symbolicNamesToNodes.size();
 199
 200                 int count = 1;
 201                 for (Node bundleNode : symbolicNamesToNodes.values()) {
 202                         processBundleArtifact(bundleNode);
 203                         bundleNode.getSession().save();
 204                         if (log.isDebugEnabled())
 205                                 log.debug(count + "/" + bundleCount + " Processed "
 206                                                 + bundleNode.getName());
 207                         count++;
 208                 }
 209
 210                 // indexes
 211                 Set<Artifact> indexes = new TreeSet<Artifact>(
 212                                 new ArtifactIdComparator());
 213                 Artifact indexArtifact = writeIndex(session,
 214                                 RepoConstants.BINARIES_ARTIFACT_ID, binaries);
 215                 indexes.add(indexArtifact);
 216                 indexArtifact = writeIndex(session, RepoConstants.SOURCES_ARTIFACT_ID,
 217                                 sources);
 218                 indexes.add(indexArtifact);
 219                 // sdk
 220                 writeIndex(session, RepoConstants.SDK_ARTIFACT_ID, indexes);
 221                 if (monitor != null)
 222                         monitor.worked(1);
 223         }
 224
 225         // Helpers
 226         private Node getArtifactLatestVersion(Node artifactBase) {
 227                 try {
 228                         Node highestAVersion = null;
 229                         for (NodeIterator aVersions = artifactBase.getNodes(); aVersions
 230                                         .hasNext();) {
 231                                 Node aVersion = aVersions.nextNode();
 232                                 if (aVersion.isNodeType(SlcTypes.SLC_ARTIFACT_VERSION_BASE)) {
 233                                         if (highestAVersion == null) {
 234                                                 highestAVersion = aVersion;
 235                                                 if (allArtifactsHighestVersion == null)
 236                                                         allArtifactsHighestVersion = aVersion;
 237                                                 // Correctly handle following arrival order:
 238                                                 // Name1 - V1, name2 - V3
 239                                                 else {
 240                                                         Version cachedHighestVersion = extractOsgiVersion(allArtifactsHighestVersion);
 241                                                         Version currVersion = extractOsgiVersion(aVersion);
 242                                                         if (currVersion.compareTo(cachedHighestVersion) > 0)
 243                                                                 allArtifactsHighestVersion = aVersion;
 244                                                 }
 245                                         } else {
 246                                                 Version currVersion = extractOsgiVersion(aVersion);
 247                                                 Version currentHighestVersion = extractOsgiVersion(highestAVersion);
 248                                                 if (currVersion.compareTo(currentHighestVersion) > 0) {
 249                                                         highestAVersion = aVersion;
 250                                                 }
 251                                                 if (currVersion
 252                                                                 .compareTo(extractOsgiVersion(allArtifactsHighestVersion)) > 0) {
 253                                                         allArtifactsHighestVersion = aVersion;
 254                                                 }
 255                                         }
 256
 257                                 }
 258                         }
 259                         return highestAVersion;
 260                 } catch (RepositoryException re) {
 261                         throw new SlcException("Unable to get latest version for node "
 262                                         + artifactBase, re);
 263                 }
 264         }
 265
 266         private Version extractOsgiVersion(Node artifactVersion)
 267                         throws RepositoryException {
 268                 String rawVersion = artifactVersion.getProperty(SLC_ARTIFACT_VERSION)
 269                                 .getString();
 270                 String cleanVersion = rawVersion.replace("-SNAPSHOT", ".SNAPSHOT");
 271                 return new Version(cleanVersion);
 272         }
 273
 274         protected void preProcessBundleArtifact(Node bundleNode)
 275                         throws RepositoryException {
 276
 277                 String symbolicName = JcrUtils.get(bundleNode, SLC_SYMBOLIC_NAME);
 278                 if (symbolicName.endsWith(".source")) {
 279                         // TODO make a shared node with classifier 'sources'?
 280                         String bundleName = RepoUtils
 281                                         .extractBundleNameFromSourceName(symbolicName);
 282                         for (String excludedSuffix : excludedSuffixes) {
 283                                 if (bundleName.endsWith(excludedSuffix))
 284                                         return;// skip adding to sources
 285                         }
 286                         sources.add(RepoUtils.asArtifact(bundleNode));
 287                         return;
 288                 }
 289
 290                 NodeIterator exportPackages = bundleNode.getNodes(SLC_
 291                                 + Constants.EXPORT_PACKAGE);
 292                 while (exportPackages.hasNext()) {
 293                         Node exportPackage = exportPackages.nextNode();
 294                         String pkg = JcrUtils.get(exportPackage, SLC_NAME);
 295                         packagesToSymbolicNames.put(pkg, symbolicName);
 296                 }
 297
 298                 symbolicNamesToNodes.put(symbolicName, bundleNode);
 299                 for (String excludedSuffix : excludedSuffixes) {
 300                         if (symbolicName.endsWith(excludedSuffix))
 301                                 return;// skip adding to binaries
 302                 }
 303                 binaries.add(RepoUtils.asArtifact(bundleNode));
 304
 305                 if (bundleNode.getSession().hasPendingChanges())
 306                         throw new SlcException("Pending changes in the session, "
 307                                         + "this should not be true here.");
 308                 // bundleNode.getSession().save();
 309         }
 310
 311         protected void processBundleArtifact(Node bundleNode)
 312                         throws RepositoryException {
 313                 Node artifactFolder = bundleNode.getParent();
 314                 String baseName = FilenameUtils.getBaseName(bundleNode.getName());
 315
 316                 // pom
 317                 String pomName = baseName + ".pom";
 318                 if (artifactFolder.hasNode(pomName) && !overridePoms)
 319                         return;// skip
 320
 321                 String pom = generatePomForBundle(bundleNode);
 322                 Node pomNode = JcrUtils.copyBytesAsFile(artifactFolder, pomName,
 323                                 pom.getBytes());
 324                 // checksum
 325                 String bundleSha = JcrUtils.checksumFile(bundleNode, "SHA-1");
 326                 JcrUtils.copyBytesAsFile(artifactFolder,
 327                                 bundleNode.getName() + ".sha1", bundleSha.getBytes());
 328                 String pomSha = JcrUtils.checksumFile(pomNode, "SHA-1");
 329                 JcrUtils.copyBytesAsFile(artifactFolder, pomNode.getName() + ".sha1",
 330                                 pomSha.getBytes());
 331         }
 332
 333         // Writers
 334         private Artifact writeIndex(Session session, String artifactId,
 335                         Set<Artifact> artifacts) throws RepositoryException {
 336                 Artifact artifact = new DefaultArtifact(groupId, artifactId, "pom",
 337                                 version);
 338                 Artifact parentArtifact = parentPomCoordinates != null ? new DefaultArtifact(
 339                                 parentPomCoordinates) : null;
 340                 String pom = MavenConventionsUtils.artifactsAsDependencyPom(artifact,
 341                                 artifacts, parentArtifact);
 342                 Node node = RepoUtils.copyBytesAsArtifact(
 343                                 session.getNode(artifactBasePath), artifact, pom.getBytes());
 344                 artifactIndexer.index(node);
 345
 346                 // TODO factorize
 347                 String pomSha = JcrUtils.checksumFile(node, "SHA-1");
 348                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".sha1",
 349                                 pomSha.getBytes());
 350                 String pomMd5 = JcrUtils.checksumFile(node, "MD5");
 351                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".md5",
 352                                 pomMd5.getBytes());
 353                 session.save();
 354                 return artifact;
 355         }
 356
 357         private String generatePomForBundle(Node n) throws RepositoryException {
 358                 String ownSymbolicName = JcrUtils.get(n, SLC_SYMBOLIC_NAME);
 359
 360                 StringBuffer p = new StringBuffer();
 361
 362                 // XML header
 363                 p.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
 364                 p.append("<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd\">\n");
 365                 p.append("<modelVersion>4.0.0</modelVersion>");
 366
 367                 // Artifact
 368                 p.append("<groupId>").append(JcrUtils.get(n, SLC_GROUP_ID))
 369                                 .append("</groupId>\n");
 370                 p.append("<artifactId>").append(JcrUtils.get(n, SLC_ARTIFACT_ID))
 371                                 .append("</artifactId>\n");
 372                 p.append("<version>").append(JcrUtils.get(n, SLC_ARTIFACT_VERSION))
 373                                 .append("</version>\n");
 374                 p.append("<packaging>pom</packaging>\n");
 375                 if (n.hasProperty(SLC_ + Constants.BUNDLE_NAME))
 376                         p.append("<name>")
 377                                         .append(JcrUtils.get(n, SLC_ + Constants.BUNDLE_NAME))
 378                                         .append("</name>\n");
 379                 if (n.hasProperty(SLC_ + Constants.BUNDLE_DESCRIPTION))
 380                         p.append("<description>")
 381                                         .append(JcrUtils
 382                                                         .get(n, SLC_ + Constants.BUNDLE_DESCRIPTION))
 383                                         .append("</description>\n");
 384
 385                 // Dependencies
 386                 Set<String> dependenciesSymbolicNames = new TreeSet<String>();
 387                 Set<String> optionalSymbolicNames = new TreeSet<String>();
 388                 NodeIterator importPackages = n.getNodes(SLC_
 389                                 + Constants.IMPORT_PACKAGE);
 390                 while (importPackages.hasNext()) {
 391                         Node importPackage = importPackages.nextNode();
 392                         String pkg = JcrUtils.get(importPackage, SLC_NAME);
 393                         if (packagesToSymbolicNames.containsKey(pkg)) {
 394                                 String dependencySymbolicName = packagesToSymbolicNames
 395                                                 .get(pkg);
 396                                 if (JcrUtils.check(importPackage, SLC_OPTIONAL))
 397                                         optionalSymbolicNames.add(dependencySymbolicName);
 398                                 else
 399                                         dependenciesSymbolicNames.add(dependencySymbolicName);
 400                         } else {
 401                                 if (!JcrUtils.check(importPackage, SLC_OPTIONAL)
 402                                                 && !systemPackages.contains(pkg))
 403                                         log.warn("No bundle found for pkg " + pkg);
 404                         }
 405                 }
 406
 407                 if (n.hasNode(SLC_ + Constants.FRAGMENT_HOST)) {
 408                         String fragmentHost = JcrUtils.get(
 409                                         n.getNode(SLC_ + Constants.FRAGMENT_HOST),
 410                                         SLC_SYMBOLIC_NAME);
 411                         dependenciesSymbolicNames.add(fragmentHost);
 412                 }
 413
 414                 // TODO require bundles
 415
 416                 List<Node> dependencyNodes = new ArrayList<Node>();
 417                 for (String depSymbName : dependenciesSymbolicNames) {
 418                         if (depSymbName.equals(ownSymbolicName))
 419                                 continue;// skip self
 420
 421                         if (symbolicNamesToNodes.containsKey(depSymbName))
 422                                 dependencyNodes.add(symbolicNamesToNodes.get(depSymbName));
 423                         else
 424                                 log.warn("Could not find node for " + depSymbName);
 425                 }
 426                 List<Node> optionalDependencyNodes = new ArrayList<Node>();
 427                 for (String depSymbName : optionalSymbolicNames) {
 428                         if (symbolicNamesToNodes.containsKey(depSymbName))
 429                                 optionalDependencyNodes.add(symbolicNamesToNodes
 430                                                 .get(depSymbName));
 431                         else
 432                                 log.warn("Could not find node for " + depSymbName);
 433                 }
 434
 435                 p.append("<dependencies>\n");
 436                 for (Node dependencyNode : dependencyNodes) {
 437                         p.append("<dependency>\n");
 438                         p.append("\t<groupId>")
 439                                         .append(JcrUtils.get(dependencyNode, SLC_GROUP_ID))
 440                                         .append("</groupId>\n");
 441                         p.append("\t<artifactId>")
 442                                         .append(JcrUtils.get(dependencyNode, SLC_ARTIFACT_ID))
 443                                         .append("</artifactId>\n");
 444                         p.append("</dependency>\n");
 445                 }
 446
 447                 if (optionalDependencyNodes.size() > 0)
 448                         p.append("<!-- OPTIONAL -->\n");
 449                 for (Node dependencyNode : optionalDependencyNodes) {
 450                         p.append("<dependency>\n");
 451                         p.append("\t<groupId>")
 452                                         .append(JcrUtils.get(dependencyNode, SLC_GROUP_ID))
 453                                         .append("</groupId>\n");
 454                         p.append("\t<artifactId>")
 455                                         .append(JcrUtils.get(dependencyNode, SLC_ARTIFACT_ID))
 456                                         .append("</artifactId>\n");
 457                         p.append("\t<optional>true</optional>\n");
 458                         p.append("</dependency>\n");
 459                 }
 460                 p.append("</dependencies>\n");
 461
 462                 // Dependency management
 463                 p.append("<dependencyManagement>\n");
 464                 p.append("<dependencies>\n");
 465                 p.append("<dependency>\n");
 466                 p.append("\t<groupId>").append(groupId).append("</groupId>\n");
 467                 p.append("\t<artifactId>")
 468                                 .append(ownSymbolicName.endsWith(".source") ? RepoConstants.SOURCES_ARTIFACT_ID
 469                                                 : RepoConstants.BINARIES_ARTIFACT_ID)
 470                                 .append("</artifactId>\n");
 471                 p.append("\t<version>").append(version).append("</version>\n");
 472                 p.append("\t<type>pom</type>\n");
 473                 p.append("\t<scope>import</scope>\n");
 474                 p.append("</dependency>\n");
 475                 p.append("</dependencies>\n");
 476                 p.append("</dependencyManagement>\n");
 477
 478                 p.append("</project>\n");
 479                 return p.toString();
 480         }
 481
 482         /* SETTERS */
 483         public void setRepository(Repository repository) {
 484                 this.repository = repository;
 485         }
 486
 487         public void setWorkspace(String workspace) {
 488                 this.workspace = workspace;
 489         }
 490
 491         public void setGroupId(String groupId) {
 492                 this.groupId = groupId;
 493         }
 494
 495         public void setParentPomCoordinates(String parentPomCoordinates) {
 496                 this.parentPomCoordinates = parentPomCoordinates;
 497         }
 498
 499         public void setArtifactBasePath(String artifactBasePath) {
 500                 this.artifactBasePath = artifactBasePath;
 501         }
 502
 503         public void setVersion(String version) {
 504                 this.version = version;
 505         }
 506
 507         public void setExcludedSuffixes(List<String> excludedSuffixes) {
 508                 this.excludedSuffixes = excludedSuffixes;
 509         }
 510
 511         public void setOverridePoms(Boolean overridePoms) {
 512                 this.overridePoms = overridePoms;
 513         }
 514
 515         public void setArtifactIndexer(ArtifactIndexer artifactIndexer) {
 516                 this.artifactIndexer = artifactIndexer;
 517         }
 518 }