runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/osgi/NormalizeGroup.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.repo.osgi;
  17
  18 import java.util.ArrayList;
  19 import java.util.HashMap;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.TreeSet;
  24
  25 import javax.jcr.Node;
  26 import javax.jcr.NodeIterator;
  27 import javax.jcr.Repository;
  28 import javax.jcr.RepositoryException;
  29 import javax.jcr.Session;
  30
  31 import org.apache.commons.io.FilenameUtils;
  32 import org.apache.commons.logging.Log;
  33 import org.apache.commons.logging.LogFactory;
  34 import org.argeo.ArgeoMonitor;
  35 import org.argeo.jcr.JcrUtils;
  36 import org.argeo.slc.SlcException;
  37 import org.argeo.slc.aether.ArtifactIdComparator;
  38 import org.argeo.slc.jcr.SlcNames;
  39 import org.argeo.slc.jcr.SlcTypes;
  40 import org.argeo.slc.repo.ArtifactIndexer;
  41 import org.argeo.slc.repo.RepoUtils;
  42 import org.argeo.slc.repo.maven.MavenConventionsUtils;
  43 import org.osgi.framework.Constants;
  44 import org.osgi.framework.Version;
  45 import org.sonatype.aether.artifact.Artifact;
  46 import org.sonatype.aether.util.artifact.DefaultArtifact;
  47
  48 /**
  49  * Make sure that all JCR metadata and Maven metadata are consistent for this
  50  * group of OSGi bundles.
  51  */
  52 public class NormalizeGroup implements Runnable, SlcNames {
  53         public final static String BINARIES_ARTIFACT_ID = "binaries";
  54         public final static String SOURCES_ARTIFACT_ID = "sources";
  55         public final static String SDK_ARTIFACT_ID = "sdk";
  56
  57         private final static Log log = LogFactory.getLog(NormalizeGroup.class);
  58
  59         private Repository repository;
  60         private String workspace;
  61         private String groupId;
  62         private Boolean overridePoms = false;
  63         private String artifactBasePath = "/";
  64         private String version = null;
  65         private String parentPomCoordinates;
  66
  67         private List<String> excludedSuffixes = new ArrayList<String>();
  68
  69         private ArtifactIndexer artifactIndexer = new ArtifactIndexer();
  70         // private JarFileIndexer jarFileIndexer = new JarFileIndexer();
  71
  72         /** TODO make it more generic */
  73         private List<String> systemPackages = OsgiProfile.PROFILE_JAVA_SE_1_6
  74                         .getSystemPackages();
  75
  76         // indexes
  77         private Map<String, String> packagesToSymbolicNames = new HashMap<String, String>();
  78         private Map<String, Node> symbolicNamesToNodes = new HashMap<String, Node>();
  79
  80         private Set<Artifact> binaries = new TreeSet<Artifact>(
  81                         new ArtifactIdComparator());
  82         private Set<Artifact> sources = new TreeSet<Artifact>(
  83                         new ArtifactIdComparator());
  84
  85         public void run() {
  86                 Session session = null;
  87                 try {
  88                         session = repository.login(workspace);
  89                         Node groupNode = session.getNode(MavenConventionsUtils.groupPath(
  90                                         artifactBasePath, groupId));
  91                         processGroupNode(groupNode, null);
  92                 } catch (Exception e) {
  93                         throw new SlcException("Cannot normalize group " + groupId + " in "
  94                                         + workspace, e);
  95                 } finally {
  96                         JcrUtils.logoutQuietly(session);
  97                 }
  98         }
  99
 100         public static void processGroupNode(Node groupNode, String version,
 101                         Boolean overridePoms, ArgeoMonitor monitor)
 102                         throws RepositoryException {
 103                 // TODO set artifactsBase based on group node
 104                 NormalizeGroup ng = new NormalizeGroup();
 105                 String groupId = groupNode.getProperty(SlcNames.SLC_GROUP_BASE_ID)
 106                                 .getString();
 107                 ng.setGroupId(groupId);
 108                 ng.setVersion(version);
 109                 ng.setOverridePoms(overridePoms);
 110                 ng.processGroupNode(groupNode, monitor);
 111         }
 112
 113         protected void processGroupNode(Node groupNode, ArgeoMonitor monitor)
 114                         throws RepositoryException {
 115                 if (monitor != null)
 116                         monitor.subTask("Group " + groupId);
 117                 Node allArtifactsHighestVersion = null;
 118                 Session session = groupNode.getSession();
 119                 aBases: for (NodeIterator aBases = groupNode.getNodes(); aBases
 120                                 .hasNext();) {
 121                         Node aBase = aBases.nextNode();
 122                         if (aBase.isNodeType(SlcTypes.SLC_ARTIFACT_BASE)) {
 123                                 Node highestAVersion = null;
 124                                 for (NodeIterator aVersions = aBase.getNodes(); aVersions
 125                                                 .hasNext();) {
 126                                         Node aVersion = aVersions.nextNode();
 127                                         if (aVersion.isNodeType(SlcTypes.SLC_ARTIFACT_VERSION_BASE)) {
 128                                                 if (highestAVersion == null) {
 129                                                         highestAVersion = aVersion;
 130                                                         if (allArtifactsHighestVersion == null)
 131                                                                 allArtifactsHighestVersion = aVersion;
 132
 133                                                 } else {
 134                                                         Version currVersion = extractOsgiVersion(aVersion);
 135                                                         Version currentHighestVersion = extractOsgiVersion(highestAVersion);
 136                                                         if (currVersion.compareTo(currentHighestVersion) > 0) {
 137                                                                 highestAVersion = aVersion;
 138                                                         }
 139                                                         if (currVersion
 140                                                                         .compareTo(extractOsgiVersion(allArtifactsHighestVersion)) > 0) {
 141                                                                 allArtifactsHighestVersion = aVersion;
 142                                                         }
 143                                                 }
 144
 145                                         }
 146
 147                                 }
 148                                 if (highestAVersion == null)
 149                                         continue aBases;
 150                                 for (NodeIterator files = highestAVersion.getNodes(); files
 151                                                 .hasNext();) {
 152                                         Node file = files.nextNode();
 153                                         if (file.isNodeType(SlcTypes.SLC_BUNDLE_ARTIFACT)) {
 154                                                 preProcessBundleArtifact(file);
 155                                                 file.getSession().save();
 156                                                 if (log.isDebugEnabled())
 157                                                         log.debug("Pre-processed " + file.getName());
 158                                         }
 159
 160                                 }
 161                         }
 162                 }
 163
 164                 // if version not set or empty, use the highets version
 165                 // useful when indexing a product maven repository where
 166                 // all artifacts have the same version for a given release
 167                 // => the version can then be left empty
 168                 if (version == null || version.trim().equals(""))
 169                         if (allArtifactsHighestVersion != null)
 170                                 version = allArtifactsHighestVersion.getProperty(
 171                                                 SLC_ARTIFACT_VERSION).getString();
 172                         else
 173                                 throw new SlcException("Group version " + version
 174                                                 + " is empty.");
 175
 176                 int bundleCount = symbolicNamesToNodes.size();
 177                 if (log.isDebugEnabled())
 178                         log.debug("Indexed " + bundleCount + " bundles");
 179
 180                 int count = 1;
 181                 for (Node bundleNode : symbolicNamesToNodes.values()) {
 182                         processBundleArtifact(bundleNode);
 183                         bundleNode.getSession().save();
 184                         if (log.isDebugEnabled())
 185                                 log.debug(count + "/" + bundleCount + " Processed "
 186                                                 + bundleNode.getName());
 187                         count++;
 188                 }
 189
 190                 // indexes
 191                 Set<Artifact> indexes = new TreeSet<Artifact>(
 192                                 new ArtifactIdComparator());
 193                 Artifact indexArtifact = writeIndex(session, BINARIES_ARTIFACT_ID,
 194                                 binaries);
 195                 indexes.add(indexArtifact);
 196                 indexArtifact = writeIndex(session, SOURCES_ARTIFACT_ID, sources);
 197                 indexes.add(indexArtifact);
 198                 // sdk
 199                 writeIndex(session, SDK_ARTIFACT_ID, indexes);
 200                 if (monitor != null)
 201                         monitor.worked(1);
 202         }
 203
 204         private Version extractOsgiVersion(Node artifactVersion)
 205                         throws RepositoryException {
 206                 String rawVersion = artifactVersion.getProperty(SLC_ARTIFACT_VERSION)
 207                                 .getString();
 208                 String cleanVersion = rawVersion.replace("-SNAPSHOT", ".SNAPSHOT");
 209                 return new Version(cleanVersion);
 210         }
 211
 212         private Artifact writeIndex(Session session, String artifactId,
 213                         Set<Artifact> artifacts) throws RepositoryException {
 214                 Artifact artifact = new DefaultArtifact(groupId, artifactId, "pom",
 215                                 version);
 216                 Artifact parentArtifact = parentPomCoordinates != null ? new DefaultArtifact(
 217                                 parentPomCoordinates) : null;
 218                 String pom = MavenConventionsUtils.artifactsAsDependencyPom(artifact,
 219                                 artifacts, parentArtifact);
 220                 Node node = RepoUtils.copyBytesAsArtifact(
 221                                 session.getNode(artifactBasePath), artifact, pom.getBytes());
 222                 artifactIndexer.index(node);
 223
 224                 // TODO factorize
 225                 String pomSha = JcrUtils.checksumFile(node, "SHA-1");
 226                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".sha1",
 227                                 pomSha.getBytes());
 228                 String pomMd5 = JcrUtils.checksumFile(node, "MD5");
 229                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".md5",
 230                                 pomMd5.getBytes());
 231                 session.save();
 232                 return artifact;
 233         }
 234
 235         protected void preProcessBundleArtifact(Node bundleNode)
 236                         throws RepositoryException {
 237                 // we assume nodes are already indexed
 238                 // artifactIndexer.index(bundleNode);
 239                 // jarFileIndexer.index(bundleNode);
 240
 241                 String symbolicName = JcrUtils.get(bundleNode, SLC_SYMBOLIC_NAME);
 242
 243                 if (symbolicName.endsWith(".source")) {
 244                         // TODO make a shared node with classifier 'sources'?
 245                         String bundleName = RepoUtils
 246                                         .extractBundleNameFromSourceName(symbolicName);
 247                         for (String excludedSuffix : excludedSuffixes) {
 248                                 if (bundleName.endsWith(excludedSuffix))
 249                                         return;// skip adding to sources
 250                         }
 251                         sources.add(RepoUtils.asArtifact(bundleNode));
 252                         return;
 253                 }
 254
 255                 NodeIterator exportPackages = bundleNode.getNodes(SLC_
 256                                 + Constants.EXPORT_PACKAGE);
 257                 while (exportPackages.hasNext()) {
 258                         Node exportPackage = exportPackages.nextNode();
 259                         String pkg = JcrUtils.get(exportPackage, SLC_NAME);
 260                         packagesToSymbolicNames.put(pkg, symbolicName);
 261                 }
 262
 263                 symbolicNamesToNodes.put(symbolicName, bundleNode);
 264                 for (String excludedSuffix : excludedSuffixes) {
 265                         if (symbolicName.endsWith(excludedSuffix))
 266                                 return;// skip adding to binaries
 267                 }
 268                 binaries.add(RepoUtils.asArtifact(bundleNode));
 269
 270                 if (bundleNode.getSession().hasPendingChanges())
 271                         bundleNode.getSession().save();
 272         }
 273
 274         protected void processBundleArtifact(Node bundleNode)
 275                         throws RepositoryException {
 276                 Node artifactFolder = bundleNode.getParent();
 277                 String baseName = FilenameUtils.getBaseName(bundleNode.getName());
 278
 279                 // pom
 280                 String pomName = baseName + ".pom";
 281                 if (artifactFolder.hasNode(pomName) && !overridePoms)
 282                         return;// skip
 283
 284                 String pom = generatePomForBundle(bundleNode);
 285                 Node pomNode = JcrUtils.copyBytesAsFile(artifactFolder, pomName,
 286                                 pom.getBytes());
 287                 // checksum
 288                 String bundleSha = JcrUtils.checksumFile(bundleNode, "SHA-1");
 289                 JcrUtils.copyBytesAsFile(artifactFolder,
 290                                 bundleNode.getName() + ".sha1", bundleSha.getBytes());
 291                 String pomSha = JcrUtils.checksumFile(pomNode, "SHA-1");
 292                 JcrUtils.copyBytesAsFile(artifactFolder, pomNode.getName() + ".sha1",
 293                                 pomSha.getBytes());
 294         }
 295
 296         private String generatePomForBundle(Node n) throws RepositoryException {
 297                 String ownSymbolicName = JcrUtils.get(n, SLC_SYMBOLIC_NAME);
 298
 299                 StringBuffer p = new StringBuffer();
 300
 301                 // XML header
 302                 p.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
 303                 p.append("<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd\">\n");
 304                 p.append("<modelVersion>4.0.0</modelVersion>");
 305
 306                 // Artifact
 307                 // p.append("<parent><groupId>org.argeo</groupId><artifactId>parent</artifactId><version>1.2.0</version></parent>\n");
 308                 p.append("<groupId>").append(JcrUtils.get(n, SLC_GROUP_ID))
 309                                 .append("</groupId>\n");
 310                 p.append("<artifactId>").append(JcrUtils.get(n, SLC_ARTIFACT_ID))
 311                                 .append("</artifactId>\n");
 312                 p.append("<version>").append(JcrUtils.get(n, SLC_ARTIFACT_VERSION))
 313                                 .append("</version>\n");
 314                 p.append("<packaging>pom</packaging>\n");
 315                 if (n.hasProperty(SLC_ + Constants.BUNDLE_NAME))
 316                         p.append("<name>")
 317                                         .append(JcrUtils.get(n, SLC_ + Constants.BUNDLE_NAME))
 318                                         .append("</name>\n");
 319                 if (n.hasProperty(SLC_ + Constants.BUNDLE_DESCRIPTION))
 320                         p.append("<description>")
 321                                         .append(JcrUtils
 322                                                         .get(n, SLC_ + Constants.BUNDLE_DESCRIPTION))
 323                                         .append("</description>\n");
 324
 325                 // Dependencies
 326                 Set<String> dependenciesSymbolicNames = new TreeSet<String>();
 327                 Set<String> optionalSymbolicNames = new TreeSet<String>();
 328                 NodeIterator importPackages = n.getNodes(SLC_
 329                                 + Constants.IMPORT_PACKAGE);
 330                 while (importPackages.hasNext()) {
 331                         Node importPackage = importPackages.nextNode();
 332                         String pkg = JcrUtils.get(importPackage, SLC_NAME);
 333                         if (packagesToSymbolicNames.containsKey(pkg)) {
 334                                 String dependencySymbolicName = packagesToSymbolicNames
 335                                                 .get(pkg);
 336                                 if (JcrUtils.check(importPackage, SLC_OPTIONAL))
 337                                         optionalSymbolicNames.add(dependencySymbolicName);
 338                                 else
 339                                         dependenciesSymbolicNames.add(dependencySymbolicName);
 340                         } else {
 341                                 if (!JcrUtils.check(importPackage, SLC_OPTIONAL)
 342                                                 && !systemPackages.contains(pkg))
 343                                         log.warn("No bundle found for pkg " + pkg);
 344                         }
 345                 }
 346
 347                 if (n.hasNode(SLC_ + Constants.FRAGMENT_HOST)) {
 348                         String fragmentHost = JcrUtils.get(
 349                                         n.getNode(SLC_ + Constants.FRAGMENT_HOST),
 350                                         SLC_SYMBOLIC_NAME);
 351                         dependenciesSymbolicNames.add(fragmentHost);
 352                 }
 353
 354                 // TODO require bundles
 355
 356                 List<Node> dependencyNodes = new ArrayList<Node>();
 357                 for (String depSymbName : dependenciesSymbolicNames) {
 358                         if (depSymbName.equals(ownSymbolicName))
 359                                 continue;// skip self
 360
 361                         if (symbolicNamesToNodes.containsKey(depSymbName))
 362                                 dependencyNodes.add(symbolicNamesToNodes.get(depSymbName));
 363                         else
 364                                 log.warn("Could not find node for " + depSymbName);
 365                 }
 366                 List<Node> optionalDependencyNodes = new ArrayList<Node>();
 367                 for (String depSymbName : optionalSymbolicNames) {
 368                         if (symbolicNamesToNodes.containsKey(depSymbName))
 369                                 optionalDependencyNodes.add(symbolicNamesToNodes
 370                                                 .get(depSymbName));
 371                         else
 372                                 log.warn("Could not find node for " + depSymbName);
 373                 }
 374
 375                 p.append("<dependencies>\n");
 376                 for (Node dependencyNode : dependencyNodes) {
 377                         p.append("<dependency>\n");
 378                         p.append("\t<groupId>")
 379                                         .append(JcrUtils.get(dependencyNode, SLC_GROUP_ID))
 380                                         .append("</groupId>\n");
 381                         p.append("\t<artifactId>")
 382                                         .append(JcrUtils.get(dependencyNode, SLC_ARTIFACT_ID))
 383                                         .append("</artifactId>\n");
 384                         p.append("</dependency>\n");
 385                 }
 386
 387                 if (optionalDependencyNodes.size() > 0)
 388                         p.append("<!-- OPTIONAL -->\n");
 389                 for (Node dependencyNode : optionalDependencyNodes) {
 390                         p.append("<dependency>\n");
 391                         p.append("\t<groupId>")
 392                                         .append(JcrUtils.get(dependencyNode, SLC_GROUP_ID))
 393                                         .append("</groupId>\n");
 394                         p.append("\t<artifactId>")
 395                                         .append(JcrUtils.get(dependencyNode, SLC_ARTIFACT_ID))
 396                                         .append("</artifactId>\n");
 397                         p.append("\t<optional>true</optional>\n");
 398                         p.append("</dependency>\n");
 399                 }
 400                 p.append("</dependencies>\n");
 401
 402                 // Dependency management
 403                 p.append("<dependencyManagement>\n");
 404                 p.append("<dependencies>\n");
 405                 p.append("<dependency>\n");
 406                 p.append("\t<groupId>").append(groupId).append("</groupId>\n");
 407                 p.append("\t<artifactId>")
 408                                 .append(ownSymbolicName.endsWith(".source") ? SOURCES_ARTIFACT_ID
 409                                                 : BINARIES_ARTIFACT_ID).append("</artifactId>\n");
 410                 p.append("\t<version>").append(version).append("</version>\n");
 411                 p.append("\t<type>pom</type>\n");
 412                 p.append("\t<scope>import</scope>\n");
 413                 p.append("</dependency>\n");
 414                 p.append("</dependencies>\n");
 415                 p.append("</dependencyManagement>\n");
 416
 417                 p.append("</project>\n");
 418                 return p.toString();
 419         }
 420
 421         public void setRepository(Repository repository) {
 422                 this.repository = repository;
 423         }
 424
 425         public void setWorkspace(String workspace) {
 426                 this.workspace = workspace;
 427         }
 428
 429         public void setGroupId(String groupId) {
 430                 this.groupId = groupId;
 431         }
 432
 433         public void setParentPomCoordinates(String parentPomCoordinates) {
 434                 this.parentPomCoordinates = parentPomCoordinates;
 435         }
 436
 437         public void setArtifactBasePath(String artifactBasePath) {
 438                 this.artifactBasePath = artifactBasePath;
 439         }
 440
 441         public void setVersion(String version) {
 442                 this.version = version;
 443         }
 444
 445         public void setExcludedSuffixes(List<String> excludedSuffixes) {
 446                 this.excludedSuffixes = excludedSuffixes;
 447         }
 448
 449         public void setOverridePoms(Boolean overridePoms) {
 450                 this.overridePoms = overridePoms;
 451         }
 452
 453         public void setArtifactIndexer(ArtifactIndexer artifactIndexer) {
 454                 this.artifactIndexer = artifactIndexer;
 455         }
 456 }