runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/osgi/NormalizeGroup.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.repo.osgi;
  17
  18 import java.util.ArrayList;
  19 import java.util.HashMap;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.StringTokenizer;
  24 import java.util.TreeSet;
  25
  26 import javax.jcr.Node;
  27 import javax.jcr.NodeIterator;
  28 import javax.jcr.Repository;
  29 import javax.jcr.RepositoryException;
  30 import javax.jcr.Session;
  31
  32 import org.apache.commons.io.FilenameUtils;
  33 import org.apache.commons.logging.Log;
  34 import org.apache.commons.logging.LogFactory;
  35 import org.argeo.ArgeoMonitor;
  36 import org.argeo.jcr.JcrUtils;
  37 import org.argeo.slc.SlcException;
  38 import org.argeo.slc.aether.ArtifactIdComparator;
  39 import org.argeo.slc.jcr.SlcNames;
  40 import org.argeo.slc.jcr.SlcTypes;
  41 import org.argeo.slc.repo.ArtifactIndexer;
  42 import org.argeo.slc.repo.RepoConstants;
  43 import org.argeo.slc.repo.RepoUtils;
  44 import org.argeo.slc.repo.maven.MavenConventionsUtils;
  45 import org.osgi.framework.Constants;
  46 import org.osgi.framework.Version;
  47 import org.sonatype.aether.artifact.Artifact;
  48 import org.sonatype.aether.util.artifact.DefaultArtifact;
  49
  50 /**
  51  * Make sure that all JCR metadata and Maven metadata are consistent for this
  52  * group of OSGi bundles.
  53  *
  54  * The job is now done via the various {@code NodeIndexer} of the
  55  * WorkspaceManager. TODO import dependencies in the workspace.
  56  */
  57 @Deprecated
  58 public class NormalizeGroup implements Runnable, SlcNames {
  59         private final static Log log = LogFactory.getLog(NormalizeGroup.class);
  60
  61         private Repository repository;
  62         private String workspace;
  63         private String groupId;
  64         private Boolean overridePoms = false;
  65         private String artifactBasePath = "/";
  66         private String version = null;
  67         private String parentPomCoordinates;
  68
  69         private List<String> excludedSuffixes = new ArrayList<String>();
  70
  71         private ArtifactIndexer artifactIndexer = new ArtifactIndexer();
  72         // private JarFileIndexer jarFileIndexer = new JarFileIndexer();
  73
  74         /** TODO make it more generic */
  75         private List<String> systemPackages = OsgiProfile.PROFILE_JAVA_SE_1_6
  76                         .getSystemPackages();
  77
  78         // indexes
  79         private Map<String, String> packagesToSymbolicNames = new HashMap<String, String>();
  80         private Map<String, Node> symbolicNamesToNodes = new HashMap<String, Node>();
  81
  82         private Set<Artifact> binaries = new TreeSet<Artifact>(
  83                         new ArtifactIdComparator());
  84         private Set<Artifact> sources = new TreeSet<Artifact>(
  85                         new ArtifactIdComparator());
  86
  87         public void run() {
  88                 Session session = null;
  89                 try {
  90                         session = repository.login(workspace);
  91                         Node groupNode = session.getNode(MavenConventionsUtils.groupPath(
  92                                         artifactBasePath, groupId));
  93                         processGroupNode(groupNode, null);
  94                 } catch (Exception e) {
  95                         throw new SlcException("Cannot normalize group " + groupId + " in "
  96                                         + workspace, e);
  97                 } finally {
  98                         JcrUtils.logoutQuietly(session);
  99                 }
 100         }
 101
 102         public static void processGroupNode(Node groupNode, String version,
 103                         Boolean overridePoms, ArgeoMonitor monitor)
 104                         throws RepositoryException {
 105                 // TODO set artifactsBase based on group node
 106                 NormalizeGroup ng = new NormalizeGroup();
 107                 String groupId = groupNode.getProperty(SlcNames.SLC_GROUP_BASE_ID)
 108                                 .getString();
 109                 ng.setGroupId(groupId);
 110                 ng.setVersion(version);
 111                 ng.setOverridePoms(overridePoms);
 112                 ng.processGroupNode(groupNode, monitor);
 113         }
 114
 115         protected void processGroupNode(Node groupNode, ArgeoMonitor monitor)
 116                         throws RepositoryException {
 117                 if (monitor != null)
 118                         monitor.subTask("Group " + groupId);
 119                 Node allArtifactsHighestVersion = null;
 120                 Session session = groupNode.getSession();
 121                 aBases: for (NodeIterator aBases = groupNode.getNodes(); aBases
 122                                 .hasNext();) {
 123                         Node aBase = aBases.nextNode();
 124                         if (aBase.isNodeType(SlcTypes.SLC_ARTIFACT_BASE)) {
 125                                 Node highestAVersion = null;
 126                                 for (NodeIterator aVersions = aBase.getNodes(); aVersions
 127                                                 .hasNext();) {
 128                                         Node aVersion = aVersions.nextNode();
 129                                         if (aVersion.isNodeType(SlcTypes.SLC_ARTIFACT_VERSION_BASE)) {
 130                                                 if (highestAVersion == null) {
 131                                                         highestAVersion = aVersion;
 132                                                         if (allArtifactsHighestVersion == null)
 133                                                                 allArtifactsHighestVersion = aVersion;
 134
 135                                                         // BS will fail if artifacts arrive in this order
 136                                                         // Name1 - V1, name2 - V3, V1 will remain the
 137                                                         // allArtifactsHighestVersion
 138                                                         // Fixed below
 139                                                         else {
 140                                                                 Version currVersion = extractOsgiVersion(aVersion);
 141                                                                 Version highestVersion = extractOsgiVersion(allArtifactsHighestVersion);
 142                                                                 if (currVersion.compareTo(highestVersion) > 0)
 143                                                                         allArtifactsHighestVersion = aVersion;
 144                                                         }
 145
 146                                                 } else {
 147                                                         Version currVersion = extractOsgiVersion(aVersion);
 148                                                         Version currentHighestVersion = extractOsgiVersion(highestAVersion);
 149                                                         if (currVersion.compareTo(currentHighestVersion) > 0) {
 150                                                                 highestAVersion = aVersion;
 151                                                         }
 152                                                         if (currVersion
 153                                                                         .compareTo(extractOsgiVersion(allArtifactsHighestVersion)) > 0) {
 154                                                                 allArtifactsHighestVersion = aVersion;
 155                                                         }
 156                                                 }
 157
 158                                         }
 159
 160                                 }
 161                                 if (highestAVersion == null)
 162                                         continue aBases;
 163                                 for (NodeIterator files = highestAVersion.getNodes(); files
 164                                                 .hasNext();) {
 165                                         Node file = files.nextNode();
 166                                         if (file.isNodeType(SlcTypes.SLC_BUNDLE_ARTIFACT)) {
 167                                                 preProcessBundleArtifact(file);
 168                                                 file.getSession().save();
 169                                                 if (log.isDebugEnabled())
 170                                                         log.debug("Pre-processed " + file.getName());
 171                                         }
 172
 173                                 }
 174                         }
 175                 }
 176
 177                 // if version not set or empty, use the highest version
 178                 // useful when indexing a product maven repository where
 179                 // all artifacts have the same version for a given release
 180                 // => the version can then be left empty
 181                 if (version == null || version.trim().equals(""))
 182                         if (allArtifactsHighestVersion != null)
 183                                 version = allArtifactsHighestVersion.getProperty(
 184                                                 SLC_ARTIFACT_VERSION).getString();
 185                         else
 186                                 version = "0.0";
 187                 // throw new SlcException("Group version " + version
 188                 // + " is empty.");
 189
 190                 int bundleCount = symbolicNamesToNodes.size();
 191                 if (log.isDebugEnabled())
 192                         log.debug("Indexed " + bundleCount + " bundles");
 193
 194                 int count = 1;
 195                 for (Node bundleNode : symbolicNamesToNodes.values()) {
 196                         processBundleArtifact(bundleNode);
 197                         bundleNode.getSession().save();
 198                         if (log.isDebugEnabled())
 199                                 log.debug(count + "/" + bundleCount + " Processed "
 200                                                 + bundleNode.getName());
 201                         count++;
 202                 }
 203
 204                 // indexes
 205                 Set<Artifact> indexes = new TreeSet<Artifact>(
 206                                 new ArtifactIdComparator());
 207                 Artifact indexArtifact = writeIndex(session,
 208                                 RepoConstants.BINARIES_ARTIFACT_ID, binaries);
 209                 indexes.add(indexArtifact);
 210                 indexArtifact = writeIndex(session, RepoConstants.SOURCES_ARTIFACT_ID,
 211                                 sources);
 212                 indexes.add(indexArtifact);
 213                 // sdk
 214                 writeIndex(session, RepoConstants.SDK_ARTIFACT_ID, indexes);
 215                 if (monitor != null)
 216                         monitor.worked(1);
 217         }
 218
 219         private Version extractOsgiVersion(Node artifactVersion)
 220                         throws RepositoryException {
 221                 String rawVersion = artifactVersion.getProperty(SLC_ARTIFACT_VERSION)
 222                                 .getString();
 223                 String cleanVersion = rawVersion.replace("-SNAPSHOT", ".SNAPSHOT");
 224                 Version osgiVersion = null;
 225                 // log invalid version value to enable tracking them
 226                 try {
 227                         osgiVersion = new Version(cleanVersion);
 228                 } catch (IllegalArgumentException e) {
 229                         log.error("Version string " + cleanVersion + " is invalid ");
 230                         String twickedVersion = twickInvalidVersion(cleanVersion);
 231                         osgiVersion = new Version(twickedVersion);
 232                         log.error("Using " + twickedVersion + " instead");
 233                         // throw e;
 234                 }
 235                 return osgiVersion;
 236         }
 237
 238         private String twickInvalidVersion(String tmpVersion) {
 239                 String[] tokens = tmpVersion.split("\\.");
 240                 if (tokens.length == 3 && tokens[2].lastIndexOf("-") > 0) {
 241                         String newSuffix = tokens[2].replaceFirst("-", ".");
 242                         tmpVersion = tmpVersion.replaceFirst(tokens[2], newSuffix);
 243                 } else if (tokens.length > 4) {
 244                         // FIXME manually remove other "."
 245                         StringTokenizer st = new StringTokenizer(tmpVersion, ".", true);
 246                         StringBuilder builder = new StringBuilder();
 247                         // Major
 248                         builder.append(st.nextToken()).append(st.nextToken());
 249                         // Minor
 250                         builder.append(st.nextToken()).append(st.nextToken());
 251                         // Micro
 252                         builder.append(st.nextToken()).append(st.nextToken());
 253                         // Qualifier
 254                         builder.append(st.nextToken());
 255                         while (st.hasMoreTokens()) {
 256                                 // consume delimiter
 257                                 st.nextToken();
 258                                 if (st.hasMoreTokens())
 259                                         builder.append("-").append(st.nextToken());
 260                         }
 261                         tmpVersion = builder.toString();
 262                 }
 263                 return tmpVersion;
 264         }
 265
 266         private Artifact writeIndex(Session session, String artifactId,
 267                         Set<Artifact> artifacts) throws RepositoryException {
 268                 Artifact artifact = new DefaultArtifact(groupId, artifactId, "pom",
 269                                 version);
 270                 Artifact parentArtifact = parentPomCoordinates != null ? new DefaultArtifact(
 271                                 parentPomCoordinates) : null;
 272                 String pom = MavenConventionsUtils.artifactsAsDependencyPom(artifact,
 273                                 artifacts, parentArtifact);
 274                 Node node = RepoUtils.copyBytesAsArtifact(
 275                                 session.getNode(artifactBasePath), artifact, pom.getBytes());
 276                 artifactIndexer.index(node);
 277
 278                 // TODO factorize
 279                 String pomSha = JcrUtils.checksumFile(node, "SHA-1");
 280                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".sha1",
 281                                 pomSha.getBytes());
 282                 String pomMd5 = JcrUtils.checksumFile(node, "MD5");
 283                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".md5",
 284                                 pomMd5.getBytes());
 285                 session.save();
 286                 return artifact;
 287         }
 288
 289         protected void preProcessBundleArtifact(Node bundleNode)
 290                         throws RepositoryException {
 291
 292                 String symbolicName = JcrUtils.get(bundleNode, SLC_SYMBOLIC_NAME);
 293                 if (symbolicName.endsWith(".source")) {
 294                         // TODO make a shared node with classifier 'sources'?
 295                         String bundleName = RepoUtils
 296                                         .extractBundleNameFromSourceName(symbolicName);
 297                         for (String excludedSuffix : excludedSuffixes) {
 298                                 if (bundleName.endsWith(excludedSuffix))
 299                                         return;// skip adding to sources
 300                         }
 301                         sources.add(RepoUtils.asArtifact(bundleNode));
 302                         return;
 303                 }
 304
 305                 NodeIterator exportPackages = bundleNode.getNodes(SLC_
 306                                 + Constants.EXPORT_PACKAGE);
 307                 while (exportPackages.hasNext()) {
 308                         Node exportPackage = exportPackages.nextNode();
 309                         String pkg = JcrUtils.get(exportPackage, SLC_NAME);
 310                         packagesToSymbolicNames.put(pkg, symbolicName);
 311                 }
 312
 313                 symbolicNamesToNodes.put(symbolicName, bundleNode);
 314                 for (String excludedSuffix : excludedSuffixes) {
 315                         if (symbolicName.endsWith(excludedSuffix))
 316                                 return;// skip adding to binaries
 317                 }
 318                 binaries.add(RepoUtils.asArtifact(bundleNode));
 319
 320                 if (bundleNode.getSession().hasPendingChanges())
 321                         bundleNode.getSession().save();
 322         }
 323
 324         protected void processBundleArtifact(Node bundleNode)
 325                         throws RepositoryException {
 326                 Node artifactFolder = bundleNode.getParent();
 327                 String baseName = FilenameUtils.getBaseName(bundleNode.getName());
 328
 329                 // pom
 330                 String pomName = baseName + ".pom";
 331                 if (artifactFolder.hasNode(pomName) && !overridePoms)
 332                         return;// skip
 333
 334                 String pom = generatePomForBundle(bundleNode);
 335                 Node pomNode = JcrUtils.copyBytesAsFile(artifactFolder, pomName,
 336                                 pom.getBytes());
 337                 // checksum
 338                 String bundleSha = JcrUtils.checksumFile(bundleNode, "SHA-1");
 339                 JcrUtils.copyBytesAsFile(artifactFolder,
 340                                 bundleNode.getName() + ".sha1", bundleSha.getBytes());
 341                 String pomSha = JcrUtils.checksumFile(pomNode, "SHA-1");
 342                 JcrUtils.copyBytesAsFile(artifactFolder, pomNode.getName() + ".sha1",
 343                                 pomSha.getBytes());
 344         }
 345
 346         private String generatePomForBundle(Node n) throws RepositoryException {
 347                 String ownSymbolicName = JcrUtils.get(n, SLC_SYMBOLIC_NAME);
 348
 349                 StringBuffer p = new StringBuffer();
 350
 351                 // XML header
 352                 p.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
 353                 p.append("<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd\">\n");
 354                 p.append("<modelVersion>4.0.0</modelVersion>");
 355
 356                 // Artifact
 357                 p.append("<groupId>").append(JcrUtils.get(n, SLC_GROUP_ID))
 358                                 .append("</groupId>\n");
 359                 p.append("<artifactId>").append(JcrUtils.get(n, SLC_ARTIFACT_ID))
 360                                 .append("</artifactId>\n");
 361                 p.append("<version>").append(JcrUtils.get(n, SLC_ARTIFACT_VERSION))
 362                                 .append("</version>\n");
 363                 p.append("<packaging>pom</packaging>\n");
 364                 if (n.hasProperty(SLC_ + Constants.BUNDLE_NAME))
 365                         p.append("<name>")
 366                                         .append(JcrUtils.get(n, SLC_ + Constants.BUNDLE_NAME))
 367                                         .append("</name>\n");
 368                 if (n.hasProperty(SLC_ + Constants.BUNDLE_DESCRIPTION))
 369                         p.append("<description>")
 370                                         .append(JcrUtils
 371                                                         .get(n, SLC_ + Constants.BUNDLE_DESCRIPTION))
 372                                         .append("</description>\n");
 373
 374                 // Dependencies
 375                 Set<String> dependenciesSymbolicNames = new TreeSet<String>();
 376                 Set<String> optionalSymbolicNames = new TreeSet<String>();
 377                 NodeIterator importPackages = n.getNodes(SLC_
 378                                 + Constants.IMPORT_PACKAGE);
 379                 while (importPackages.hasNext()) {
 380                         Node importPackage = importPackages.nextNode();
 381                         String pkg = JcrUtils.get(importPackage, SLC_NAME);
 382                         if (packagesToSymbolicNames.containsKey(pkg)) {
 383                                 String dependencySymbolicName = packagesToSymbolicNames
 384                                                 .get(pkg);
 385                                 if (JcrUtils.check(importPackage, SLC_OPTIONAL))
 386                                         optionalSymbolicNames.add(dependencySymbolicName);
 387                                 else
 388                                         dependenciesSymbolicNames.add(dependencySymbolicName);
 389                         } else {
 390                                 if (!JcrUtils.check(importPackage, SLC_OPTIONAL)
 391                                                 && !systemPackages.contains(pkg))
 392                                         log.warn("No bundle found for pkg " + pkg);
 393                         }
 394                 }
 395
 396                 if (n.hasNode(SLC_ + Constants.FRAGMENT_HOST)) {
 397                         String fragmentHost = JcrUtils.get(
 398                                         n.getNode(SLC_ + Constants.FRAGMENT_HOST),
 399                                         SLC_SYMBOLIC_NAME);
 400                         dependenciesSymbolicNames.add(fragmentHost);
 401                 }
 402
 403                 // TODO require bundles
 404
 405                 List<Node> dependencyNodes = new ArrayList<Node>();
 406                 for (String depSymbName : dependenciesSymbolicNames) {
 407                         if (depSymbName.equals(ownSymbolicName))
 408                                 continue;// skip self
 409
 410                         if (symbolicNamesToNodes.containsKey(depSymbName))
 411                                 dependencyNodes.add(symbolicNamesToNodes.get(depSymbName));
 412                         else
 413                                 log.warn("Could not find node for " + depSymbName);
 414                 }
 415                 List<Node> optionalDependencyNodes = new ArrayList<Node>();
 416                 for (String depSymbName : optionalSymbolicNames) {
 417                         if (symbolicNamesToNodes.containsKey(depSymbName))
 418                                 optionalDependencyNodes.add(symbolicNamesToNodes
 419                                                 .get(depSymbName));
 420                         else
 421                                 log.warn("Could not find node for " + depSymbName);
 422                 }
 423
 424                 p.append("<dependencies>\n");
 425                 for (Node dependencyNode : dependencyNodes) {
 426                         p.append("<dependency>\n");
 427                         p.append("\t<groupId>")
 428                                         .append(JcrUtils.get(dependencyNode, SLC_GROUP_ID))
 429                                         .append("</groupId>\n");
 430                         p.append("\t<artifactId>")
 431                                         .append(JcrUtils.get(dependencyNode, SLC_ARTIFACT_ID))
 432                                         .append("</artifactId>\n");
 433                         p.append("</dependency>\n");
 434                 }
 435
 436                 if (optionalDependencyNodes.size() > 0)
 437                         p.append("<!-- OPTIONAL -->\n");
 438                 for (Node dependencyNode : optionalDependencyNodes) {
 439                         p.append("<dependency>\n");
 440                         p.append("\t<groupId>")
 441                                         .append(JcrUtils.get(dependencyNode, SLC_GROUP_ID))
 442                                         .append("</groupId>\n");
 443                         p.append("\t<artifactId>")
 444                                         .append(JcrUtils.get(dependencyNode, SLC_ARTIFACT_ID))
 445                                         .append("</artifactId>\n");
 446                         p.append("\t<optional>true</optional>\n");
 447                         p.append("</dependency>\n");
 448                 }
 449                 p.append("</dependencies>\n");
 450
 451                 // Dependency management
 452                 p.append("<dependencyManagement>\n");
 453                 p.append("<dependencies>\n");
 454                 p.append("<dependency>\n");
 455                 p.append("\t<groupId>").append(groupId).append("</groupId>\n");
 456                 p.append("\t<artifactId>")
 457                                 .append(ownSymbolicName.endsWith(".source") ? RepoConstants.SOURCES_ARTIFACT_ID
 458                                                 : RepoConstants.BINARIES_ARTIFACT_ID)
 459                                 .append("</artifactId>\n");
 460                 p.append("\t<version>").append(version).append("</version>\n");
 461                 p.append("\t<type>pom</type>\n");
 462                 p.append("\t<scope>import</scope>\n");
 463                 p.append("</dependency>\n");
 464                 p.append("</dependencies>\n");
 465                 p.append("</dependencyManagement>\n");
 466
 467                 p.append("</project>\n");
 468                 return p.toString();
 469         }
 470
 471         /* DEPENDENCY INJECTION */
 472         public void setRepository(Repository repository) {
 473                 this.repository = repository;
 474         }
 475
 476         public void setWorkspace(String workspace) {
 477                 this.workspace = workspace;
 478         }
 479
 480         public void setGroupId(String groupId) {
 481                 this.groupId = groupId;
 482         }
 483
 484         public void setParentPomCoordinates(String parentPomCoordinates) {
 485                 this.parentPomCoordinates = parentPomCoordinates;
 486         }
 487
 488         public void setArtifactBasePath(String artifactBasePath) {
 489                 this.artifactBasePath = artifactBasePath;
 490         }
 491
 492         public void setVersion(String version) {
 493                 this.version = version;
 494         }
 495
 496         public void setExcludedSuffixes(List<String> excludedSuffixes) {
 497                 this.excludedSuffixes = excludedSuffixes;
 498         }
 499
 500         public void setOverridePoms(Boolean overridePoms) {
 501                 this.overridePoms = overridePoms;
 502         }
 503
 504         public void setArtifactIndexer(ArtifactIndexer artifactIndexer) {
 505                 this.artifactIndexer = artifactIndexer;
 506         }
 507 }