runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoUtils.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.repo;
  17
  18 import java.io.ByteArrayOutputStream;
  19 import java.io.File;
  20 import java.io.FileInputStream;
  21 import java.io.IOException;
  22 import java.io.InputStream;
  23 import java.io.OutputStream;
  24 import java.util.Enumeration;
  25 import java.util.Iterator;
  26 import java.util.Set;
  27 import java.util.StringTokenizer;
  28 import java.util.TreeSet;
  29 import java.util.jar.Attributes;
  30 import java.util.jar.JarEntry;
  31 import java.util.jar.JarFile;
  32 import java.util.jar.JarInputStream;
  33 import java.util.jar.JarOutputStream;
  34 import java.util.jar.Manifest;
  35
  36 import javax.jcr.Credentials;
  37 import javax.jcr.GuestCredentials;
  38 import javax.jcr.Node;
  39 import javax.jcr.NodeIterator;
  40 import javax.jcr.Property;
  41 import javax.jcr.PropertyIterator;
  42 import javax.jcr.Repository;
  43 import javax.jcr.RepositoryException;
  44 import javax.jcr.RepositoryFactory;
  45 import javax.jcr.Session;
  46 import javax.jcr.SimpleCredentials;
  47 import javax.jcr.nodetype.NodeType;
  48
  49 import org.apache.commons.io.FilenameUtils;
  50 import org.apache.commons.io.IOUtils;
  51 import org.apache.commons.logging.Log;
  52 import org.apache.commons.logging.LogFactory;
  53 import org.argeo.jcr.ArgeoJcrUtils;
  54 import org.argeo.jcr.ArgeoNames;
  55 import org.argeo.jcr.ArgeoTypes;
  56 import org.argeo.jcr.JcrUtils;
  57 import org.argeo.slc.DefaultNameVersion;
  58 import org.argeo.slc.NameVersion;
  59 import org.argeo.slc.SlcException;
  60 import org.argeo.slc.aether.ArtifactIdComparator;
  61 import org.argeo.slc.jcr.SlcNames;
  62 import org.argeo.slc.jcr.SlcTypes;
  63 import org.argeo.slc.repo.maven.MavenConventionsUtils;
  64 import org.argeo.util.security.Keyring;
  65 import org.osgi.framework.Constants;
  66 import org.sonatype.aether.artifact.Artifact;
  67 import org.sonatype.aether.util.artifact.DefaultArtifact;
  68
  69 /** Utilities around repo */
  70 public class RepoUtils implements ArgeoNames, SlcNames {
  71         private final static Log log = LogFactory.getLog(RepoUtils.class);
  72
  73         /** Packages a regular sources jar as PDE source. */
  74         public static void packagesAsPdeSource(File sourceFile,
  75                         NameVersion nameVersion, OutputStream out) throws IOException {
  76                 if (isAlreadyPdeSource(sourceFile)) {
  77                         FileInputStream in = new FileInputStream(sourceFile);
  78                         IOUtils.copy(in, out);
  79                         IOUtils.closeQuietly(in);
  80                 } else {
  81                         String sourceSymbolicName = nameVersion.getName() + ".source";
  82
  83                         Manifest sourceManifest = null;
  84                         sourceManifest = new Manifest();
  85                         sourceManifest.getMainAttributes().put(
  86                                         Attributes.Name.MANIFEST_VERSION, "1.0");
  87                         sourceManifest.getMainAttributes().putValue("Bundle-SymbolicName",
  88                                         sourceSymbolicName);
  89                         sourceManifest.getMainAttributes().putValue("Bundle-Version",
  90                                         nameVersion.getVersion());
  91                         sourceManifest.getMainAttributes().putValue(
  92                                         "Eclipse-SourceBundle",
  93                                         nameVersion.getName() + ";version="
  94                                                         + nameVersion.getVersion());
  95                         copyJar(sourceFile, out, sourceManifest);
  96                 }
  97         }
  98
  99         public static byte[] packageAsPdeSource(InputStream sourceJar,
 100                         NameVersion nameVersion) {
 101                 String sourceSymbolicName = nameVersion.getName() + ".source";
 102
 103                 Manifest sourceManifest = null;
 104                 sourceManifest = new Manifest();
 105                 sourceManifest.getMainAttributes().put(
 106                                 Attributes.Name.MANIFEST_VERSION, "1.0");
 107                 sourceManifest.getMainAttributes().putValue("Bundle-SymbolicName",
 108                                 sourceSymbolicName);
 109                 sourceManifest.getMainAttributes().putValue("Bundle-Version",
 110                                 nameVersion.getVersion());
 111                 sourceManifest.getMainAttributes().putValue("Eclipse-SourceBundle",
 112                                 nameVersion.getName() + ";version=" + nameVersion.getVersion());
 113
 114                 return modifyManifest(sourceJar, sourceManifest);
 115         }
 116
 117         /**
 118          * Check whether the file as already been packaged as PDE source, in order
 119          * not to mess with Jar signing
 120          */
 121         private static boolean isAlreadyPdeSource(File sourceFile) {
 122                 JarInputStream jarInputStream = null;
 123
 124                 try {
 125                         jarInputStream = new JarInputStream(new FileInputStream(sourceFile));
 126
 127                         Manifest manifest = jarInputStream.getManifest();
 128                         Iterator<?> it = manifest.getMainAttributes().keySet().iterator();
 129                         boolean res = false;
 130                         // containsKey() does not work, iterating...
 131                         while (it.hasNext())
 132                                 if (it.next().toString().equals("Eclipse-SourceBundle")) {
 133                                         res = true;
 134                                         break;
 135                                 }
 136                         // boolean res = manifest.getMainAttributes().get(
 137                         // "Eclipse-SourceBundle") != null;
 138                         if (res)
 139                                 log.info(sourceFile + " is already a PDE source");
 140                         return res;
 141                 } catch (Exception e) {
 142                         // probably not a jar, skipping
 143                         if (log.isDebugEnabled())
 144                                 log.debug("Skipping " + sourceFile + " because of "
 145                                                 + e.getMessage());
 146                         return false;
 147                 } finally {
 148                         IOUtils.closeQuietly(jarInputStream);
 149                 }
 150         }
 151
 152         /**
 153          * Copy a jar, replacing its manifest with the provided one
 154          *
 155          * @param manifest
 156          *            can be null
 157          */
 158         private static void copyJar(File source, OutputStream out, Manifest manifest)
 159                         throws IOException {
 160                 JarFile sourceJar = null;
 161                 JarOutputStream output = null;
 162                 try {
 163                         output = manifest != null ? new JarOutputStream(out, manifest)
 164                                         : new JarOutputStream(out);
 165                         sourceJar = new JarFile(source);
 166
 167                         entries: for (Enumeration<?> entries = sourceJar.entries(); entries
 168                                         .hasMoreElements();) {
 169                                 JarEntry entry = (JarEntry) entries.nextElement();
 170                                 if (manifest != null
 171                                                 && entry.getName().equals("META-INF/MANIFEST.MF"))
 172                                         continue entries;
 173
 174                                 InputStream entryStream = sourceJar.getInputStream(entry);
 175                                 JarEntry newEntry = new JarEntry(entry.getName());
 176                                 // newEntry.setMethod(JarEntry.DEFLATED);
 177                                 output.putNextEntry(newEntry);
 178                                 IOUtils.copy(entryStream, output);
 179                         }
 180                 } finally {
 181                         IOUtils.closeQuietly(output);
 182                         try {
 183                                 if (sourceJar != null)
 184                                         sourceJar.close();
 185                         } catch (IOException e) {
 186                                 // silent
 187                         }
 188                 }
 189         }
 190
 191         /** Copy a jar changing onlythe manifest */
 192         public static void copyJar(InputStream in, OutputStream out,
 193                         Manifest manifest) {
 194                 JarInputStream jarIn = null;
 195                 JarOutputStream jarOut = null;
 196                 try {
 197                         jarIn = new JarInputStream(in);
 198                         jarOut = new JarOutputStream(out, manifest);
 199                         JarEntry jarEntry = null;
 200                         while ((jarEntry = jarIn.getNextJarEntry()) != null) {
 201                                 jarOut.putNextEntry(jarEntry);
 202                                 IOUtils.copy(jarIn, jarOut);
 203                                 jarIn.closeEntry();
 204                                 jarOut.closeEntry();
 205                         }
 206                 } catch (IOException e) {
 207                         throw new SlcException("Could not copy jar with MANIFEST "
 208                                         + manifest.getMainAttributes(), e);
 209                 } finally {
 210                         IOUtils.closeQuietly(jarIn);
 211                         IOUtils.closeQuietly(jarOut);
 212                 }
 213         }
 214
 215         /** Reads a jar file, modify its manifest */
 216         public static byte[] modifyManifest(InputStream in, Manifest manifest) {
 217                 ByteArrayOutputStream out = new ByteArrayOutputStream(200 * 1024);
 218                 try {
 219                         copyJar(in, out, manifest);
 220                         return out.toByteArray();
 221                 } finally {
 222                         IOUtils.closeQuietly(out);
 223                 }
 224         }
 225
 226         /** Read the OSGi {@link NameVersion} */
 227         public static NameVersion readNameVersion(Artifact artifact) {
 228                 File artifactFile = artifact.getFile();
 229                 if (artifact.getExtension().equals("pom")) {
 230                         // hack to process jars which weirdly appear as POMs
 231                         File jarFile = new File(artifactFile.getParentFile(),
 232                                         FilenameUtils.getBaseName(artifactFile.getPath()) + ".jar");
 233                         if (jarFile.exists()) {
 234                                 log.warn("Use " + jarFile + " instead of " + artifactFile
 235                                                 + " for " + artifact);
 236                                 artifactFile = jarFile;
 237                         }
 238                 }
 239                 return readNameVersion(artifactFile);
 240         }
 241
 242         /** Read the OSGi {@link NameVersion} */
 243         public static NameVersion readNameVersion(File artifactFile) {
 244                 try {
 245                         return readNameVersion(new FileInputStream(artifactFile));
 246                 } catch (Exception e) {
 247                         // probably not a jar, skipping
 248                         if (log.isDebugEnabled()) {
 249                                 log.debug("Skipping " + artifactFile + " because of " + e);
 250                                 // e.printStackTrace();
 251                         }
 252                 }
 253                 return null;
 254         }
 255
 256         /** Read the OSGi {@link NameVersion} */
 257         public static NameVersion readNameVersion(InputStream in) {
 258                 JarInputStream jarInputStream = null;
 259                 try {
 260                         jarInputStream = new JarInputStream(in);
 261                         return readNameVersion(jarInputStream.getManifest());
 262                 } catch (Exception e) {
 263                         // probably not a jar, skipping
 264                         if (log.isDebugEnabled()) {
 265                                 log.debug("Skipping because of " + e);
 266                                 // e.printStackTrace();
 267                         }
 268                 } finally {
 269                         IOUtils.closeQuietly(jarInputStream);
 270                 }
 271                 return null;
 272         }
 273
 274         /** Read the OSGi {@link NameVersion} */
 275         public static NameVersion readNameVersion(Manifest manifest) {
 276                 DefaultNameVersion nameVersion = new DefaultNameVersion();
 277                 nameVersion.setName(manifest.getMainAttributes().getValue(
 278                                 Constants.BUNDLE_SYMBOLICNAME));
 279
 280                 // Skip additional specs such as
 281                 // ; singleton:=true
 282                 if (nameVersion.getName().indexOf(';') > -1) {
 283                         nameVersion
 284                                         .setName(new StringTokenizer(nameVersion.getName(), " ;")
 285                                                         .nextToken());
 286                 }
 287
 288                 nameVersion.setVersion(manifest.getMainAttributes().getValue(
 289                                 Constants.BUNDLE_VERSION));
 290
 291                 return nameVersion;
 292         }
 293
 294         /*
 295          * DATA MODEL
 296          */
 297         /** The artifact described by this node */
 298         public static Artifact asArtifact(Node node) throws RepositoryException {
 299                 if (node.isNodeType(SlcTypes.SLC_ARTIFACT_VERSION_BASE)) {
 300                         // FIXME update data model to store packaging at this level
 301                         String extension = "jar";
 302                         return new DefaultArtifact(node.getProperty(SLC_GROUP_ID)
 303                                         .getString(),
 304                                         node.getProperty(SLC_ARTIFACT_ID).getString(), extension,
 305                                         node.getProperty(SLC_ARTIFACT_VERSION).getString());
 306                 } else if (node.isNodeType(SlcTypes.SLC_ARTIFACT)) {
 307                         return new DefaultArtifact(node.getProperty(SLC_GROUP_ID)
 308                                         .getString(),
 309                                         node.getProperty(SLC_ARTIFACT_ID).getString(), node
 310                                                         .getProperty(SLC_ARTIFACT_CLASSIFIER).getString(),
 311                                         node.getProperty(SLC_ARTIFACT_EXTENSION).getString(), node
 312                                                         .getProperty(SLC_ARTIFACT_VERSION).getString());
 313                 } else {
 314                         throw new SlcException("Unsupported node type for " + node);
 315                 }
 316         }
 317
 318         /**
 319          * The path to the PDE source related to this artifact (or artifact version
 320          * base). There may or there may not be a node at this location (the
 321          * returned path will typically be used to test whether PDE sources are
 322          * attached to this artifact).
 323          */
 324         public static String relatedPdeSourcePath(String artifactBasePath,
 325                         Node artifactNode) throws RepositoryException {
 326                 Artifact artifact = asArtifact(artifactNode);
 327                 Artifact pdeSourceArtifact = new DefaultArtifact(artifact.getGroupId(),
 328                                 artifact.getArtifactId() + ".source", artifact.getExtension(),
 329                                 artifact.getVersion());
 330                 return MavenConventionsUtils.artifactPath(artifactBasePath,
 331                                 pdeSourceArtifact);
 332         }
 333
 334         /**
 335          * Copy this bytes array as an artifact, relative to the root of the
 336          * repository (typically the workspace root node)
 337          */
 338         public static Node copyBytesAsArtifact(Node artifactsBase,
 339                         Artifact artifact, byte[] bytes) throws RepositoryException {
 340                 String parentPath = MavenConventionsUtils.artifactParentPath(
 341                                 artifactsBase.getPath(), artifact);
 342                 Node folderNode = JcrUtils.mkfolders(artifactsBase.getSession(),
 343                                 parentPath);
 344                 return JcrUtils.copyBytesAsFile(folderNode,
 345                                 MavenConventionsUtils.artifactFileName(artifact), bytes);
 346         }
 347
 348         private RepoUtils() {
 349         }
 350
 351         /** If a source return the base bundle name, does not change otherwise */
 352         public static String extractBundleNameFromSourceName(String sourceBundleName) {
 353                 if (sourceBundleName.endsWith(".source"))
 354                         return sourceBundleName.substring(0, sourceBundleName.length()
 355                                         - ".source".length());
 356                 else
 357                         return sourceBundleName;
 358         }
 359
 360         /*
 361          * SOFTWARE REPOSITORIES
 362          */
 363
 364         /** Retrieve repository based on information in the repo node */
 365         public static Repository getRepository(RepositoryFactory repositoryFactory,
 366                         Keyring keyring, Node repoNode) {
 367                 try {
 368                         Repository repository;
 369                         if (repoNode.isNodeType(ArgeoTypes.ARGEO_REMOTE_REPOSITORY)) {
 370                                 String uri = repoNode.getProperty(ARGEO_URI).getString();
 371                                 if (uri.startsWith("http")) {// http, https
 372                                         repository = ArgeoJcrUtils.getRepositoryByUri(
 373                                                         repositoryFactory, uri);
 374                                 } else if (uri.startsWith("vm:")) {// alias
 375                                         repository = ArgeoJcrUtils.getRepositoryByUri(
 376                                                         repositoryFactory, uri);
 377                                 } else {
 378                                         throw new SlcException("Unsupported repository uri " + uri);
 379                                 }
 380                                 return repository;
 381                         } else {
 382                                 throw new SlcException("Unsupported node type " + repoNode);
 383                         }
 384                 } catch (RepositoryException e) {
 385                         throw new SlcException("Cannot connect to repository " + repoNode,
 386                                         e);
 387                 }
 388
 389         }
 390
 391         /**
 392          * Reads credentials from node, using keyring if there is a password. Cann
 393          * return null if no credentials needed (local repo) at all, but returns
 394          * {@link GuestCredentials} if user id is 'anonymous' .
 395          */
 396         public static Credentials getRepositoryCredentials(Keyring keyring,
 397                         Node repoNode) {
 398                 try {
 399                         if (repoNode.isNodeType(ArgeoTypes.ARGEO_REMOTE_REPOSITORY)) {
 400                                 if (!repoNode.hasProperty(ARGEO_USER_ID))
 401                                         return null;
 402
 403                                 String userId = repoNode.getProperty(ARGEO_USER_ID).getString();
 404                                 if (userId.equals("anonymous"))// FIXME hardcoded userId
 405                                         return new GuestCredentials();
 406                                 char[] password = keyring.getAsChars(repoNode.getPath() + '/'
 407                                                 + ARGEO_PASSWORD);
 408                                 Credentials credentials = new SimpleCredentials(userId,
 409                                                 password);
 410                                 return credentials;
 411                         } else {
 412                                 throw new SlcException("Unsupported node type " + repoNode);
 413                         }
 414                 } catch (RepositoryException e) {
 415                         throw new SlcException("Cannot connect to repository " + repoNode,
 416                                         e);
 417                 }
 418         }
 419
 420         /**
 421          * Write group indexes: 'binaries' lists all bundles and their versions,
 422          * 'sources' list theire sources, and 'sdk' aggregates both.
 423          */
 424         public static void writeGroupIndexes(Session session,
 425                         String artifactBasePath, String groupId, String version,
 426                         Set<Artifact> binaries, Set<Artifact> sources) {
 427                 try {
 428                         Set<Artifact> indexes = new TreeSet<Artifact>(
 429                                         new ArtifactIdComparator());
 430                         Artifact binariesArtifact = writeIndex(session, artifactBasePath,
 431                                         groupId, RepoConstants.BINARIES_ARTIFACT_ID, version,
 432                                         binaries);
 433                         indexes.add(binariesArtifact);
 434                         if (sources != null) {
 435                                 Artifact sourcesArtifact = writeIndex(session,
 436                                                 artifactBasePath, groupId,
 437                                                 RepoConstants.SOURCES_ARTIFACT_ID, version, sources);
 438                                 indexes.add(sourcesArtifact);
 439                         }
 440                         // sdk
 441                         writeIndex(session, artifactBasePath, groupId,
 442                                         RepoConstants.SDK_ARTIFACT_ID, version, indexes);
 443                         session.save();
 444                 } catch (RepositoryException e) {
 445                         throw new SlcException("Cannot write indexes for group " + groupId,
 446                                         e);
 447                 }
 448         }
 449
 450         /** Write a group index. */
 451         private static Artifact writeIndex(Session session,
 452                         String artifactBasePath, String groupId, String artifactId,
 453                         String version, Set<Artifact> artifacts) throws RepositoryException {
 454                 Artifact artifact = new DefaultArtifact(groupId, artifactId, "pom",
 455                                 version);
 456                 String pom = MavenConventionsUtils.artifactsAsDependencyPom(artifact,
 457                                 artifacts, null);
 458                 Node node = RepoUtils.copyBytesAsArtifact(
 459                                 session.getNode(artifactBasePath), artifact, pom.getBytes());
 460                 addMavenChecksums(node);
 461                 return artifact;
 462         }
 463
 464         /** Add files containing the SHA-1 and MD5 checksums. */
 465         public static void addMavenChecksums(Node node) throws RepositoryException {
 466                 // TODO optimize
 467                 String sha = JcrUtils.checksumFile(node, "SHA-1");
 468                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".sha1",
 469                                 sha.getBytes());
 470                 String md5 = JcrUtils.checksumFile(node, "MD5");
 471                 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".md5",
 472                                 md5.getBytes());
 473         }
 474
 475         /**
 476          * Custom copy since the one in commons does not fit the needs when copying
 477          * a workspace completely.
 478          */
 479         public static void copy(Node fromNode, Node toNode) {
 480                 try {
 481                         if (log.isDebugEnabled())
 482                                 log.debug("copy node :" + fromNode.getPath());
 483
 484                         // FIXME : small hack to enable specific workspace copy
 485                         if (fromNode.isNodeType("rep:ACL")
 486                                         || fromNode.isNodeType("rep:system")) {
 487                                 if (log.isTraceEnabled())
 488                                         log.trace("node " + fromNode + " skipped");
 489                                 return;
 490                         }
 491
 492                         // add mixins
 493                         for (NodeType mixinType : fromNode.getMixinNodeTypes()) {
 494                                 toNode.addMixin(mixinType.getName());
 495                         }
 496
 497                         // Double check
 498                         for (NodeType mixinType : toNode.getMixinNodeTypes()) {
 499                                 if (log.isDebugEnabled())
 500                                         log.debug(mixinType.getName());
 501                         }
 502
 503                         // process properties
 504                         PropertyIterator pit = fromNode.getProperties();
 505                         properties: while (pit.hasNext()) {
 506                                 Property fromProperty = pit.nextProperty();
 507                                 String propName = fromProperty.getName();
 508                                 try {
 509                                         String propertyName = fromProperty.getName();
 510                                         if (toNode.hasProperty(propertyName)
 511                                                         && toNode.getProperty(propertyName).getDefinition()
 512                                                                         .isProtected())
 513                                                 continue properties;
 514
 515                                         if (fromProperty.getDefinition().isProtected())
 516                                                 continue properties;
 517
 518                                         if (propertyName.equals("jcr:created")
 519                                                         || propertyName.equals("jcr:createdBy")
 520                                                         || propertyName.equals("jcr:lastModified")
 521                                                         || propertyName.equals("jcr:lastModifiedBy"))
 522                                                 continue properties;
 523
 524                                         if (fromProperty.isMultiple()) {
 525                                                 toNode.setProperty(propertyName,
 526                                                                 fromProperty.getValues());
 527                                         } else {
 528                                                 toNode.setProperty(propertyName,
 529                                                                 fromProperty.getValue());
 530                                         }
 531                                 } catch (RepositoryException e) {
 532                                         throw new SlcException("Cannot property " + propName, e);
 533                                 }
 534                         }
 535
 536                         // recursively process children nodes
 537                         NodeIterator nit = fromNode.getNodes();
 538                         while (nit.hasNext()) {
 539                                 Node fromChild = nit.nextNode();
 540                                 Integer index = fromChild.getIndex();
 541                                 String nodeRelPath = fromChild.getName() + "[" + index + "]";
 542                                 Node toChild;
 543                                 if (toNode.hasNode(nodeRelPath))
 544                                         toChild = toNode.getNode(nodeRelPath);
 545                                 else
 546                                         toChild = toNode.addNode(fromChild.getName(), fromChild
 547                                                         .getPrimaryNodeType().getName());
 548                                 copy(fromChild, toChild);
 549                         }
 550
 551                         // update jcr:lastModified and jcr:lastModifiedBy in toNode in
 552                         // case
 553                         // they existed
 554                         if (!toNode.getDefinition().isProtected()
 555                                         && toNode.isNodeType(NodeType.MIX_LAST_MODIFIED))
 556                                 JcrUtils.updateLastModified(toNode);
 557
 558                         // Workaround to reduce session size: artifact is a saveable
 559                         // unity
 560                         if (toNode.isNodeType(SlcTypes.SLC_ARTIFACT))
 561                                 toNode.getSession().save();
 562
 563                 } catch (RepositoryException e) {
 564                         throw new SlcException("Cannot copy " + fromNode + " to " + toNode,
 565                                         e);
 566                 }
 567         }
 568
 569 }