runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/osgi/ArchiveWrapper.java

   1 package org.argeo.slc.repo.osgi;
   2
   3 import java.io.ByteArrayInputStream;
   4 import java.io.ByteArrayOutputStream;
   5 import java.io.InputStream;
   6 import java.util.ArrayList;
   7 import java.util.HashMap;
   8 import java.util.Iterator;
   9 import java.util.List;
  10 import java.util.Map;
  11 import java.util.Set;
  12 import java.util.TreeSet;
  13 import java.util.jar.JarInputStream;
  14 import java.util.zip.ZipEntry;
  15 import java.util.zip.ZipInputStream;
  16 import java.util.zip.ZipOutputStream;
  17
  18 import javax.jcr.Node;
  19 import javax.jcr.Property;
  20 import javax.jcr.RepositoryException;
  21 import javax.jcr.Session;
  22
  23 import org.apache.commons.io.FilenameUtils;
  24 import org.apache.commons.io.IOUtils;
  25 import org.apache.commons.logging.Log;
  26 import org.apache.commons.logging.LogFactory;
  27 import org.argeo.jcr.JcrUtils;
  28 import org.argeo.slc.CategorizedNameVersion;
  29 import org.argeo.slc.DefaultNameVersion;
  30 import org.argeo.slc.ModuleSet;
  31 import org.argeo.slc.NameVersion;
  32 import org.argeo.slc.SlcException;
  33 import org.argeo.slc.aether.ArtifactIdComparator;
  34 import org.argeo.slc.build.Distribution;
  35 import org.argeo.slc.build.License;
  36 import org.argeo.slc.repo.OsgiFactory;
  37 import org.argeo.slc.repo.RepoUtils;
  38 import org.sonatype.aether.artifact.Artifact;
  39 import org.sonatype.aether.util.artifact.DefaultArtifact;
  40 import org.springframework.util.AntPathMatcher;
  41 import org.springframework.util.PathMatcher;
  42
  43 import aQute.lib.osgi.Jar;
  44
  45 /**
  46  * Download a software distribution and generates the related OSGi bundles from
  47  * the jars, or import them directly if they are already OSGi bundles and don't
  48  * need further modification.
  49  */
  50 public class ArchiveWrapper implements Runnable, ModuleSet, Distribution {
  51         private final static Log log = LogFactory.getLog(ArchiveWrapper.class);
  52
  53         private OsgiFactory osgiFactory;
  54         private String version;
  55         private License license;
  56
  57         private String uri;
  58
  59         /** Jars to wrap as OSGi bundles */
  60         private Map<String, BndWrapper> wrappers = new HashMap<String, BndWrapper>();
  61
  62         private SourcesProvider sourcesProvider;
  63
  64         // pattern of OSGi bundles to import
  65         private PathMatcher pathMatcher = new AntPathMatcher();
  66         private Map<String, String> includes = new HashMap<String, String>();
  67         private List<String> excludes = new ArrayList<String>();
  68
  69         private Boolean mavenGroupIndexes = false;
  70
  71         public void init() {
  72                 for (BndWrapper wrapper : wrappers.values()) {
  73                         wrapper.setFactory(this);
  74                         if (version != null && wrapper.getVersion() == null)
  75                                 wrapper.setVersion(version);
  76                         if (license != null && wrapper.getLicense() == null)
  77                                 wrapper.setLicense(license);
  78                 }
  79         }
  80
  81         public void destroy() {
  82
  83         }
  84
  85         public String getDistributionId() {
  86                 return uri;
  87         }
  88
  89         public Iterator<? extends NameVersion> nameVersions() {
  90                 if (wrappers.size() > 0)
  91                         return wrappers.values().iterator();
  92                 else
  93                         return osgiNameVersions();
  94         }
  95
  96         @SuppressWarnings("resource")
  97         protected Iterator<? extends NameVersion> osgiNameVersions() {
  98                 List<CategorizedNameVersion> nvs = new ArrayList<CategorizedNameVersion>();
  99
 100                 Session distSession = null;
 101                 ZipInputStream zin = null;
 102                 try {
 103                         distSession = osgiFactory.openDistSession();
 104
 105                         Node distNode = osgiFactory.getDist(distSession, uri);
 106                         zin = new ZipInputStream(distNode.getNode(Node.JCR_CONTENT)
 107                                         .getProperty(Property.JCR_DATA).getBinary().getStream());
 108
 109                         ZipEntry zentry = null;
 110                         entries: while ((zentry = zin.getNextEntry()) != null) {
 111                                 String name = zentry.getName();
 112                                 // log.debug(name);
 113                                 for (String exclude : excludes)
 114                                         if (pathMatcher.match(exclude, name))
 115                                                 continue entries;
 116
 117                                 for (String include : includes.keySet()) {
 118                                         if (pathMatcher.match(include, name)) {
 119                                                 String groupId = includes.get(include);
 120                                                 JarInputStream jis = new JarInputStream(zin);
 121                                                 NameVersion nv = RepoUtils.readNameVersion(jis
 122                                                                 .getManifest());
 123                                                 if (nv != null) {
 124                                                         if (nv.getName().endsWith(".source"))
 125                                                                 continue entries;
 126                                                         CategorizedNameVersion cnv = new OsgiCategorizedNV(
 127                                                                         groupId, nv.getName(), nv.getVersion(),
 128                                                                         this);
 129                                                         nvs.add(cnv);
 130                                                 }
 131                                         }
 132                                 }
 133                         }
 134                         return nvs.iterator();
 135                 } catch (Exception e) {
 136                         throw new SlcException("Cannot wrap distribution " + uri, e);
 137                 } finally {
 138                         IOUtils.closeQuietly(zin);
 139                         JcrUtils.logoutQuietly(distSession);
 140                 }
 141         }
 142
 143         public void run() {
 144                 if (mavenGroupIndexes && (version == null))
 145                         throw new SlcException(
 146                                         "'mavenGroupIndexes' requires 'version' to be set");
 147
 148                 Map<String, Set<Artifact>> binaries = new HashMap<String, Set<Artifact>>();
 149                 Map<String, Set<Artifact>> sources = new HashMap<String, Set<Artifact>>();
 150
 151                 Session distSession = null;
 152                 Session javaSession = null;
 153                 ZipInputStream zin = null;
 154                 try {
 155                         javaSession = osgiFactory.openJavaSession();
 156                         distSession = osgiFactory.openDistSession();
 157
 158                         if (log.isDebugEnabled())
 159                                 log.debug("Wrapping " + uri);
 160                         boolean nothingWasDone = true;
 161
 162                         Node distNode = osgiFactory.getDist(distSession, uri);
 163                         zin = new ZipInputStream(distNode.getNode(Node.JCR_CONTENT)
 164                                         .getProperty(Property.JCR_DATA).getBinary().getStream());
 165
 166                         ZipEntry zentry = null;
 167                         entries: while ((zentry = zin.getNextEntry()) != null) {
 168                                 String name = zentry.getName();
 169
 170                                 // sources autodetect
 171                                 String baseName = FilenameUtils.getBaseName(name);
 172                                 if (baseName.endsWith("-sources")) {
 173                                         String bundle = baseName.substring(0, baseName.length()
 174                                                         - "-sources".length());
 175                                         // log.debug(name + "," + baseName + ", " + bundle);
 176                                         String bundlePath = FilenameUtils.getPath(name) + bundle
 177                                                         + ".jar";
 178                                         if (wrappers.containsKey(bundlePath)) {
 179                                                 BndWrapper wrapper = wrappers.get(bundlePath);
 180                                                 NameVersion bundleNv = new DefaultNameVersion(
 181                                                                 wrapper.getName(), wrapper.getVersion());
 182                                                 byte[] pdeSource = RepoUtils.packageAsPdeSource(zin,
 183                                                                 bundleNv);
 184                                                 Artifact sourcesArtifact = new DefaultArtifact(
 185                                                                 wrapper.getCategory(), wrapper.getName()
 186                                                                                 + ".source", "jar",
 187                                                                 wrapper.getVersion());
 188                                                 Node pdeSourceNode = RepoUtils.copyBytesAsArtifact(
 189                                                                 javaSession.getRootNode(), sourcesArtifact,
 190                                                                 pdeSource);
 191                                                 osgiFactory.indexNode(pdeSourceNode);
 192                                                 pdeSourceNode.getSession().save();
 193                                                 if (log.isDebugEnabled())
 194                                                         log.debug("Added sources " + sourcesArtifact
 195                                                                         + " for bundle " + wrapper.getArtifact()
 196                                                                         + "from " + name + " in binary archive.");
 197                                         }
 198
 199                                 }
 200                                 // else if (baseName.endsWith(".source")) {
 201                                 // }
 202
 203                                 // binaries
 204                                 if (wrappers.containsKey(name)) {
 205                                         BndWrapper wrapper = (BndWrapper) wrappers.get(name);
 206                                         // we must copy since the stream is closed by BND
 207                                         byte[] origJarBytes = IOUtils.toByteArray(zin);
 208                                         Artifact artifact = wrapZipEntry(javaSession, zentry,
 209                                                         origJarBytes, wrapper);
 210                                         nothingWasDone = false;
 211                                         addArtifactToIndex(binaries, wrapper.getGroupId(), artifact);
 212                                 } else {
 213                                         for (String wrapperKey : wrappers.keySet())
 214                                                 if (pathMatcher.match(wrapperKey, name)) {
 215                                                         // first matched is taken
 216                                                         BndWrapper wrapper = (BndWrapper) wrappers
 217                                                                         .get(wrapperKey);
 218                                                         // we must copy since the stream is closed by BND
 219                                                         byte[] origJarBytes = IOUtils.toByteArray(zin);
 220                                                         Artifact artifact = wrapZipEntry(javaSession,
 221                                                                         zentry, origJarBytes, wrapper);
 222                                                         nothingWasDone = false;
 223                                                         addArtifactToIndex(binaries, wrapper.getGroupId(),
 224                                                                         artifact);
 225                                                         continue entries;
 226                                                 } else {
 227                                                         if (log.isTraceEnabled())
 228                                                                 log.trace(name + " not matched by "
 229                                                                                 + wrapperKey);
 230                                                 }
 231
 232                                         for (String exclude : excludes)
 233                                                 if (pathMatcher.match(exclude, name))
 234                                                         continue entries;
 235
 236                                         for (String include : includes.keySet()) {
 237                                                 if (pathMatcher.match(include, name)) {
 238                                                         String groupId = includes.get(include);
 239                                                         byte[] origJarBytes = IOUtils.toByteArray(zin);
 240                                                         Artifact artifact = importZipEntry(javaSession,
 241                                                                         zentry, origJarBytes, groupId);
 242                                                         if (artifact == null) {
 243                                                                 log.warn("Skipped non identified " + zentry);
 244                                                                 continue entries;
 245                                                         }
 246                                                         nothingWasDone = false;
 247                                                         if (artifact.getArtifactId().endsWith(".source"))
 248                                                                 addArtifactToIndex(sources, groupId, artifact);
 249                                                         else
 250                                                                 addArtifactToIndex(binaries, groupId, artifact);
 251                                                 }
 252                                         }
 253                                 }
 254                         }
 255
 256                         // indexes
 257                         if (mavenGroupIndexes && version != null) {
 258                                 for (String groupId : binaries.keySet()) {
 259                                         RepoUtils.writeGroupIndexes(javaSession, "/", groupId,
 260                                                         version, binaries.get(groupId),
 261                                                         sources.containsKey(groupId) ? sources.get(groupId)
 262                                                                         : null);
 263                                 }
 264                         }
 265
 266                         if (nothingWasDone)
 267                                 throw new SlcException("Nothing was done");
 268
 269                         // FIXME Fail if not all wrappers matched
 270                 } catch (Exception e) {
 271                         throw new SlcException("Cannot wrap distribution " + uri, e);
 272                 } finally {
 273                         IOUtils.closeQuietly(zin);
 274                         JcrUtils.logoutQuietly(distSession);
 275                         JcrUtils.logoutQuietly(javaSession);
 276                 }
 277         }
 278
 279         protected Artifact wrapZipEntry(Session javaSession, ZipEntry zentry,
 280                         byte[] origJarBytes, BndWrapper wrapper) throws RepositoryException {
 281                 ByteArrayOutputStream out = null;
 282                 ByteArrayInputStream in = null;
 283                 Node newJarNode;
 284                 Jar jar = null;
 285                 try {
 286                         out = new ByteArrayOutputStream((int) zentry.getSize());
 287                         in = new ByteArrayInputStream(origJarBytes);
 288                         wrapper.wrapJar(in, out);
 289
 290                         Artifact artifact = wrapper.getArtifact();
 291                         newJarNode = RepoUtils.copyBytesAsArtifact(
 292                                         javaSession.getRootNode(), artifact, out.toByteArray());
 293                         osgiFactory.indexNode(newJarNode);
 294                         newJarNode.getSession().save();
 295                         if (log.isDebugEnabled())
 296                                 log.debug("Wrapped jar " + zentry.getName() + " to "
 297                                                 + newJarNode.getPath());
 298
 299                         if (sourcesProvider != null)
 300                                 addSource(javaSession, artifact, out.toByteArray());
 301
 302                         return artifact;
 303                 } finally {
 304                         IOUtils.closeQuietly(in);
 305                         IOUtils.closeQuietly(out);
 306                         if (jar != null)
 307                                 jar.close();
 308                 }
 309         }
 310
 311         protected void addSource(Session javaSession, Artifact artifact,
 312                         byte[] binaryJarBytes) {
 313                 InputStream in = null;
 314                 ByteArrayOutputStream out = null;
 315                 Jar jar = null;
 316                 try {
 317                         in = new ByteArrayInputStream(binaryJarBytes);
 318                         jar = new Jar(null, in);
 319                         List<String> packages = jar.getPackages();
 320
 321                         out = new ByteArrayOutputStream();
 322                         sourcesProvider.writeSources(packages, new ZipOutputStream(out));
 323
 324                         IOUtils.closeQuietly(in);
 325                         in = new ByteArrayInputStream(out.toByteArray());
 326                         byte[] sourcesJar = RepoUtils.packageAsPdeSource(
 327                                         in,
 328                                         new DefaultNameVersion(artifact.getArtifactId(), artifact
 329                                                         .getVersion()));
 330                         Artifact sourcesArtifact = new DefaultArtifact(
 331                                         artifact.getGroupId(),
 332                                         artifact.getArtifactId() + ".source", "jar",
 333                                         artifact.getVersion());
 334                         Node sourcesJarNode = RepoUtils.copyBytesAsArtifact(
 335                                         javaSession.getRootNode(), sourcesArtifact, sourcesJar);
 336                         sourcesJarNode.getSession().save();
 337
 338                         if (log.isDebugEnabled())
 339                                 log.debug("Added sources " + sourcesArtifact + " for bundle "
 340                                                 + artifact + "from source provider " + sourcesProvider);
 341                 } catch (Exception e) {
 342                         throw new SlcException("Cannot get sources for " + artifact, e);
 343                 } finally {
 344                         IOUtils.closeQuietly(in);
 345                         IOUtils.closeQuietly(out);
 346                         if (jar != null)
 347                                 jar.close();
 348                 }
 349         }
 350
 351         protected Artifact importZipEntry(Session javaSession, ZipEntry zentry,
 352                         byte[] binaryJarBytes, String groupId) throws RepositoryException {
 353                 ByteArrayInputStream in = null;
 354                 Node newJarNode;
 355                 try {
 356                         in = new ByteArrayInputStream(binaryJarBytes);
 357                         NameVersion nameVersion = RepoUtils.readNameVersion(in);
 358                         if (nameVersion == null) {
 359                                 return null;
 360                         }
 361                         Artifact artifact = new DefaultArtifact(groupId,
 362                                         nameVersion.getName(), "jar", nameVersion.getVersion());
 363                         newJarNode = RepoUtils.copyBytesAsArtifact(
 364                                         javaSession.getRootNode(), artifact, binaryJarBytes);
 365                         osgiFactory.indexNode(newJarNode);
 366                         newJarNode.getSession().save();
 367                         if (log.isDebugEnabled())
 368                                 log.debug("Imported OSGi bundle " + zentry.getName() + " to "
 369                                                 + newJarNode.getPath());
 370
 371                         if (sourcesProvider != null)
 372                                 addSource(javaSession, artifact, binaryJarBytes);
 373
 374                         return artifact;
 375                 } finally {
 376                         IOUtils.closeQuietly(in);
 377                 }
 378         }
 379
 380         private void addArtifactToIndex(Map<String, Set<Artifact>> index,
 381                         String groupId, Artifact artifact) {
 382                 if (!index.containsKey(groupId))
 383                         index.put(groupId,
 384                                         new TreeSet<Artifact>(new ArtifactIdComparator()));
 385                 index.get(groupId).add(artifact);
 386         }
 387
 388         public void setUri(String uri) {
 389                 this.uri = uri;
 390         }
 391
 392         public void setWrappers(Map<String, BndWrapper> wrappers) {
 393                 this.wrappers = wrappers;
 394         }
 395
 396         public void setOsgiFactory(OsgiFactory osgiFactory) {
 397                 this.osgiFactory = osgiFactory;
 398         }
 399
 400         public void setVersion(String version) {
 401                 this.version = version;
 402         }
 403
 404         public void setLicense(License license) {
 405                 this.license = license;
 406         }
 407
 408         public void setPathMatcher(PathMatcher pathMatcher) {
 409                 this.pathMatcher = pathMatcher;
 410         }
 411
 412         public void setIncludes(Map<String, String> includes) {
 413                 this.includes = includes;
 414         }
 415
 416         public void setExcludes(List<String> excludes) {
 417                 this.excludes = excludes;
 418         }
 419
 420         public void setMavenGroupIndexes(Boolean mavenGroupIndexes) {
 421                 this.mavenGroupIndexes = mavenGroupIndexes;
 422         }
 423
 424         public void setSourcesProvider(SourcesProvider sourcesProvider) {
 425                 this.sourcesProvider = sourcesProvider;
 426         }
 427
 428 }