org.argeo.slc.repo/src/org/argeo/slc/repo/osgi/ArchiveWrapper.java

   1 package org.argeo.slc.repo.osgi;
   2
   3 import java.io.ByteArrayInputStream;
   4 import java.io.ByteArrayOutputStream;
   5 import java.io.InputStream;
   6 import java.util.ArrayList;
   7 import java.util.HashMap;
   8 import java.util.Iterator;
   9 import java.util.List;
  10 import java.util.Map;
  11 import java.util.Set;
  12 import java.util.TreeSet;
  13 import java.util.jar.JarInputStream;
  14 import java.util.zip.ZipEntry;
  15 import java.util.zip.ZipInputStream;
  16 import java.util.zip.ZipOutputStream;
  17
  18 import javax.jcr.Node;
  19 import javax.jcr.Property;
  20 import javax.jcr.RepositoryException;
  21 import javax.jcr.Session;
  22
  23 import org.apache.commons.io.FilenameUtils;
  24 import org.apache.commons.io.IOUtils;
  25 import org.argeo.api.cms.CmsLog;
  26 import org.argeo.jcr.JcrUtils;
  27 import org.argeo.slc.CategoryNameVersion;
  28 import org.argeo.slc.DefaultNameVersion;
  29 import org.argeo.slc.ModuleSet;
  30 import org.argeo.slc.NameVersion;
  31 import org.argeo.slc.SlcException;
  32 import org.argeo.slc.build.Distribution;
  33 import org.argeo.slc.build.License;
  34 import org.argeo.slc.repo.OsgiFactory;
  35 import org.argeo.slc.repo.RepoUtils;
  36 import org.argeo.slc.repo.internal.springutil.AntPathMatcher;
  37 import org.argeo.slc.repo.internal.springutil.PathMatcher;
  38 import org.argeo.slc.repo.maven.ArtifactIdComparator;
  39 import org.eclipse.aether.artifact.Artifact;
  40 import org.eclipse.aether.artifact.DefaultArtifact;
  41
  42 import aQute.bnd.osgi.Jar;
  43
  44 /**
  45  * Download a software distribution and generates the related OSGi bundles from
  46  * the jars, or import them directly if they are already OSGi bundles and don't
  47  * need further modification.
  48  */
  49 public class ArchiveWrapper implements Runnable, ModuleSet, Distribution {
  50         private final static CmsLog log = CmsLog.getLog(ArchiveWrapper.class);
  51
  52         private OsgiFactory osgiFactory;
  53         private String version;
  54         private License license;
  55
  56         private String uri;
  57
  58         /** Jars to wrap as OSGi bundles */
  59         private Map<String, BndWrapper> wrappers = new HashMap<String, BndWrapper>();
  60
  61         private SourcesProvider sourcesProvider;
  62
  63         // pattern of OSGi bundles to import
  64         private PathMatcher pathMatcher = new AntPathMatcher();
  65         private Map<String, String> includes = new HashMap<String, String>();
  66         private List<String> excludes = new ArrayList<String>();
  67
  68         private Boolean mavenGroupIndexes = false;
  69
  70         public void init() {
  71                 for (BndWrapper wrapper : wrappers.values()) {
  72                         wrapper.setFactory(this);
  73                         if (version != null && wrapper.getVersion() == null)
  74                                 wrapper.setVersion(version);
  75                         if (license != null && wrapper.getLicense() == null)
  76                                 wrapper.setLicense(license);
  77                 }
  78         }
  79
  80         public void destroy() {
  81
  82         }
  83
  84         public String getDistributionId() {
  85                 return uri;
  86         }
  87
  88         public String getVersion() {
  89                 return version;
  90         }
  91
  92         public License getLicense() {
  93                 return license;
  94         }
  95
  96         public String getUri() {
  97                 return uri;
  98         }
  99
 100         public Iterator<? extends NameVersion> nameVersions() {
 101                 if (wrappers.size() > 0)
 102                         return wrappers.values().iterator();
 103                 else
 104                         return osgiNameVersions();
 105         }
 106
 107         @SuppressWarnings("resource")
 108         protected Iterator<? extends NameVersion> osgiNameVersions() {
 109                 List<CategoryNameVersion> nvs = new ArrayList<CategoryNameVersion>();
 110
 111                 Session distSession = null;
 112                 ZipInputStream zin = null;
 113                 try {
 114                         distSession = osgiFactory.openDistSession();
 115
 116                         Node distNode = osgiFactory.getDist(distSession, uri);
 117                         zin = new ZipInputStream(
 118                                         distNode.getNode(Node.JCR_CONTENT).getProperty(Property.JCR_DATA).getBinary().getStream());
 119
 120                         ZipEntry zentry = null;
 121                         entries: while ((zentry = zin.getNextEntry()) != null) {
 122                                 String name = zentry.getName();
 123                                 if (log.isTraceEnabled())
 124                                         log.trace("Zip entry " + name);
 125                                 for (String exclude : excludes)
 126                                         if (pathMatcher.match(exclude, name))
 127                                                 continue entries;
 128
 129                                 for (String include : includes.keySet()) {
 130                                         if (pathMatcher.match(include, name)) {
 131                                                 String groupId = includes.get(include);
 132                                                 JarInputStream jis = new JarInputStream(zin);
 133                                                 if (jis.getManifest() == null) {
 134                                                         log.warn("No MANIFEST in entry " + name + ", skipping...");
 135                                                         continue entries;
 136                                                 }
 137                                                 NameVersion nv = RepoUtils.readNameVersion(jis.getManifest());
 138                                                 if (nv != null) {
 139                                                         if (nv.getName().endsWith(".source"))
 140                                                                 continue entries;
 141                                                         CategoryNameVersion cnv = new ArchiveWrapperCNV(groupId, nv.getName(), nv.getVersion(),
 142                                                                         this);
 143                                                         nvs.add(cnv);
 144                                                         // no need to process further includes
 145                                                         continue entries;
 146                                                 }
 147                                         }
 148                                 }
 149                         }
 150                         return nvs.iterator();
 151                 } catch (Exception e) {
 152                         throw new SlcException("Cannot wrap distribution " + uri, e);
 153                 } finally {
 154                         IOUtils.closeQuietly(zin);
 155                         JcrUtils.logoutQuietly(distSession);
 156                 }
 157         }
 158
 159         public void run() {
 160                 if (mavenGroupIndexes && (version == null))
 161                         throw new SlcException("'mavenGroupIndexes' requires 'version' to be set");
 162
 163                 Map<String, Set<Artifact>> binaries = new HashMap<String, Set<Artifact>>();
 164                 Map<String, Set<Artifact>> sources = new HashMap<String, Set<Artifact>>();
 165
 166                 Session distSession = null;
 167                 Session javaSession = null;
 168                 ZipInputStream zin = null;
 169                 try {
 170                         javaSession = osgiFactory.openJavaSession();
 171                         distSession = osgiFactory.openDistSession();
 172
 173                         if (log.isDebugEnabled())
 174                                 log.debug("Wrapping " + uri);
 175                         boolean nothingWasDone = true;
 176
 177                         Node distNode = osgiFactory.getDist(distSession, uri);
 178                         zin = new ZipInputStream(
 179                                         distNode.getNode(Node.JCR_CONTENT).getProperty(Property.JCR_DATA).getBinary().getStream());
 180
 181                         ZipEntry zentry = null;
 182                         entries: while ((zentry = zin.getNextEntry()) != null) {
 183                                 String name = zentry.getName();
 184
 185                                 // sources autodetect
 186                                 String baseName = FilenameUtils.getBaseName(name);
 187                                 if (baseName.endsWith("-sources")) {
 188                                         String bundle = baseName.substring(0, baseName.length() - "-sources".length());
 189                                         // log.debug(name + "," + baseName + ", " + bundle);
 190                                         String bundlePath = FilenameUtils.getPath(name) + bundle + ".jar";
 191                                         if (wrappers.containsKey(bundlePath)) {
 192                                                 BndWrapper wrapper = wrappers.get(bundlePath);
 193                                                 NameVersion bundleNv = new DefaultNameVersion(wrapper.getName(), wrapper.getVersion());
 194                                                 byte[] pdeSource = RepoUtils.packageAsPdeSource(zin, bundleNv);
 195                                                 Artifact sourcesArtifact = new DefaultArtifact(wrapper.getCategory(),
 196                                                                 wrapper.getName() + ".source", "jar", wrapper.getVersion());
 197                                                 Node pdeSourceNode = RepoUtils.copyBytesAsArtifact(javaSession.getRootNode(), sourcesArtifact,
 198                                                                 pdeSource);
 199                                                 osgiFactory.indexNode(pdeSourceNode);
 200                                                 pdeSourceNode.getSession().save();
 201                                                 if (log.isDebugEnabled())
 202                                                         log.debug("Added sources " + sourcesArtifact + " for bundle " + wrapper.getArtifact()
 203                                                                         + "from " + name + " in binary archive.");
 204                                         }
 205
 206                                 }
 207                                 // else if (baseName.endsWith(".source")) {
 208                                 // }
 209
 210                                 // binaries
 211                                 if (wrappers.containsKey(name)) {
 212                                         BndWrapper wrapper = (BndWrapper) wrappers.get(name);
 213                                         // we must copy since the stream is closed by BND
 214                                         byte[] origJarBytes = IOUtils.toByteArray(zin);
 215                                         Artifact artifact = wrapZipEntry(javaSession, zentry, origJarBytes, wrapper);
 216                                         nothingWasDone = false;
 217                                         addArtifactToIndex(binaries, wrapper.getGroupId(), artifact);
 218                                 } else {
 219                                         for (String wrapperKey : wrappers.keySet())
 220                                                 if (pathMatcher.match(wrapperKey, name)) {
 221                                                         // first matched is taken
 222                                                         BndWrapper wrapper = (BndWrapper) wrappers.get(wrapperKey);
 223                                                         // we must copy since the stream is closed by BND
 224                                                         byte[] origJarBytes = IOUtils.toByteArray(zin);
 225                                                         Artifact artifact = wrapZipEntry(javaSession, zentry, origJarBytes, wrapper);
 226                                                         nothingWasDone = false;
 227                                                         addArtifactToIndex(binaries, wrapper.getGroupId(), artifact);
 228                                                         continue entries;
 229                                                 } else {
 230                                                         if (log.isTraceEnabled())
 231                                                                 log.trace(name + " not matched by " + wrapperKey);
 232                                                 }
 233
 234                                         for (String exclude : excludes)
 235                                                 if (pathMatcher.match(exclude, name))
 236                                                         continue entries;
 237
 238                                         for (String include : includes.keySet()) {
 239                                                 if (pathMatcher.match(include, name)) {
 240                                                         String groupId = includes.get(include);
 241                                                         byte[] origJarBytes = IOUtils.toByteArray(zin);
 242                                                         Artifact artifact = importZipEntry(javaSession, zentry, origJarBytes, groupId);
 243                                                         if (artifact == null) {
 244                                                                 log.warn("Skipped non identified " + zentry);
 245                                                                 continue entries;
 246                                                         }
 247                                                         nothingWasDone = false;
 248                                                         if (artifact.getArtifactId().endsWith(".source"))
 249                                                                 addArtifactToIndex(sources, groupId, artifact);
 250                                                         else
 251                                                                 addArtifactToIndex(binaries, groupId, artifact);
 252                                                         // no need to process this entry further
 253                                                         continue entries;
 254                                                 }
 255                                         }
 256                                 }
 257                         }
 258
 259                         // indexes
 260                         if (mavenGroupIndexes && version != null) {
 261                                 for (String groupId : binaries.keySet()) {
 262                                         RepoUtils.writeGroupIndexes(javaSession, "/", groupId, version, binaries.get(groupId),
 263                                                         sources.containsKey(groupId) ? sources.get(groupId) : null);
 264                                 }
 265                         }
 266
 267                         if (nothingWasDone) {
 268                                 log.error("Nothing was done when wrapping " + uri + ". THE DISTRIBUTION IS INCONSISTENT.");
 269                                 // throw new SlcException("Nothing was done");
 270                                 // TODO Fail if not all wrappers matched
 271                         }
 272
 273                 } catch (Exception e) {
 274                         throw new SlcException("Cannot wrap distribution " + uri, e);
 275                 } finally {
 276                         IOUtils.closeQuietly(zin);
 277                         JcrUtils.logoutQuietly(distSession);
 278                         JcrUtils.logoutQuietly(javaSession);
 279                 }
 280         }
 281
 282         protected Artifact wrapZipEntry(Session javaSession, ZipEntry zentry, byte[] origJarBytes, BndWrapper wrapper)
 283                         throws RepositoryException {
 284                 ByteArrayOutputStream out = null;
 285                 ByteArrayInputStream in = null;
 286                 Node newJarNode;
 287                 Jar jar = null;
 288                 try {
 289                         out = new ByteArrayOutputStream((int) zentry.getSize());
 290                         in = new ByteArrayInputStream(origJarBytes);
 291                         wrapper.wrapJar(in, out);
 292
 293                         Artifact artifact = wrapper.getArtifact();
 294                         newJarNode = RepoUtils.copyBytesAsArtifact(javaSession.getRootNode(), artifact, out.toByteArray());
 295                         osgiFactory.indexNode(newJarNode);
 296                         newJarNode.getSession().save();
 297                         if (log.isDebugEnabled())
 298                                 log.debug("Wrapped jar " + zentry.getName() + " to " + newJarNode.getPath());
 299
 300                         if (sourcesProvider != null)
 301                                 addSource(javaSession, artifact, out.toByteArray());
 302
 303                         return artifact;
 304                 } finally {
 305                         IOUtils.closeQuietly(in);
 306                         IOUtils.closeQuietly(out);
 307                         if (jar != null)
 308                                 jar.close();
 309                 }
 310         }
 311
 312         protected void addSource(Session javaSession, Artifact artifact, byte[] binaryJarBytes) {
 313                 InputStream in = null;
 314                 ByteArrayOutputStream out = null;
 315                 Jar jar = null;
 316                 try {
 317                         in = new ByteArrayInputStream(binaryJarBytes);
 318                         jar = new Jar(null, in);
 319                         List<String> packages = jar.getPackages();
 320
 321                         out = new ByteArrayOutputStream();
 322                         sourcesProvider.writeSources(packages, new ZipOutputStream(out));
 323
 324                         IOUtils.closeQuietly(in);
 325                         in = new ByteArrayInputStream(out.toByteArray());
 326                         byte[] sourcesJar = RepoUtils.packageAsPdeSource(in,
 327                                         new DefaultNameVersion(artifact.getArtifactId(), artifact.getVersion()));
 328                         Artifact sourcesArtifact = new DefaultArtifact(artifact.getGroupId(), artifact.getArtifactId() + ".source",
 329                                         "jar", artifact.getVersion());
 330                         Node sourcesJarNode = RepoUtils.copyBytesAsArtifact(javaSession.getRootNode(), sourcesArtifact, sourcesJar);
 331                         sourcesJarNode.getSession().save();
 332
 333                         if (log.isDebugEnabled())
 334                                 log.debug("Added sources " + sourcesArtifact + " for bundle " + artifact + "from source provider "
 335                                                 + sourcesProvider);
 336                 } catch (Exception e) {
 337                         throw new SlcException("Cannot get sources for " + artifact, e);
 338                 } finally {
 339                         IOUtils.closeQuietly(in);
 340                         IOUtils.closeQuietly(out);
 341                         if (jar != null)
 342                                 jar.close();
 343                 }
 344         }
 345
 346         protected Artifact importZipEntry(Session javaSession, ZipEntry zentry, byte[] binaryJarBytes, String groupId)
 347                         throws RepositoryException {
 348                 ByteArrayInputStream in = null;
 349                 Node newJarNode;
 350                 try {
 351                         in = new ByteArrayInputStream(binaryJarBytes);
 352                         NameVersion nameVersion = RepoUtils.readNameVersion(in);
 353                         if (nameVersion == null) {
 354                                 log.warn("Cannot identify " + zentry.getName());
 355                                 return null;
 356                         }
 357                         Artifact artifact = new DefaultArtifact(groupId, nameVersion.getName(), "jar", nameVersion.getVersion());
 358                         newJarNode = RepoUtils.copyBytesAsArtifact(javaSession.getRootNode(), artifact, binaryJarBytes);
 359                         osgiFactory.indexNode(newJarNode);
 360                         newJarNode.getSession().save();
 361                         if (log.isDebugEnabled()) {
 362                                 log.debug(zentry.getName() + " => " + artifact);
 363                         }
 364
 365                         if (sourcesProvider != null)
 366                                 addSource(javaSession, artifact, binaryJarBytes);
 367
 368                         return artifact;
 369                 } finally {
 370                         IOUtils.closeQuietly(in);
 371                 }
 372         }
 373
 374         private void addArtifactToIndex(Map<String, Set<Artifact>> index, String groupId, Artifact artifact) {
 375                 if (!index.containsKey(groupId))
 376                         index.put(groupId, new TreeSet<Artifact>(new ArtifactIdComparator()));
 377                 index.get(groupId).add(artifact);
 378         }
 379
 380         public void setUri(String uri) {
 381                 this.uri = uri;
 382         }
 383
 384         public void setWrappers(Map<String, BndWrapper> wrappers) {
 385                 this.wrappers = wrappers;
 386         }
 387
 388         public void setOsgiFactory(OsgiFactory osgiFactory) {
 389                 this.osgiFactory = osgiFactory;
 390         }
 391
 392         public void setVersion(String version) {
 393                 this.version = version;
 394         }
 395
 396         public void setLicense(License license) {
 397                 this.license = license;
 398         }
 399
 400         public void setPathMatcher(PathMatcher pathMatcher) {
 401                 this.pathMatcher = pathMatcher;
 402         }
 403
 404         public void setIncludes(Map<String, String> includes) {
 405                 this.includes = includes;
 406         }
 407
 408         public void setExcludes(List<String> excludes) {
 409                 this.excludes = excludes;
 410         }
 411
 412         public void setMavenGroupIndexes(Boolean mavenGroupIndexes) {
 413                 this.mavenGroupIndexes = mavenGroupIndexes;
 414         }
 415
 416         public void setSourcesProvider(SourcesProvider sourcesProvider) {
 417                 this.sourcesProvider = sourcesProvider;
 418         }
 419
 420         public Map<String, BndWrapper> getWrappers() {
 421                 return wrappers;
 422         }
 423
 424         public Map<String, String> getIncludes() {
 425                 return includes;
 426         }
 427
 428         public List<String> getExcludes() {
 429                 return excludes;
 430         }
 431
 432 }