src/org/argeo/build/Repackage.java

   1 package org.argeo.build;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static java.lang.System.Logger.Level.ERROR;
   5 import static java.lang.System.Logger.Level.INFO;
   6 import static java.lang.System.Logger.Level.TRACE;
   7 import static java.lang.System.Logger.Level.WARNING;
   8 import static java.nio.file.FileVisitResult.CONTINUE;
   9 import static java.util.jar.Attributes.Name.MANIFEST_VERSION;
  10 import static org.argeo.build.Repackage.ManifestConstants.ARGEO_ORIGIN_EMBED;
  11 import static org.argeo.build.Repackage.ManifestConstants.ARGEO_ORIGIN_M2;
  12 import static org.argeo.build.Repackage.ManifestConstants.ARGEO_ORIGIN_M2_MERGE;
  13 import static org.argeo.build.Repackage.ManifestConstants.ARGEO_ORIGIN_M2_REPO;
  14 import static org.argeo.build.Repackage.ManifestConstants.ARGEO_ORIGIN_MANIFEST_NOT_MODIFIED;
  15 import static org.argeo.build.Repackage.ManifestConstants.ARGEO_ORIGIN_URI;
  16 import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_LICENSE;
  17 import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_SYMBOLICNAME;
  18 import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_VERSION;
  19 import static org.argeo.build.Repackage.ManifestConstants.ECLIPSE_SOURCE_BUNDLE;
  20 import static org.argeo.build.Repackage.ManifestConstants.EXPORT_PACKAGE;
  21 import static org.argeo.build.Repackage.ManifestConstants.IMPORT_PACKAGE;
  22 import static org.argeo.build.Repackage.ManifestConstants.SPDX_LICENSE_IDENTIFIER;
  23
  24 import java.io.BufferedWriter;
  25 import java.io.File;
  26 import java.io.FileNotFoundException;
  27 import java.io.IOException;
  28 import java.io.InputStream;
  29 import java.io.OutputStream;
  30 import java.lang.System.Logger;
  31 import java.net.MalformedURLException;
  32 import java.net.URL;
  33 import java.nio.charset.StandardCharsets;
  34 import java.nio.file.DirectoryStream;
  35 import java.nio.file.FileSystem;
  36 import java.nio.file.FileSystems;
  37 import java.nio.file.FileVisitResult;
  38 import java.nio.file.Files;
  39 import java.nio.file.Path;
  40 import java.nio.file.PathMatcher;
  41 import java.nio.file.Paths;
  42 import java.nio.file.SimpleFileVisitor;
  43 import java.nio.file.StandardCopyOption;
  44 import java.nio.file.StandardOpenOption;
  45 import java.nio.file.attribute.BasicFileAttributes;
  46 import java.util.ArrayList;
  47 import java.util.HashMap;
  48 import java.util.Iterator;
  49 import java.util.List;
  50 import java.util.Map;
  51 import java.util.Objects;
  52 import java.util.Properties;
  53 import java.util.Set;
  54 import java.util.TreeMap;
  55 import java.util.TreeSet;
  56 import java.util.concurrent.CompletableFuture;
  57 import java.util.jar.Attributes;
  58 import java.util.jar.JarEntry;
  59 import java.util.jar.JarInputStream;
  60 import java.util.jar.JarOutputStream;
  61 import java.util.jar.Manifest;
  62 import java.util.zip.Deflater;
  63
  64 import aQute.bnd.osgi.Analyzer;
  65 import aQute.bnd.osgi.Jar;
  66
  67 /**
  68  * Simple tool repackaging existing jar files into OSGi bundles in an A2
  69  * repository.
  70  */
  71 public class Repackage {
  72         final static Logger logger = System.getLogger(Repackage.class.getName());
  73
  74         /**
  75          * Environment variable on whether sources should be packaged separately or
  76          * integrated in the bundles.
  77          */
  78         final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES";
  79
  80         /** Whether repackaging should run in parallel or sequentially. */
  81         final static boolean parallel = true;
  82
  83         // cache
  84         /** Summary of all license seen during the repackaging. */
  85         final static Map<String, Set<String>> licensesUsed = new TreeMap<>();
  86
  87         /** Main entry point. */
  88         public static void main(String[] args) {
  89                 if (args.length < 2) {
  90                         System.err.println("Usage: <path to a2 output dir> <category1> <category2> ...");
  91                         System.exit(1);
  92                 }
  93                 Path a2Base = Paths.get(args[0]).toAbsolutePath().normalize();
  94                 Path descriptorsBase = Paths.get(".").toAbsolutePath().normalize();
  95                 Repackage factory = new Repackage(a2Base, descriptorsBase);
  96
  97                 List<CompletableFuture<Void>> toDos = new ArrayList<>();
  98                 for (int i = 1; i < args.length; i++) {
  99                         Path p = Paths.get(args[i]);
 100                         if (parallel)
 101                                 toDos.add(CompletableFuture.runAsync(() -> factory.processCategory(p)));
 102                         else
 103                                 factory.processCategory(p);
 104                 }
 105                 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
 106
 107                 // Summary
 108                 StringBuilder sb = new StringBuilder();
 109                 for (String licenseId : licensesUsed.keySet())
 110                         for (String name : licensesUsed.get(licenseId))
 111                                 sb.append((licenseId.equals("") ? "Proprietary" : licenseId) + "\t\t" + name + "\n");
 112                 logger.log(INFO, "# License summary:\n" + sb);
 113         }
 114
 115         /** Name of the file centralising information for multiple M2 artifacts. */
 116         final static String COMMON_BND = "common.bnd";
 117         /** Name of the file centralising information for mergin M2 artifacts. */
 118         final static String MERGE_BND = "merge.bnd";
 119         /**
 120          * Subdirectory of the jar file where origin informations (changes, legal
 121          * notices etc. are stored)
 122          */
 123         final static String A2_ORIGIN = "A2-ORIGIN";
 124
 125         /** Directory where to download archives */
 126         final Path originBase;
 127         /** Directory where to download Maven artifacts */
 128         final Path mavenBase;
 129
 130         /** A2 repository base for binary bundles */
 131         final Path a2Base;
 132         /** A2 repository base for source bundles */
 133         final Path a2SrcBase;
 134         /** A2 base for native components */
 135         final Path a2LibBase;
 136         /** Location of the descriptors driving the packaging */
 137         final Path descriptorsBase;
 138         /** URIs of archives to download */
 139         final Properties uris = new Properties();
 140         /** Mirrors for archive download. Key is URI prefix, value list of base URLs */
 141         final Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
 142
 143         /** Whether sources should be packaged separately */
 144         final boolean sourceBundles;
 145
 146         /** Constructor initialises the various variables */
 147         public Repackage(Path a2Base, Path descriptorsBase) {
 148                 sourceBundles = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES));
 149                 if (sourceBundles)
 150                         logger.log(INFO, "Sources will be packaged separately");
 151
 152                 Objects.requireNonNull(a2Base);
 153                 Objects.requireNonNull(descriptorsBase);
 154                 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/build/origin");
 155                 this.mavenBase = Paths.get(System.getProperty("user.home"), ".m2", "repository");
 156
 157                 // TODO define and use a build base
 158                 this.a2Base = a2Base;
 159                 this.a2SrcBase = a2Base.getParent().resolve(a2Base.getFileName() + ".src");
 160                 this.a2LibBase = a2Base.resolve("lib");
 161                 this.descriptorsBase = descriptorsBase;
 162                 if (!Files.exists(this.descriptorsBase))
 163                         throw new IllegalArgumentException(this.descriptorsBase + " does not exist");
 164
 165                 // URIs mapping
 166                 Path urisPath = this.descriptorsBase.resolve("uris.properties");
 167                 if (Files.exists(urisPath)) {
 168                         try (InputStream in = Files.newInputStream(urisPath)) {
 169                                 uris.load(in);
 170                         } catch (IOException e) {
 171                                 throw new IllegalStateException("Cannot load " + urisPath, e);
 172                         }
 173                 }
 174
 175                 // Eclipse mirrors
 176                 Path eclipseMirrorsPath = this.descriptorsBase.resolve("eclipse.mirrors.txt");
 177                 List<String> eclipseMirrors = new ArrayList<>();
 178                 if (Files.exists(eclipseMirrorsPath)) {
 179                         try {
 180                                 eclipseMirrors = Files.readAllLines(eclipseMirrorsPath, StandardCharsets.UTF_8);
 181                         } catch (IOException e) {
 182                                 throw new IllegalStateException("Cannot load " + eclipseMirrorsPath, e);
 183                         }
 184                         for (Iterator<String> it = eclipseMirrors.iterator(); it.hasNext();) {
 185                                 String value = it.next();
 186                                 if (value.strip().equals(""))
 187                                         it.remove();
 188                         }
 189                 }
 190                 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
 191         }
 192
 193         /*
 194          * MAVEN ORIGIN
 195          */
 196         /** Process a whole category/group id. */
 197         void processCategory(Path categoryRelativePath) {
 198                 try {
 199                         Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath);
 200                         DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
 201                                         (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
 202                                                         && !p.getFileName().toString().equals(MERGE_BND));
 203                         for (Path p : bnds) {
 204                                 processSingleM2ArtifactDistributionUnit(p);
 205                         }
 206
 207                         DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
 208                         for (Path duDir : dus) {
 209                                 if (duDir.getFileName().toString().startsWith("eclipse-")) {
 210                                         processEclipseArchive(duDir);
 211                                 } else {
 212                                         processM2BasedDistributionUnit(duDir);
 213                                 }
 214                         }
 215                 } catch (IOException e) {
 216                         throw new RuntimeException("Cannot process category " + categoryRelativePath, e);
 217                 }
 218         }
 219
 220         /** Process a standalone Maven artifact. */
 221         void processSingleM2ArtifactDistributionUnit(Path bndFile) {
 222                 try {
 223                         Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent());
 224                         Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
 225
 226                         Properties fileProps = new Properties();
 227                         try (InputStream in = Files.newInputStream(bndFile)) {
 228                                 fileProps.load(in);
 229                         }
 230                         String repoStr = fileProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
 231                                         ? fileProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
 232                                         : null;
 233
 234                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 235                                 // use file name as symbolic name
 236                                 String symbolicName = bndFile.getFileName().toString();
 237                                 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 238                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 239                         }
 240
 241                         String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
 242                         if (m2Coordinates == null)
 243                                 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
 244                         M2Artifact artifact = new M2Artifact(m2Coordinates);
 245                         URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
 246                         Path downloaded = downloadMaven(url, artifact);
 247
 248                         // some proprietary artifacts do not allow any modification
 249                         // when releasing (with separate sources) we just copy it
 250                         boolean doNotModify = Boolean.parseBoolean(
 251                                         fileProps.getOrDefault(ManifestConstants.ARGEO_DO_NOT_MODIFY.toString(), "false").toString());
 252                         if (doNotModify && sourceBundles) {
 253                                 Path unmodifiedTarget = targetCategoryBase.resolve(
 254                                                 fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch() + ".jar");
 255                                 Files.copy(downloaded, unmodifiedTarget, StandardCopyOption.REPLACE_EXISTING);
 256                                 Path bundleDir = targetCategoryBase
 257                                                 .resolve(fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch());
 258                                 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
 259                                 Manifest manifest;
 260                                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(unmodifiedTarget))) {
 261                                         manifest = jarIn.getManifest();
 262                                 }
 263                                 createSourceJar(bundleDir, manifest);
 264                                 return;
 265                         }
 266
 267                         // regular processing
 268                         A2Origin origin = new A2Origin();
 269                         Path bundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact, origin);
 270                         downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
 271                         createJar(bundleDir, origin);
 272                 } catch (Exception e) {
 273                         throw new RuntimeException("Cannot process " + bndFile, e);
 274                 }
 275         }
 276
 277         /**
 278          * Process multiple Maven artifacts coming from a same project and therefore
 279          * with information in common (typically the version), generating single bundles
 280          * or merging them if necessary.
 281          */
 282         void processM2BasedDistributionUnit(Path duDir) {
 283                 try {
 284                         Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
 285                         Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
 286
 287                         Path mergeBnd = duDir.resolve(MERGE_BND);
 288                         if (Files.exists(mergeBnd)) // merge
 289                                 mergeM2Artifacts(mergeBnd);
 290
 291                         Path commonBnd = duDir.resolve(COMMON_BND);
 292                         if (!Files.exists(commonBnd))
 293                                 return;
 294
 295                         Properties commonProps = new Properties();
 296                         try (InputStream in = Files.newInputStream(commonBnd)) {
 297                                 commonProps.load(in);
 298                         }
 299
 300                         String m2Version = commonProps.getProperty(ARGEO_ORIGIN_M2.toString());
 301                         if (m2Version == null) {
 302                                 logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
 303                                 return;// ignore, this is probably an Eclipse archive
 304                         }
 305                         if (!m2Version.startsWith(":")) {
 306                                 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 307                         }
 308                         m2Version = m2Version.substring(1);
 309
 310                         DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
 311                                         (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
 312                                                         && !p.getFileName().toString().equals(MERGE_BND));
 313                         for (Path p : ds) {
 314                                 Properties fileProps = new Properties();
 315                                 try (InputStream in = Files.newInputStream(p)) {
 316                                         fileProps.load(in);
 317                                 }
 318                                 String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
 319                                 M2Artifact artifact = new M2Artifact(m2Coordinates);
 320                                 artifact.setVersion(m2Version);
 321
 322                                 // prepare manifest entries
 323                                 Properties mergeProps = new Properties();
 324                                 mergeProps.putAll(commonProps);
 325
 326                                 fileEntries: for (Object key : fileProps.keySet()) {
 327                                         if (ARGEO_ORIGIN_M2.toString().equals(key))
 328                                                 continue fileEntries;
 329                                         String value = fileProps.getProperty(key.toString());
 330                                         Object previousValue = mergeProps.put(key.toString(), value);
 331                                         if (previousValue != null) {
 332                                                 logger.log(WARNING,
 333                                                                 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
 334                                         }
 335                                 }
 336                                 mergeProps.put(ARGEO_ORIGIN_M2.toString(), artifact.toM2Coordinates());
 337                                 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 338                                         // use file name as symbolic name
 339                                         String symbolicName = p.getFileName().toString();
 340                                         symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 341                                         mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 342                                 }
 343
 344                                 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
 345                                                 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
 346                                                 : null;
 347
 348                                 // download
 349                                 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
 350                                 Path downloaded = downloadMaven(url, artifact);
 351
 352                                 A2Origin origin = new A2Origin();
 353                                 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact, origin);
 354                                 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir, false);
 355                                 createJar(targetBundleDir, origin);
 356                         }
 357                 } catch (IOException e) {
 358                         throw new RuntimeException("Cannot process " + duDir, e);
 359                 }
 360         }
 361
 362         /** Merge multiple Maven artifacts. */
 363         void mergeM2Artifacts(Path mergeBnd) throws IOException {
 364                 Path duDir = mergeBnd.getParent();
 365                 String category = duDir.getParent().getFileName().toString();
 366                 Path targetCategoryBase = a2Base.resolve(category);
 367
 368                 Properties mergeProps = new Properties();
 369                 try (InputStream in = Files.newInputStream(mergeBnd)) {
 370                         mergeProps.load(in);
 371                 }
 372
 373                 String m2Version = mergeProps.getProperty(ARGEO_ORIGIN_M2.toString());
 374                 if (m2Version == null) {
 375                         logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
 376                         return;// ignore, this is probably an Eclipse archive
 377                 }
 378                 if (!m2Version.startsWith(":")) {
 379                         throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 380                 }
 381                 m2Version = m2Version.substring(1);
 382                 mergeProps.put(BUNDLE_VERSION.toString(), m2Version);
 383
 384                 String artifactsStr = mergeProps.getProperty(ARGEO_ORIGIN_M2_MERGE.toString());
 385                 if (artifactsStr == null)
 386                         throw new IllegalArgumentException(mergeBnd + ": " + ARGEO_ORIGIN_M2_MERGE + " must be set");
 387
 388                 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
 389                                 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
 390                                 : null;
 391
 392                 String bundleSymbolicName = mergeProps.getProperty(BUNDLE_SYMBOLICNAME.toString());
 393                 if (bundleSymbolicName == null)
 394                         throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
 395                 CategoryNameVersion nameVersion = new M2Artifact(category + ":" + bundleSymbolicName + ":" + m2Version);
 396
 397                 A2Origin origin = new A2Origin();
 398                 Path bundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
 399
 400                 String[] artifacts = artifactsStr.split(",");
 401                 artifacts: for (String str : artifacts) {
 402                         String m2Coordinates = str.trim();
 403                         if ("".equals(m2Coordinates))
 404                                 continue artifacts;
 405                         M2Artifact artifact = new M2Artifact(m2Coordinates.trim());
 406                         if (artifact.getVersion() == null)
 407                                 artifact.setVersion(m2Version);
 408                         URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
 409                         Path downloaded = downloadMaven(url, artifact);
 410                         JarEntry entry;
 411                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
 412                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 413                                         if (entry.isDirectory())
 414                                                 continue entries;
 415                                         if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
 416                                                         || entry.getName().endsWith(".SF")) {
 417                                                 origin.deleted.add("cryptographic signatures from " + artifact);
 418                                                 continue entries;
 419                                         }
 420                                         if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
 421                                                 origin.deleted.add("Java module information (module-info.class) from " + artifact);
 422                                                 continue entries;
 423                                         }
 424                                         if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
 425                                                 origin.deleted.add("additional Java versions (META-INF/versions) from " + artifact);
 426                                                 continue entries;
 427                                         }
 428                                         if (entry.getName().startsWith("META-INF/maven/")) {
 429                                                 origin.deleted.add("Maven information (META-INF/maven) from " + artifact);
 430                                                 continue entries;
 431                                         }
 432                                         if (entry.getName().startsWith(".cache/")) { // Apache SSHD
 433                                                 origin.deleted.add("cache directory (.cache) from " + artifact);
 434                                                 continue entries;
 435                                         }
 436                                         if (entry.getName().equals("META-INF/DEPENDENCIES")) {
 437                                                 origin.deleted.add("Dependencies (META-INF/DEPENDENCIES) from " + artifact);
 438                                                 continue entries;
 439                                         }
 440                                         if (entry.getName().equals("META-INF/MANIFEST.MF")) {
 441                                                 Path originalManifest = bundleDir.resolve(A2_ORIGIN).resolve(artifact.getGroupId())
 442                                                                 .resolve(artifact.getArtifactId()).resolve("MANIFEST.MF");
 443                                                 Files.createDirectories(originalManifest.getParent());
 444                                                 try (OutputStream out = Files.newOutputStream(originalManifest)) {
 445                                                         Files.copy(jarIn, originalManifest);
 446                                                 }
 447                                                 origin.added.add(
 448                                                                 "original MANIFEST (" + bundleDir.relativize(originalManifest) + ") from " + artifact);
 449                                                 continue entries;
 450                                         }
 451
 452                                         if (entry.getName().endsWith("NOTICE") || entry.getName().endsWith("NOTICE.txt")
 453                                                         || entry.getName().endsWith("LICENSE") || entry.getName().endsWith("LICENSE.md")
 454                                                         || entry.getName().endsWith("LICENSE-notice.md") || entry.getName().endsWith("COPYING")
 455                                                         || entry.getName().endsWith("COPYING.LESSER")) {
 456                                                 Path artifactOriginDir = bundleDir.resolve(A2_ORIGIN).resolve(artifact.getGroupId())
 457                                                                 .resolve(artifact.getArtifactId());
 458                                                 Path target = artifactOriginDir.resolve(entry.getName());
 459                                                 Files.createDirectories(target.getParent());
 460                                                 Files.copy(jarIn, target);
 461                                                 origin.moved.add(entry.getName() + " in " + artifact + " to " + bundleDir.relativize(target));
 462                                                 continue entries;
 463                                         }
 464                                         Path target = bundleDir.resolve(entry.getName());
 465                                         Files.createDirectories(target.getParent());
 466                                         if (!Files.exists(target)) {
 467                                                 Files.copy(jarIn, target);
 468                                         } else {
 469                                                 if (entry.getName().startsWith("META-INF/services/")) {
 470                                                         try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
 471                                                                 out.write("\n".getBytes());
 472                                                                 jarIn.transferTo(out);
 473                                                                 logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName());
 474                                                         }
 475                                                         origin.modified.add(entry.getName() + ", merging from " + artifact);
 476                                                 } else if (entry.getName().startsWith("org/apache/batik/")) {
 477                                                         logger.log(TRACE, "Skip " + entry.getName());
 478                                                         continue entries;
 479                                                 } else {
 480                                                         throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
 481                                                 }
 482                                         }
 483                                         logger.log(TRACE, () -> "Copied " + target);
 484                                 }
 485                         }
 486                         origin.added.add("binary content of " + artifact);
 487
 488                         // process sources
 489                         downloadAndProcessM2Sources(repoStr, artifact, bundleDir, true);
 490                 }
 491
 492                 // additional service files
 493                 Path servicesDir = duDir.resolve("services");
 494                 if (Files.exists(servicesDir)) {
 495                         for (Path p : Files.newDirectoryStream(servicesDir)) {
 496                                 Path target = bundleDir.resolve("META-INF/services/").resolve(p.getFileName());
 497                                 try (InputStream in = Files.newInputStream(p);
 498                                                 OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) {
 499                                         out.write("\n".getBytes());
 500                                         in.transferTo(out);
 501                                         logger.log(DEBUG, "Appended " + p);
 502                                 }
 503                                 origin.added.add(bundleDir.relativize(target).toString());
 504                         }
 505                 }
 506
 507                 Map<String, String> entries = new TreeMap<>();
 508                 try (Analyzer bndAnalyzer = new Analyzer()) {
 509                         bndAnalyzer.setProperties(mergeProps);
 510                         Jar jar = new Jar(bundleDir.toFile());
 511                         bndAnalyzer.setJar(jar);
 512                         Manifest manifest = bndAnalyzer.calcManifest();
 513
 514                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 515                                 Object value = manifest.getMainAttributes().get(key);
 516
 517                                 switch (key.toString()) {
 518                                 case "Tool":
 519                                 case "Bnd-LastModified":
 520                                 case "Created-By":
 521                                         continue keys;
 522                                 }
 523                                 if ("Require-Capability".equals(key.toString())
 524                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
 525                                         continue keys;// hack for very old classes
 526                                 entries.put(key.toString(), value.toString());
 527                         }
 528                 } catch (Exception e) {
 529                         throw new RuntimeException("Cannot process " + mergeBnd, e);
 530                 }
 531
 532                 Manifest manifest = new Manifest();
 533                 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
 534                 Files.createDirectories(manifestPath.getParent());
 535                 for (String key : entries.keySet()) {
 536                         String value = entries.get(key);
 537                         manifest.getMainAttributes().putValue(key, value);
 538                 }
 539
 540                 try (OutputStream out = Files.newOutputStream(manifestPath)) {
 541                         manifest.write(out);
 542                 }
 543                 createJar(bundleDir, origin);
 544         }
 545
 546         /** Generate MANIFEST using BND. */
 547         Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, M2Artifact artifact,
 548                         A2Origin origin) {
 549                 try {
 550                         Map<String, String> additionalEntries = new TreeMap<>();
 551                         boolean doNotModifyManifest = Boolean.parseBoolean(
 552                                         fileProps.getOrDefault(ARGEO_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString());
 553
 554                         // Note: we always force the symbolic name
 555                         if (doNotModifyManifest) {
 556                                 fileEntries: for (Object key : fileProps.keySet()) {
 557                                         if (ARGEO_ORIGIN_M2.toString().equals(key))
 558                                                 continue fileEntries;
 559                                         String value = fileProps.getProperty(key.toString());
 560                                         additionalEntries.put(key.toString(), value);
 561                                 }
 562                         } else {
 563                                 if (artifact != null) {
 564                                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 565                                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
 566                                         }
 567                                         if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
 568                                                 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
 569                                         }
 570                                 }
 571
 572                                 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
 573                                         fileProps.put(EXPORT_PACKAGE.toString(),
 574                                                         "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
 575                                 }
 576
 577                                 try (Analyzer bndAnalyzer = new Analyzer()) {
 578                                         bndAnalyzer.setProperties(fileProps);
 579                                         Jar jar = new Jar(downloaded.toFile());
 580                                         bndAnalyzer.setJar(jar);
 581                                         Manifest manifest = bndAnalyzer.calcManifest();
 582
 583                                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 584                                                 Object value = manifest.getMainAttributes().get(key);
 585
 586                                                 switch (key.toString()) {
 587                                                 case "Tool":
 588                                                 case "Bnd-LastModified":
 589                                                 case "Created-By":
 590                                                         continue keys;
 591                                                 }
 592                                                 if ("Require-Capability".equals(key.toString())
 593                                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
 594                                                         continue keys;// !! hack for very old classes
 595                                                 additionalEntries.put(key.toString(), value.toString());
 596                                         }
 597                                 }
 598                         }
 599                         Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries, origin);
 600                         logger.log(DEBUG, () -> "Processed " + downloaded);
 601                         return targetBundleDir;
 602                 } catch (Exception e) {
 603                         throw new RuntimeException("Cannot BND process " + downloaded, e);
 604                 }
 605
 606         }
 607
 608         /** Download and integrates sources for a single Maven artifact. */
 609         void downloadAndProcessM2Sources(String repoStr, M2Artifact artifact, Path targetBundleDir, boolean merging)
 610                         throws IOException {
 611                 try {
 612                         M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources");
 613                         URL sourcesUrl = M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
 614                         Path sourcesDownloaded = downloadMaven(sourcesUrl, artifact, true);
 615                         processM2SourceJar(sourcesDownloaded, targetBundleDir, merging ? artifact : null);
 616                         logger.log(TRACE, () -> "Processed source " + sourcesDownloaded);
 617                 } catch (Exception e) {
 618                         logger.log(ERROR, () -> "Cannot download source for  " + artifact);
 619                 }
 620
 621         }
 622
 623         /** Integrate sources from a downloaded jar file. */
 624         void processM2SourceJar(Path file, Path bundleDir, M2Artifact mergingFrom) throws IOException {
 625                 A2Origin origin = new A2Origin();
 626                 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
 627                                 : bundleDir.resolve("OSGI-OPT/src");
 628                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 629
 630                         String mergingMsg = "";
 631                         if (mergingFrom != null)
 632                                 mergingMsg = " of " + mergingFrom;
 633
 634                         Files.createDirectories(sourceDir);
 635                         JarEntry entry;
 636                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 637                                 if (entry.isDirectory())
 638                                         continue entries;
 639                                 if (entry.getName().startsWith("META-INF")) {// skip META-INF entries
 640                                         origin.deleted.add("META-INF directory from the sources" + mergingMsg);
 641                                         continue entries;
 642                                 }
 643                                 if (entry.getName().startsWith("module-info.java")) {// skip Java module information
 644                                         origin.deleted.add("Java module information from the sources (module-info.java)" + mergingMsg);
 645                                         continue entries;
 646                                 }
 647                                 if (entry.getName().startsWith("/")) { // absolute paths
 648                                         // TODO does it really happen?
 649                                         logger.log(WARNING, entry.getName() + " has an absolute path");
 650                                         origin.deleted.add(entry.getName() + " from the sources" + mergingMsg);
 651                                         continue entries;
 652                                 }
 653                                 Path target = sourceDir.resolve(entry.getName());
 654                                 Files.createDirectories(target.getParent());
 655                                 if (!Files.exists(target)) {
 656                                         Files.copy(jarIn, target);
 657                                         logger.log(TRACE, () -> "Copied source " + target);
 658                                 } else {
 659                                         logger.log(TRACE, () -> target + " already exists, skipping...");
 660                                 }
 661                         }
 662                 }
 663                 // write the changes
 664                 if (sourceBundles) {
 665                         origin.appendChanges(sourceDir);
 666                 } else {
 667                         origin.added.add("source code under OSGI-OPT/src");
 668                         origin.appendChanges(bundleDir);
 669                 }
 670         }
 671
 672         /** Download a Maven artifact. */
 673         Path downloadMaven(URL url, M2Artifact artifact) throws IOException {
 674                 return downloadMaven(url, artifact, false);
 675         }
 676
 677         /** Download a Maven artifact. */
 678         Path downloadMaven(URL url, M2Artifact artifact, boolean sources) throws IOException {
 679                 return download(url, mavenBase, artifact.getGroupId().replace(".", "/") //
 680                                 + '/' + artifact.getArtifactId() + '/' + artifact.getVersion() //
 681                                 + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() + (sources ? "-sources" : "") + ".jar");
 682         }
 683
 684         /*
 685          * ECLIPSE ORIGIN
 686          */
 687         /** Process an archive in Eclipse format. */
 688         void processEclipseArchive(Path duDir) {
 689                 try {
 690                         Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
 691                         Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
 692                         Files.createDirectories(targetCategoryBase);
 693                         // first delete all directories from previous builds
 694                         for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)))
 695                                 deleteDirectory(dir);
 696
 697                         Files.createDirectories(originBase);
 698
 699                         Path commonBnd = duDir.resolve(COMMON_BND);
 700                         Properties commonProps = new Properties();
 701                         try (InputStream in = Files.newInputStream(commonBnd)) {
 702                                 commonProps.load(in);
 703                         }
 704                         String url = commonProps.getProperty(ARGEO_ORIGIN_URI.toString());
 705                         if (url == null) {
 706                                 url = uris.getProperty(duDir.getFileName().toString());
 707                                 if (url == null)
 708                                         throw new IllegalStateException("No url available for " + duDir);
 709                                 commonProps.put(ARGEO_ORIGIN_URI.toString(), url);
 710                         }
 711                         Path downloaded = tryDownloadArchive(url, originBase);
 712
 713                         FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
 714
 715                         // filters
 716                         List<PathMatcher> includeMatchers = new ArrayList<>();
 717                         Properties includes = new Properties();
 718                         try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
 719                                 includes.load(in);
 720                         }
 721                         for (Object pattern : includes.keySet()) {
 722                                 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 723                                 includeMatchers.add(pathMatcher);
 724                         }
 725
 726                         List<PathMatcher> excludeMatchers = new ArrayList<>();
 727                         Path excludeFile = duDir.resolve("excludes.properties");
 728                         if (Files.exists(excludeFile)) {
 729                                 Properties excludes = new Properties();
 730                                 try (InputStream in = Files.newInputStream(excludeFile)) {
 731                                         excludes.load(in);
 732                                 }
 733                                 for (Object pattern : excludes.keySet()) {
 734                                         PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 735                                         excludeMatchers.add(pathMatcher);
 736                                 }
 737                         }
 738
 739                         // keys are the bundle directories
 740                         Map<Path, A2Origin> origins = new HashMap<>();
 741                         Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
 742
 743                                 @Override
 744                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 745                                         includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
 746                                                 if (includeMatcher.matches(file)) {
 747                                                         for (PathMatcher excludeMatcher : excludeMatchers) {
 748                                                                 if (excludeMatcher.matches(file)) {
 749                                                                         logger.log(TRACE, "Skipping excluded " + file);
 750                                                                         return FileVisitResult.CONTINUE;
 751                                                                 }
 752                                                         }
 753                                                         if (file.getFileName().toString().contains(".source_")) {
 754                                                                 processEclipseSourceJar(file, targetCategoryBase);
 755                                                                 logger.log(DEBUG, () -> "Processed source " + file);
 756                                                         } else {
 757                                                                 Map<String, String> map = new HashMap<>();
 758                                                                 for (Object key : commonProps.keySet())
 759                                                                         map.put(key.toString(), commonProps.getProperty(key.toString()));
 760                                                                 A2Origin origin = new A2Origin();
 761                                                                 Path bundleDir = processBundleJar(file, targetCategoryBase, map, origin);
 762                                                                 origins.put(bundleDir, origin);
 763                                                                 logger.log(DEBUG, () -> "Processed " + file);
 764                                                         }
 765                                                         break includeMatchers;
 766                                                 }
 767                                         }
 768                                         return FileVisitResult.CONTINUE;
 769                                 }
 770                         });
 771
 772                         DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)
 773                                         && p.getFileName().toString().indexOf('.') >= 0 && !p.getFileName().toString().endsWith(".src"));
 774                         for (Path bundleDir : dirs) {
 775                                 A2Origin origin = origins.get(bundleDir);
 776                                 Objects.requireNonNull(origin, "No A2 origin found for " + bundleDir);
 777                                 createJar(bundleDir, origin);
 778                         }
 779                 } catch (IOException e) {
 780                         throw new RuntimeException("Cannot process " + duDir, e);
 781                 }
 782
 783         }
 784
 785         /** Process sources in Eclipse format. */
 786         void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
 787                 try {
 788                         A2Origin origin = new A2Origin();
 789                         Path bundleDir;
 790                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 791                                 Manifest manifest = jarIn.getManifest();
 792
 793                                 String[] relatedBundle = manifest.getMainAttributes().getValue(ECLIPSE_SOURCE_BUNDLE.toString())
 794                                                 .split(";");
 795                                 String version = relatedBundle[1].substring("version=\"".length());
 796                                 version = version.substring(0, version.length() - 1);
 797                                 NameVersion nameVersion = new NameVersion(relatedBundle[0], version);
 798                                 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 799
 800                                 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
 801                                                 : bundleDir.resolve("OSGI-OPT/src");
 802
 803                                 Files.createDirectories(sourceDir);
 804                                 JarEntry entry;
 805                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 806                                         if (entry.isDirectory())
 807                                                 continue entries;
 808                                         if (entry.getName().startsWith("META-INF"))// skip META-INF entries
 809                                                 continue entries;
 810                                         Path target = sourceDir.resolve(entry.getName());
 811                                         Files.createDirectories(target.getParent());
 812                                         Files.copy(jarIn, target);
 813                                         logger.log(TRACE, () -> "Copied source " + target);
 814                                 }
 815
 816                                 // write the changes
 817                                 if (sourceBundles) {
 818                                         origin.appendChanges(sourceDir);
 819                                 } else {
 820                                         origin.added.add("source code under OSGI-OPT/src");
 821                                         origin.appendChanges(bundleDir);
 822                                 }
 823                         }
 824                 } catch (IOException e) {
 825                         throw new IllegalStateException("Cannot process " + file, e);
 826                 }
 827         }
 828
 829         /*
 830          * COMMON PROCESSING
 831          */
 832         /** Normalise a bundle. */
 833         Path processBundleJar(Path file, Path targetBase, Map<String, String> entries, A2Origin origin) throws IOException {
 834                 boolean embed = Boolean.parseBoolean(entries.getOrDefault(ARGEO_ORIGIN_EMBED.toString(), "false").toString());
 835                 NameVersion nameVersion;
 836                 Path bundleDir;
 837                 // singleton
 838                 boolean isSingleton = false;
 839                 Manifest manifest;
 840                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 841                         Manifest sourceManifest = jarIn.getManifest();
 842                         manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
 843
 844                         String rawSourceSymbolicName = manifest.getMainAttributes().getValue(BUNDLE_SYMBOLICNAME.toString());
 845                         if (rawSourceSymbolicName != null) {
 846                                 // make sure there is no directive
 847                                 String[] arr = rawSourceSymbolicName.split(";");
 848                                 for (int i = 1; i < arr.length; i++) {
 849                                         if (arr[i].trim().equals("singleton:=true"))
 850                                                 isSingleton = true;
 851                                         logger.log(DEBUG, file.getFileName() + " is a singleton");
 852                                 }
 853                         }
 854                         // remove problematic entries in MANIFEST
 855                         manifest.getEntries().clear();
 856
 857                         String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString());
 858                         String ourVersion = entries.get(BUNDLE_VERSION.toString());
 859
 860                         if (ourSymbolicName != null && ourVersion != null) {
 861                                 nameVersion = new NameVersion(ourSymbolicName, ourVersion);
 862                         } else {
 863                                 nameVersion = nameVersionFromManifest(manifest);
 864                                 if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) {
 865                                         logger.log(WARNING,
 866                                                         "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion);
 867                                         entries.put(BUNDLE_VERSION.toString(), ourVersion);
 868                                 }
 869                                 if (ourSymbolicName != null) {
 870                                         // we always force our symbolic name
 871                                         nameVersion.setName(ourSymbolicName);
 872                                 }
 873                         }
 874                         bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 875
 876                         if (sourceManifest != null) {// copy original MANIFEST
 877                                 Path originalManifest = bundleDir.resolve(A2_ORIGIN).resolve("MANIFEST.MF");
 878                                 Files.createDirectories(originalManifest.getParent());
 879                                 try (OutputStream out = Files.newOutputStream(originalManifest)) {
 880                                         sourceManifest.write(out);
 881                                 }
 882                                 origin.added.add("original MANIFEST (" + bundleDir.relativize(originalManifest) + ")");
 883                         }
 884
 885                         // force Java 9 module name
 886                         entries.put(ManifestConstants.AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName());
 887
 888                         boolean isNative = false;
 889                         String os = null;
 890                         String arch = null;
 891                         if (bundleDir.startsWith(a2LibBase)) {
 892                                 isNative = true;
 893                                 Path libRelativePath = a2LibBase.relativize(bundleDir);
 894                                 os = libRelativePath.getName(0).toString();
 895                                 arch = libRelativePath.getName(1).toString();
 896                         }
 897
 898                         if (!embed) {
 899                                 // copy entries
 900                                 JarEntry entry;
 901                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 902                                         if (entry.isDirectory())
 903                                                 continue entries;
 904                                         if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
 905                                                         || entry.getName().endsWith(".SF")) {
 906                                                 origin.deleted.add("cryptographic signatures");
 907                                                 continue entries;
 908                                         }
 909                                         if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
 910                                                 origin.deleted.add("Java module information (module-info.class)");
 911                                                 continue entries;
 912                                         }
 913                                         if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
 914                                                 origin.deleted.add("additional Java versions (META-INF/versions)");
 915                                                 continue entries;
 916                                         }
 917                                         if (entry.getName().startsWith("META-INF/maven/")) {
 918                                                 origin.deleted.add("Maven information (META-INF/maven)");
 919                                                 continue entries;
 920                                         }
 921                                         // skip file system providers as they cause issues with native image
 922                                         if (entry.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider")) {
 923                                                 origin.deleted
 924                                                                 .add("file system providers (META-INF/services/java.nio.file.spi.FileSystemProvider)");
 925                                                 continue entries;
 926                                         }
 927                                         if (entry.getName().startsWith("OSGI-OPT/src/")) { // skip embedded sources
 928                                                 origin.deleted.add("embedded sources");
 929                                                 continue entries;
 930                                         }
 931                                         Path target = bundleDir.resolve(entry.getName());
 932                                         Files.createDirectories(target.getParent());
 933                                         Files.copy(jarIn, target);
 934
 935                                         // native libraries
 936                                         if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll")
 937                                                         || entry.getName().endsWith(".jnilib"))) {
 938                                                 Path categoryDir = bundleDir.getParent();
 939                                                 boolean copyDll = false;
 940                                                 Path targetDll = categoryDir.resolve(bundleDir.relativize(target));
 941                                                 if (nameVersion.getName().equals("com.sun.jna")) {
 942                                                         if (arch.equals("x86_64"))
 943                                                                 arch = "x86-64";
 944                                                         if (os.equals("macosx"))
 945                                                                 os = "darwin";
 946                                                         if (target.getParent().getFileName().toString().equals(os + "-" + arch)) {
 947                                                                 copyDll = true;
 948                                                         }
 949                                                         targetDll = categoryDir.resolve(target.getFileName());
 950                                                 } else {
 951                                                         copyDll = true;
 952                                                 }
 953                                                 if (copyDll) {
 954                                                         Files.createDirectories(targetDll.getParent());
 955                                                         if (Files.exists(targetDll))
 956                                                                 Files.delete(targetDll);
 957                                                         Files.copy(target, targetDll);
 958                                                 }
 959                                                 Files.delete(target);
 960                                                 origin.deleted.add(bundleDir.relativize(target).toString());
 961                                         }
 962                                         logger.log(TRACE, () -> "Copied " + target);
 963                                 }
 964                         }
 965                 }
 966
 967                 // copy MANIFEST
 968                 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
 969                 Files.createDirectories(manifestPath.getParent());
 970
 971                 if (isSingleton && entries.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 972                         entries.put(BUNDLE_SYMBOLICNAME.toString(),
 973                                         entries.get(BUNDLE_SYMBOLICNAME.toString()) + ";singleton:=true");
 974                 }
 975
 976                 if (embed) {// copy embedded jar
 977                         Files.copy(file, bundleDir.resolve(file.getFileName()));
 978                         entries.put(ManifestConstants.BUNDLE_CLASSPATH.toString(), file.getFileName().toString());
 979                 }
 980
 981                 // Final MANIFEST decisions
 982                 // This also where we check the original OSGi metadata and compare with our
 983                 // changes
 984                 for (String key : entries.keySet()) {
 985                         String value = entries.get(key);
 986                         String previousValue = manifest.getMainAttributes().getValue(key);
 987                         boolean wasDifferent = previousValue != null && !previousValue.equals(value);
 988                         boolean keepPrevious = false;
 989                         if (wasDifferent) {
 990                                 if (SPDX_LICENSE_IDENTIFIER.toString().equals(key) && previousValue != null)
 991                                         keepPrevious = true;
 992                                 else if (BUNDLE_VERSION.toString().equals(key) && wasDifferent)
 993                                         if (previousValue.equals(value + ".0")) // typically a Maven first release
 994                                                 keepPrevious = true;
 995
 996                                 if (keepPrevious) {
 997                                         if (logger.isLoggable(DEBUG))
 998                                                 logger.log(DEBUG, file.getFileName() + ": " + key + " was NOT modified, value kept is "
 999                                                                 + previousValue + ", not overriden with " + value);
1000                                         value = previousValue;
1001                                 }
1002                         }
1003
1004                         manifest.getMainAttributes().putValue(key, value);
1005                         if (wasDifferent && !keepPrevious) {
1006                                 if (IMPORT_PACKAGE.toString().equals(key) || EXPORT_PACKAGE.toString().equals(key))
1007                                         logger.log(TRACE, () -> file.getFileName() + ": " + key + " was modified");
1008                                 else
1009                                         logger.log(WARNING,
1010                                                         file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value);
1011                         }
1012
1013                         // !! hack to remove unresolvable
1014                         if (key.equals("Provide-Capability") || key.equals("Require-Capability"))
1015                                 if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) {
1016                                         manifest.getMainAttributes().remove(key);
1017                                 }
1018                 }
1019
1020                 // de-pollute MANIFEST
1021                 for (Iterator<Map.Entry<Object, Object>> manifestEntries = manifest.getMainAttributes().entrySet()
1022                                 .iterator(); manifestEntries.hasNext();) {
1023                         Map.Entry<Object, Object> manifestEntry = manifestEntries.next();
1024                         switch (manifestEntry.getKey().toString()) {
1025                         case "Archiver-Version":
1026                         case "Build-By":
1027                         case "Created-By":
1028                         case "Originally-Created-By":
1029                         case "Tool":
1030                         case "Bnd-LastModified":
1031                                 manifestEntries.remove();
1032                                 break;
1033                         default: // do nothing
1034                         }
1035                 }
1036
1037                 // license checks
1038                 String spdxLicenceId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString());
1039                 String bundleLicense = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString());
1040                 if (spdxLicenceId == null) {
1041                         logger.log(ERROR, file.getFileName() + ": " + SPDX_LICENSE_IDENTIFIER + " not available, " + BUNDLE_LICENSE
1042                                         + " is " + bundleLicense);
1043                 } else {
1044                         if (!licensesUsed.containsKey(spdxLicenceId))
1045                                 licensesUsed.put(spdxLicenceId, new TreeSet<>());
1046                         licensesUsed.get(spdxLicenceId).add(nameVersion.toString());
1047                 }
1048
1049                 origin.modified.add("jar MANIFEST (META-INF/MANIFEST.MF)");
1050                 // write the MANIFEST
1051                 try (OutputStream out = Files.newOutputStream(manifestPath)) {
1052                         manifest.write(out);
1053                 }
1054                 return bundleDir;
1055         }
1056
1057         /*
1058          * UTILITIES
1059          */
1060         /** Recursively deletes a directory. */
1061         static void deleteDirectory(Path path) throws IOException {
1062                 if (!Files.exists(path))
1063                         return;
1064                 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
1065                         @Override
1066                         public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
1067                                 if (e != null)
1068                                         throw e;
1069                                 Files.delete(directory);
1070                                 return CONTINUE;
1071                         }
1072
1073                         @Override
1074                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1075                                 Files.delete(file);
1076                                 return CONTINUE;
1077                         }
1078                 });
1079         }
1080
1081         /** Extract name/version from a MANIFEST. */
1082         NameVersion nameVersionFromManifest(Manifest manifest) {
1083                 Attributes attrs = manifest.getMainAttributes();
1084                 // symbolic name
1085                 String symbolicName = attrs.getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
1086                 if (symbolicName == null)
1087                         return null;
1088                 // make sure there is no directive
1089                 symbolicName = symbolicName.split(";")[0];
1090
1091                 String version = attrs.getValue(ManifestConstants.BUNDLE_VERSION.toString());
1092                 return new NameVersion(symbolicName, version);
1093         }
1094
1095         /** Try to download from an URI. */
1096         Path tryDownloadArchive(String uri, Path dir) throws IOException {
1097                 // find mirror
1098                 List<String> urlBases = null;
1099                 String uriPrefix = null;
1100                 uriPrefixes: for (String uriPref : mirrors.keySet()) {
1101                         if (uri.startsWith(uriPref)) {
1102                                 if (mirrors.get(uriPref).size() > 0) {
1103                                         urlBases = mirrors.get(uriPref);
1104                                         uriPrefix = uriPref;
1105                                         break uriPrefixes;
1106                                 }
1107                         }
1108                 }
1109                 if (urlBases == null)
1110                         try {
1111                                 return downloadArchive(new URL(uri), dir);
1112                         } catch (FileNotFoundException e) {
1113                                 throw new FileNotFoundException("Cannot find " + uri);
1114                         }
1115
1116                 // try to download
1117                 for (String urlBase : urlBases) {
1118                         String relativePath = uri.substring(uriPrefix.length());
1119                         URL url = new URL(urlBase + relativePath);
1120                         try {
1121                                 return downloadArchive(url, dir);
1122                         } catch (FileNotFoundException e) {
1123                                 logger.log(WARNING, "Cannot download " + url + ", trying another mirror");
1124                         }
1125                 }
1126                 throw new FileNotFoundException("Cannot find " + uri);
1127         }
1128
1129         /**
1130          * Effectively download. Synchronised in order to avoid downloading twice in
1131          * parallel.
1132          */
1133         synchronized Path downloadArchive(URL url, Path dir) throws IOException {
1134                 return download(url, dir, (String) null);
1135         }
1136
1137         /** Effectively download. */
1138         Path download(URL url, Path dir, String name) throws IOException {
1139
1140                 Path dest;
1141                 if (name == null) {
1142                         // We use also use parent directory in case the archive itself has a fixed name
1143                         String[] segments = url.getPath().split("/");
1144                         name = segments.length > 1 ? segments[segments.length - 2] + '-' + segments[segments.length - 1]
1145                                         : segments[segments.length - 1];
1146                 }
1147
1148                 dest = dir.resolve(name);
1149                 if (Files.exists(dest)) {
1150                         logger.log(TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
1151                         return dest;
1152                 } else {
1153                         Files.createDirectories(dest.getParent());
1154                 }
1155
1156                 try (InputStream in = url.openStream()) {
1157                         Files.copy(in, dest);
1158                         logger.log(DEBUG, () -> "Downloaded " + dest + " from " + url);
1159                 }
1160                 return dest;
1161         }
1162
1163         /** Create a JAR file from a directory. */
1164         Path createJar(Path bundleDir, A2Origin origin) throws IOException {
1165                 // write changes
1166                 origin.appendChanges(bundleDir);
1167                 // Create the jar
1168                 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
1169                 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1170                 Manifest manifest;
1171                 try (InputStream in = Files.newInputStream(manifestPath)) {
1172                         manifest = new Manifest(in);
1173                 }
1174                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
1175                         jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
1176                         Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
1177
1178                                 @Override
1179                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1180                                         if (file.getFileName().toString().equals("MANIFEST.MF"))
1181                                                 return super.visitFile(file, attrs);
1182                                         JarEntry entry = new JarEntry(
1183                                                         bundleDir.relativize(file).toString().replace(File.separatorChar, '/'));
1184                                         jarOut.putNextEntry(entry);
1185                                         Files.copy(file, jarOut);
1186                                         return super.visitFile(file, attrs);
1187                                 }
1188
1189                         });
1190                 }
1191                 deleteDirectory(bundleDir);
1192
1193                 if (sourceBundles)
1194                         createSourceJar(bundleDir, manifest);
1195
1196                 return jarPath;
1197         }
1198
1199         void createSourceJar(Path bundleDir, Manifest manifest) throws IOException {
1200                 Path bundleCategoryDir = bundleDir.getParent();
1201                 Path sourceDir = bundleCategoryDir.resolve(bundleDir.toString() + ".src");
1202                 if (!Files.exists(sourceDir)) {
1203                         logger.log(WARNING, sourceDir + " does not exist, skipping...");
1204                         return;
1205
1206                 }
1207
1208                 Path relPath = a2Base.relativize(bundleCategoryDir);
1209                 Path srcCategoryDir = a2SrcBase.resolve(relPath);
1210                 Path srcJarP = srcCategoryDir.resolve(sourceDir.getFileName() + ".jar");
1211                 Files.createDirectories(srcJarP.getParent());
1212
1213                 String bundleSymbolicName = manifest.getMainAttributes().getValue("Bundle-SymbolicName").toString();
1214                 // in case there are additional directives
1215                 bundleSymbolicName = bundleSymbolicName.split(";")[0];
1216                 Manifest srcManifest = new Manifest();
1217                 srcManifest.getMainAttributes().put(MANIFEST_VERSION, "1.0");
1218                 srcManifest.getMainAttributes().putValue(BUNDLE_SYMBOLICNAME.toString(), bundleSymbolicName + ".src");
1219                 srcManifest.getMainAttributes().putValue(BUNDLE_VERSION.toString(),
1220                                 manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()).toString());
1221                 srcManifest.getMainAttributes().putValue(ECLIPSE_SOURCE_BUNDLE.toString(), bundleSymbolicName + ";version=\""
1222                                 + manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()) + "\"");
1223
1224                 try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
1225                         srcJarOut.setLevel(Deflater.BEST_COMPRESSION);
1226                         Files.walkFileTree(sourceDir, new SimpleFileVisitor<Path>() {
1227
1228                                 @Override
1229                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1230                                         if (file.getFileName().toString().equals("MANIFEST.MF"))
1231                                                 return super.visitFile(file, attrs);
1232                                         JarEntry entry = new JarEntry(
1233                                                         sourceDir.relativize(file).toString().replace(File.separatorChar, '/'));
1234                                         srcJarOut.putNextEntry(entry);
1235                                         Files.copy(file, srcJarOut);
1236                                         return super.visitFile(file, attrs);
1237                                 }
1238
1239                         });
1240                 }
1241                 deleteDirectory(sourceDir);
1242         }
1243
1244         /** MANIFEST headers. */
1245         enum ManifestConstants {
1246                 // OSGi
1247                 /** OSGi bundle symbolic name. */
1248                 BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), //
1249                 /** OSGi bundle version. */
1250                 BUNDLE_VERSION("Bundle-Version"), //
1251                 /** OSGi bundle license. */
1252                 BUNDLE_LICENSE("Bundle-License"), //
1253                 /** OSGi exported packages list. */
1254                 EXPORT_PACKAGE("Export-Package"), //
1255                 /** OSGi imported packages list. */
1256                 IMPORT_PACKAGE("Import-Package"), //
1257                 /** OSGi path to embedded jar. */
1258                 BUNDLE_CLASSPATH("Bundle-Classpath"), //
1259                 // Java
1260                 /** Java module name. */
1261                 AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), //
1262                 // Eclipse
1263                 /** Eclipse source bundle. */
1264                 ECLIPSE_SOURCE_BUNDLE("Eclipse-SourceBundle"), //
1265                 // SPDX
1266                 /**
1267                  * SPDX license identifier.
1268                  *
1269                  * @see https://spdx.org/licenses/
1270                  */
1271                 SPDX_LICENSE_IDENTIFIER("SPDX-License-Identifier"), //
1272                 // Argeo Origin
1273                 /**
1274                  * Maven coordinates of the origin, possibly partial when using common.bnd or
1275                  * merge.bnd.
1276                  */
1277                 ARGEO_ORIGIN_M2("Argeo-Origin-M2"), //
1278                 /** List of Maven coordinates to merge. */
1279                 ARGEO_ORIGIN_M2_MERGE("Argeo-Origin-M2-Merge"), //
1280                 /** Maven repository, if not the default one. */
1281                 ARGEO_ORIGIN_M2_REPO("Argeo-Origin-M2-Repo"), //
1282                 /**
1283                  * Do not perform BND analysis of the origin component. Typically IMport_package
1284                  * and Export-Package will be kept untouched.
1285                  */
1286                 ARGEO_ORIGIN_MANIFEST_NOT_MODIFIED("Argeo-Origin-ManifestNotModified"), //
1287                 /**
1288                  * Embed the original jar without modifying it (may be required by some
1289                  * proprietary licenses, such as JCR Day License).
1290                  */
1291                 ARGEO_ORIGIN_EMBED("Argeo-Origin-Embed"), //
1292                 /**
1293                  * Do not modify original jar (may be required by some proprietary licenses,
1294                  * such as JCR Day License).
1295                  */
1296                 ARGEO_DO_NOT_MODIFY("Argeo-Origin-Do-Not-Modify"), //
1297                 /**
1298                  * Origin (non-Maven) URI of the component. It may be anything (jar, archive,
1299                  * etc.).
1300                  */
1301                 ARGEO_ORIGIN_URI("Argeo-Origin-URI"), //
1302                 ;
1303
1304                 final String value;
1305
1306                 private ManifestConstants(String value) {
1307                         this.value = value;
1308                 }
1309
1310                 @Override
1311                 public String toString() {
1312                         return value;
1313                 }
1314         }
1315 }
1316
1317 /**
1318  * Gathers modifications performed on the original binaries and sources,
1319  * especially in order to comply with their license requirements.
1320  */
1321 class A2Origin {
1322         Set<String> modified = new TreeSet<>();
1323         Set<String> deleted = new TreeSet<>();
1324         Set<String> added = new TreeSet<>();
1325         Set<String> moved = new TreeSet<>();
1326
1327         /** Append changes to the A2-ORIGIN/changes file. */
1328         void appendChanges(Path baseDirectory) throws IOException {
1329                 Path changesFile = baseDirectory.resolve("A2-ORIGIN/changes");
1330                 Files.createDirectories(changesFile.getParent());
1331                 try (BufferedWriter writer = Files.newBufferedWriter(changesFile, StandardOpenOption.APPEND,
1332                                 StandardOpenOption.CREATE)) {
1333                         for (String msg : added)
1334                                 writer.write("- Added " + msg + ".\n");
1335                         for (String msg : modified)
1336                                 writer.write("- Modified " + msg + ".\n");
1337                         for (String msg : moved)
1338                                 writer.write("- Moved " + msg + ".\n");
1339                         for (String msg : deleted)
1340                                 writer.write("- Deleted " + msg + ".\n");
1341                 }
1342         }
1343 }
1344
1345 /** Simple representation of an M2 artifact. */
1346 class M2Artifact extends CategoryNameVersion {
1347         private String classifier;
1348
1349         M2Artifact(String m2coordinates) {
1350                 this(m2coordinates, null);
1351         }
1352
1353         M2Artifact(String m2coordinates, String classifier) {
1354                 String[] parts = m2coordinates.split(":");
1355                 setCategory(parts[0]);
1356                 setName(parts[1]);
1357                 if (parts.length > 2) {
1358                         setVersion(parts[2]);
1359                 }
1360                 this.classifier = classifier;
1361         }
1362
1363         String getGroupId() {
1364                 return super.getCategory();
1365         }
1366
1367         String getArtifactId() {
1368                 return super.getName();
1369         }
1370
1371         String toM2Coordinates() {
1372                 return getCategory() + ":" + getName() + (getVersion() != null ? ":" + getVersion() : "");
1373         }
1374
1375         String getClassifier() {
1376                 return classifier != null ? classifier : "";
1377         }
1378
1379         String getExtension() {
1380                 return "jar";
1381         }
1382 }
1383
1384 /** Utilities around Maven (conventions based). */
1385 class M2ConventionsUtils {
1386         final static String MAVEN_CENTRAL_BASE_URL = "https://repo1.maven.org/maven2/";
1387
1388         /** The file name of this artifact when stored */
1389         static String artifactFileName(M2Artifact artifact) {
1390                 return artifact.getArtifactId() + '-' + artifact.getVersion()
1391                                 + (artifact.getClassifier().equals("") ? "" : '-' + artifact.getClassifier()) + '.'
1392                                 + artifact.getExtension();
1393         }
1394
1395         /** Absolute path to the file */
1396         static String artifactPath(String artifactBasePath, M2Artifact artifact) {
1397                 return artifactParentPath(artifactBasePath, artifact) + '/' + artifactFileName(artifact);
1398         }
1399
1400         /** Absolute path to the file */
1401         static String artifactUrl(String repoUrl, M2Artifact artifact) {
1402                 if (repoUrl.endsWith("/"))
1403                         return repoUrl + artifactPath("/", artifact).substring(1);
1404                 else
1405                         return repoUrl + artifactPath("/", artifact);
1406         }
1407
1408         /** Absolute path to the file */
1409         static URL mavenRepoUrl(String repoBase, M2Artifact artifact) {
1410                 String url = artifactUrl(repoBase == null ? MAVEN_CENTRAL_BASE_URL : repoBase, artifact);
1411                 try {
1412                         return new URL(url);
1413                 } catch (MalformedURLException e) {
1414                         // it should not happen
1415                         throw new IllegalStateException(e);
1416                 }
1417         }
1418
1419         /** Absolute path to the directories where the files will be stored */
1420         static String artifactParentPath(String artifactBasePath, M2Artifact artifact) {
1421                 return artifactBasePath + (artifactBasePath.endsWith("/") ? "" : "/") + artifactParentPath(artifact);
1422         }
1423
1424         /** Relative path to the directories where the files will be stored */
1425         static String artifactParentPath(M2Artifact artifact) {
1426                 return artifact.getGroupId().replace('.', '/') + '/' + artifact.getArtifactId() + '/' + artifact.getVersion();
1427         }
1428
1429         /** Singleton */
1430         private M2ConventionsUtils() {
1431         }
1432 }
1433
1434 /** Combination of a category, a name and a version. */
1435 class CategoryNameVersion extends NameVersion {
1436         private String category;
1437
1438         CategoryNameVersion() {
1439         }
1440
1441         CategoryNameVersion(String category, String name, String version) {
1442                 super(name, version);
1443                 this.category = category;
1444         }
1445
1446         CategoryNameVersion(String category, NameVersion nameVersion) {
1447                 super(nameVersion);
1448                 this.category = category;
1449         }
1450
1451         String getCategory() {
1452                 return category;
1453         }
1454
1455         void setCategory(String category) {
1456                 this.category = category;
1457         }
1458
1459         @Override
1460         public String toString() {
1461                 return category + ":" + super.toString();
1462         }
1463
1464 }
1465
1466 /** Combination of a name and a version. */
1467 class NameVersion implements Comparable<NameVersion> {
1468         private String name;
1469         private String version;
1470
1471         NameVersion() {
1472         }
1473
1474         /** Interprets string in OSGi-like format my.module.name;version=0.0.0 */
1475         NameVersion(String nameVersion) {
1476                 int index = nameVersion.indexOf(";version=");
1477                 if (index < 0) {
1478                         setName(nameVersion);
1479                         setVersion(null);
1480                 } else {
1481                         setName(nameVersion.substring(0, index));
1482                         setVersion(nameVersion.substring(index + ";version=".length()));
1483                 }
1484         }
1485
1486         NameVersion(String name, String version) {
1487                 this.name = name;
1488                 this.version = version;
1489         }
1490
1491         NameVersion(NameVersion nameVersion) {
1492                 this.name = nameVersion.getName();
1493                 this.version = nameVersion.getVersion();
1494         }
1495
1496         String getName() {
1497                 return name;
1498         }
1499
1500         void setName(String name) {
1501                 this.name = name;
1502         }
1503
1504         String getVersion() {
1505                 return version;
1506         }
1507
1508         void setVersion(String version) {
1509                 this.version = version;
1510         }
1511
1512         String getBranch() {
1513                 String[] parts = getVersion().split("\\.");
1514                 if (parts.length < 2)
1515                         throw new IllegalStateException("Version " + getVersion() + " cannot be interpreted as branch.");
1516                 return parts[0] + "." + parts[1];
1517         }
1518
1519         @Override
1520         public boolean equals(Object obj) {
1521                 if (obj instanceof NameVersion) {
1522                         NameVersion nameVersion = (NameVersion) obj;
1523                         return name.equals(nameVersion.getName()) && version.equals(nameVersion.getVersion());
1524                 } else
1525                         return false;
1526         }
1527
1528         @Override
1529         public int hashCode() {
1530                 return name.hashCode();
1531         }
1532
1533         @Override
1534         public String toString() {
1535                 return name + ":" + version;
1536         }
1537
1538         public int compareTo(NameVersion o) {
1539                 if (o.getName().equals(name))
1540                         return version.compareTo(o.getVersion());
1541                 else
1542                         return name.compareTo(o.getName());
1543         }
1544 }