src/org/argeo/build/Repackage.java

   1 package org.argeo.build;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static java.lang.System.Logger.Level.ERROR;
   5 import static java.lang.System.Logger.Level.INFO;
   6 import static java.lang.System.Logger.Level.TRACE;
   7 import static java.lang.System.Logger.Level.WARNING;
   8 import static java.nio.file.FileVisitResult.CONTINUE;
   9 import static java.nio.file.StandardOpenOption.APPEND;
  10 import static java.nio.file.StandardOpenOption.CREATE;
  11 import static java.util.jar.Attributes.Name.MANIFEST_VERSION;
  12 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_DO_NOT_MODIFY;
  13 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2;
  14 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_MERGE;
  15 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_REPO;
  16 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_NO_METADATA_GENERATION;
  17 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_SOURCES_URI;
  18 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_URI;
  19 import static org.argeo.build.Repackage.ManifestHeader.AUTOMATIC_MODULE_NAME;
  20 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_LICENSE;
  21 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_SYMBOLICNAME;
  22 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_VERSION;
  23 import static org.argeo.build.Repackage.ManifestHeader.ECLIPSE_SOURCE_BUNDLE;
  24 import static org.argeo.build.Repackage.ManifestHeader.EXPORT_PACKAGE;
  25 import static org.argeo.build.Repackage.ManifestHeader.IMPORT_PACKAGE;
  26 import static org.argeo.build.Repackage.ManifestHeader.REQUIRE_CAPABILITY;
  27 //import static org.argeo.build.Repackage.ManifestHeader.REQUIRE_BUNDLE;
  28 import static org.argeo.build.Repackage.ManifestHeader.SPDX_LICENSE_IDENTIFIER;
  29
  30 import java.io.BufferedWriter;
  31 import java.io.File;
  32 import java.io.FileNotFoundException;
  33 import java.io.IOException;
  34 import java.io.InputStream;
  35 import java.io.OutputStream;
  36 import java.lang.System.Logger;
  37 import java.net.URI;
  38 import java.net.URISyntaxException;
  39 import java.nio.charset.StandardCharsets;
  40 import java.nio.file.DirectoryStream;
  41 import java.nio.file.FileSystem;
  42 import java.nio.file.FileSystems;
  43 import java.nio.file.FileVisitResult;
  44 import java.nio.file.Files;
  45 import java.nio.file.Path;
  46 import java.nio.file.PathMatcher;
  47 import java.nio.file.Paths;
  48 import java.nio.file.SimpleFileVisitor;
  49 import java.nio.file.StandardCopyOption;
  50 import java.nio.file.StandardOpenOption;
  51 import java.nio.file.attribute.BasicFileAttributes;
  52 import java.util.ArrayList;
  53 import java.util.HashMap;
  54 import java.util.Iterator;
  55 import java.util.List;
  56 import java.util.Map;
  57 import java.util.Objects;
  58 import java.util.Properties;
  59 import java.util.Set;
  60 import java.util.StringJoiner;
  61 import java.util.TreeMap;
  62 import java.util.TreeSet;
  63 import java.util.concurrent.CompletableFuture;
  64 import java.util.jar.Attributes;
  65 import java.util.jar.JarEntry;
  66 import java.util.jar.JarInputStream;
  67 import java.util.jar.JarOutputStream;
  68 import java.util.jar.Manifest;
  69 import java.util.zip.Deflater;
  70
  71 import aQute.bnd.osgi.Analyzer;
  72 import aQute.bnd.osgi.Jar;
  73
  74 /** Repackages existing jar files into OSGi bundles in an A2 repository. */
  75 public class Repackage {
  76         final static Logger logger = System.getLogger(Repackage.class.getName());
  77
  78         /**
  79          * Environment variable on whether sources should be packaged separately or
  80          * integrated in the bundles.
  81          */
  82         final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES";
  83         /** Environment variable on whether operations should be parallelised. */
  84         final static String ENV_ARGEO_BUILD_SEQUENTIAL = "ARGEO_BUILD_SEQUENTIAL";
  85
  86         /** Whether repackaging should run in parallel (default) or sequentially. */
  87         final static boolean sequential = Boolean.parseBoolean(System.getenv(ENV_ARGEO_BUILD_SEQUENTIAL));
  88
  89         /** Main entry point. */
  90         public static void main(String[] args) {
  91                 if (sequential)
  92                         logger.log(INFO, "Build will be sequential");
  93                 if (args.length < 2) {
  94                         System.err.println("Usage: <path to a2 output dir> <category1> <category2> ...");
  95                         System.exit(1);
  96                 }
  97                 Path a2Base = Paths.get(args[0]).toAbsolutePath().normalize();
  98                 Path descriptorsBase = Paths.get(".").toAbsolutePath().normalize();
  99                 Repackage factory = new Repackage(a2Base, descriptorsBase);
 100
 101                 List<CompletableFuture<Void>> toDos = new ArrayList<>();
 102                 for (int i = 1; i < args.length; i++) {
 103                         Path categoryPath = Paths.get(args[i]);
 104                         factory.cleanPreviousFailedBuild(categoryPath);
 105                         if (sequential) // sequential processing happens here
 106                                 factory.processCategory(categoryPath);
 107                         else
 108                                 toDos.add(CompletableFuture.runAsync(() -> factory.processCategory(categoryPath)));
 109                 }
 110                 if (!sequential)// parallel processing
 111                         CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
 112
 113                 // Summary
 114                 StringBuilder sb = new StringBuilder();
 115                 for (String licenseId : licensesUsed.keySet())
 116                         for (String name : licensesUsed.get(licenseId))
 117                                 sb.append((licenseId.equals("") ? "Proprietary" : licenseId) + "\t\t" + name + "\n");
 118                 logger.log(INFO, "# License summary:\n" + sb);
 119         }
 120
 121         /** Deletes remaining sub directories. */
 122         void cleanPreviousFailedBuild(Path categoryPath) {
 123                 Path outputCategoryPath = a2Base.resolve(categoryPath);
 124                 if (!Files.exists(outputCategoryPath))
 125                         return;
 126                 // clean previous failed build
 127                 try {
 128                         for (Path subDir : Files.newDirectoryStream(outputCategoryPath, (d) -> Files.isDirectory(d))) {
 129                                 if (Files.exists(subDir)) {
 130                                         logger.log(WARNING, "Bundle dir " + subDir
 131                                                         + " already exists, probably from a previous failed build, deleting it...");
 132                                         deleteDirectory(subDir);
 133                                 }
 134                         }
 135                 } catch (IOException e) {
 136                         logger.log(ERROR, "Cannot clean previous build", e);
 137                 }
 138         }
 139
 140         /** MANIFEST headers. */
 141         enum ManifestHeader {
 142                 // OSGi
 143                 /** OSGi bundle symbolic name. */
 144                 BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), //
 145                 /** OSGi bundle version. */
 146                 BUNDLE_VERSION("Bundle-Version"), //
 147                 /** OSGi bundle license. */
 148                 BUNDLE_LICENSE("Bundle-License"), //
 149                 /** OSGi exported packages list. */
 150                 EXPORT_PACKAGE("Export-Package"), //
 151                 /** OSGi imported packages list. */
 152                 IMPORT_PACKAGE("Import-Package"), //
 153                 /** Require capability. */
 154                 REQUIRE_CAPABILITY("Require-Capability"), //
 155 //              /** OSGi required bundles. */
 156 //              REQUIRE_BUNDLE("Require-Bundle"), //
 157 //              /** OSGi path to embedded jar. */
 158 //              BUNDLE_CLASSPATH("Bundle-Classpath"), //
 159                 // Java
 160                 /** Java module name. */
 161                 AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), //
 162                 // Eclipse
 163                 /** Eclipse source bundle. */
 164                 ECLIPSE_SOURCE_BUNDLE("Eclipse-SourceBundle"), //
 165                 // SPDX
 166                 /**
 167                  * SPDX license identifier.
 168                  *
 169                  * @see https://spdx.org/licenses/
 170                  */
 171                 SPDX_LICENSE_IDENTIFIER("SPDX-License-Identifier"), //
 172                 // Argeo Origin
 173                 /**
 174                  * Maven coordinates of the origin, possibly partial when using common.bnd or
 175                  * merge.bnd.
 176                  */
 177                 ARGEO_ORIGIN_M2("Argeo-Origin-M2"), //
 178                 /** List of Maven coordinates to merge. */
 179                 ARGEO_ORIGIN_M2_MERGE("Argeo-Origin-M2-Merge"), //
 180                 /** Maven repository, if not the default one. */
 181                 ARGEO_ORIGIN_M2_REPO("Argeo-Origin-M2-Repo"), //
 182                 /**
 183                  * Do not perform BND analysis of the origin component. Typically Import-Package
 184                  * and Export-Package will be kept untouched.
 185                  */
 186                 ARGEO_ORIGIN_NO_METADATA_GENERATION("Argeo-Origin-NoMetadataGeneration"), //
 187                 /** Keep JPMS module-info */
 188                 ARGEO_ORIGIN_KEEP_MODULE_INFO("Argeo-Origin-KeepModuleInfo"), //
 189 //              /**
 190 //               * Embed the original jar without modifying it (may be required by some
 191 //               * proprietary licenses, such as JCR Day License).
 192 //               */
 193 //              ARGEO_ORIGIN_EMBED("Argeo-Origin-Embed"), //
 194                 /**
 195                  * Do not modify original jar (may be required by some proprietary licenses,
 196                  * such as JCR Day License).
 197                  */
 198                 ARGEO_ORIGIN_DO_NOT_MODIFY("Argeo-Origin-Do-Not-Modify"), //
 199                 /**
 200                  * Origin (non-Maven) URI of the component. It may be anything (jar, archive,
 201                  * etc.).
 202                  */
 203                 ARGEO_ORIGIN_URI("Argeo-Origin-URI"), //
 204                 /**
 205                  * Origin (non-Maven) URI of the source of the component. It may be anything
 206                  * (jar, archive, code repository, etc.).
 207                  */
 208                 ARGEO_ORIGIN_SOURCES_URI("Argeo-Origin-Sources-URI"), //
 209                 ;
 210
 211                 final String headerName;
 212
 213                 private ManifestHeader(String headerName) {
 214                         this.headerName = headerName;
 215                 }
 216
 217                 @Override
 218                 public String toString() {
 219                         return headerName;
 220                 }
 221
 222                 /** Get the value from either a {@link Manifest} or a {@link Properties}. */
 223                 String get(Object map) {
 224                         if (map instanceof Manifest manifest)
 225                                 return manifest.getMainAttributes().getValue(headerName);
 226                         else if (map instanceof Properties props)
 227                                 return props.getProperty(headerName);
 228                         else
 229                                 throw new IllegalArgumentException("Unsupported mapping " + map.getClass());
 230                 }
 231
 232                 /** Put the value into either a {@link Manifest} or a {@link Properties}. */
 233                 void put(Object map, String value) {
 234                         if (map instanceof Manifest manifest)
 235                                 manifest.getMainAttributes().putValue(headerName, value);
 236                         else if (map instanceof Properties props)
 237                                 props.setProperty(headerName, value);
 238                         else
 239                                 throw new IllegalArgumentException("Unsupported mapping " + map.getClass());
 240                 }
 241         }
 242
 243         /** Name of the file centralising information for multiple M2 artifacts. */
 244         final static String COMMON_BND = "common.bnd";
 245         /** Name of the file centralising information for mergin M2 artifacts. */
 246         final static String MERGE_BND = "merge.bnd";
 247         /**
 248          * Subdirectory of the jar file where origin informations (changes, legal
 249          * notices etc. are stored)
 250          */
 251         final static String ARGEO_ORIGIN = "ARGEO-ORIGIN";
 252         /** File detailing modifications to the original component. */
 253         final static String CHANGES = ARGEO_ORIGIN + "/changes";
 254         /**
 255          * Name of the file at the root of the repackaged jar, which prominently
 256          * notifies that the component has be repackaged.
 257          */
 258         final static String README_REPACKAGED = "README.repackaged";
 259
 260         // cache
 261         /** Summary of all license seen during the repackaging. */
 262         final static Map<String, Set<String>> licensesUsed = new TreeMap<>();
 263
 264         /** Directory where to download archives */
 265         final Path originBase;
 266         /** Directory where to download Maven artifacts */
 267         final Path mavenBase;
 268
 269         /** A2 repository base for binary bundles */
 270         final Path a2Base;
 271         /** A2 repository base for source bundles */
 272         final Path a2SrcBase;
 273         /** A2 base for native components */
 274         final Path a2LibBase;
 275         /** Location of the descriptors driving the packaging */
 276         final Path descriptorsBase;
 277         /** URIs of archives to download */
 278         final Properties uris = new Properties();
 279         /** Mirrors for archive download. Key is URI prefix, value list of base URLs */
 280         final Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
 281
 282         /** Whether sources should be packaged separately */
 283         final boolean separateSources;
 284
 285         /** Constructor initialises the various variables */
 286         public Repackage(Path a2Base, Path descriptorsBase) {
 287                 separateSources = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES));
 288                 if (separateSources)
 289                         logger.log(INFO, "Sources will be packaged separately");
 290
 291                 Objects.requireNonNull(a2Base);
 292                 Objects.requireNonNull(descriptorsBase);
 293                 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/build/origin");
 294                 this.mavenBase = Paths.get(System.getProperty("user.home"), ".m2", "repository");
 295
 296                 // TODO define and use a build base
 297                 this.a2Base = a2Base;
 298                 this.a2SrcBase = separateSources ? a2Base.getParent().resolve(a2Base.getFileName() + ".src") : a2Base;
 299                 this.a2LibBase = a2Base.resolve("lib");
 300                 this.descriptorsBase = descriptorsBase;
 301                 if (!Files.exists(this.descriptorsBase))
 302                         throw new IllegalArgumentException(this.descriptorsBase + " does not exist");
 303
 304                 // URIs mapping
 305                 Path urisPath = this.descriptorsBase.resolve("uris.properties");
 306                 if (Files.exists(urisPath)) {
 307                         try (InputStream in = Files.newInputStream(urisPath)) {
 308                                 uris.load(in);
 309                         } catch (IOException e) {
 310                                 throw new IllegalStateException("Cannot load " + urisPath, e);
 311                         }
 312                 }
 313
 314                 // Eclipse mirrors
 315                 Path eclipseMirrorsPath = this.descriptorsBase.resolve("eclipse.mirrors.txt");
 316                 List<String> eclipseMirrors = new ArrayList<>();
 317                 if (Files.exists(eclipseMirrorsPath)) {
 318                         try {
 319                                 eclipseMirrors = Files.readAllLines(eclipseMirrorsPath, StandardCharsets.UTF_8);
 320                         } catch (IOException e) {
 321                                 throw new IllegalStateException("Cannot load " + eclipseMirrorsPath, e);
 322                         }
 323                         for (Iterator<String> it = eclipseMirrors.iterator(); it.hasNext();) {
 324                                 String value = it.next();
 325                                 if (value.strip().equals(""))
 326                                         it.remove();
 327                         }
 328                 }
 329                 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
 330         }
 331
 332         /*
 333          * MAVEN ORIGIN
 334          */
 335         /** Process a whole category/group id. */
 336         void processCategory(Path categoryRelativePath) {
 337                 try {
 338                         Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath);
 339                         try (DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
 340                                         (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
 341                                                         && !p.getFileName().toString().equals(MERGE_BND))) {
 342                                 for (Path p : bnds) {
 343                                         processSingleM2ArtifactDistributionUnit(p);
 344                                 }
 345                         }
 346
 347                         try (DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase,
 348                                         (p) -> Files.isDirectory(p))) {
 349                                 for (Path duDir : dus) {
 350                                         if (duDir.getFileName().toString().startsWith("eclipse-")) {
 351                                                 processEclipseArchive(duDir);
 352                                         } else {
 353                                                 processM2BasedDistributionUnit(duDir);
 354                                         }
 355                                 }
 356                         }
 357                 } catch (IOException e) {
 358                         throw new RuntimeException("Cannot process category " + categoryRelativePath, e);
 359                 }
 360         }
 361
 362         /** Process a standalone Maven artifact. */
 363         void processSingleM2ArtifactDistributionUnit(Path bndFile) {
 364                 try {
 365                         Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent());
 366                         Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
 367
 368                         Properties fileProps = new Properties();
 369                         try (InputStream in = Files.newInputStream(bndFile)) {
 370                                 fileProps.load(in);
 371                         }
 372                         // use file name as symbolic name
 373                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 374                                 String symbolicName = bndFile.getFileName().toString();
 375                                 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 376                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 377                         }
 378
 379                         String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
 380                         if (m2Coordinates == null)
 381                                 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
 382                         M2Artifact artifact = new M2Artifact(m2Coordinates);
 383
 384                         Path downloaded = downloadMaven(fileProps, artifact);
 385
 386                         boolean doNotModify = Boolean
 387                                         .parseBoolean(fileProps.getOrDefault(ARGEO_ORIGIN_DO_NOT_MODIFY.toString(), "false").toString());
 388                         if (doNotModify) {
 389                                 processNotModified(targetCategoryBase, downloaded, fileProps, artifact);
 390                                 return;
 391                         }
 392
 393                         // regular processing
 394                         A2Origin origin = new A2Origin();
 395                         Path bundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact, origin);
 396                         downloadAndProcessM2Sources(fileProps, artifact, bundleDir, false, false);
 397                         createJar(bundleDir, origin);
 398                 } catch (Exception e) {
 399                         throw new RuntimeException("Cannot process " + bndFile, e);
 400                 }
 401         }
 402
 403         /**
 404          * Process multiple Maven artifacts coming from a same project and therefore
 405          * with information in common (typically the version), generating single bundles
 406          * or merging them if necessary.
 407          *
 408          * @see #COMMON_BND
 409          * @see #MERGE_BND
 410          */
 411         void processM2BasedDistributionUnit(Path duDir) {
 412                 try {
 413                         Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
 414                         Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
 415
 416                         Path mergeBnd = duDir.resolve(MERGE_BND);
 417                         if (Files.exists(mergeBnd)) // merge
 418                                 mergeM2Artifacts(mergeBnd);
 419
 420                         Path commonBnd = duDir.resolve(COMMON_BND);
 421                         if (!Files.exists(commonBnd))
 422                                 return;
 423
 424                         Properties commonProps = new Properties();
 425                         try (InputStream in = Files.newInputStream(commonBnd)) {
 426                                 commonProps.load(in);
 427                         }
 428
 429                         String m2Version = commonProps.getProperty(ARGEO_ORIGIN_M2.toString());
 430                         if (m2Version == null) {
 431                                 logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
 432                                 return;// ignore, this is probably an Eclipse archive
 433                         }
 434                         if (!m2Version.startsWith(":")) {
 435                                 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 436                         }
 437                         m2Version = m2Version.substring(1);
 438
 439                         try (DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
 440                                         (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
 441                                                         && !p.getFileName().toString().equals(MERGE_BND))) {
 442                                 for (Path p : ds) {
 443                                         Properties fileProps = new Properties();
 444                                         try (InputStream in = Files.newInputStream(p)) {
 445                                                 fileProps.load(in);
 446                                         }
 447                                         String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
 448                                         M2Artifact artifact = new M2Artifact(m2Coordinates);
 449                                         if (artifact.getVersion() == null) {
 450                                                 artifact.setVersion(m2Version);
 451                                         } else {
 452                                                 logger.log(DEBUG, p.getFileName() + " : Using version " + artifact.getVersion()
 453                                                                 + " specified in descriptor rather than " + m2Version + " specified in " + COMMON_BND);
 454                                         }
 455
 456                                         // prepare manifest entries
 457                                         Properties mergedProps = new Properties();
 458                                         mergedProps.putAll(commonProps);
 459
 460                                         fileEntries: for (Object key : fileProps.keySet()) {
 461                                                 if (ARGEO_ORIGIN_M2.toString().equals(key))
 462                                                         continue fileEntries;
 463                                                 String value = fileProps.getProperty(key.toString());
 464                                                 Object previousValue = mergedProps.put(key.toString(), value);
 465                                                 if (previousValue != null) {
 466                                                         logger.log(WARNING,
 467                                                                         commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
 468                                                 }
 469                                         }
 470                                         mergedProps.put(ARGEO_ORIGIN_M2.toString(), artifact.toM2Coordinates());
 471                                         if (!mergedProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 472                                                 // use file name as symbolic name
 473                                                 String symbolicName = p.getFileName().toString();
 474                                                 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 475                                                 mergedProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 476                                         }
 477
 478                                         // download
 479                                         Path downloaded = downloadMaven(mergedProps, artifact);
 480
 481                                         boolean doNotModify = Boolean.parseBoolean(
 482                                                         mergedProps.getOrDefault(ARGEO_ORIGIN_DO_NOT_MODIFY.toString(), "false").toString());
 483                                         if (doNotModify) {
 484                                                 processNotModified(targetCategoryBase, downloaded, mergedProps, artifact);
 485                                         } else {
 486                                                 A2Origin origin = new A2Origin();
 487                                                 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergedProps, artifact,
 488                                                                 origin);
 489                                                 downloadAndProcessM2Sources(mergedProps, artifact, targetBundleDir, false, false);
 490                                                 createJar(targetBundleDir, origin);
 491                                         }
 492                                 }
 493                         }
 494                 } catch (IOException e) {
 495                         throw new RuntimeException("Cannot process " + duDir, e);
 496                 }
 497         }
 498
 499         /** Merge multiple Maven artifacts. */
 500         void mergeM2Artifacts(Path mergeBnd) throws IOException {
 501                 Path duDir = mergeBnd.getParent();
 502                 String category = duDir.getParent().getFileName().toString();
 503                 Path targetCategoryBase = a2Base.resolve(category);
 504
 505                 Properties mergeProps = new Properties();
 506                 // first, load common properties
 507                 Path commonBnd = duDir.resolve(COMMON_BND);
 508                 if (Files.exists(commonBnd))
 509                         try (InputStream in = Files.newInputStream(commonBnd)) {
 510                                 mergeProps.load(in);
 511                         }
 512                 // then, the merge properties themselves
 513                 try (InputStream in = Files.newInputStream(mergeBnd)) {
 514                         mergeProps.load(in);
 515                 }
 516
 517                 String m2Version = mergeProps.getProperty(ARGEO_ORIGIN_M2.toString());
 518                 if (m2Version == null) {
 519                         logger.log(WARNING, "Ignoring merging in " + duDir + " as it is not an M2-based distribution unit");
 520                         return;// ignore, this is probably an Eclipse archive
 521                 }
 522                 if (!m2Version.startsWith(":")) {
 523                         throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 524                 }
 525                 m2Version = m2Version.substring(1);
 526                 mergeProps.put(BUNDLE_VERSION.toString(), m2Version);
 527
 528                 String artifactsStr = mergeProps.getProperty(ARGEO_ORIGIN_M2_MERGE.toString());
 529                 if (artifactsStr == null)
 530                         throw new IllegalArgumentException(mergeBnd + ": " + ARGEO_ORIGIN_M2_MERGE + " must be set");
 531
 532                 String bundleSymbolicName = mergeProps.getProperty(BUNDLE_SYMBOLICNAME.toString());
 533                 if (bundleSymbolicName == null)
 534                         throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
 535                 CategoryNameVersion nameVersion = new M2Artifact(category + ":" + bundleSymbolicName + ":" + m2Version);
 536
 537                 A2Origin origin = new A2Origin();
 538                 Path bundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
 539
 540                 StringJoiner originDesc = new StringJoiner(",");
 541                 String[] artifacts = artifactsStr.split(",");
 542                 artifacts: for (String str : artifacts) {
 543                         String m2Coordinates = str.trim();
 544                         if ("".equals(m2Coordinates))
 545                                 continue artifacts;
 546                         M2Artifact artifact = new M2Artifact(m2Coordinates.trim());
 547                         if (artifact.getVersion() == null)
 548                                 artifact.setVersion(m2Version);
 549                         originDesc.add(artifact.toString());
 550                         Path downloaded = downloadMaven(mergeProps, artifact);
 551                         JarEntry entry;
 552                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
 553                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 554                                         if (entry.isDirectory())
 555                                                 continue entries;
 556                                         if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
 557                                                         || entry.getName().endsWith(".SF")) {
 558                                                 origin.deleted.add("cryptographic signatures from " + artifact);
 559                                                 continue entries;
 560                                         }
 561                                         if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
 562                                                 origin.deleted.add("Java module information (module-info.class) from " + artifact);
 563                                                 continue entries;
 564                                         }
 565                                         if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
 566                                                 origin.deleted.add("additional Java versions (META-INF/versions) from " + artifact);
 567                                                 continue entries;
 568                                         }
 569                                         if (entry.getName().startsWith("META-INF/maven/")) {
 570                                                 origin.deleted.add("Maven information (META-INF/maven) from " + artifact);
 571                                                 continue entries;
 572                                         }
 573                                         if (entry.getName().startsWith(".cache/")) { // Apache SSHD
 574                                                 origin.deleted.add("cache directory (.cache) from " + artifact);
 575                                                 continue entries;
 576                                         }
 577                                         if (entry.getName().equals("META-INF/DEPENDENCIES")) {
 578                                                 origin.deleted.add("Dependencies (META-INF/DEPENDENCIES) from " + artifact);
 579                                                 continue entries;
 580                                         }
 581                                         if (entry.getName().equals("META-INF/MANIFEST.MF")) {
 582                                                 Path originalManifest = bundleDir.resolve(ARGEO_ORIGIN).resolve(artifact.getGroupId())
 583                                                                 .resolve(artifact.getArtifactId()).resolve("MANIFEST.MF");
 584                                                 Files.createDirectories(originalManifest.getParent());
 585                                                 try (OutputStream out = Files.newOutputStream(originalManifest)) {
 586                                                         Files.copy(jarIn, originalManifest);
 587                                                 }
 588                                                 origin.added.add(
 589                                                                 "original MANIFEST (" + bundleDir.relativize(originalManifest) + ") from " + artifact);
 590                                                 continue entries;
 591                                         }
 592
 593                                         if (entry.getName().endsWith("NOTICE") || entry.getName().endsWith("NOTICE.txt")
 594                                                         || entry.getName().endsWith("NOTICE.md") || entry.getName().endsWith("LICENSE")
 595                                                         || entry.getName().endsWith("LICENSE.md") || entry.getName().endsWith("LICENSE-notice.md")
 596                                                         || entry.getName().endsWith("COPYING") || entry.getName().endsWith("COPYING.LESSER")) {
 597                                                 Path artifactOriginDir = bundleDir.resolve(ARGEO_ORIGIN).resolve(artifact.getGroupId())
 598                                                                 .resolve(artifact.getArtifactId());
 599                                                 Path target = artifactOriginDir.resolve(entry.getName());
 600                                                 Files.createDirectories(target.getParent());
 601                                                 Files.copy(jarIn, target);
 602                                                 origin.moved.add(entry.getName() + " in " + artifact + " to " + bundleDir.relativize(target));
 603                                                 continue entries;
 604                                         }
 605                                         Path target = bundleDir.resolve(entry.getName());
 606                                         Files.createDirectories(target.getParent());
 607                                         if (!Files.exists(target)) {
 608                                                 Files.copy(jarIn, target);
 609                                         } else {
 610                                                 if (entry.getName().startsWith("META-INF/services/")) {
 611                                                         try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
 612                                                                 out.write("\n".getBytes());
 613                                                                 jarIn.transferTo(out);
 614                                                                 logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName());
 615                                                         }
 616                                                         origin.modified.add(entry.getName() + ", merging from " + artifact);
 617                                                 } else if (entry.getName().startsWith("org/apache/batik/")) {
 618                                                         logger.log(TRACE, "Skip " + entry.getName());
 619                                                         continue entries;
 620                                                 } else if (entry.getName().startsWith("META-INF/NOTICE")) {
 621                                                         logger.log(WARNING, "Skip " + entry.getName() + " from " + artifact);
 622                                                         // TODO merge them?
 623                                                         continue entries;
 624                                                 } else {
 625                                                         throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
 626                                                 }
 627                                         }
 628                                         logger.log(TRACE, () -> "Copied " + target);
 629                                 }
 630                         }
 631                         origin.added.add("binary content of " + artifact);
 632
 633                         // process sources
 634                         downloadAndProcessM2Sources(mergeProps, artifact, bundleDir, true, false);
 635                 }
 636
 637                 // additional service files
 638                 Path servicesDir = duDir.resolve("services");
 639                 if (Files.exists(servicesDir)) {
 640                         for (Path p : Files.newDirectoryStream(servicesDir)) {
 641                                 Path target = bundleDir.resolve("META-INF/services/").resolve(p.getFileName());
 642                                 try (InputStream in = Files.newInputStream(p);
 643                                                 OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) {
 644                                         out.write("\n".getBytes());
 645                                         in.transferTo(out);
 646                                         logger.log(DEBUG, "Appended " + p);
 647                                 }
 648                                 origin.added.add(bundleDir.relativize(target).toString());
 649                         }
 650                 }
 651
 652                 // BND analysis
 653                 Map<String, String> entries = new TreeMap<>();
 654                 try (Analyzer bndAnalyzer = new Analyzer()) {
 655                         bndAnalyzer.setProperties(mergeProps);
 656                         Jar jar = new Jar(bundleDir.toFile());
 657                         bndAnalyzer.setJar(jar);
 658                         Manifest manifest = bndAnalyzer.calcManifest();
 659
 660                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 661                                 Object value = manifest.getMainAttributes().get(key);
 662
 663                                 switch (key.toString()) {
 664                                 case "Tool":
 665                                 case "Bnd-LastModified":
 666                                 case "Created-By":
 667                                         continue keys;
 668                                 }
 669                                 if (REQUIRE_CAPABILITY.toString().equals(key.toString())
 670                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) {
 671                                         origin.deleted.add("MANIFEST header " + key);
 672                                         continue keys;// hack for very old classes
 673                                 }
 674                                 entries.put(key.toString(), value.toString());
 675                         }
 676                 } catch (Exception e) {
 677                         throw new RuntimeException("Cannot process " + mergeBnd, e);
 678                 }
 679
 680                 Manifest manifest = new Manifest();
 681                 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
 682                 Files.createDirectories(manifestPath.getParent());
 683                 for (String key : entries.keySet()) {
 684                         String value = entries.get(key);
 685                         manifest.getMainAttributes().putValue(key, value);
 686                 }
 687                 manifest.getMainAttributes().putValue(ARGEO_ORIGIN_M2.toString(), originDesc.toString());
 688
 689                 processLicense(bundleDir, manifest);
 690
 691                 // write MANIFEST
 692                 try (OutputStream out = Files.newOutputStream(manifestPath)) {
 693                         manifest.write(out);
 694                 }
 695                 createJar(bundleDir, origin);
 696         }
 697
 698         /** Generates MANIFEST using BND. */
 699         Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, M2Artifact artifact,
 700                         A2Origin origin) {
 701                 try {
 702                         Map<String, String> additionalEntries = new TreeMap<>();
 703                         boolean doNotModifyManifest = Boolean.parseBoolean(
 704                                         fileProps.getOrDefault(ARGEO_ORIGIN_NO_METADATA_GENERATION.toString(), "false").toString());
 705
 706                         // Note: we always force the symbolic name
 707                         if (doNotModifyManifest) {
 708                                 for (Object key : fileProps.keySet()) {
 709                                         String value = fileProps.getProperty(key.toString());
 710                                         additionalEntries.put(key.toString(), value);
 711                                 }
 712                         } else {
 713                                 if (artifact != null) {
 714                                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 715                                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
 716                                         }
 717                                         if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
 718                                                 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
 719                                         }
 720                                 }
 721
 722                                 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
 723                                         fileProps.put(EXPORT_PACKAGE.toString(),
 724                                                         "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
 725                                 }
 726
 727                                 // BND analysis
 728                                 try (Analyzer bndAnalyzer = new Analyzer()) {
 729                                         bndAnalyzer.setProperties(fileProps);
 730                                         Jar jar = new Jar(downloaded.toFile());
 731                                         bndAnalyzer.setJar(jar);
 732                                         Manifest manifest = bndAnalyzer.calcManifest();
 733
 734                                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 735                                                 Object value = manifest.getMainAttributes().get(key);
 736
 737                                                 switch (key.toString()) {
 738                                                 case "Tool":
 739                                                 case "Bnd-LastModified":
 740                                                 case "Created-By":
 741                                                         continue keys;
 742                                                 }
 743                                                 if (REQUIRE_CAPABILITY.toString().equals(key.toString())
 744                                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) {
 745                                                         origin.deleted.add("MANIFEST header " + key);
 746                                                         continue keys;// !! hack for very old classes
 747                                                 }
 748                                                 additionalEntries.put(key.toString(), value.toString());
 749                                         }
 750                                 }
 751                         }
 752                         Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries, origin);
 753                         logger.log(DEBUG, () -> "Processed " + downloaded);
 754                         return targetBundleDir;
 755                 } catch (Exception e) {
 756                         throw new RuntimeException("Cannot BND process " + downloaded, e);
 757                 }
 758
 759         }
 760
 761         /** Process an artifact that should not be modified. */
 762         void processNotModified(Path targetCategoryBase, Path downloaded, Properties fileProps, M2Artifact artifact)
 763                         throws IOException {
 764                 // Some proprietary or signed artifacts do not allow any modification
 765                 // When releasing (with separate sources), we just copy it
 766                 Path unmodifiedTarget = targetCategoryBase
 767                                 .resolve(fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch() + ".jar");
 768                 Files.createDirectories(unmodifiedTarget.getParent());
 769                 Files.copy(downloaded, unmodifiedTarget, StandardCopyOption.REPLACE_EXISTING);
 770                 Path bundleDir = targetCategoryBase
 771                                 .resolve(fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch());
 772                 downloadAndProcessM2Sources(fileProps, artifact, bundleDir, false, true);
 773                 Manifest manifest;
 774                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(unmodifiedTarget))) {
 775                         manifest = jarIn.getManifest();
 776                 }
 777                 createSourceJar(bundleDir, manifest, fileProps);
 778         }
 779
 780         /** Download and integrates sources for a single Maven artifact. */
 781         void downloadAndProcessM2Sources(Properties props, M2Artifact artifact, Path targetBundleDir, boolean merging,
 782                         boolean unmodified) throws IOException {
 783                 try {
 784                         String repoStr = props.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
 785                                         ? props.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
 786                                         : null;
 787                         String alternateUri = props.getProperty(ARGEO_ORIGIN_SOURCES_URI.toString());
 788                         M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources");
 789                         URI sourcesUrl = alternateUri != null ? new URI(alternateUri)
 790                                         : M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
 791                         Path sourcesDownloaded = downloadMaven(sourcesUrl, sourcesArtifact);
 792                         processM2SourceJar(sourcesDownloaded, targetBundleDir, merging ? artifact : null, unmodified);
 793                         logger.log(TRACE, () -> "Processed source " + sourcesDownloaded);
 794                 } catch (Exception e) {
 795                         logger.log(ERROR, () -> "Cannot download source for  " + artifact);
 796                 }
 797
 798         }
 799
 800         /** Integrate sources from a downloaded jar file. */
 801         void processM2SourceJar(Path file, Path bundleDir, M2Artifact mergingFrom, boolean unmodified) throws IOException {
 802                 A2Origin origin = new A2Origin();
 803                 Path sourceDir = separateSources || unmodified ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
 804                                 : bundleDir.resolve("OSGI-OPT/src");
 805                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 806
 807                         String mergingMsg = "";
 808                         if (mergingFrom != null)
 809                                 mergingMsg = " of " + mergingFrom;
 810
 811                         Files.createDirectories(sourceDir);
 812                         JarEntry entry;
 813                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 814                                 String relPath = entry.getName();
 815                                 if (entry.isDirectory())
 816                                         continue entries;
 817                                 if (entry.getName().equals("META-INF/MANIFEST.MF")) {// skip META-INF entries
 818                                         origin.deleted.add("MANIFEST.MF from the sources" + mergingMsg);
 819                                         continue entries;
 820                                 }
 821                                 if (!unmodified) {
 822                                         if (entry.getName().startsWith("module-info.java")) {// skip Java module information
 823                                                 origin.deleted.add("Java module information from the sources (module-info.java)" + mergingMsg);
 824                                                 continue entries;
 825                                         }
 826                                         if (entry.getName().startsWith("/")) { // absolute paths
 827                                                 int metaInfIndex = entry.getName().indexOf("META-INF");
 828                                                 if (metaInfIndex >= 0) {
 829                                                         relPath = entry.getName().substring(metaInfIndex);
 830                                                         origin.moved.add(" to " + relPath + " entry with absolute path " + entry.getName());
 831                                                 } else {
 832                                                         logger.log(WARNING, entry.getName() + " has an absolute path");
 833                                                         origin.deleted.add(entry.getName() + " from the sources" + mergingMsg);
 834                                                 }
 835                                                 continue entries;
 836                                         }
 837                                 }
 838                                 Path target = sourceDir.resolve(relPath);
 839                                 Files.createDirectories(target.getParent());
 840                                 if (!Files.exists(target)) {
 841                                         Files.copy(jarIn, target);
 842                                         logger.log(TRACE, () -> "Copied source " + target);
 843                                 } else {
 844                                         logger.log(TRACE, () -> target + " already exists, skipping...");
 845                                 }
 846                         }
 847                 }
 848                 // write the changes
 849                 if (separateSources || unmodified) {
 850                         origin.appendChanges(sourceDir);
 851                 } else {
 852                         origin.added.add("source code under OSGI-OPT/src");
 853                         origin.appendChanges(bundleDir);
 854                 }
 855         }
 856
 857         /** Download a Maven artifact. */
 858         Path downloadMaven(Properties props, M2Artifact artifact) throws IOException {
 859                 String repoStr = props.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
 860                                 ? props.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
 861                                 : null;
 862                 String alternateUri = props.getProperty(ARGEO_ORIGIN_URI.toString());
 863                 try {
 864                         URI uri = alternateUri != null ? new URI(alternateUri) : M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
 865                         return downloadMaven(uri, artifact);
 866                 } catch (URISyntaxException e) {
 867                         throw new IllegalArgumentException("Wrong aritfact URI", e);
 868                 }
 869         }
 870
 871         /** Download a Maven artifact. */
 872         Path downloadMaven(URI uri, M2Artifact artifact) throws IOException {
 873                 return download(uri, mavenBase, M2ConventionsUtils.artifactPath("", artifact));
 874         }
 875
 876         /*
 877          * ECLIPSE ORIGIN
 878          */
 879         /** Process an archive in Eclipse format. */
 880         void processEclipseArchive(Path duDir) {
 881                 try {
 882                         Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
 883                         Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
 884                         Files.createDirectories(targetCategoryBase);
 885                         // first delete all directories from previous builds
 886                         for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)))
 887                                 deleteDirectory(dir);
 888
 889                         Files.createDirectories(originBase);
 890
 891                         Path commonBnd = duDir.resolve(COMMON_BND);
 892                         Properties commonProps = new Properties();
 893                         try (InputStream in = Files.newInputStream(commonBnd)) {
 894                                 commonProps.load(in);
 895                         }
 896                         String url = commonProps.getProperty(ARGEO_ORIGIN_URI.toString());
 897                         if (url == null) {
 898                                 url = uris.getProperty(duDir.getFileName().toString());
 899                                 if (url == null)
 900                                         throw new IllegalStateException("No url available for " + duDir);
 901                                 commonProps.put(ARGEO_ORIGIN_URI.toString(), url);
 902                         }
 903                         Path downloaded = tryDownloadArchive(url, originBase);
 904
 905                         FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
 906
 907                         // filters
 908                         List<PathMatcher> includeMatchers = new ArrayList<>();
 909                         Properties includes = new Properties();
 910                         try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
 911                                 includes.load(in);
 912                         }
 913                         for (Object pattern : includes.keySet()) {
 914                                 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 915                                 includeMatchers.add(pathMatcher);
 916                         }
 917
 918                         List<PathMatcher> excludeMatchers = new ArrayList<>();
 919                         Path excludeFile = duDir.resolve("excludes.properties");
 920                         if (Files.exists(excludeFile)) {
 921                                 Properties excludes = new Properties();
 922                                 try (InputStream in = Files.newInputStream(excludeFile)) {
 923                                         excludes.load(in);
 924                                 }
 925                                 for (Object pattern : excludes.keySet()) {
 926                                         PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 927                                         excludeMatchers.add(pathMatcher);
 928                                 }
 929                         }
 930
 931                         // keys are the bundle directories
 932                         Map<Path, A2Origin> origins = new HashMap<>();
 933                         Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
 934
 935                                 @Override
 936                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 937                                         includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
 938                                                 if (includeMatcher.matches(file)) {
 939                                                         for (PathMatcher excludeMatcher : excludeMatchers) {
 940                                                                 if (excludeMatcher.matches(file)) {
 941                                                                         logger.log(TRACE, "Skipping excluded " + file);
 942                                                                         return FileVisitResult.CONTINUE;
 943                                                                 }
 944                                                         }
 945                                                         if (file.getFileName().toString().contains(".source_")) {
 946                                                                 processEclipseSourceJar(file, targetCategoryBase);
 947                                                                 logger.log(DEBUG, () -> "Processed source " + file);
 948                                                         } else {
 949                                                                 Map<String, String> map = new HashMap<>();
 950                                                                 for (Object key : commonProps.keySet())
 951                                                                         map.put(key.toString(), commonProps.getProperty(key.toString()));
 952                                                                 A2Origin origin = new A2Origin();
 953                                                                 Path bundleDir = processBundleJar(file, targetCategoryBase, map, origin);
 954                                                                 if (bundleDir == null) {
 955                                                                         logger.log(WARNING, "No bundle dir created for " + file + ", skipping...");
 956                                                                         return FileVisitResult.CONTINUE;
 957                                                                 }
 958                                                                 origins.put(bundleDir, origin);
 959                                                                 logger.log(DEBUG, () -> "Processed " + file);
 960                                                         }
 961                                                         break includeMatchers;
 962                                                 }
 963                                         }
 964                                         return FileVisitResult.CONTINUE;
 965                                 }
 966                         });
 967
 968                         try (DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)
 969                                         && p.getFileName().toString().indexOf('.') >= 0 && !p.getFileName().toString().endsWith(".src"))) {
 970                                 for (Path bundleDir : dirs) {
 971                                         A2Origin origin = origins.get(bundleDir);
 972                                         Objects.requireNonNull(origin, "No A2 origin found for " + bundleDir);
 973                                         createJar(bundleDir, origin);
 974                                 }
 975                         }
 976                 } catch (IOException e) {
 977                         throw new RuntimeException("Cannot process " + duDir, e);
 978                 }
 979
 980         }
 981
 982         /** Process sources in Eclipse format. */
 983         void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
 984                 try {
 985                         A2Origin origin = new A2Origin();
 986                         Path bundleDir;
 987                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 988                                 Manifest manifest = jarIn.getManifest();
 989
 990                                 String[] relatedBundle = manifest.getMainAttributes().getValue(ECLIPSE_SOURCE_BUNDLE.toString())
 991                                                 .split(";");
 992                                 String version = relatedBundle[1].substring("version=\"".length());
 993                                 version = version.substring(0, version.length() - 1);
 994                                 NameVersion nameVersion = new NameVersion(relatedBundle[0], version);
 995                                 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 996
 997                                 Path sourceDir = separateSources ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
 998                                                 : bundleDir.resolve("OSGI-OPT/src");
 999
1000                                 Files.createDirectories(sourceDir);
1001                                 JarEntry entry;
1002                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
1003                                         if (entry.isDirectory())
1004                                                 continue entries;
1005                                         if (entry.getName().startsWith("META-INF"))// skip META-INF entries
1006                                                 continue entries;
1007                                         Path target = sourceDir.resolve(entry.getName());
1008                                         Files.createDirectories(target.getParent());
1009                                         Files.copy(jarIn, target);
1010                                         logger.log(TRACE, () -> "Copied source " + target);
1011                                 }
1012
1013                                 // write the changes
1014                                 if (separateSources) {
1015                                         origin.appendChanges(sourceDir);
1016                                 } else {
1017                                         origin.added.add("source code under OSGI-OPT/src");
1018                                         origin.appendChanges(bundleDir);
1019                                 }
1020                         }
1021                 } catch (IOException e) {
1022                         throw new IllegalStateException("Cannot process " + file, e);
1023                 }
1024         }
1025
1026         /*
1027          * COMMON PROCESSING
1028          */
1029         /** Normalise a single (that is, non-merged) bundle. */
1030         Path processBundleJar(Path file, Path targetBase, Map<String, String> entries, A2Origin origin) throws IOException {
1031 //              boolean embed = Boolean.parseBoolean(entries.getOrDefault(ARGEO_ORIGIN_EMBED.toString(), "false").toString());
1032                 boolean doNotModify = Boolean.parseBoolean(
1033                                 entries.getOrDefault(ManifestHeader.ARGEO_ORIGIN_DO_NOT_MODIFY.toString(), "false").toString());
1034                 boolean keepModuleInfo = Boolean.parseBoolean(
1035                                 entries.getOrDefault(ManifestHeader.ARGEO_ORIGIN_KEEP_MODULE_INFO.toString(), "false").toString());
1036                 NameVersion nameVersion;
1037                 Path bundleDir;
1038                 // singleton
1039                 boolean isSingleton = false;
1040                 Manifest manifest;
1041                 Manifest sourceManifest;
1042                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
1043                         sourceManifest = jarIn.getManifest();
1044                         if (sourceManifest == null)
1045                                 logger.log(WARNING, file + " has no manifest");
1046                         manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
1047
1048                         String rawSourceSymbolicName = manifest.getMainAttributes().getValue(BUNDLE_SYMBOLICNAME.toString());
1049                         if (rawSourceSymbolicName != null) {
1050                                 // make sure there is no directive
1051                                 String[] arr = rawSourceSymbolicName.split(";");
1052                                 for (int i = 1; i < arr.length; i++) {
1053                                         if (arr[i].trim().equals("singleton:=true"))
1054                                                 isSingleton = true;
1055                                         logger.log(DEBUG, file.getFileName() + " is a singleton");
1056                                 }
1057                         }
1058                         // remove problematic entries in MANIFEST
1059                         manifest.getEntries().clear();
1060
1061                         String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString());
1062                         String ourVersion = entries.get(BUNDLE_VERSION.toString());
1063
1064                         if (ourSymbolicName != null && ourVersion != null) {
1065                                 nameVersion = new NameVersion(ourSymbolicName, ourVersion);
1066                         } else {
1067                                 nameVersion = nameVersionFromManifest(manifest);
1068                                 if (nameVersion == null)
1069                                         throw new IllegalStateException("Could not compute name/version from Manifest");
1070                                 if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) {
1071                                         logger.log(WARNING,
1072                                                         "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion);
1073                                         entries.put(BUNDLE_VERSION.toString(), ourVersion);
1074                                 }
1075                                 if (ourSymbolicName != null) {
1076                                         // we always force our symbolic name
1077                                         nameVersion.setName(ourSymbolicName);
1078                                 }
1079                         }
1080
1081                         bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
1082
1083                         // copy original MANIFEST
1084                         if (sourceManifest != null) {
1085                                 Path originalManifest = bundleDir.resolve(ARGEO_ORIGIN).resolve("MANIFEST.MF");
1086                                 Files.createDirectories(originalManifest.getParent());
1087                                 try (OutputStream out = Files.newOutputStream(originalManifest)) {
1088                                         sourceManifest.write(out);
1089                                 }
1090                                 origin.moved.add("original MANIFEST to " + bundleDir.relativize(originalManifest));
1091                         }
1092
1093                         // force Java 9 module name
1094                         entries.put(AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName());
1095
1096                         boolean isNative = false;
1097                         String os = null;
1098                         String arch = null;
1099                         if (bundleDir.startsWith(a2LibBase)) {
1100                                 isNative = true;
1101                                 Path libRelativePath = a2LibBase.relativize(bundleDir);
1102                                 os = libRelativePath.getName(0).toString();
1103                                 arch = libRelativePath.getName(1).toString();
1104                         }
1105
1106                         // copy entries
1107                         JarEntry entry;
1108                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
1109                                 if (entry.isDirectory())
1110                                         continue entries;
1111                                 if (!doNotModify) {
1112                                         if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
1113                                                         || entry.getName().endsWith(".SF")) {
1114                                                 origin.deleted.add("cryptographic signatures");
1115                                                 continue entries;
1116                                         }
1117                                         if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
1118                                                 if (keepModuleInfo) {
1119                                                         entries.remove(AUTOMATIC_MODULE_NAME.toString());
1120                                                 } else {
1121                                                         origin.deleted.add("Java module information (module-info.class)");
1122                                                         continue entries;
1123                                                 }
1124                                         }
1125                                         if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
1126                                                 origin.deleted.add("additional Java versions (META-INF/versions)");
1127                                                 continue entries;
1128                                         }
1129                                         if (entry.getName().startsWith("META-INF/maven/")) {
1130                                                 origin.deleted.add("Maven information (META-INF/maven)");
1131                                                 continue entries;
1132                                         }
1133                                         // skip file system providers as they cause issues with native image
1134                                         if (entry.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider")) {
1135                                                 origin.deleted
1136                                                                 .add("file system providers (META-INF/services/java.nio.file.spi.FileSystemProvider)");
1137                                                 continue entries;
1138                                         }
1139                                 }
1140                                 if (entry.getName().startsWith("OSGI-OPT/src/")) { // skip embedded sources
1141                                         origin.deleted.add("embedded sources");
1142                                         continue entries;
1143                                 }
1144                                 Path target = bundleDir.resolve(entry.getName());
1145                                 Files.createDirectories(target.getParent());
1146                                 Files.copy(jarIn, target);
1147
1148                                 // native libraries
1149                                 boolean removeDllFromJar = true;
1150                                 if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll")
1151                                                 || entry.getName().endsWith(".jnilib") || entry.getName().endsWith(".a"))) {
1152                                         Path categoryDir = bundleDir.getParent();
1153                                         boolean copyDll = false;
1154                                         Path targetDll = categoryDir.resolve(bundleDir.relativize(target));
1155                                         if (nameVersion.getName().equals("com.sun.jna")) {
1156                                                 if (arch.equals("x86_64"))
1157                                                         arch = "x86-64";
1158                                                 if (os.equals("macosx"))
1159                                                         os = "darwin";
1160                                                 if (target.getParent().getFileName().toString().equals(os + "-" + arch)) {
1161                                                         copyDll = true;
1162                                                 }
1163                                                 targetDll = categoryDir.resolve(target.getFileName());
1164                                         } else {
1165                                                 copyDll = true;
1166                                         }
1167                                         if (copyDll) {
1168                                                 Files.createDirectories(targetDll.getParent());
1169                                                 if (Files.exists(targetDll))
1170                                                         Files.delete(targetDll);
1171                                                 Files.copy(target, targetDll);
1172                                         }
1173
1174                                         if (removeDllFromJar) {
1175                                                 Files.delete(target);
1176                                                 origin.deleted.add(bundleDir.relativize(target).toString());
1177                                         }
1178                                 }
1179                                 logger.log(TRACE, () -> "Copied " + target);
1180                         }
1181                 }
1182
1183                 // copy MANIFEST
1184                 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1185                 Files.createDirectories(manifestPath.getParent());
1186
1187                 if (isSingleton && entries.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
1188                         entries.put(BUNDLE_SYMBOLICNAME.toString(),
1189                                         entries.get(BUNDLE_SYMBOLICNAME.toString()) + ";singleton:=true");
1190                 }
1191
1192                 // Final MANIFEST decisions
1193                 // We also check the original OSGi metadata and compare with our changes
1194                 for (String key : entries.keySet()) {
1195                         String value = entries.get(key);
1196                         String previousValue = manifest.getMainAttributes().getValue(key);
1197                         boolean wasDifferent = previousValue != null && !previousValue.equals(value);
1198                         boolean keepPrevious = false;
1199                         if (wasDifferent) {
1200                                 if (SPDX_LICENSE_IDENTIFIER.toString().equals(key) && previousValue != null)
1201                                         keepPrevious = true;
1202                                 if (REQUIRE_CAPABILITY.toString().equals(key) && previousValue != null)
1203                                         keepPrevious = true;
1204                                 else if (BUNDLE_VERSION.toString().equals(key) && wasDifferent)
1205                                         if (previousValue.equals(value + ".0")) // typically a Maven first release
1206                                                 keepPrevious = true;
1207
1208                                 if (keepPrevious) {
1209                                         if (logger.isLoggable(DEBUG))
1210                                                 logger.log(DEBUG, file.getFileName() + ": " + key + " was NOT modified, value kept is "
1211                                                                 + previousValue + ", not overriden with " + value);
1212                                         value = previousValue;
1213                                 }
1214                         }
1215
1216                         manifest.getMainAttributes().putValue(key, value);
1217                         if (wasDifferent && !keepPrevious) {
1218                                 if (IMPORT_PACKAGE.toString().equals(key) || EXPORT_PACKAGE.toString().equals(key))
1219                                         logger.log(TRACE, () -> file.getFileName() + ": " + key + " was modified");
1220                                 else if (BUNDLE_SYMBOLICNAME.toString().equals(key) || AUTOMATIC_MODULE_NAME.toString().equals(key))
1221                                         logger.log(DEBUG,
1222                                                         file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value);
1223                                 else
1224                                         logger.log(WARNING,
1225                                                         file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value);
1226                                 origin.modified.add("MANIFEST header " + key);
1227                         }
1228
1229                         // !! hack to remove unresolvable
1230                         if (key.equals("Provide-Capability") || key.equals(REQUIRE_CAPABILITY.toString()))
1231                                 if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) {
1232                                         manifest.getMainAttributes().remove(key);
1233                                         origin.deleted.add("MANIFEST header " + key);
1234                                 }
1235                 }
1236
1237                 // de-pollute MANIFEST
1238                 for (Iterator<Map.Entry<Object, Object>> manifestEntries = manifest.getMainAttributes().entrySet()
1239                                 .iterator(); manifestEntries.hasNext();) {
1240                         Map.Entry<Object, Object> manifestEntry = manifestEntries.next();
1241                         String key = manifestEntry.getKey().toString();
1242                         // TODO make it more generic
1243 //                      if (key.equals(REQUIRE_BUNDLE.toString()) && nameVersion.getName().equals("com.sun.jna.platform"))
1244 //                              manifestEntries.remove();
1245                         switch (key) {
1246                         case "Archiver-Version":
1247                         case "Build-By":
1248                         case "Created-By":
1249                         case "Originally-Created-By":
1250                         case "Tool":
1251                         case "Bnd-LastModified":
1252                                 manifestEntries.remove();
1253                                 origin.deleted.add("MANIFEST header " + manifestEntry.getKey());
1254                                 break;
1255                         default:
1256                                 if (sourceManifest != null && !sourceManifest.getMainAttributes().containsKey(manifestEntry.getKey()))
1257                                         origin.added.add("MANIFEST header " + manifestEntry.getKey());
1258                         }
1259                 }
1260
1261                 processLicense(bundleDir, manifest);
1262
1263                 origin.modified.add("MANIFEST (META-INF/MANIFEST.MF)");
1264                 // write the MANIFEST
1265                 try (OutputStream out = Files.newOutputStream(manifestPath)) {
1266                         manifest.write(out);
1267                 }
1268                 return bundleDir;
1269         }
1270
1271         /** Process SPDX license identifier. */
1272         void processLicense(Path bundleDir, Manifest manifest) {
1273                 String spdxLicenceId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString());
1274                 String bundleLicense = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString());
1275                 if (spdxLicenceId == null) {
1276                         logger.log(ERROR, bundleDir.getFileName() + ": " + SPDX_LICENSE_IDENTIFIER + " not available, "
1277                                         + BUNDLE_LICENSE + " is " + bundleLicense);
1278                 } else {
1279                         // only use the first licensing option
1280                         int orIndex = spdxLicenceId.indexOf(" OR ");
1281                         if (orIndex >= 0)
1282                                 spdxLicenceId = spdxLicenceId.substring(0, orIndex).trim();
1283
1284                         String bundleDirName = bundleDir.getFileName().toString();
1285                         // force licenses of some well-known components
1286                         // even if we say otherwise (typically because from an Eclipse archive)
1287                         if (bundleDirName.startsWith("org.apache."))
1288                                 spdxLicenceId = "Apache-2.0";
1289                         if (bundleDirName.startsWith("com.sun.jna."))
1290                                 spdxLicenceId = "Apache-2.0";
1291                         if (bundleDirName.startsWith("com.ibm.icu."))
1292                                 spdxLicenceId = "ICU";
1293                         if (bundleDirName.startsWith("javax.annotation."))
1294                                 spdxLicenceId = "GPL-2.0-only WITH Classpath-exception-2.0";
1295                         if (bundleDirName.startsWith("javax.inject."))
1296                                 spdxLicenceId = "Apache-2.0";
1297                         if (bundleDirName.startsWith("org.osgi."))
1298                                 spdxLicenceId = "Apache-2.0";
1299
1300                         manifest.getMainAttributes().putValue(SPDX_LICENSE_IDENTIFIER.toString(), spdxLicenceId);
1301                         if (!licensesUsed.containsKey(spdxLicenceId))
1302                                 licensesUsed.put(spdxLicenceId, new TreeSet<>());
1303                         licensesUsed.get(spdxLicenceId).add(bundleDir.getParent().getFileName() + "/" + bundleDir.getFileName());
1304                 }
1305         }
1306
1307         /*
1308          * UTILITIES
1309          */
1310         /** Recursively deletes a directory. */
1311         static void deleteDirectory(Path path) throws IOException {
1312                 if (!Files.exists(path))
1313                         return;
1314                 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
1315                         @Override
1316                         public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
1317                                 if (e != null)
1318                                         throw e;
1319                                 Files.delete(directory);
1320                                 return CONTINUE;
1321                         }
1322
1323                         @Override
1324                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1325                                 Files.delete(file);
1326                                 return CONTINUE;
1327                         }
1328                 });
1329         }
1330
1331         /** Extract name/version from a MANIFEST. */
1332         NameVersion nameVersionFromManifest(Manifest manifest) {
1333                 Attributes attrs = manifest.getMainAttributes();
1334                 // symbolic name
1335                 String symbolicName = attrs.getValue(ManifestHeader.BUNDLE_SYMBOLICNAME.toString());
1336                 if (symbolicName == null)
1337                         return null;
1338                 // make sure there is no directive
1339                 symbolicName = symbolicName.split(";")[0];
1340
1341                 String version = attrs.getValue(ManifestHeader.BUNDLE_VERSION.toString());
1342                 return new NameVersion(symbolicName, version);
1343         }
1344
1345         /** Try to download from an URI. */
1346         Path tryDownloadArchive(String uriStr, Path dir) throws IOException {
1347                 // find mirror
1348                 List<String> urlBases = null;
1349                 String uriPrefix = null;
1350                 uriPrefixes: for (String uriPref : mirrors.keySet()) {
1351                         if (uriStr.startsWith(uriPref)) {
1352                                 if (mirrors.get(uriPref).size() > 0) {
1353                                         urlBases = mirrors.get(uriPref);
1354                                         uriPrefix = uriPref;
1355                                         break uriPrefixes;
1356                                 }
1357                         }
1358                 }
1359                 if (urlBases == null)
1360                         try {
1361                                 return downloadArchive(new URI(uriStr), dir);
1362                         } catch (FileNotFoundException | URISyntaxException e) {
1363                                 throw new FileNotFoundException("Cannot find " + uriStr);
1364                         }
1365
1366                 // try to download
1367                 for (String urlBase : urlBases) {
1368                         String relativePath = uriStr.substring(uriPrefix.length());
1369                         String uStr = urlBase + relativePath;
1370                         try {
1371                                 return downloadArchive(new URI(uStr), dir);
1372                         } catch (FileNotFoundException | URISyntaxException e) {
1373                                 logger.log(WARNING, "Cannot download " + uStr + ", trying another mirror");
1374                         }
1375                 }
1376                 throw new FileNotFoundException("Cannot find " + uriStr);
1377         }
1378
1379         /**
1380          * Effectively download an archive.
1381          */
1382         Path downloadArchive(URI uri, Path dir) throws IOException {
1383                 return download(uri, dir, (String) null);
1384         }
1385
1386         /**
1387          * Effectively download. Synchronised in order to avoid downloading twice in
1388          * parallel.
1389          */
1390         synchronized Path download(URI uri, Path dir, String name) throws IOException {
1391
1392                 Path dest;
1393                 if (name == null) {
1394                         // We use also use parent directory in case the archive itself has a fixed name
1395                         String[] segments = uri.getPath().split("/");
1396                         name = segments.length > 1 ? segments[segments.length - 2] + '-' + segments[segments.length - 1]
1397                                         : segments[segments.length - 1];
1398                 }
1399
1400                 dest = dir.resolve(name);
1401                 if (Files.exists(dest)) {
1402                         logger.log(TRACE, () -> "File " + dest + " already exists for " + uri + ", not downloading again");
1403                         return dest;
1404                 } else {
1405                         Files.createDirectories(dest.getParent());
1406                 }
1407
1408                 try (InputStream in = uri.toURL().openStream()) {
1409                         Files.copy(in, dest);
1410                         logger.log(DEBUG, () -> "Downloaded " + dest + " from " + uri);
1411                 }
1412                 return dest;
1413         }
1414
1415         /** Create a JAR file from a directory. */
1416         Path createJar(Path bundleDir, A2Origin origin) throws IOException {
1417                 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1418                 Manifest manifest;
1419                 try (InputStream in = Files.newInputStream(manifestPath)) {
1420                         manifest = new Manifest(in);
1421                 }
1422                 // legal requirements
1423                 origin.appendChanges(bundleDir);
1424                 createReadMe(bundleDir, manifest);
1425
1426                 // create the jar
1427                 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
1428                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
1429                         jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
1430                         Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
1431
1432                                 @Override
1433                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1434                                         if (file.getFileName().toString().equals("MANIFEST.MF"))
1435                                                 return super.visitFile(file, attrs);
1436                                         JarEntry entry = new JarEntry(
1437                                                         bundleDir.relativize(file).toString().replace(File.separatorChar, '/'));
1438                                         jarOut.putNextEntry(entry);
1439                                         Files.copy(file, jarOut);
1440                                         return super.visitFile(file, attrs);
1441                                 }
1442
1443                         });
1444                 }
1445                 deleteDirectory(bundleDir);
1446
1447                 if (separateSources)
1448                         createSourceJar(bundleDir, manifest, null);
1449
1450                 return jarPath;
1451         }
1452
1453         /** Package sources separately, in the Eclipse-SourceBundle format. */
1454         void createSourceJar(Path bundleDir, Manifest manifest, Properties props) throws IOException {
1455                 boolean unmodified = props != null;
1456                 Path bundleCategoryDir = bundleDir.getParent();
1457                 Path sourceDir = bundleCategoryDir.resolve(bundleDir.toString() + ".src");
1458                 if (!Files.exists(sourceDir)) {
1459                         logger.log(WARNING, sourceDir + " does not exist, skipping...");
1460                         return;
1461                 }
1462
1463                 Path relPath = a2Base.relativize(bundleCategoryDir);
1464                 Path srcCategoryDir = a2SrcBase.resolve(relPath);
1465                 Path srcJarP = srcCategoryDir.resolve(sourceDir.getFileName() + ".jar");
1466                 Files.createDirectories(srcJarP.getParent());
1467
1468                 String bundleSymbolicName = manifest.getMainAttributes().getValue("Bundle-SymbolicName").toString();
1469                 // in case there are additional directives
1470                 bundleSymbolicName = bundleSymbolicName.split(";")[0];
1471                 Manifest srcManifest = new Manifest();
1472                 srcManifest.getMainAttributes().put(MANIFEST_VERSION, "1.0");
1473                 BUNDLE_SYMBOLICNAME.put(srcManifest, bundleSymbolicName + ".src");
1474                 BUNDLE_VERSION.put(srcManifest, BUNDLE_VERSION.get(manifest));
1475                 ECLIPSE_SOURCE_BUNDLE.put(srcManifest,
1476                                 bundleSymbolicName + ";version=\"" + BUNDLE_VERSION.get(manifest) + "\"");
1477
1478                 // metadata
1479                 createReadMe(sourceDir, unmodified ? props : manifest);
1480                 // create jar
1481                 try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
1482                         // srcJarOut.setLevel(Deflater.BEST_COMPRESSION);
1483                         Files.walkFileTree(sourceDir, new SimpleFileVisitor<Path>() {
1484
1485                                 @Override
1486                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1487                                         if (file.getFileName().toString().equals("MANIFEST.MF"))
1488                                                 return super.visitFile(file, attrs);
1489                                         JarEntry entry = new JarEntry(
1490                                                         sourceDir.relativize(file).toString().replace(File.separatorChar, '/'));
1491                                         srcJarOut.putNextEntry(entry);
1492                                         Files.copy(file, srcJarOut);
1493                                         return super.visitFile(file, attrs);
1494                                 }
1495
1496                         });
1497                 }
1498                 deleteDirectory(sourceDir);
1499         }
1500
1501         /**
1502          * Generate a readme clarifying and prominently notifying of the repackaging and
1503          * modifications.
1504          */
1505         void createReadMe(Path jarDir, Object mapping) throws IOException {
1506                 // write repackaged README
1507                 try (BufferedWriter writer = Files.newBufferedWriter(jarDir.resolve(README_REPACKAGED))) {
1508                         boolean merged = ARGEO_ORIGIN_M2_MERGE.get(mapping) != null;
1509                         if (merged)
1510                                 writer.append("This component is a merging of third party components"
1511                                                 + " in order to comply with A2 packaging standards.\n");
1512                         else
1513                                 writer.append("This component is a repackaging of a third party component"
1514                                                 + " in order to comply with A2 packaging standards.\n");
1515
1516                         // license
1517                         String spdxLicenseId = SPDX_LICENSE_IDENTIFIER.get(mapping);
1518                         if (spdxLicenseId == null)
1519                                 throw new IllegalStateException("An SPDX license id must have beend defined at this stage.");
1520                         writer.append("\nIt is redistributed under the following license:\n\n");
1521                         writer.append("SPDX-Identifier: " + spdxLicenseId + "\n\n");
1522
1523                         if (!spdxLicenseId.startsWith("LicenseRef")) {// standard
1524                                 int withIndex = spdxLicenseId.indexOf(" WITH ");
1525                                 if (withIndex >= 0) {
1526                                         String simpleId = spdxLicenseId.substring(0, withIndex).trim();
1527                                         String exception = spdxLicenseId.substring(withIndex + " WITH ".length());
1528                                         writer.append("which are available here: https://spdx.org/licenses/" + simpleId
1529                                                         + "\nand here: https://spdx.org/licenses/" + exception + "\n");
1530                                 } else {
1531                                         writer.append("which is available here: https://spdx.org/licenses/" + spdxLicenseId + "\n");
1532                                 }
1533                         } else {
1534                                 String url = BUNDLE_LICENSE.get(mapping);
1535                                 if (url != null) {
1536                                         writer.write("which is available here: " + url + "\n");
1537                                 } else {
1538                                         logger.log(ERROR, "No licence URL for " + jarDir);
1539                                 }
1540                         }
1541
1542                         // origin
1543                         String originDesc = ARGEO_ORIGIN_URI.get(mapping);
1544                         if (originDesc != null)
1545                                 writer.append("\nThe original component comes from " + originDesc + ".\n");
1546                         else {
1547                                 String m2Repo = ARGEO_ORIGIN_M2_REPO.get(mapping);
1548                                 originDesc = ARGEO_ORIGIN_M2.get(mapping);
1549                                 if (originDesc != null)
1550                                         writer.append("\nThe original component has M2 coordinates:\n" + originDesc.replace(',', '\n')
1551                                                         + "\n" + (m2Repo != null ? "\nin M2 repository " + m2Repo + "\n" : ""));
1552                                 else
1553                                         logger.log(ERROR, "Cannot find origin information in " + jarDir);
1554                         }
1555                         String originSources = ARGEO_ORIGIN_SOURCES_URI.get(mapping);
1556                         if (originSources != null)
1557                                 writer.append("\nThe original sources come from " + originSources + ".\n");
1558
1559                         if (Files.exists(jarDir.resolve(CHANGES)))
1560                                 writer.append("\nA detailed list of changes is available under " + CHANGES + ".\n");
1561
1562                         if (!jarDir.getFileName().toString().endsWith(".src")) {// binary archive
1563                                 if (separateSources)
1564                                         writer.append("Corresponding sources are available in the related archive named "
1565                                                         + jarDir.toString() + ".src.jar.\n");
1566                                 else
1567                                         writer.append("Corresponding sources are available under OSGI-OPT/src.\n");
1568                         }
1569                 }
1570         }
1571
1572         /**
1573          * Gathers modifications performed on the original binaries and sources,
1574          * especially in order to comply with their license requirements.
1575          */
1576         class A2Origin {
1577                 A2Origin() {
1578
1579                 }
1580
1581                 Set<String> modified = new TreeSet<>();
1582                 Set<String> deleted = new TreeSet<>();
1583                 Set<String> added = new TreeSet<>();
1584                 Set<String> moved = new TreeSet<>();
1585
1586                 /** Append changes to the A2-ORIGIN/changes file. */
1587                 void appendChanges(Path baseDirectory) throws IOException {
1588                         if (modified.isEmpty() && deleted.isEmpty() && added.isEmpty() && moved.isEmpty())
1589                                 return; // no changes
1590                         Path changesFile = baseDirectory.resolve(CHANGES);
1591                         Files.createDirectories(changesFile.getParent());
1592                         try (BufferedWriter writer = Files.newBufferedWriter(changesFile, APPEND, CREATE)) {
1593                                 for (String msg : added)
1594                                         writer.write("- Added " + msg + ".\n");
1595                                 for (String msg : modified)
1596                                         writer.write("- Modified " + msg + ".\n");
1597                                 for (String msg : moved)
1598                                         writer.write("- Moved " + msg + ".\n");
1599                                 for (String msg : deleted)
1600                                         writer.write("- Deleted " + msg + ".\n");
1601                         }
1602                 }
1603         }
1604 }
1605
1606 /** Simple representation of an M2 artifact. */
1607 class M2Artifact extends CategoryNameVersion {
1608         private String classifier;
1609
1610         M2Artifact(String m2coordinates) {
1611                 this(m2coordinates, null);
1612         }
1613
1614         M2Artifact(String m2coordinates, String classifier) {
1615                 String[] parts = m2coordinates.split(":");
1616                 setCategory(parts[0]);
1617                 setName(parts[1]);
1618                 if (parts.length > 2) {
1619                         setVersion(parts[2]);
1620                 }
1621                 this.classifier = classifier;
1622         }
1623
1624         String getGroupId() {
1625                 return super.getCategory();
1626         }
1627
1628         String getArtifactId() {
1629                 return super.getName();
1630         }
1631
1632         String toM2Coordinates() {
1633                 return getCategory() + ":" + getName() + (getVersion() != null ? ":" + getVersion() : "");
1634         }
1635
1636         String getClassifier() {
1637                 return classifier != null ? classifier : "";
1638         }
1639
1640         String getExtension() {
1641                 return "jar";
1642         }
1643 }
1644
1645 /** Utilities around Maven (conventions based). */
1646 class M2ConventionsUtils {
1647         final static String MAVEN_CENTRAL_BASE_URL = "https://repo1.maven.org/maven2/";
1648
1649         /** The file name of this artifact when stored */
1650         static String artifactFileName(M2Artifact artifact) {
1651                 return artifact.getArtifactId() + '-' + artifact.getVersion()
1652                                 + (artifact.getClassifier().equals("") ? "" : '-' + artifact.getClassifier()) + '.'
1653                                 + artifact.getExtension();
1654         }
1655
1656         /** Absolute path to the file */
1657         static String artifactPath(String artifactBasePath, M2Artifact artifact) {
1658                 return artifactParentPath(artifactBasePath, artifact) + '/' + artifactFileName(artifact);
1659         }
1660
1661         /** Absolute path to the file */
1662         static String artifactUrl(String repoUrl, M2Artifact artifact) {
1663                 if (repoUrl.endsWith("/"))
1664                         return repoUrl + artifactPath("/", artifact).substring(1);
1665                 else
1666                         return repoUrl + artifactPath("/", artifact);
1667         }
1668
1669         /** Absolute path to the file */
1670         static URI mavenRepoUrl(String repoBase, M2Artifact artifact) throws URISyntaxException {
1671                 String uri = artifactUrl(repoBase == null ? MAVEN_CENTRAL_BASE_URL : repoBase, artifact);
1672                 return new URI(uri);
1673         }
1674
1675         /** Absolute path to the directories where the files will be stored */
1676         static String artifactParentPath(String artifactBasePath, M2Artifact artifact) {
1677                 return artifactBasePath + (artifactBasePath.endsWith("/") || artifactBasePath.equals("") ? "" : "/")
1678                                 + artifactParentPath(artifact);
1679         }
1680
1681         /** Relative path to the directories where the files will be stored */
1682         static String artifactParentPath(M2Artifact artifact) {
1683                 return artifact.getGroupId().replace('.', '/') + '/' + artifact.getArtifactId() + '/' + artifact.getVersion();
1684         }
1685
1686         /** Singleton */
1687         private M2ConventionsUtils() {
1688         }
1689 }
1690
1691 /** Combination of a category, a name and a version. */
1692 class CategoryNameVersion extends NameVersion {
1693         private String category;
1694
1695         CategoryNameVersion() {
1696         }
1697
1698         CategoryNameVersion(String category, String name, String version) {
1699                 super(name, version);
1700                 this.category = category;
1701         }
1702
1703         CategoryNameVersion(String category, NameVersion nameVersion) {
1704                 super(nameVersion);
1705                 this.category = category;
1706         }
1707
1708         String getCategory() {
1709                 return category;
1710         }
1711
1712         void setCategory(String category) {
1713                 this.category = category;
1714         }
1715
1716         @Override
1717         public String toString() {
1718                 return category + ":" + super.toString();
1719         }
1720
1721 }
1722
1723 /** Combination of a name and a version. */
1724 class NameVersion implements Comparable<NameVersion> {
1725         private String name;
1726         private String version;
1727
1728         NameVersion() {
1729         }
1730
1731         /** Interprets string in OSGi-like format my.module.name;version=0.0.0 */
1732         NameVersion(String nameVersion) {
1733                 int index = nameVersion.indexOf(";version=");
1734                 if (index < 0) {
1735                         setName(nameVersion);
1736                         setVersion(null);
1737                 } else {
1738                         setName(nameVersion.substring(0, index));
1739                         setVersion(nameVersion.substring(index + ";version=".length()));
1740                 }
1741         }
1742
1743         NameVersion(String name, String version) {
1744                 this.name = name;
1745                 this.version = version;
1746         }
1747
1748         NameVersion(NameVersion nameVersion) {
1749                 this.name = nameVersion.getName();
1750                 this.version = nameVersion.getVersion();
1751         }
1752
1753         String getName() {
1754                 return name;
1755         }
1756
1757         void setName(String name) {
1758                 this.name = name;
1759         }
1760
1761         String getVersion() {
1762                 return version;
1763         }
1764
1765         void setVersion(String version) {
1766                 this.version = version;
1767         }
1768
1769         String getBranch() {
1770                 String[] parts = getVersion().split("\\.");
1771                 if (parts.length < 2)
1772                         throw new IllegalStateException("Version " + getVersion() + " cannot be interpreted as branch.");
1773                 return parts[0] + "." + parts[1];
1774         }
1775
1776         @Override
1777         public boolean equals(Object obj) {
1778                 if (obj instanceof NameVersion) {
1779                         NameVersion nameVersion = (NameVersion) obj;
1780                         return name.equals(nameVersion.getName()) && version.equals(nameVersion.getVersion());
1781                 } else
1782                         return false;
1783         }
1784
1785         @Override
1786         public int hashCode() {
1787                 return name.hashCode();
1788         }
1789
1790         @Override
1791         public String toString() {
1792                 return name + ":" + version;
1793         }
1794
1795         public int compareTo(NameVersion o) {
1796                 if (o.getName().equals(name))
1797                         return version.compareTo(o.getVersion());
1798                 else
1799                         return name.compareTo(o.getName());
1800         }
1801 }