org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java

   1 package org.argeo.slc.factory;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static org.argeo.slc.ManifestConstants.BUNDLE_SYMBOLICNAME;
   5 import static org.argeo.slc.ManifestConstants.BUNDLE_VERSION;
   6 import static org.argeo.slc.ManifestConstants.EXPORT_PACKAGE;
   7 import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2;
   8 import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2_REPO;
   9
  10 import java.io.FileNotFoundException;
  11 import java.io.IOException;
  12 import java.io.InputStream;
  13 import java.io.OutputStream;
  14 import java.lang.System.Logger;
  15 import java.lang.System.Logger.Level;
  16 import java.net.URL;
  17 import java.nio.file.DirectoryStream;
  18 import java.nio.file.FileSystem;
  19 import java.nio.file.FileSystems;
  20 import java.nio.file.FileVisitResult;
  21 import java.nio.file.Files;
  22 import java.nio.file.Path;
  23 import java.nio.file.PathMatcher;
  24 import java.nio.file.Paths;
  25 import java.nio.file.SimpleFileVisitor;
  26 import java.nio.file.StandardOpenOption;
  27 import java.nio.file.attribute.BasicFileAttributes;
  28 import java.util.ArrayList;
  29 import java.util.HashMap;
  30 import java.util.List;
  31 import java.util.Map;
  32 import java.util.Properties;
  33 import java.util.TreeMap;
  34 import java.util.jar.Attributes;
  35 import java.util.jar.JarEntry;
  36 import java.util.jar.JarInputStream;
  37 import java.util.jar.JarOutputStream;
  38 import java.util.jar.Manifest;
  39 import java.util.zip.Deflater;
  40
  41 import org.argeo.slc.DefaultCategoryNameVersion;
  42 import org.argeo.slc.DefaultNameVersion;
  43 import org.argeo.slc.ManifestConstants;
  44 import org.argeo.slc.NameVersion;
  45 import org.argeo.slc.factory.m2.Artifact;
  46 import org.argeo.slc.factory.m2.DefaultArtifact;
  47 import org.argeo.slc.factory.m2.MavenConventionsUtils;
  48
  49 import aQute.bnd.osgi.Analyzer;
  50 import aQute.bnd.osgi.Jar;
  51
  52 /** The central class for A2 packaging. */
  53 public class A2Factory {
  54         private final static Logger logger = System.getLogger(A2Factory.class.getName());
  55
  56         private final static String COMMON_BND = "common.bnd";
  57         private final static String MERGE_BND = "merge.bnd";
  58
  59         private Path originBase;
  60         private Path a2Base;
  61
  62         /** key is URI prefix, value list of base URLs */
  63         private Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
  64
  65         public A2Factory(Path a2Base) {
  66                 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/slc/origin");
  67                 this.a2Base = a2Base;
  68
  69                 // TODO make it configurable
  70                 List<String> eclipseMirrors = new ArrayList<>();
  71                 eclipseMirrors.add("https://archive.eclipse.org/");
  72                 eclipseMirrors.add("http://ftp-stud.hs-esslingen.de/Mirrors/eclipse/");
  73                 eclipseMirrors.add("http://ftp.fau.de/eclipse/");
  74
  75                 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
  76         }
  77
  78         /*
  79          * MAVEN ORIGIN
  80          */
  81
  82         /** Process a whole category/group id. */
  83         public void processCategory(Path targetCategoryBase) {
  84                 try {
  85                         DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
  86                                         (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
  87                                                         && !p.getFileName().toString().equals(MERGE_BND));
  88                         for (Path p : bnds) {
  89                                 processSingleM2ArtifactDistributionUnit(p);
  90                         }
  91
  92                         DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
  93                         for (Path duDir : dus) {
  94                                 processM2BasedDistributionUnit(duDir);
  95                         }
  96                 } catch (IOException e) {
  97                         throw new RuntimeException("Cannot process category " + targetCategoryBase, e);
  98                 }
  99         }
 100
 101         /** Process a standalone Maven artifact. */
 102         public void processSingleM2ArtifactDistributionUnit(Path bndFile) {
 103                 try {
 104                         String category = bndFile.getParent().getFileName().toString();
 105                         Path targetCategoryBase = a2Base.resolve(category);
 106                         Properties fileProps = new Properties();
 107                         try (InputStream in = Files.newInputStream(bndFile)) {
 108                                 fileProps.load(in);
 109                         }
 110                         String repoStr = fileProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
 111                                         ? fileProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
 112                                         : null;
 113
 114                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())
 115                                         && !fileProps.containsKey(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString())) {
 116                                 // use file name as symbolic name
 117                                 String symbolicName = bndFile.getFileName().toString();
 118                                 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 119                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 120                         }
 121
 122                         String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
 123                         if (m2Coordinates == null)
 124                                 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
 125                         DefaultArtifact artifact = new DefaultArtifact(m2Coordinates);
 126                         URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
 127                         Path downloaded = download(url, originBase, artifact);
 128
 129                         Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact);
 130
 131                         downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
 132
 133                         createJar(targetBundleDir);
 134                 } catch (Exception e) {
 135                         throw new RuntimeException("Cannot process " + bndFile, e);
 136                 }
 137         }
 138
 139         /** Process multiple Maven artifacts. */
 140         public void processM2BasedDistributionUnit(Path duDir) {
 141                 try {
 142                         String category = duDir.getParent().getFileName().toString();
 143                         Path targetCategoryBase = a2Base.resolve(category);
 144
 145                         // merge
 146                         Path mergeBnd = duDir.resolve(MERGE_BND);
 147                         if (Files.exists(mergeBnd)) {
 148                                 mergeM2Artifacts(mergeBnd);
 149 //                              return;
 150                         }
 151
 152                         Path commonBnd = duDir.resolve(COMMON_BND);
 153                         if (!Files.exists(commonBnd)) {
 154                                 return;
 155                         }
 156                         Properties commonProps = new Properties();
 157                         try (InputStream in = Files.newInputStream(commonBnd)) {
 158                                 commonProps.load(in);
 159                         }
 160
 161                         String m2Version = commonProps.getProperty(SLC_ORIGIN_M2.toString());
 162                         if (m2Version == null) {
 163                                 logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
 164                                 return;// ignore, this is probably an Eclipse archive
 165                         }
 166                         if (!m2Version.startsWith(":")) {
 167                                 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 168                         }
 169                         m2Version = m2Version.substring(1);
 170
 171                         DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
 172                                         (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
 173                                                         && !p.getFileName().toString().equals(MERGE_BND));
 174                         for (Path p : ds) {
 175                                 Properties fileProps = new Properties();
 176                                 try (InputStream in = Files.newInputStream(p)) {
 177                                         fileProps.load(in);
 178                                 }
 179                                 String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
 180                                 DefaultArtifact artifact = new DefaultArtifact(m2Coordinates);
 181
 182                                 // temporary rewrite, for migration
 183 //                              String localLicense = fileProps.getProperty(BUNDLE_LICENSE.toString());
 184 //                              if (localLicense != null || artifact.getVersion() != null) {
 185 //                                      fileProps.remove(BUNDLE_LICENSE.toString());
 186 //                                      fileProps.put(SLC_ORIGIN_M2.toString(), artifact.getGroupId() + ":" + artifact.getArtifactId());
 187 //                                      try (Writer writer = Files.newBufferedWriter(p)) {
 188 //                                              for (Object key : fileProps.keySet()) {
 189 //                                                      String value = fileProps.getProperty(key.toString());
 190 //                                                      writer.write(key + ": " + value + '\n');
 191 //                                              }
 192 //                                              logger.log(DEBUG, () -> "Migrated  " + p);
 193 //                                      }
 194 //                              }
 195
 196                                 artifact.setVersion(m2Version);
 197
 198                                 // prepare manifest entries
 199                                 Properties mergeProps = new Properties();
 200                                 mergeProps.putAll(commonProps);
 201
 202                                 fileEntries: for (Object key : fileProps.keySet()) {
 203                                         if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
 204                                                 continue fileEntries;
 205                                         String value = fileProps.getProperty(key.toString());
 206                                         Object previousValue = mergeProps.put(key.toString(), value);
 207                                         if (previousValue != null) {
 208                                                 logger.log(Level.WARNING,
 209                                                                 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
 210                                         }
 211                                 }
 212                                 mergeProps.put(ManifestConstants.SLC_ORIGIN_M2.toString(), artifact.toM2Coordinates());
 213                                 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())
 214                                                 && !mergeProps.containsKey(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString())) {
 215                                         // use file name as symbolic name
 216                                         String symbolicName = p.getFileName().toString();
 217                                         symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 218                                         mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 219                                 }
 220
 221                                 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
 222                                                 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
 223                                                 : null;
 224
 225                                 // download
 226                                 URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
 227                                 Path downloaded = download(url, originBase, artifact);
 228
 229                                 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact);
 230 //                              logger.log(Level.DEBUG, () -> "Processed " + downloaded);
 231
 232                                 // sources
 233                                 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
 234
 235                                 createJar(targetBundleDir);
 236                         }
 237                 } catch (IOException e) {
 238                         throw new RuntimeException("Cannot process " + duDir, e);
 239                 }
 240
 241         }
 242
 243         /** Merge multiple Maven artifacts. */
 244         protected void mergeM2Artifacts(Path mergeBnd) throws IOException {
 245                 Path duDir = mergeBnd.getParent();
 246                 String category = duDir.getParent().getFileName().toString();
 247                 Path targetCategoryBase = a2Base.resolve(category);
 248
 249                 Properties mergeProps = new Properties();
 250                 try (InputStream in = Files.newInputStream(mergeBnd)) {
 251                         mergeProps.load(in);
 252                 }
 253
 254                 // Version
 255                 String m2Version = mergeProps.getProperty(SLC_ORIGIN_M2.toString());
 256                 if (m2Version == null) {
 257                         logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
 258                         return;// ignore, this is probably an Eclipse archive
 259                 }
 260                 if (!m2Version.startsWith(":")) {
 261                         throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 262                 }
 263                 m2Version = m2Version.substring(1);
 264                 mergeProps.put(ManifestConstants.BUNDLE_VERSION.toString(), m2Version);
 265
 266                 String artifactsStr = mergeProps.getProperty(ManifestConstants.SLC_ORIGIN_M2_MERGE.toString());
 267                 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
 268                                 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
 269                                 : null;
 270
 271                 String bundleSymbolicName = mergeProps.getProperty(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
 272                 if (bundleSymbolicName == null)
 273                         throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
 274                 DefaultCategoryNameVersion nameVersion = new DefaultArtifact(
 275                                 category + ":" + bundleSymbolicName + ":" + m2Version);
 276                 Path targetBundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
 277
 278                 String[] artifacts = artifactsStr.split(",");
 279                 artifacts: for (String str : artifacts) {
 280                         String m2Coordinates = str.trim();
 281                         if ("".equals(m2Coordinates))
 282                                 continue artifacts;
 283                         DefaultArtifact artifact = new DefaultArtifact(m2Coordinates.trim());
 284                         if (artifact.getVersion() == null)
 285                                 artifact.setVersion(m2Version);
 286                         URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
 287                         Path downloaded = download(url, originBase, artifact);
 288                         JarEntry entry;
 289                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
 290                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 291                                         if (entry.isDirectory())
 292                                                 continue entries;
 293                                         else if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
 294                                                 continue entries;
 295                                         else if (entry.getName().startsWith("META-INF/versions/"))
 296                                                 continue entries;
 297                                         else if (entry.getName().startsWith("META-INF/maven/"))
 298                                                 continue entries;
 299                                         else if (entry.getName().equals("module-info.class"))
 300                                                 continue entries;
 301                                         else if (entry.getName().equals("META-INF/NOTICE"))
 302                                                 continue entries;
 303                                         else if (entry.getName().equals("META-INF/NOTICE.txt"))
 304                                                 continue entries;
 305                                         else if (entry.getName().equals("META-INF/LICENSE"))
 306                                                 continue entries;
 307                                         else if (entry.getName().equals("META-INF/LICENSE.md"))
 308                                                 continue entries;
 309                                         else if (entry.getName().equals("META-INF/LICENSE-notice.md"))
 310                                                 continue entries;
 311                                         Path target = targetBundleDir.resolve(entry.getName());
 312                                         Files.createDirectories(target.getParent());
 313                                         if (!Files.exists(target)) {
 314                                                 Files.copy(jarIn, target);
 315                                         } else {
 316                                                 if (entry.getName().startsWith("META-INF/services/")) {
 317                                                         try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
 318                                                                 out.write("\n".getBytes());
 319                                                                 jarIn.transferTo(out);
 320                                                                 if (logger.isLoggable(DEBUG))
 321                                                                         logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName());
 322                                                         }
 323                                                 } else if (entry.getName().startsWith("org/apache/batik/")) {
 324                                                         logger.log(Level.WARNING, "Skip " + entry.getName());
 325                                                         continue entries;
 326                                                 } else {
 327                                                         throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
 328                                                 }
 329                                         }
 330                                         logger.log(Level.TRACE, () -> "Copied " + target);
 331                                 }
 332
 333                         }
 334                         downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
 335                 }
 336
 337                 // additional service files
 338                 Path servicesDir = duDir.resolve("services");
 339                 if (Files.exists(servicesDir)) {
 340                         for (Path p : Files.newDirectoryStream(servicesDir)) {
 341                                 Path target = targetBundleDir.resolve("META-INF/services/").resolve(p.getFileName());
 342                                 try (InputStream in = Files.newInputStream(p);
 343                                                 OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) {
 344                                         out.write("\n".getBytes());
 345                                         in.transferTo(out);
 346                                         if (logger.isLoggable(DEBUG))
 347                                                 logger.log(DEBUG, "Appended " + p);
 348                                 }
 349                         }
 350                 }
 351
 352                 Map<String, String> entries = new TreeMap<>();
 353                 try (Analyzer bndAnalyzer = new Analyzer()) {
 354                         bndAnalyzer.setProperties(mergeProps);
 355                         Jar jar = new Jar(targetBundleDir.toFile());
 356                         bndAnalyzer.setJar(jar);
 357                         Manifest manifest = bndAnalyzer.calcManifest();
 358
 359                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 360                                 Object value = manifest.getMainAttributes().get(key);
 361
 362                                 switch (key.toString()) {
 363                                 case "Tool":
 364                                 case "Bnd-LastModified":
 365                                 case "Created-By":
 366                                         continue keys;
 367                                 }
 368                                 if ("Require-Capability".equals(key.toString())
 369                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
 370                                         continue keys;// hack for very old classes
 371                                 entries.put(key.toString(), value.toString());
 372                                 // logger.log(DEBUG, () -> key + "=" + value);
 373
 374                         }
 375                 } catch (Exception e) {
 376                         throw new RuntimeException("Cannot process " + mergeBnd, e);
 377                 }
 378
 379                 Manifest manifest = new Manifest();
 380                 Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 381                 Files.createDirectories(manifestPath.getParent());
 382                 for (String key : entries.keySet()) {
 383                         String value = entries.get(key);
 384                         manifest.getMainAttributes().putValue(key, value);
 385                 }
 386
 387 //              // Use Maven version as Bundle-Version
 388 //              String bundleVersion = manifest.getMainAttributes().getValue(ManifestConstants.BUNDLE_VERSION.toString());
 389 //              if (bundleVersion == null || bundleVersion.trim().equals("0")) {
 390 //                      // TODO check why it is sometimes set to "0"
 391 //                      manifest.getMainAttributes().putValue(ManifestConstants.BUNDLE_VERSION.toString(), m2Version);
 392 //              }
 393                 try (OutputStream out = Files.newOutputStream(manifestPath)) {
 394                         manifest.write(out);
 395                 }
 396
 397                 createJar(targetBundleDir);
 398
 399         }
 400
 401         /** Generate MANIFEST using BND. */
 402         protected Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps,
 403                         DefaultArtifact artifact) {
 404
 405                 try {
 406                         Map<String, String> additionalEntries = new TreeMap<>();
 407                         boolean doNotModify = Boolean.parseBoolean(fileProps
 408                                         .getOrDefault(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString());
 409
 410                         // we always force the symbolic name
 411
 412                         if (doNotModify) {
 413                                 fileEntries: for (Object key : fileProps.keySet()) {
 414                                         if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
 415                                                 continue fileEntries;
 416                                         String value = fileProps.getProperty(key.toString());
 417                                         additionalEntries.put(key.toString(), value);
 418                                 }
 419                         } else {
 420                                 if (artifact != null) {
 421                                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 422                                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
 423                                         }
 424                                         if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
 425                                                 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
 426                                         }
 427                                 }
 428
 429                                 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
 430                                         fileProps.put(EXPORT_PACKAGE.toString(),
 431                                                         "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
 432                                 }
 433 //                              if (!fileProps.contains(IMPORT_PACKAGE.toString())) {
 434 //                                      fileProps.put(IMPORT_PACKAGE.toString(), "*");
 435 //                              }
 436
 437                                 try (Analyzer bndAnalyzer = new Analyzer()) {
 438                                         bndAnalyzer.setProperties(fileProps);
 439                                         Jar jar = new Jar(downloaded.toFile());
 440                                         bndAnalyzer.setJar(jar);
 441                                         Manifest manifest = bndAnalyzer.calcManifest();
 442
 443                                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 444                                                 Object value = manifest.getMainAttributes().get(key);
 445
 446                                                 switch (key.toString()) {
 447                                                 case "Tool":
 448                                                 case "Bnd-LastModified":
 449                                                 case "Created-By":
 450                                                         continue keys;
 451                                                 }
 452                                                 if ("Require-Capability".equals(key.toString())
 453                                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
 454                                                         continue keys;// hack for very old classes
 455                                                 additionalEntries.put(key.toString(), value.toString());
 456                                                 // logger.log(DEBUG, () -> key + "=" + value);
 457
 458                                         }
 459                                 }
 460
 461 //                              try (Builder bndBuilder = new Builder()) {
 462 //                                      Jar jar = new Jar(downloaded.toFile());
 463 //                                      bndBuilder.addClasspath(jar);
 464 //                                      Path targetBundleDir = targetCategoryBase.resolve(artifact.getName() + "." + artifact.getBranch());
 465 //
 466 //                                      Jar target = new Jar(targetBundleDir.toFile());
 467 //                                      bndBuilder.setJar(target);
 468 //                                      return targetBundleDir;
 469 //                              }
 470                         }
 471                         Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries);
 472                         logger.log(Level.DEBUG, () -> "Processed " + downloaded);
 473                         return targetBundleDir;
 474                 } catch (Exception e) {
 475                         throw new RuntimeException("Cannot BND process " + downloaded, e);
 476                 }
 477
 478         }
 479
 480         /** Download and integrates sources for a single Maven artifact. */
 481         protected void downloadAndProcessM2Sources(String repoStr, DefaultArtifact artifact, Path targetBundleDir)
 482                         throws IOException {
 483                 DefaultArtifact sourcesArtifact = new DefaultArtifact(artifact.toM2Coordinates(), "sources");
 484                 URL sourcesUrl = MavenConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
 485                 Path sourcesDownloaded = download(sourcesUrl, originBase, artifact, true);
 486                 processM2SourceJar(sourcesDownloaded, targetBundleDir);
 487                 logger.log(Level.TRACE, () -> "Processed source " + sourcesDownloaded);
 488
 489         }
 490
 491         /** Integrate sources from a downloaded jar file. */
 492         protected void processM2SourceJar(Path file, Path targetBundleDir) throws IOException {
 493                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 494                         Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
 495
 496                         // TODO make it less dangerous?
 497                         if (Files.exists(targetSourceDir)) {
 498 //                              deleteDirectory(targetSourceDir);
 499                         } else {
 500                                 Files.createDirectories(targetSourceDir);
 501                         }
 502
 503                         // copy entries
 504                         JarEntry entry;
 505                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 506                                 if (entry.isDirectory())
 507                                         continue entries;
 508                                 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
 509                                         continue entries;
 510                                 if (entry.getName().startsWith("module-info.java"))// skip META-INF entries
 511                                         continue entries;
 512                                 if (entry.getName().startsWith("/")) // absolute paths
 513                                         continue entries;
 514                                 Path target = targetSourceDir.resolve(entry.getName());
 515                                 Files.createDirectories(target.getParent());
 516                                 if (!Files.exists(target)) {
 517                                         Files.copy(jarIn, target);
 518                                         logger.log(Level.TRACE, () -> "Copied source " + target);
 519                                 } else {
 520                                         logger.log(Level.WARNING, () -> target + " already exists, skipping...");
 521                                 }
 522                         }
 523                 }
 524
 525         }
 526
 527         /** Download a Maven artifact. */
 528         protected Path download(URL url, Path dir, Artifact artifact) throws IOException {
 529                 return download(url, dir, artifact, false);
 530         }
 531
 532         /** Download a Maven artifact. */
 533         protected Path download(URL url, Path dir, Artifact artifact, boolean sources) throws IOException {
 534                 return download(url, dir, artifact.getGroupId() + '/' + artifact.getArtifactId() + "-" + artifact.getVersion()
 535                                 + (sources ? "-sources" : "") + ".jar");
 536         }
 537
 538         /*
 539          * ECLIPSE ORIGIN
 540          */
 541
 542         /** Process an archive in Eclipse format. */
 543         public void processEclipseArchive(Path duDir) {
 544                 try {
 545                         String category = duDir.getParent().getFileName().toString();
 546                         Path targetCategoryBase = a2Base.resolve(category);
 547                         Files.createDirectories(targetCategoryBase);
 548                         // first delete all directories from previous builds
 549                         for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) {
 550                                 deleteDirectory(dir);
 551                         }
 552
 553                         Files.createDirectories(originBase);
 554
 555                         Path commonBnd = duDir.resolve(COMMON_BND);
 556                         Properties commonProps = new Properties();
 557                         try (InputStream in = Files.newInputStream(commonBnd)) {
 558                                 commonProps.load(in);
 559                         }
 560                         String url = commonProps.getProperty(ManifestConstants.SLC_ORIGIN_URI.toString());
 561                         Path downloaded = tryDownload(url, originBase);
 562
 563                         FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
 564
 565                         // filters
 566                         List<PathMatcher> includeMatchers = new ArrayList<>();
 567                         Properties includes = new Properties();
 568                         try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
 569                                 includes.load(in);
 570                         }
 571                         for (Object pattern : includes.keySet()) {
 572                                 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 573                                 includeMatchers.add(pathMatcher);
 574                         }
 575
 576                         List<PathMatcher> excludeMatchers = new ArrayList<>();
 577                         Path excludeFile = duDir.resolve("excludes.properties");
 578                         if (Files.exists(excludeFile)) {
 579                                 Properties excludes = new Properties();
 580                                 try (InputStream in = Files.newInputStream(excludeFile)) {
 581                                         excludes.load(in);
 582                                 }
 583                                 for (Object pattern : excludes.keySet()) {
 584                                         PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 585                                         excludeMatchers.add(pathMatcher);
 586                                 }
 587                         }
 588
 589                         Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
 590
 591                                 @Override
 592                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 593                                         includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
 594                                                 if (includeMatcher.matches(file)) {
 595                                                         for (PathMatcher excludeMatcher : excludeMatchers) {
 596                                                                 if (excludeMatcher.matches(file)) {
 597                                                                         logger.log(Level.WARNING, "Skipping excluded " + file);
 598                                                                         return FileVisitResult.CONTINUE;
 599                                                                 }
 600                                                         }
 601                                                         if (file.getFileName().toString().contains(".source_")) {
 602                                                                 processEclipseSourceJar(file, targetCategoryBase);
 603                                                                 logger.log(Level.DEBUG, () -> "Processed source " + file);
 604
 605                                                         } else {
 606                                                                 processBundleJar(file, targetCategoryBase, new HashMap<>());
 607                                                                 logger.log(Level.DEBUG, () -> "Processed " + file);
 608                                                         }
 609                                                         break includeMatchers;
 610                                                 }
 611                                         }
 612                                         return FileVisitResult.CONTINUE;
 613                                 }
 614                         });
 615
 616                         DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
 617                         for (Path dir : dirs) {
 618                                 createJar(dir);
 619                         }
 620                 } catch (IOException e) {
 621                         throw new RuntimeException("Cannot process " + duDir, e);
 622                 }
 623
 624         }
 625
 626         /** Process sources in Eclipse format. */
 627         protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
 628                 try {
 629                         Path targetBundleDir;
 630                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 631                                 Manifest manifest = jarIn.getManifest();
 632
 633                                 String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
 634                                 String version = relatedBundle[1].substring("version=\"".length());
 635                                 version = version.substring(0, version.length() - 1);
 636                                 NameVersion nameVersion = new DefaultNameVersion(relatedBundle[0], version);
 637                                 targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 638
 639                                 Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
 640
 641                                 // TODO make it less dangerous?
 642                                 if (Files.exists(targetSourceDir)) {
 643 //                              deleteDirectory(targetSourceDir);
 644                                 } else {
 645                                         Files.createDirectories(targetSourceDir);
 646                                 }
 647
 648                                 // copy entries
 649                                 JarEntry entry;
 650                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 651                                         if (entry.isDirectory())
 652                                                 continue entries;
 653                                         if (entry.getName().startsWith("META-INF"))// skip META-INF entries
 654                                                 continue entries;
 655                                         Path target = targetSourceDir.resolve(entry.getName());
 656                                         Files.createDirectories(target.getParent());
 657                                         Files.copy(jarIn, target);
 658                                         logger.log(Level.TRACE, () -> "Copied source " + target);
 659                                 }
 660
 661                                 // copy MANIFEST
 662 //                      Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 663 //                      Files.createDirectories(manifestPath.getParent());
 664 //                      try (OutputStream out = Files.newOutputStream(manifestPath)) {
 665 //                              manifest.write(out);
 666 //                      }
 667                         }
 668                 } catch (IOException e) {
 669                         throw new IllegalStateException("Cannot process " + file, e);
 670                 }
 671
 672         }
 673
 674         /*
 675          * COMMON PROCESSING
 676          */
 677         /** Normalise a bundle. */
 678         protected Path processBundleJar(Path file, Path targetBase, Map<String, String> entries) throws IOException {
 679                 DefaultNameVersion nameVersion;
 680                 Path targetBundleDir;
 681                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 682                         Manifest sourceManifest = jarIn.getManifest();
 683                         Manifest manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
 684
 685                         // remove problematic entries in MANIFEST
 686                         manifest.getEntries().clear();
 687 //                      Set<String> entriesToDelete = new HashSet<>();
 688 //                      for (String key : manifest.getEntries().keySet()) {
 689 ////                            logger.log(DEBUG, "## " + key);
 690 //                              Attributes attrs = manifest.getAttributes(key);
 691 //                              for (Object attrName : attrs.keySet()) {
 692 ////                                    logger.log(DEBUG, attrName + "=" + attrs.get(attrName));
 693 //                                      if ("Specification-Version".equals(attrName.toString())
 694 //                                                      || "Implementation-Version".equals(attrName.toString())) {
 695 //                                              entriesToDelete.add(key);
 696 //
 697 //                                      }
 698 //                              }
 699 //                      }
 700 //                      for (String key : entriesToDelete) {
 701 //                              manifest.getEntries().remove(key);
 702 //                      }
 703
 704                         String symbolicNameFromEntries = entries.get(BUNDLE_SYMBOLICNAME.toString());
 705                         String versionFromEntries = entries.get(BUNDLE_VERSION.toString());
 706
 707                         if (symbolicNameFromEntries != null && versionFromEntries != null) {
 708                                 nameVersion = new DefaultNameVersion(symbolicNameFromEntries, versionFromEntries);
 709                         } else {
 710                                 nameVersion = nameVersionFromManifest(manifest);
 711                                 if (versionFromEntries != null && !nameVersion.getVersion().equals(versionFromEntries)) {
 712                                         logger.log(Level.WARNING, "Original version is " + nameVersion.getVersion()
 713                                                         + " while new version is " + versionFromEntries);
 714                                 }
 715                                 if (symbolicNameFromEntries != null) {
 716                                         // we always force our symbolic name
 717                                         nameVersion.setName(symbolicNameFromEntries);
 718                                 }
 719                         }
 720                         targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 721
 722                         // TODO make it less dangerous?
 723 //                      if (Files.exists(targetBundleDir)) {
 724 //                              deleteDirectory(targetBundleDir);
 725 //                      }
 726
 727                         // copy entries
 728                         JarEntry entry;
 729                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 730                                 if (entry.isDirectory())
 731                                         continue entries;
 732                                 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
 733                                         continue entries;
 734                                 Path target = targetBundleDir.resolve(entry.getName());
 735                                 Files.createDirectories(target.getParent());
 736                                 Files.copy(jarIn, target);
 737                                 logger.log(Level.TRACE, () -> "Copied " + target);
 738                         }
 739
 740                         // copy MANIFEST
 741                         Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 742                         Files.createDirectories(manifestPath.getParent());
 743                         for (String key : entries.keySet()) {
 744                                 String value = entries.get(key);
 745                                 Object previousValue = manifest.getMainAttributes().putValue(key, value);
 746                                 if (previousValue != null && !previousValue.equals(value)) {
 747                                         if (ManifestConstants.IMPORT_PACKAGE.toString().equals(key)
 748                                                         || ManifestConstants.EXPORT_PACKAGE.toString().equals(key))
 749                                                 logger.log(Level.WARNING, file.getFileName() + ": " + key + " was modified");
 750
 751                                         else
 752                                                 logger.log(Level.WARNING, file.getFileName() + ": " + key + " was " + previousValue
 753                                                                 + ", overridden with " + value);
 754                                 }
 755                         }
 756                         try (OutputStream out = Files.newOutputStream(manifestPath)) {
 757                                 manifest.write(out);
 758                         }
 759                 }
 760                 return targetBundleDir;
 761         }
 762
 763         /*
 764          * UTILITIES
 765          */
 766
 767         /** Recursively deletes a directory. */
 768         private static void deleteDirectory(Path path) throws IOException {
 769                 if (!Files.exists(path))
 770                         return;
 771                 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
 772                         @Override
 773                         public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
 774                                 if (e != null)
 775                                         throw e;
 776                                 Files.delete(directory);
 777                                 return FileVisitResult.CONTINUE;
 778                         }
 779
 780                         @Override
 781                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 782                                 Files.delete(file);
 783                                 return FileVisitResult.CONTINUE;
 784                         }
 785                 });
 786         }
 787
 788         /** Extract name/version from a MANIFEST. */
 789         protected DefaultNameVersion nameVersionFromManifest(Manifest manifest) {
 790                 Attributes attrs = manifest.getMainAttributes();
 791                 // symbolic name
 792                 String symbolicName = attrs.getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
 793                 if (symbolicName == null)
 794                         return null;
 795                 // make sure there is no directive
 796                 symbolicName = symbolicName.split(";")[0];
 797
 798                 String version = attrs.getValue(ManifestConstants.BUNDLE_VERSION.toString());
 799                 return new DefaultNameVersion(symbolicName, version);
 800         }
 801
 802         /** Try to download from an URI. */
 803         protected Path tryDownload(String uri, Path dir) throws IOException {
 804                 // find mirror
 805                 List<String> urlBases = null;
 806                 String uriPrefix = null;
 807                 uriPrefixes: for (String uriPref : mirrors.keySet()) {
 808                         if (uri.startsWith(uriPref)) {
 809                                 if (mirrors.get(uriPref).size() > 0) {
 810                                         urlBases = mirrors.get(uriPref);
 811                                         uriPrefix = uriPref;
 812                                         break uriPrefixes;
 813                                 }
 814                         }
 815                 }
 816                 if (urlBases == null)
 817                         try {
 818                                 return download(new URL(uri), dir);
 819                         } catch (FileNotFoundException e) {
 820                                 throw new FileNotFoundException("Cannot find " + uri);
 821                         }
 822
 823                 // try to download
 824                 for (String urlBase : urlBases) {
 825                         String relativePath = uri.substring(uriPrefix.length());
 826                         URL url = new URL(urlBase + relativePath);
 827                         try {
 828                                 return download(url, dir);
 829                         } catch (FileNotFoundException e) {
 830                                 logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror");
 831                         }
 832                 }
 833                 throw new FileNotFoundException("Cannot find " + uri);
 834         }
 835
 836 //      protected String simplifyName(URL u) {
 837 //      String  name = u.getPath().substring(u.getPath().lastIndexOf('/') + 1);
 838 //
 839 //      }
 840
 841         /** Effectively download. */
 842         protected Path download(URL url, Path dir) throws IOException {
 843                 return download(url, dir, (String) null);
 844         }
 845
 846         /** Effectively download. */
 847         protected Path download(URL url, Path dir, String name) throws IOException {
 848
 849                 Path dest;
 850                 if (name == null) {
 851                         name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1);
 852                 }
 853
 854                 dest = dir.resolve(name);
 855                 if (Files.exists(dest)) {
 856                         logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
 857                         return dest;
 858                 } else {
 859                         Files.createDirectories(dest.getParent());
 860                 }
 861
 862                 try (InputStream in = url.openStream()) {
 863                         Files.copy(in, dest);
 864                         logger.log(Level.DEBUG, () -> "Downloaded " + dest + " from " + url);
 865                 }
 866                 return dest;
 867         }
 868
 869         /** Create a JAR file from a directory. */
 870         protected Path createJar(Path bundleDir) throws IOException {
 871                 // Remove from source the resources already in the jar file
 872 //              Path srcDir = bundleDir.resolve("OSGI-OPT/src");
 873 //              Files.walkFileTree(srcDir, new SimpleFileVisitor<Path>() {
 874 //
 875 //                      @Override
 876 //                      public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 877 //                              // always keep .java file
 878 //                              if (file.getFileName().toString().endsWith(".java"))
 879 //                                      return super.visitFile(file, attrs);
 880 //
 881 //                              Path relPath = srcDir.relativize(file);
 882 //                              Path bundlePath = bundleDir.resolve(relPath);
 883 //                              if (Files.exists(bundlePath)) {
 884 //                                      Files.delete(file);
 885 //                                      logger.log(DEBUG, () -> "Removed " + file + " from sources.");
 886 //                              }
 887 //                              return super.visitFile(file, attrs);
 888 //                      }
 889 //
 890 //              });
 891
 892                 // Create the jar
 893                 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
 894                 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
 895                 Manifest manifest;
 896                 try (InputStream in = Files.newInputStream(manifestPath)) {
 897                         manifest = new Manifest(in);
 898                 }
 899                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
 900                         jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
 901                         Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
 902
 903                                 @Override
 904                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 905                                         if (file.getFileName().toString().equals("MANIFEST.MF"))
 906                                                 return super.visitFile(file, attrs);
 907                                         JarEntry entry = new JarEntry(bundleDir.relativize(file).toString());
 908                                         jarOut.putNextEntry(entry);
 909                                         Files.copy(file, jarOut);
 910                                         return super.visitFile(file, attrs);
 911                                 }
 912
 913                         });
 914                 }
 915                 deleteDirectory(bundleDir);
 916                 return jarPath;
 917         }
 918
 919         /** For development purproses. */
 920         public static void main(String[] args) {
 921                 Path factoryBase = Paths.get("../../output/a2").toAbsolutePath().normalize();
 922                 A2Factory factory = new A2Factory(factoryBase);
 923
 924                 Path descriptorsBase = Paths.get("../tp").toAbsolutePath().normalize();
 925
 926 //              factory.processSingleM2ArtifactDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache").resolve("org.apache.xml.resolver.bnd"));
 927 //              factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache/apache-sshd"));
 928 //              factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty"));
 929                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk"));
 930 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp"));
 931 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp"));
 932 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache"));
 933 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats"));
 934 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis"));
 935                 System.exit(0);
 936
 937                 // Eclipse
 938                 factory.processEclipseArchive(
 939                                 descriptorsBase.resolve("org.argeo.tp.eclipse.equinox").resolve("eclipse-equinox"));
 940                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rwt").resolve("eclipse-rwt"));
 941                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rap").resolve("eclipse-rap"));
 942                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.swt").resolve("eclipse-swt"));
 943                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.swt").resolve("eclipse-nebula"));
 944                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.swt").resolve("eclipse-equinox"));
 945                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp").resolve("eclipse-rcp"));
 946                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp"));
 947
 948                 // Maven
 949                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk"));
 950                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp"));
 951                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache"));
 952                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jetty"));
 953                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jcr"));
 954                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats"));
 955                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.poi"));
 956                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis"));
 957         }
 958 }