org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java

   1 package org.argeo.slc.factory;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static org.argeo.slc.ManifestConstants.BUNDLE_SYMBOLICNAME;
   5 import static org.argeo.slc.ManifestConstants.BUNDLE_VERSION;
   6 import static org.argeo.slc.ManifestConstants.EXPORT_PACKAGE;
   7 import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2;
   8 import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2_REPO;
   9
  10 import java.io.FileNotFoundException;
  11 import java.io.IOException;
  12 import java.io.InputStream;
  13 import java.io.OutputStream;
  14 import java.lang.System.Logger;
  15 import java.lang.System.Logger.Level;
  16 import java.net.URL;
  17 import java.nio.file.DirectoryStream;
  18 import java.nio.file.FileSystem;
  19 import java.nio.file.FileSystems;
  20 import java.nio.file.FileVisitResult;
  21 import java.nio.file.Files;
  22 import java.nio.file.Path;
  23 import java.nio.file.PathMatcher;
  24 import java.nio.file.Paths;
  25 import java.nio.file.SimpleFileVisitor;
  26 import java.nio.file.StandardOpenOption;
  27 import java.nio.file.attribute.BasicFileAttributes;
  28 import java.util.ArrayList;
  29 import java.util.HashMap;
  30 import java.util.List;
  31 import java.util.Map;
  32 import java.util.Properties;
  33 import java.util.TreeMap;
  34 import java.util.jar.Attributes;
  35 import java.util.jar.JarEntry;
  36 import java.util.jar.JarInputStream;
  37 import java.util.jar.JarOutputStream;
  38 import java.util.jar.Manifest;
  39
  40 import org.argeo.slc.DefaultCategoryNameVersion;
  41 import org.argeo.slc.DefaultNameVersion;
  42 import org.argeo.slc.ManifestConstants;
  43 import org.argeo.slc.NameVersion;
  44 import org.argeo.slc.factory.m2.Artifact;
  45 import org.argeo.slc.factory.m2.DefaultArtifact;
  46 import org.argeo.slc.factory.m2.MavenConventionsUtils;
  47
  48 import aQute.bnd.osgi.Analyzer;
  49 import aQute.bnd.osgi.Jar;
  50
  51 /** The central class for A2 packaging. */
  52 public class A2Factory {
  53         private final static Logger logger = System.getLogger(A2Factory.class.getName());
  54
  55         private final static String COMMON_BND = "common.bnd";
  56         private final static String MERGE_BND = "merge.bnd";
  57
  58         private Path originBase;
  59         private Path a2Base;
  60
  61         /** key is URI prefix, value list of base URLs */
  62         private Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
  63
  64         public A2Factory(Path a2Base) {
  65                 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/slc/origin");
  66                 this.a2Base = a2Base;
  67
  68                 // TODO make it configurable
  69                 List<String> eclipseMirrors = new ArrayList<>();
  70                 eclipseMirrors.add("https://archive.eclipse.org/");
  71                 eclipseMirrors.add("http://ftp-stud.hs-esslingen.de/Mirrors/eclipse/");
  72                 eclipseMirrors.add("http://ftp.fau.de/eclipse/");
  73
  74                 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
  75         }
  76
  77         /*
  78          * MAVEN ORIGIN
  79          */
  80
  81         /** Process a whole category/group id. */
  82         public void processCategory(Path targetCategoryBase) {
  83                 try {
  84                         DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
  85                                         (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
  86                                                         && !p.getFileName().toString().equals(MERGE_BND));
  87                         for (Path p : bnds) {
  88                                 processSingleM2ArtifactDistributionUnit(p);
  89                         }
  90
  91                         DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
  92                         for (Path duDir : dus) {
  93                                 processM2BasedDistributionUnit(duDir);
  94                         }
  95                 } catch (IOException e) {
  96                         throw new RuntimeException("Cannot process category " + targetCategoryBase, e);
  97                 }
  98         }
  99
 100         /** Process a standalone Maven artifact. */
 101         public void processSingleM2ArtifactDistributionUnit(Path bndFile) {
 102                 try {
 103                         String category = bndFile.getParent().getFileName().toString();
 104                         Path targetCategoryBase = a2Base.resolve(category);
 105                         Properties fileProps = new Properties();
 106                         try (InputStream in = Files.newInputStream(bndFile)) {
 107                                 fileProps.load(in);
 108                         }
 109                         String repoStr = fileProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
 110                                         ? fileProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
 111                                         : null;
 112
 113                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())
 114                                         && !fileProps.containsKey(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString())) {
 115                                 // use file name as symbolic name
 116                                 String symbolicName = bndFile.getFileName().toString();
 117                                 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 118                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 119                         }
 120
 121                         String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
 122                         if (m2Coordinates == null)
 123                                 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
 124                         DefaultArtifact artifact = new DefaultArtifact(m2Coordinates);
 125                         URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
 126                         Path downloaded = download(url, originBase, artifact);
 127
 128                         Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact);
 129
 130                         downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
 131
 132                         createJar(targetBundleDir);
 133                 } catch (Exception e) {
 134                         throw new RuntimeException("Cannot process " + bndFile, e);
 135                 }
 136         }
 137
 138         /** Process multiple Maven artifacts. */
 139         public void processM2BasedDistributionUnit(Path duDir) {
 140                 try {
 141                         String category = duDir.getParent().getFileName().toString();
 142                         Path targetCategoryBase = a2Base.resolve(category);
 143
 144                         // merge
 145                         Path mergeBnd = duDir.resolve(MERGE_BND);
 146                         if (Files.exists(mergeBnd)) {
 147                                 mergeM2Artifacts(mergeBnd);
 148 //                              return;
 149                         }
 150
 151                         Path commonBnd = duDir.resolve(COMMON_BND);
 152                         if (!Files.exists(commonBnd)) {
 153                                 return;
 154                         }
 155                         Properties commonProps = new Properties();
 156                         try (InputStream in = Files.newInputStream(commonBnd)) {
 157                                 commonProps.load(in);
 158                         }
 159
 160                         String m2Version = commonProps.getProperty(SLC_ORIGIN_M2.toString());
 161                         if (m2Version == null) {
 162                                 logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
 163                                 return;// ignore, this is probably an Eclipse archive
 164                         }
 165                         if (!m2Version.startsWith(":")) {
 166                                 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 167                         }
 168                         m2Version = m2Version.substring(1);
 169
 170                         DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
 171                                         (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
 172                                                         && !p.getFileName().toString().equals(MERGE_BND));
 173                         for (Path p : ds) {
 174                                 Properties fileProps = new Properties();
 175                                 try (InputStream in = Files.newInputStream(p)) {
 176                                         fileProps.load(in);
 177                                 }
 178                                 String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
 179                                 DefaultArtifact artifact = new DefaultArtifact(m2Coordinates);
 180
 181                                 // temporary rewrite, for migration
 182 //                              String localLicense = fileProps.getProperty(BUNDLE_LICENSE.toString());
 183 //                              if (localLicense != null || artifact.getVersion() != null) {
 184 //                                      fileProps.remove(BUNDLE_LICENSE.toString());
 185 //                                      fileProps.put(SLC_ORIGIN_M2.toString(), artifact.getGroupId() + ":" + artifact.getArtifactId());
 186 //                                      try (Writer writer = Files.newBufferedWriter(p)) {
 187 //                                              for (Object key : fileProps.keySet()) {
 188 //                                                      String value = fileProps.getProperty(key.toString());
 189 //                                                      writer.write(key + ": " + value + '\n');
 190 //                                              }
 191 //                                              logger.log(DEBUG, () -> "Migrated  " + p);
 192 //                                      }
 193 //                              }
 194
 195                                 artifact.setVersion(m2Version);
 196
 197                                 // prepare manifest entries
 198                                 Properties mergeProps = new Properties();
 199                                 mergeProps.putAll(commonProps);
 200
 201                                 fileEntries: for (Object key : fileProps.keySet()) {
 202                                         if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
 203                                                 continue fileEntries;
 204                                         String value = fileProps.getProperty(key.toString());
 205                                         Object previousValue = mergeProps.put(key.toString(), value);
 206                                         if (previousValue != null) {
 207                                                 logger.log(Level.WARNING,
 208                                                                 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
 209                                         }
 210                                 }
 211                                 mergeProps.put(ManifestConstants.SLC_ORIGIN_M2.toString(), artifact.toM2Coordinates());
 212                                 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())
 213                                                 && !mergeProps.containsKey(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString())) {
 214                                         // use file name as symbolic name
 215                                         String symbolicName = p.getFileName().toString();
 216                                         symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 217                                         mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 218                                 }
 219
 220                                 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
 221                                                 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
 222                                                 : null;
 223
 224                                 // download
 225                                 URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
 226                                 Path downloaded = download(url, originBase, artifact);
 227
 228                                 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact);
 229 //                              logger.log(Level.DEBUG, () -> "Processed " + downloaded);
 230
 231                                 // sources
 232                                 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
 233
 234                                 createJar(targetBundleDir);
 235                         }
 236                 } catch (IOException e) {
 237                         throw new RuntimeException("Cannot process " + duDir, e);
 238                 }
 239
 240         }
 241
 242         /** Merge multiple Maven artifacts. */
 243         protected void mergeM2Artifacts(Path mergeBnd) throws IOException {
 244                 Path duDir = mergeBnd.getParent();
 245                 String category = duDir.getParent().getFileName().toString();
 246                 Path targetCategoryBase = a2Base.resolve(category);
 247
 248                 Properties mergeProps = new Properties();
 249                 try (InputStream in = Files.newInputStream(mergeBnd)) {
 250                         mergeProps.load(in);
 251                 }
 252                 String m2Version = mergeProps.getProperty(SLC_ORIGIN_M2.toString());
 253                 if (m2Version == null) {
 254                         logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
 255                         return;// ignore, this is probably an Eclipse archive
 256                 }
 257                 if (!m2Version.startsWith(":")) {
 258                         throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 259                 }
 260                 m2Version = m2Version.substring(1);
 261
 262                 String artifactsStr = mergeProps.getProperty(ManifestConstants.SLC_ORIGIN_M2_MERGE.toString());
 263                 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
 264                                 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
 265                                 : null;
 266
 267                 String bundleSymbolicName = mergeProps.getProperty(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
 268                 if (bundleSymbolicName == null)
 269                         throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
 270                 DefaultCategoryNameVersion nameVersion = new DefaultArtifact(
 271                                 category + ":" + bundleSymbolicName + ":" + m2Version);
 272                 Path targetBundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
 273
 274                 String[] artifacts = artifactsStr.split(",");
 275                 artifacts: for (String str : artifacts) {
 276                         String m2Coordinates = str.trim();
 277                         if ("".equals(m2Coordinates))
 278                                 continue artifacts;
 279                         DefaultArtifact artifact = new DefaultArtifact(m2Coordinates.trim());
 280                         if (artifact.getVersion() == null)
 281                                 artifact.setVersion(m2Version);
 282                         URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
 283                         Path downloaded = download(url, originBase, artifact);
 284                         JarEntry entry;
 285                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
 286                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 287                                         if (entry.isDirectory())
 288                                                 continue entries;
 289                                         if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
 290                                                 continue entries;
 291                                         if (entry.getName().startsWith("META-INF/versions/"))
 292                                                 continue entries;
 293                                         if (entry.getName().startsWith("META-INF/maven/"))
 294                                                 continue entries;
 295                                         if (entry.getName().equals("module-info.class"))
 296                                                 continue entries;
 297                                         if (entry.getName().equals("META-INF/NOTICE"))
 298                                                 continue entries;
 299                                         if (entry.getName().equals("META-INF/LICENSE"))
 300                                                 continue entries;
 301                                         Path target = targetBundleDir.resolve(entry.getName());
 302                                         Files.createDirectories(target.getParent());
 303                                         if (!Files.exists(target)) {
 304                                                 Files.copy(jarIn, target);
 305                                         } else {
 306                                                 if (entry.getName().startsWith("META-INF/services/")) {
 307                                                         try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
 308                                                                 out.write("\n".getBytes());
 309                                                                 jarIn.transferTo(out);
 310                                                                 if (logger.isLoggable(DEBUG))
 311                                                                         logger.log(DEBUG, "Appended " + entry.getName());
 312                                                         }
 313                                                 } else if (entry.getName().startsWith("org/apache/batik/")) {
 314                                                         logger.log(Level.WARNING, "Skip " + entry.getName());
 315                                                         continue entries;
 316                                                 } else {
 317                                                         throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
 318                                                 }
 319                                         }
 320                                         logger.log(Level.TRACE, () -> "Copied " + target);
 321                                 }
 322
 323                         }
 324                         downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
 325                 }
 326
 327                 Map<String, String> entries = new TreeMap<>();
 328                 try (Analyzer bndAnalyzer = new Analyzer()) {
 329                         bndAnalyzer.setProperties(mergeProps);
 330                         Jar jar = new Jar(targetBundleDir.toFile());
 331                         bndAnalyzer.setJar(jar);
 332                         Manifest manifest = bndAnalyzer.calcManifest();
 333
 334                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 335                                 Object value = manifest.getMainAttributes().get(key);
 336
 337                                 switch (key.toString()) {
 338                                 case "Tool":
 339                                 case "Bnd-LastModified":
 340                                 case "Created-By":
 341                                         continue keys;
 342                                 }
 343                                 if ("Require-Capability".equals(key.toString())
 344                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
 345                                         continue keys;// hack for very old classes
 346                                 entries.put(key.toString(), value.toString());
 347                                 logger.log(DEBUG, () -> key + "=" + value);
 348
 349                         }
 350                 } catch (Exception e) {
 351                         throw new RuntimeException("Cannot process " + mergeBnd, e);
 352                 }
 353
 354                 Manifest manifest = new Manifest();
 355                 Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 356                 Files.createDirectories(manifestPath.getParent());
 357                 for (String key : entries.keySet()) {
 358                         String value = entries.get(key);
 359                         manifest.getMainAttributes().putValue(key, value);
 360                 }
 361                 try (OutputStream out = Files.newOutputStream(manifestPath)) {
 362                         manifest.write(out);
 363                 }
 364
 365                 createJar(targetBundleDir);
 366
 367         }
 368
 369         /** Generate MANIFEST using BND. */
 370         protected Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps,
 371                         DefaultArtifact artifact) {
 372
 373                 try {
 374                         Map<String, String> additionalEntries = new TreeMap<>();
 375                         boolean doNotModify = Boolean.parseBoolean(fileProps
 376                                         .getOrDefault(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString());
 377
 378                         // we always force the symbolic name
 379
 380                         if (doNotModify) {
 381                                 fileEntries: for (Object key : fileProps.keySet()) {
 382                                         if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
 383                                                 continue fileEntries;
 384                                         String value = fileProps.getProperty(key.toString());
 385                                         additionalEntries.put(key.toString(), value);
 386                                 }
 387                         } else {
 388                                 if (artifact != null) {
 389                                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 390                                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
 391                                         }
 392                                         if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
 393                                                 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
 394                                         }
 395                                 }
 396
 397                                 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
 398                                         fileProps.put(EXPORT_PACKAGE.toString(),
 399                                                         "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
 400                                 }
 401 //                              if (!fileProps.contains(IMPORT_PACKAGE.toString())) {
 402 //                                      fileProps.put(IMPORT_PACKAGE.toString(), "*");
 403 //                              }
 404
 405                                 try (Analyzer bndAnalyzer = new Analyzer()) {
 406                                         bndAnalyzer.setProperties(fileProps);
 407                                         Jar jar = new Jar(downloaded.toFile());
 408                                         bndAnalyzer.setJar(jar);
 409                                         Manifest manifest = bndAnalyzer.calcManifest();
 410
 411                                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 412                                                 Object value = manifest.getMainAttributes().get(key);
 413
 414                                                 switch (key.toString()) {
 415                                                 case "Tool":
 416                                                 case "Bnd-LastModified":
 417                                                 case "Created-By":
 418                                                         continue keys;
 419                                                 }
 420                                                 if ("Require-Capability".equals(key.toString())
 421                                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
 422                                                         continue keys;// hack for very old classes
 423                                                 additionalEntries.put(key.toString(), value.toString());
 424                                                 logger.log(DEBUG, () -> key + "=" + value);
 425
 426                                         }
 427                                 }
 428
 429 //                              try (Builder bndBuilder = new Builder()) {
 430 //                                      Jar jar = new Jar(downloaded.toFile());
 431 //                                      bndBuilder.addClasspath(jar);
 432 //                                      Path targetBundleDir = targetCategoryBase.resolve(artifact.getName() + "." + artifact.getBranch());
 433 //
 434 //                                      Jar target = new Jar(targetBundleDir.toFile());
 435 //                                      bndBuilder.setJar(target);
 436 //                                      return targetBundleDir;
 437 //                              }
 438                         }
 439                         Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries);
 440                         logger.log(Level.DEBUG, () -> "Processed " + downloaded);
 441                         return targetBundleDir;
 442                 } catch (Exception e) {
 443                         throw new RuntimeException("Cannot BND process " + downloaded, e);
 444                 }
 445
 446         }
 447
 448         /** Download and integrates sources for a single Maven artifact. */
 449         protected void downloadAndProcessM2Sources(String repoStr, DefaultArtifact artifact, Path targetBundleDir)
 450                         throws IOException {
 451                 DefaultArtifact sourcesArtifact = new DefaultArtifact(artifact.toM2Coordinates(), "sources");
 452                 URL sourcesUrl = MavenConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
 453                 Path sourcesDownloaded = download(sourcesUrl, originBase, artifact, true);
 454                 processM2SourceJar(sourcesDownloaded, targetBundleDir);
 455                 logger.log(Level.DEBUG, () -> "Processed source " + sourcesDownloaded);
 456
 457         }
 458
 459         /** Integrate sources from a downloaded jar file. */
 460         protected void processM2SourceJar(Path file, Path targetBundleDir) throws IOException {
 461                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 462                         Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
 463
 464                         // TODO make it less dangerous?
 465                         if (Files.exists(targetSourceDir)) {
 466 //                              deleteDirectory(targetSourceDir);
 467                         } else {
 468                                 Files.createDirectories(targetSourceDir);
 469                         }
 470
 471                         // copy entries
 472                         JarEntry entry;
 473                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 474                                 if (entry.isDirectory())
 475                                         continue entries;
 476                                 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
 477                                         continue entries;
 478                                 if (entry.getName().startsWith("module-info.java"))// skip META-INF entries
 479                                         continue entries;
 480                                 if (entry.getName().startsWith("/")) // absolute paths
 481                                         continue entries;
 482                                 Path target = targetSourceDir.resolve(entry.getName());
 483                                 Files.createDirectories(target.getParent());
 484                                 if (!Files.exists(target)) {
 485                                         Files.copy(jarIn, target);
 486                                         logger.log(Level.TRACE, () -> "Copied source " + target);
 487                                 } else {
 488                                         logger.log(Level.WARNING, () -> target + " already exists, skipping...");
 489                                 }
 490                         }
 491                 }
 492
 493         }
 494
 495         /** Download a Maven artifact. */
 496         protected Path download(URL url, Path dir, Artifact artifact) throws IOException {
 497                 return download(url, dir, artifact, false);
 498         }
 499
 500         /** Download a Maven artifact. */
 501         protected Path download(URL url, Path dir, Artifact artifact, boolean sources) throws IOException {
 502                 return download(url, dir, artifact.getGroupId() + '/' + artifact.getArtifactId() + "-" + artifact.getVersion()
 503                                 + (sources ? "-sources" : "") + ".jar");
 504         }
 505
 506         /*
 507          * ECLIPSE ORIGIN
 508          */
 509
 510         /** Process an archive in Eclipse format. */
 511         public void processEclipseArchive(Path duDir) {
 512                 try {
 513                         String category = duDir.getParent().getFileName().toString();
 514                         Path targetCategoryBase = a2Base.resolve(category);
 515                         Files.createDirectories(targetCategoryBase);
 516                         // first delete all directories from previous builds
 517                         for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) {
 518                                 deleteDirectory(dir);
 519                         }
 520
 521                         Files.createDirectories(originBase);
 522
 523                         Path commonBnd = duDir.resolve(COMMON_BND);
 524                         Properties commonProps = new Properties();
 525                         try (InputStream in = Files.newInputStream(commonBnd)) {
 526                                 commonProps.load(in);
 527                         }
 528                         String url = commonProps.getProperty(ManifestConstants.SLC_ORIGIN_URI.toString());
 529                         Path downloaded = tryDownload(url, originBase);
 530
 531                         FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
 532
 533                         // filters
 534                         List<PathMatcher> includeMatchers = new ArrayList<>();
 535                         Properties includes = new Properties();
 536                         try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
 537                                 includes.load(in);
 538                         }
 539                         for (Object pattern : includes.keySet()) {
 540                                 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 541                                 includeMatchers.add(pathMatcher);
 542                         }
 543
 544                         List<PathMatcher> excludeMatchers = new ArrayList<>();
 545                         Path excludeFile = duDir.resolve("excludes.properties");
 546                         if (Files.exists(excludeFile)) {
 547                                 Properties excludes = new Properties();
 548                                 try (InputStream in = Files.newInputStream(excludeFile)) {
 549                                         excludes.load(in);
 550                                 }
 551                                 for (Object pattern : excludes.keySet()) {
 552                                         PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 553                                         excludeMatchers.add(pathMatcher);
 554                                 }
 555                         }
 556
 557                         Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
 558
 559                                 @Override
 560                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 561                                         includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
 562                                                 if (includeMatcher.matches(file)) {
 563                                                         for (PathMatcher excludeMatcher : excludeMatchers) {
 564                                                                 if (excludeMatcher.matches(file)) {
 565                                                                         logger.log(Level.WARNING, "Skipping excluded " + file);
 566                                                                         return FileVisitResult.CONTINUE;
 567                                                                 }
 568                                                         }
 569                                                         if (file.getFileName().toString().contains(".source_")) {
 570                                                                 processEclipseSourceJar(file, targetCategoryBase);
 571                                                                 logger.log(Level.DEBUG, () -> "Processed source " + file);
 572
 573                                                         } else {
 574                                                                 processBundleJar(file, targetCategoryBase, new HashMap<>());
 575                                                                 logger.log(Level.DEBUG, () -> "Processed " + file);
 576                                                         }
 577                                                         break includeMatchers;
 578                                                 }
 579                                         }
 580                                         return FileVisitResult.CONTINUE;
 581                                 }
 582                         });
 583
 584                         DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
 585                         for (Path dir : dirs) {
 586                                 createJar(dir);
 587                         }
 588                 } catch (IOException e) {
 589                         throw new RuntimeException("Cannot process " + duDir, e);
 590                 }
 591
 592         }
 593
 594         /** Process sources in Eclipse format. */
 595         protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
 596                 try {
 597                         Path targetBundleDir;
 598                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 599                                 Manifest manifest = jarIn.getManifest();
 600
 601                                 String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
 602                                 String version = relatedBundle[1].substring("version=\"".length());
 603                                 version = version.substring(0, version.length() - 1);
 604                                 NameVersion nameVersion = new DefaultNameVersion(relatedBundle[0], version);
 605                                 targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 606
 607                                 Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
 608
 609                                 // TODO make it less dangerous?
 610                                 if (Files.exists(targetSourceDir)) {
 611 //                              deleteDirectory(targetSourceDir);
 612                                 } else {
 613                                         Files.createDirectories(targetSourceDir);
 614                                 }
 615
 616                                 // copy entries
 617                                 JarEntry entry;
 618                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 619                                         if (entry.isDirectory())
 620                                                 continue entries;
 621                                         if (entry.getName().startsWith("META-INF"))// skip META-INF entries
 622                                                 continue entries;
 623                                         Path target = targetSourceDir.resolve(entry.getName());
 624                                         Files.createDirectories(target.getParent());
 625                                         Files.copy(jarIn, target);
 626                                         logger.log(Level.TRACE, () -> "Copied source " + target);
 627                                 }
 628
 629                                 // copy MANIFEST
 630 //                      Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 631 //                      Files.createDirectories(manifestPath.getParent());
 632 //                      try (OutputStream out = Files.newOutputStream(manifestPath)) {
 633 //                              manifest.write(out);
 634 //                      }
 635                         }
 636                 } catch (IOException e) {
 637                         throw new IllegalStateException("Cannot process " + file, e);
 638                 }
 639
 640         }
 641
 642         /*
 643          * COMMON PROCESSING
 644          */
 645         /** Normalise a bundle. */
 646         protected Path processBundleJar(Path file, Path targetBase, Map<String, String> entries) throws IOException {
 647                 DefaultNameVersion nameVersion;
 648                 Path targetBundleDir;
 649                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 650                         Manifest sourceManifest = jarIn.getManifest();
 651                         Manifest manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
 652
 653                         // remove problematic entries in MANIFEST
 654                         manifest.getEntries().clear();
 655 //                      Set<String> entriesToDelete = new HashSet<>();
 656 //                      for (String key : manifest.getEntries().keySet()) {
 657 ////                            logger.log(DEBUG, "## " + key);
 658 //                              Attributes attrs = manifest.getAttributes(key);
 659 //                              for (Object attrName : attrs.keySet()) {
 660 ////                                    logger.log(DEBUG, attrName + "=" + attrs.get(attrName));
 661 //                                      if ("Specification-Version".equals(attrName.toString())
 662 //                                                      || "Implementation-Version".equals(attrName.toString())) {
 663 //                                              entriesToDelete.add(key);
 664 //
 665 //                                      }
 666 //                              }
 667 //                      }
 668 //                      for (String key : entriesToDelete) {
 669 //                              manifest.getEntries().remove(key);
 670 //                      }
 671
 672                         String symbolicNameFromEntries = entries.get(BUNDLE_SYMBOLICNAME.toString());
 673                         String versionFromEntries = entries.get(BUNDLE_VERSION.toString());
 674
 675                         if (symbolicNameFromEntries != null && versionFromEntries != null) {
 676                                 nameVersion = new DefaultNameVersion(symbolicNameFromEntries, versionFromEntries);
 677                         } else {
 678                                 nameVersion = nameVersionFromManifest(manifest);
 679                                 if (versionFromEntries != null && !nameVersion.getVersion().equals(versionFromEntries)) {
 680                                         logger.log(Level.WARNING, "Original version is " + nameVersion.getVersion()
 681                                                         + " while new version is " + versionFromEntries);
 682                                 }
 683                                 if (symbolicNameFromEntries != null) {
 684                                         // we always force our symbolic name
 685                                         nameVersion.setName(symbolicNameFromEntries);
 686                                 }
 687                         }
 688                         targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 689
 690                         // TODO make it less dangerous?
 691 //                      if (Files.exists(targetBundleDir)) {
 692 //                              deleteDirectory(targetBundleDir);
 693 //                      }
 694
 695                         // copy entries
 696                         JarEntry entry;
 697                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 698                                 if (entry.isDirectory())
 699                                         continue entries;
 700                                 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
 701                                         continue entries;
 702                                 Path target = targetBundleDir.resolve(entry.getName());
 703                                 Files.createDirectories(target.getParent());
 704                                 Files.copy(jarIn, target);
 705                                 logger.log(Level.TRACE, () -> "Copied " + target);
 706                         }
 707
 708                         // copy MANIFEST
 709                         Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 710                         Files.createDirectories(manifestPath.getParent());
 711                         for (String key : entries.keySet()) {
 712                                 String value = entries.get(key);
 713                                 Object previousValue = manifest.getMainAttributes().putValue(key, value);
 714                                 if (previousValue != null && !previousValue.equals(value)) {
 715                                         if (ManifestConstants.IMPORT_PACKAGE.toString().equals(key)
 716                                                         || ManifestConstants.EXPORT_PACKAGE.toString().equals(key))
 717                                                 logger.log(Level.WARNING, file.getFileName() + ": " + key + " was modified");
 718
 719                                         else
 720                                                 logger.log(Level.WARNING, file.getFileName() + ": " + key + " was " + previousValue
 721                                                                 + ", overridden with " + value);
 722                                 }
 723                         }
 724                         try (OutputStream out = Files.newOutputStream(manifestPath)) {
 725                                 manifest.write(out);
 726                         }
 727                 }
 728                 return targetBundleDir;
 729         }
 730
 731         /*
 732          * UTILITIES
 733          */
 734
 735         /** Recursively deletes a directory. */
 736         private static void deleteDirectory(Path path) throws IOException {
 737                 if (!Files.exists(path))
 738                         return;
 739                 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
 740                         @Override
 741                         public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
 742                                 if (e != null)
 743                                         throw e;
 744                                 Files.delete(directory);
 745                                 return FileVisitResult.CONTINUE;
 746                         }
 747
 748                         @Override
 749                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 750                                 Files.delete(file);
 751                                 return FileVisitResult.CONTINUE;
 752                         }
 753                 });
 754         }
 755
 756         /** Extract name/version from a MANIFEST. */
 757         protected DefaultNameVersion nameVersionFromManifest(Manifest manifest) {
 758                 Attributes attrs = manifest.getMainAttributes();
 759                 // symbolic name
 760                 String symbolicName = attrs.getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
 761                 if (symbolicName == null)
 762                         return null;
 763                 // make sure there is no directive
 764                 symbolicName = symbolicName.split(";")[0];
 765
 766                 String version = attrs.getValue(ManifestConstants.BUNDLE_VERSION.toString());
 767                 return new DefaultNameVersion(symbolicName, version);
 768         }
 769
 770         /** Try to download from an URI. */
 771         protected Path tryDownload(String uri, Path dir) throws IOException {
 772                 // find mirror
 773                 List<String> urlBases = null;
 774                 String uriPrefix = null;
 775                 uriPrefixes: for (String uriPref : mirrors.keySet()) {
 776                         if (uri.startsWith(uriPref)) {
 777                                 if (mirrors.get(uriPref).size() > 0) {
 778                                         urlBases = mirrors.get(uriPref);
 779                                         uriPrefix = uriPref;
 780                                         break uriPrefixes;
 781                                 }
 782                         }
 783                 }
 784                 if (urlBases == null)
 785                         try {
 786                                 return download(new URL(uri), dir);
 787                         } catch (FileNotFoundException e) {
 788                                 throw new FileNotFoundException("Cannot find " + uri);
 789                         }
 790
 791                 // try to download
 792                 for (String urlBase : urlBases) {
 793                         String relativePath = uri.substring(uriPrefix.length());
 794                         URL url = new URL(urlBase + relativePath);
 795                         try {
 796                                 return download(url, dir);
 797                         } catch (FileNotFoundException e) {
 798                                 logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror");
 799                         }
 800                 }
 801                 throw new FileNotFoundException("Cannot find " + uri);
 802         }
 803
 804 //      protected String simplifyName(URL u) {
 805 //      String  name = u.getPath().substring(u.getPath().lastIndexOf('/') + 1);
 806 //
 807 //      }
 808
 809         /** Effectively download. */
 810         protected Path download(URL url, Path dir) throws IOException {
 811                 return download(url, dir, (String) null);
 812         }
 813
 814         /** Effectively download. */
 815         protected Path download(URL url, Path dir, String name) throws IOException {
 816
 817                 Path dest;
 818                 if (name == null) {
 819                         name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1);
 820                 }
 821
 822                 dest = dir.resolve(name);
 823                 if (Files.exists(dest)) {
 824                         logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
 825                         return dest;
 826                 } else {
 827                         Files.createDirectories(dest.getParent());
 828                 }
 829
 830                 try (InputStream in = url.openStream()) {
 831                         Files.copy(in, dest);
 832                         logger.log(Level.DEBUG, () -> "Downloaded " + dest + " from " + url);
 833                 }
 834                 return dest;
 835         }
 836
 837         /** Create a JAR file from a directory. */
 838         protected Path createJar(Path bundleDir) throws IOException {
 839                 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
 840                 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
 841                 Manifest manifest;
 842                 try (InputStream in = Files.newInputStream(manifestPath)) {
 843                         manifest = new Manifest(in);
 844                 }
 845                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
 846                         Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
 847
 848                                 @Override
 849                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 850                                         if (file.getFileName().toString().equals("MANIFEST.MF"))
 851                                                 return super.visitFile(file, attrs);
 852                                         JarEntry entry = new JarEntry(bundleDir.relativize(file).toString());
 853                                         jarOut.putNextEntry(entry);
 854                                         Files.copy(file, jarOut);
 855                                         return super.visitFile(file, attrs);
 856                                 }
 857
 858                         });
 859                 }
 860                 deleteDirectory(bundleDir);
 861                 return jarPath;
 862         }
 863
 864         /** For development purproses. */
 865         public static void main(String[] args) {
 866                 Path factoryBase = Paths.get("../../output/a2").toAbsolutePath().normalize();
 867                 A2Factory factory = new A2Factory(factoryBase);
 868
 869                 Path descriptorsBase = Paths.get("../tp").toAbsolutePath().normalize();
 870
 871 //              factory.processSingleM2ArtifactDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache").resolve("org.apache.xml.resolver.bnd"));
 872 //              factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache/apache-sshd"));
 873 //              factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty"));
 874 //              factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty-websocket"));
 875 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp"));
 876 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp"));
 877 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache"));
 878 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats"));
 879                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis"));
 880                 System.exit(0);
 881
 882                 // Eclipse
 883                 factory.processEclipseArchive(
 884                                 descriptorsBase.resolve("org.argeo.tp.eclipse.equinox").resolve("eclipse-equinox"));
 885                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rwt").resolve("eclipse-rwt"));
 886                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rap").resolve("eclipse-rap"));
 887                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.swt").resolve("eclipse-swt"));
 888                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.swt").resolve("eclipse-nebula"));
 889                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.swt").resolve("eclipse-equinox"));
 890                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp").resolve("eclipse-rcp"));
 891                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp"));
 892
 893                 // Maven
 894                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk"));
 895                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp"));
 896                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache"));
 897                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jetty"));
 898                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jcr"));
 899                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats"));
 900                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.poi"));
 901                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis"));
 902         }
 903 }