org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java

   1 package org.argeo.slc.factory;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static org.argeo.slc.ManifestConstants.BUNDLE_SYMBOLICNAME;
   5 import static org.argeo.slc.ManifestConstants.BUNDLE_VERSION;
   6 import static org.argeo.slc.ManifestConstants.EXPORT_PACKAGE;
   7 import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2;
   8 import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2_REPO;
   9
  10 import java.io.FileNotFoundException;
  11 import java.io.IOException;
  12 import java.io.InputStream;
  13 import java.io.OutputStream;
  14 import java.lang.System.Logger;
  15 import java.lang.System.Logger.Level;
  16 import java.net.URL;
  17 import java.nio.file.DirectoryStream;
  18 import java.nio.file.FileSystem;
  19 import java.nio.file.FileSystems;
  20 import java.nio.file.FileVisitResult;
  21 import java.nio.file.Files;
  22 import java.nio.file.Path;
  23 import java.nio.file.PathMatcher;
  24 import java.nio.file.Paths;
  25 import java.nio.file.SimpleFileVisitor;
  26 import java.nio.file.StandardOpenOption;
  27 import java.nio.file.attribute.BasicFileAttributes;
  28 import java.util.ArrayList;
  29 import java.util.HashMap;
  30 import java.util.List;
  31 import java.util.Map;
  32 import java.util.Properties;
  33 import java.util.TreeMap;
  34 import java.util.jar.Attributes;
  35 import java.util.jar.JarEntry;
  36 import java.util.jar.JarInputStream;
  37 import java.util.jar.JarOutputStream;
  38 import java.util.jar.Manifest;
  39
  40 import org.argeo.slc.DefaultCategoryNameVersion;
  41 import org.argeo.slc.DefaultNameVersion;
  42 import org.argeo.slc.ManifestConstants;
  43 import org.argeo.slc.NameVersion;
  44 import org.argeo.slc.factory.m2.DefaultArtifact;
  45 import org.argeo.slc.factory.m2.MavenConventionsUtils;
  46
  47 import aQute.bnd.osgi.Analyzer;
  48 import aQute.bnd.osgi.Jar;
  49
  50 /** The central class for A2 packaging. */
  51 public class A2Factory {
  52         private final static Logger logger = System.getLogger(A2Factory.class.getName());
  53
  54         private final static String COMMON_BND = "common.bnd";
  55         private final static String MERGE_BND = "merge.bnd";
  56
  57         private Path originBase;
  58         private Path a2Base;
  59
  60         /** key is URI prefix, value list of base URLs */
  61         private Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
  62
  63         public A2Factory(Path a2Base) {
  64                 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/slc/origin");
  65                 this.a2Base = a2Base;
  66
  67                 // TODO make it configurable
  68                 List<String> eclipseMirrors = new ArrayList<>();
  69                 eclipseMirrors.add("https://archive.eclipse.org/");
  70                 eclipseMirrors.add("http://ftp-stud.hs-esslingen.de/Mirrors/eclipse/");
  71                 eclipseMirrors.add("http://ftp.fau.de/eclipse/");
  72
  73                 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
  74         }
  75
  76         public void processCategory(Path targetCategoryBase) {
  77                 try {
  78                         DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
  79                                         (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
  80                                                         && !p.getFileName().toString().equals(MERGE_BND));
  81                         for (Path p : bnds) {
  82                                 processSingleM2ArtifactDistributionUnit(p);
  83                         }
  84
  85                         DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
  86                         for (Path duDir : dus) {
  87                                 processM2BasedDistributionUnit(duDir);
  88                         }
  89                 } catch (IOException e) {
  90                         throw new RuntimeException("Cannot process category " + targetCategoryBase, e);
  91                 }
  92         }
  93
  94         public void processSingleM2ArtifactDistributionUnit(Path bndFile) {
  95                 try {
  96                         String category = bndFile.getParent().getFileName().toString();
  97                         Path targetCategoryBase = a2Base.resolve(category);
  98                         Properties fileProps = new Properties();
  99                         try (InputStream in = Files.newInputStream(bndFile)) {
 100                                 fileProps.load(in);
 101                         }
 102                         String repoStr = fileProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
 103                                         ? fileProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
 104                                         : null;
 105
 106                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())
 107                                         && !fileProps.containsKey(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString())) {
 108                                 // use file name as symbolic name
 109                                 String symbolicName = bndFile.getFileName().toString();
 110                                 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 111                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 112                         }
 113
 114                         String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
 115                         if (m2Coordinates == null)
 116                                 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
 117                         DefaultArtifact artifact = new DefaultArtifact(m2Coordinates);
 118                         URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
 119                         Path downloaded = download(url, originBase, artifact.toM2Coordinates() + ".jar");
 120
 121                         Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact);
 122
 123                         downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
 124
 125                         createJar(targetBundleDir);
 126                 } catch (Exception e) {
 127                         throw new RuntimeException("Cannot process " + bndFile, e);
 128                 }
 129         }
 130
 131         public void processM2BasedDistributionUnit(Path duDir) {
 132                 try {
 133                         String category = duDir.getParent().getFileName().toString();
 134                         Path targetCategoryBase = a2Base.resolve(category);
 135
 136                         // merge
 137                         Path mergeBnd = duDir.resolve(MERGE_BND);
 138                         if (Files.exists(mergeBnd)) {
 139                                 mergeM2Artifacts(mergeBnd);
 140                                 return;
 141                         }
 142
 143                         Path commonBnd = duDir.resolve(COMMON_BND);
 144                         Properties commonProps = new Properties();
 145                         try (InputStream in = Files.newInputStream(commonBnd)) {
 146                                 commonProps.load(in);
 147                         }
 148
 149                         String m2Version = commonProps.getProperty(SLC_ORIGIN_M2.toString());
 150                         if (m2Version == null) {
 151                                 logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
 152                                 return;// ignore, this is probably an Eclipse archive
 153                         }
 154                         if (!m2Version.startsWith(":")) {
 155                                 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 156                         }
 157                         m2Version = m2Version.substring(1);
 158
 159                         DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
 160                                         (p) -> p.getFileName().toString().endsWith(".bnd")
 161                                                         && !p.getFileName().toString().equals(COMMON_BND));
 162                         for (Path p : ds) {
 163                                 Properties fileProps = new Properties();
 164                                 try (InputStream in = Files.newInputStream(p)) {
 165                                         fileProps.load(in);
 166                                 }
 167                                 String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
 168                                 DefaultArtifact artifact = new DefaultArtifact(m2Coordinates);
 169
 170                                 // temporary rewrite, for migration
 171 //                              String localLicense = fileProps.getProperty(BUNDLE_LICENSE.toString());
 172 //                              if (localLicense != null || artifact.getVersion() != null) {
 173 //                                      fileProps.remove(BUNDLE_LICENSE.toString());
 174 //                                      fileProps.put(SLC_ORIGIN_M2.toString(), artifact.getGroupId() + ":" + artifact.getArtifactId());
 175 //                                      try (Writer writer = Files.newBufferedWriter(p)) {
 176 //                                              for (Object key : fileProps.keySet()) {
 177 //                                                      String value = fileProps.getProperty(key.toString());
 178 //                                                      writer.write(key + ": " + value + '\n');
 179 //                                              }
 180 //                                              logger.log(DEBUG, () -> "Migrated  " + p);
 181 //                                      }
 182 //                              }
 183
 184                                 artifact.setVersion(m2Version);
 185
 186                                 // prepare manifest entries
 187                                 Properties mergeProps = new Properties();
 188                                 mergeProps.putAll(commonProps);
 189
 190                                 fileEntries: for (Object key : fileProps.keySet()) {
 191                                         if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
 192                                                 continue fileEntries;
 193                                         String value = fileProps.getProperty(key.toString());
 194                                         Object previousValue = mergeProps.put(key.toString(), value);
 195                                         if (previousValue != null) {
 196                                                 logger.log(Level.WARNING,
 197                                                                 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
 198                                         }
 199                                 }
 200                                 mergeProps.put(ManifestConstants.SLC_ORIGIN_M2.toString(), artifact.toM2Coordinates());
 201                                 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())
 202                                                 && !mergeProps.containsKey(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString())) {
 203                                         // use file name as symbolic name
 204                                         String symbolicName = p.getFileName().toString();
 205                                         symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
 206                                         mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
 207                                 }
 208
 209                                 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
 210                                                 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
 211                                                 : null;
 212
 213                                 // download
 214                                 URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
 215                                 Path downloaded = download(url, originBase, artifact.toM2Coordinates() + ".jar");
 216
 217                                 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact);
 218 //                              logger.log(Level.DEBUG, () -> "Processed " + downloaded);
 219
 220                                 // sources
 221                                 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
 222
 223                                 createJar(targetBundleDir);
 224                         }
 225                 } catch (IOException e) {
 226                         throw new RuntimeException("Cannot process " + duDir, e);
 227                 }
 228
 229         }
 230
 231         protected void mergeM2Artifacts(Path mergeBnd) throws IOException {
 232                 Path duDir = mergeBnd.getParent();
 233                 String category = duDir.getParent().getFileName().toString();
 234                 Path targetCategoryBase = a2Base.resolve(category);
 235
 236                 Properties mergeProps = new Properties();
 237                 try (InputStream in = Files.newInputStream(mergeBnd)) {
 238                         mergeProps.load(in);
 239                 }
 240                 String m2Version = mergeProps.getProperty(SLC_ORIGIN_M2.toString());
 241                 if (m2Version == null) {
 242                         logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
 243                         return;// ignore, this is probably an Eclipse archive
 244                 }
 245                 if (!m2Version.startsWith(":")) {
 246                         throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
 247                 }
 248                 m2Version = m2Version.substring(1);
 249
 250                 String artifactsStr = mergeProps.getProperty(ManifestConstants.SLC_ORIGIN_M2_MERGE.toString());
 251                 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
 252                                 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
 253                                 : null;
 254
 255                 String bundleSymbolicName = mergeProps.getProperty(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
 256                 DefaultCategoryNameVersion nameVersion = new DefaultArtifact(
 257                                 category + ":" + bundleSymbolicName + ":" + m2Version);
 258                 Path targetBundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
 259
 260                 String[] artifacts = artifactsStr.split(",");
 261                 artifacts: for (String str : artifacts) {
 262                         String m2Coordinates = str.trim();
 263                         if ("".equals(m2Coordinates))
 264                                 continue artifacts;
 265                         DefaultArtifact artifact = new DefaultArtifact(m2Coordinates.trim());
 266                         if (artifact.getVersion() == null)
 267                                 artifact.setVersion(m2Version);
 268                         URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
 269                         Path downloaded = download(url, originBase, artifact.toM2Coordinates() + ".jar");
 270                         JarEntry entry;
 271                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
 272                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 273                                         if (entry.isDirectory())
 274                                                 continue entries;
 275                                         if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
 276                                                 continue entries;
 277                                         if (entry.getName().startsWith("META-INF/versions/"))
 278                                                 continue entries;
 279                                         if (entry.getName().startsWith("META-INF/maven/"))
 280                                                 continue entries;
 281                                         if (entry.getName().equals("module-info.class"))
 282                                                 continue entries;
 283                                         if (entry.getName().equals("META-INF/NOTICE"))
 284                                                 continue entries;
 285                                         if (entry.getName().equals("META-INF/LICENSE"))
 286                                                 continue entries;
 287                                         Path target = targetBundleDir.resolve(entry.getName());
 288                                         Files.createDirectories(target.getParent());
 289                                         if (!Files.exists(target)) {
 290                                                 Files.copy(jarIn, target);
 291                                         } else {
 292                                                 if (entry.getName().startsWith("META-INF/services/")) {
 293                                                         try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
 294                                                                 out.write("\n".getBytes());
 295                                                                 jarIn.transferTo(out);
 296                                                                 if (logger.isLoggable(DEBUG))
 297                                                                         logger.log(DEBUG, "Appended " + entry.getName());
 298                                                         }
 299                                                 } else if (entry.getName().startsWith("org/apache/batik/")) {
 300                                                         logger.log(Level.WARNING, "Skip " + entry.getName());
 301                                                         continue entries;
 302                                                 } else {
 303                                                         throw new IllegalStateException("File " + target + " already exists");
 304                                                 }
 305                                         }
 306                                         logger.log(Level.TRACE, () -> "Copied " + target);
 307                                 }
 308
 309                         }
 310                         downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
 311                 }
 312
 313                 Map<String, String> entries = new TreeMap<>();
 314                 try (Analyzer bndAnalyzer = new Analyzer()) {
 315                         bndAnalyzer.setProperties(mergeProps);
 316                         Jar jar = new Jar(targetBundleDir.toFile());
 317                         bndAnalyzer.setJar(jar);
 318                         Manifest manifest = bndAnalyzer.calcManifest();
 319
 320                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 321                                 Object value = manifest.getMainAttributes().get(key);
 322
 323                                 switch (key.toString()) {
 324                                 case "Tool":
 325                                 case "Bnd-LastModified":
 326                                 case "Created-By":
 327                                         continue keys;
 328                                 }
 329                                 if ("Require-Capability".equals(key.toString())
 330                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
 331                                         continue keys;// hack for very old classes
 332                                 entries.put(key.toString(), value.toString());
 333                                 logger.log(DEBUG, () -> key + "=" + value);
 334
 335                         }
 336                 } catch (Exception e) {
 337                         throw new RuntimeException("Cannot process " + mergeBnd, e);
 338                 }
 339
 340                 Manifest manifest = new Manifest();
 341                 Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 342                 Files.createDirectories(manifestPath.getParent());
 343                 for (String key : entries.keySet()) {
 344                         String value = entries.get(key);
 345                         manifest.getMainAttributes().putValue(key, value);
 346                 }
 347                 try (OutputStream out = Files.newOutputStream(manifestPath)) {
 348                         manifest.write(out);
 349                 }
 350
 351                 createJar(targetBundleDir);
 352
 353         }
 354
 355         protected void downloadAndProcessM2Sources(String repoStr, DefaultArtifact artifact, Path targetBundleDir)
 356                         throws IOException {
 357                 DefaultArtifact sourcesArtifact = new DefaultArtifact(artifact.toM2Coordinates(), "sources");
 358                 URL sourcesUrl = MavenConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
 359                 Path sourcesDownloaded = download(sourcesUrl, originBase, artifact.toM2Coordinates() + ".sources.jar");
 360                 processM2SourceJar(sourcesDownloaded, targetBundleDir);
 361                 logger.log(Level.DEBUG, () -> "Processed source " + sourcesDownloaded);
 362
 363         }
 364
 365         protected Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps,
 366                         DefaultArtifact artifact) {
 367
 368                 try {
 369                         Map<String, String> additionalEntries = new TreeMap<>();
 370                         boolean doNotModify = Boolean.parseBoolean(fileProps
 371                                         .getOrDefault(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString());
 372
 373                         // we always force the symbolic name
 374
 375                         if (doNotModify) {
 376                                 fileEntries: for (Object key : fileProps.keySet()) {
 377                                         if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
 378                                                 continue fileEntries;
 379                                         String value = fileProps.getProperty(key.toString());
 380                                         additionalEntries.put(key.toString(), value);
 381                                 }
 382                         } else {
 383                                 if (artifact != null) {
 384                                         if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
 385                                                 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
 386                                         }
 387                                         if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
 388                                                 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
 389                                         }
 390                                 }
 391
 392                                 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
 393                                         fileProps.put(EXPORT_PACKAGE.toString(),
 394                                                         "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
 395                                 }
 396 //                              if (!fileProps.contains(IMPORT_PACKAGE.toString())) {
 397 //                                      fileProps.put(IMPORT_PACKAGE.toString(), "*");
 398 //                              }
 399
 400                                 try (Analyzer bndAnalyzer = new Analyzer()) {
 401                                         bndAnalyzer.setProperties(fileProps);
 402                                         Jar jar = new Jar(downloaded.toFile());
 403                                         bndAnalyzer.setJar(jar);
 404                                         Manifest manifest = bndAnalyzer.calcManifest();
 405
 406                                         keys: for (Object key : manifest.getMainAttributes().keySet()) {
 407                                                 Object value = manifest.getMainAttributes().get(key);
 408
 409                                                 switch (key.toString()) {
 410                                                 case "Tool":
 411                                                 case "Bnd-LastModified":
 412                                                 case "Created-By":
 413                                                         continue keys;
 414                                                 }
 415                                                 if ("Require-Capability".equals(key.toString())
 416                                                                 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
 417                                                         continue keys;// hack for very old classes
 418                                                 additionalEntries.put(key.toString(), value.toString());
 419                                                 logger.log(DEBUG, () -> key + "=" + value);
 420
 421                                         }
 422                                 }
 423
 424 //                              try (Builder bndBuilder = new Builder()) {
 425 //                                      Jar jar = new Jar(downloaded.toFile());
 426 //                                      bndBuilder.addClasspath(jar);
 427 //                                      Path targetBundleDir = targetCategoryBase.resolve(artifact.getName() + "." + artifact.getBranch());
 428 //
 429 //                                      Jar target = new Jar(targetBundleDir.toFile());
 430 //                                      bndBuilder.setJar(target);
 431 //                                      return targetBundleDir;
 432 //                              }
 433                         }
 434                         Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries);
 435                         logger.log(Level.DEBUG, () -> "Processed " + downloaded);
 436                         return targetBundleDir;
 437                 } catch (Exception e) {
 438                         throw new RuntimeException("Cannot BND process " + downloaded, e);
 439                 }
 440
 441         }
 442
 443         protected void processM2SourceJar(Path file, Path targetBundleDir) throws IOException {
 444                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 445                         Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
 446
 447                         // TODO make it less dangerous?
 448                         if (Files.exists(targetSourceDir)) {
 449 //                              deleteDirectory(targetSourceDir);
 450                         } else {
 451                                 Files.createDirectories(targetSourceDir);
 452                         }
 453
 454                         // copy entries
 455                         JarEntry entry;
 456                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 457                                 if (entry.isDirectory())
 458                                         continue entries;
 459                                 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
 460                                         continue entries;
 461                                 if (entry.getName().startsWith("module-info.java"))// skip META-INF entries
 462                                         continue entries;
 463                                 Path target = targetSourceDir.resolve(entry.getName());
 464                                 Files.createDirectories(target.getParent());
 465                                 if (!Files.exists(target)) {
 466                                         Files.copy(jarIn, target);
 467                                         logger.log(Level.TRACE, () -> "Copied source " + target);
 468                                 } else {
 469                                         logger.log(Level.WARNING, () -> target + " already exists, skipping...");
 470                                 }
 471                         }
 472                 }
 473
 474         }
 475
 476         public void processEclipseArchive(Path duDir) {
 477                 try {
 478                         String category = duDir.getParent().getFileName().toString();
 479                         Path targetCategoryBase = a2Base.resolve(category);
 480                         Files.createDirectories(targetCategoryBase);
 481                         // first delete all directories from previous builds
 482                         for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) {
 483                                 deleteDirectory(dir);
 484                         }
 485
 486                         Files.createDirectories(originBase);
 487
 488                         Path commonBnd = duDir.resolve(COMMON_BND);
 489                         Properties commonProps = new Properties();
 490                         try (InputStream in = Files.newInputStream(commonBnd)) {
 491                                 commonProps.load(in);
 492                         }
 493                         Properties includes = new Properties();
 494                         try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
 495                                 includes.load(in);
 496                         }
 497                         String url = commonProps.getProperty(ManifestConstants.SLC_ORIGIN_URI.toString());
 498                         Path downloaded = tryDownload(url, originBase);
 499
 500                         FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
 501
 502                         List<PathMatcher> pathMatchers = new ArrayList<>();
 503                         for (Object pattern : includes.keySet()) {
 504                                 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 505                                 pathMatchers.add(pathMatcher);
 506                         }
 507
 508                         Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
 509
 510                                 @Override
 511                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 512                                         pathMatchers: for (PathMatcher pathMatcher : pathMatchers) {
 513                                                 if (pathMatcher.matches(file)) {
 514                                                         if (file.getFileName().toString().contains(".source_")) {
 515                                                                 processEclipseSourceJar(file, targetCategoryBase);
 516                                                                 logger.log(Level.DEBUG, () -> "Processed source " + file);
 517
 518                                                         } else {
 519                                                                 processBundleJar(file, targetCategoryBase, new HashMap<>());
 520                                                                 logger.log(Level.DEBUG, () -> "Processed " + file);
 521                                                         }
 522                                                         break pathMatchers;
 523                                                 }
 524                                         }
 525                                         return FileVisitResult.CONTINUE;
 526                                 }
 527                         });
 528
 529                         DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
 530                         for (Path dir : dirs) {
 531                                 createJar(dir);
 532                         }
 533                 } catch (IOException e) {
 534                         throw new RuntimeException("Cannot process " + duDir, e);
 535                 }
 536
 537         }
 538
 539         protected Path processBundleJar(Path file, Path targetBase, Map<String, String> entries) throws IOException {
 540                 DefaultNameVersion nameVersion;
 541                 Path targetBundleDir;
 542                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 543                         Manifest sourceManifest = jarIn.getManifest();
 544                         Manifest manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
 545
 546                         // remove problematic entries in MANIFEST
 547                         manifest.getEntries().clear();
 548 //                      Set<String> entriesToDelete = new HashSet<>();
 549 //                      for (String key : manifest.getEntries().keySet()) {
 550 ////                            logger.log(DEBUG, "## " + key);
 551 //                              Attributes attrs = manifest.getAttributes(key);
 552 //                              for (Object attrName : attrs.keySet()) {
 553 ////                                    logger.log(DEBUG, attrName + "=" + attrs.get(attrName));
 554 //                                      if ("Specification-Version".equals(attrName.toString())
 555 //                                                      || "Implementation-Version".equals(attrName.toString())) {
 556 //                                              entriesToDelete.add(key);
 557 //
 558 //                                      }
 559 //                              }
 560 //                      }
 561 //                      for (String key : entriesToDelete) {
 562 //                              manifest.getEntries().remove(key);
 563 //                      }
 564
 565                         String symbolicNameFromEntries = entries.get(BUNDLE_SYMBOLICNAME.toString());
 566                         String versionFromEntries = entries.get(BUNDLE_VERSION.toString());
 567
 568                         if (symbolicNameFromEntries != null && versionFromEntries != null) {
 569                                 nameVersion = new DefaultNameVersion(symbolicNameFromEntries, versionFromEntries);
 570                         } else {
 571                                 nameVersion = nameVersionFromManifest(manifest);
 572                                 if (versionFromEntries != null && !nameVersion.getVersion().equals(versionFromEntries)) {
 573                                         logger.log(Level.WARNING, "Original version is " + nameVersion.getVersion()
 574                                                         + " while new version is " + versionFromEntries);
 575                                 }
 576                                 if (symbolicNameFromEntries != null) {
 577                                         // we always force our symbolic name
 578                                         nameVersion.setName(symbolicNameFromEntries);
 579                                 }
 580                         }
 581                         targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 582
 583                         // TODO make it less dangerous?
 584 //                      if (Files.exists(targetBundleDir)) {
 585 //                              deleteDirectory(targetBundleDir);
 586 //                      }
 587
 588                         // copy entries
 589                         JarEntry entry;
 590                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 591                                 if (entry.isDirectory())
 592                                         continue entries;
 593                                 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
 594                                         continue entries;
 595                                 Path target = targetBundleDir.resolve(entry.getName());
 596                                 Files.createDirectories(target.getParent());
 597                                 Files.copy(jarIn, target);
 598                                 logger.log(Level.TRACE, () -> "Copied " + target);
 599                         }
 600
 601                         // copy MANIFEST
 602                         Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 603                         Files.createDirectories(manifestPath.getParent());
 604                         for (String key : entries.keySet()) {
 605                                 String value = entries.get(key);
 606                                 Object previousValue = manifest.getMainAttributes().putValue(key, value);
 607                                 if (previousValue != null && !previousValue.equals(value)) {
 608                                         if (ManifestConstants.IMPORT_PACKAGE.toString().equals(key)
 609                                                         || ManifestConstants.EXPORT_PACKAGE.toString().equals(key))
 610                                                 logger.log(Level.WARNING, file.getFileName() + ": " + key + " was modified");
 611
 612                                         else
 613                                                 logger.log(Level.WARNING, file.getFileName() + ": " + key + " was " + previousValue
 614                                                                 + ", overridden with " + value);
 615                                 }
 616                         }
 617                         try (OutputStream out = Files.newOutputStream(manifestPath)) {
 618                                 manifest.write(out);
 619                         }
 620                 }
 621                 return targetBundleDir;
 622         }
 623
 624         protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
 625                 try {
 626                         Path targetBundleDir;
 627                         try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 628                                 Manifest manifest = jarIn.getManifest();
 629
 630                                 String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
 631                                 String version = relatedBundle[1].substring("version=\"".length());
 632                                 version = version.substring(0, version.length() - 1);
 633                                 NameVersion nameVersion = new DefaultNameVersion(relatedBundle[0], version);
 634                                 targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 635
 636                                 Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
 637
 638                                 // TODO make it less dangerous?
 639                                 if (Files.exists(targetSourceDir)) {
 640 //                              deleteDirectory(targetSourceDir);
 641                                 } else {
 642                                         Files.createDirectories(targetSourceDir);
 643                                 }
 644
 645                                 // copy entries
 646                                 JarEntry entry;
 647                                 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 648                                         if (entry.isDirectory())
 649                                                 continue entries;
 650                                         if (entry.getName().startsWith("META-INF"))// skip META-INF entries
 651                                                 continue entries;
 652                                         Path target = targetSourceDir.resolve(entry.getName());
 653                                         Files.createDirectories(target.getParent());
 654                                         Files.copy(jarIn, target);
 655                                         logger.log(Level.TRACE, () -> "Copied source " + target);
 656                                 }
 657
 658                                 // copy MANIFEST
 659 //                      Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 660 //                      Files.createDirectories(manifestPath.getParent());
 661 //                      try (OutputStream out = Files.newOutputStream(manifestPath)) {
 662 //                              manifest.write(out);
 663 //                      }
 664                         }
 665                 } catch (IOException e) {
 666                         throw new IllegalStateException("Cannot process " + file, e);
 667                 }
 668
 669         }
 670
 671         static void deleteDirectory(Path path) throws IOException {
 672                 if (!Files.exists(path))
 673                         return;
 674                 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
 675                         @Override
 676                         public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
 677                                 if (e != null)
 678                                         throw e;
 679                                 Files.delete(directory);
 680                                 return FileVisitResult.CONTINUE;
 681                         }
 682
 683                         @Override
 684                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 685                                 Files.delete(file);
 686                                 return FileVisitResult.CONTINUE;
 687                         }
 688                 });
 689         }
 690
 691         protected DefaultNameVersion nameVersionFromManifest(Manifest manifest) {
 692                 Attributes attrs = manifest.getMainAttributes();
 693                 // symbolic name
 694                 String symbolicName = attrs.getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
 695                 if (symbolicName == null)
 696                         return null;
 697                 // make sure there is no directive
 698                 symbolicName = symbolicName.split(";")[0];
 699
 700                 String version = attrs.getValue(ManifestConstants.BUNDLE_VERSION.toString());
 701                 return new DefaultNameVersion(symbolicName, version);
 702         }
 703
 704         protected Path tryDownload(String uri, Path dir) throws IOException {
 705                 // find mirror
 706                 List<String> urlBases = null;
 707                 String uriPrefix = null;
 708                 uriPrefixes: for (String uriPref : mirrors.keySet()) {
 709                         if (uri.startsWith(uriPref)) {
 710                                 if (mirrors.get(uriPref).size() > 0) {
 711                                         urlBases = mirrors.get(uriPref);
 712                                         uriPrefix = uriPref;
 713                                         break uriPrefixes;
 714                                 }
 715                         }
 716                 }
 717                 if (urlBases == null)
 718                         try {
 719                                 return download(new URL(uri), dir, null);
 720                         } catch (FileNotFoundException e) {
 721                                 throw new FileNotFoundException("Cannot find " + uri);
 722                         }
 723
 724                 // try to download
 725                 for (String urlBase : urlBases) {
 726                         String relativePath = uri.substring(uriPrefix.length());
 727                         URL url = new URL(urlBase + relativePath);
 728                         try {
 729                                 return download(url, dir, null);
 730                         } catch (FileNotFoundException e) {
 731                                 logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror");
 732                         }
 733                 }
 734                 throw new FileNotFoundException("Cannot find " + uri);
 735         }
 736
 737 //      protected String simplifyName(URL u) {
 738 //      String  name = u.getPath().substring(u.getPath().lastIndexOf('/') + 1);
 739 //
 740 //      }
 741
 742         protected Path download(URL url, Path dir, String name) throws IOException {
 743
 744                 Path dest;
 745                 if (name == null) {
 746                         name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1);
 747                 }
 748
 749                 dest = dir.resolve(name);
 750                 if (Files.exists(dest)) {
 751                         logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
 752                         return dest;
 753                 }
 754
 755                 try (InputStream in = url.openStream()) {
 756                         Files.copy(in, dest);
 757                         logger.log(Level.DEBUG, () -> "Downloaded " + dest + " from " + url);
 758                 }
 759                 return dest;
 760         }
 761
 762         protected Path createJar(Path bundleDir) throws IOException {
 763                 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
 764                 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
 765                 Manifest manifest;
 766                 try (InputStream in = Files.newInputStream(manifestPath)) {
 767                         manifest = new Manifest(in);
 768                 }
 769                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
 770                         Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
 771
 772                                 @Override
 773                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 774                                         if (file.getFileName().toString().equals("MANIFEST.MF"))
 775                                                 return super.visitFile(file, attrs);
 776                                         JarEntry entry = new JarEntry(bundleDir.relativize(file).toString());
 777                                         jarOut.putNextEntry(entry);
 778                                         Files.copy(file, jarOut);
 779                                         return super.visitFile(file, attrs);
 780                                 }
 781
 782                         });
 783                 }
 784                 deleteDirectory(bundleDir);
 785                 return jarPath;
 786         }
 787
 788         public static void main(String[] args) {
 789                 Path factoryBase = Paths.get("../../output/a2").toAbsolutePath().normalize();
 790                 A2Factory factory = new A2Factory(factoryBase);
 791
 792                 Path descriptorsBase = Paths.get("../tp").toAbsolutePath().normalize();
 793
 794 //              factory.processSingleM2ArtifactDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache").resolve("org.apache.xml.resolver.bnd"));
 795 //              factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache/apache-sshd"));
 796 //              factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty"));
 797 //              factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty-websocket"));
 798 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp"));
 799 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp"));
 800 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache"));
 801 //              factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats"));
 802                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats"));
 803                 System.exit(0);
 804
 805                 // Eclipse
 806                 factory.processEclipseArchive(
 807                                 descriptorsBase.resolve("org.argeo.tp.eclipse.equinox").resolve("eclipse-equinox"));
 808                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rap").resolve("eclipse-rap"));
 809                 factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp").resolve("eclipse-rcp"));
 810
 811                 System.exit(0);
 812
 813                 // Maven
 814                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk"));
 815                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp"));
 816                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache"));
 817                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jetty"));
 818                 factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jcr"));
 819         }
 820 }