org.argeo.slc.build/src/org/argeo/slc/build/A2Factory.java

   1 package org.argeo.slc.build;
   2
   3 import static org.argeo.slc.ManifestConstants.BUNDLE_LICENSE;
   4 import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2;
   5
   6 import java.io.FileNotFoundException;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.io.OutputStream;
  10 import java.io.Writer;
  11 import java.lang.System.Logger;
  12 import java.lang.System.Logger.Level;
  13 import java.net.URL;
  14 import java.nio.file.DirectoryStream;
  15 import java.nio.file.FileSystem;
  16 import java.nio.file.FileSystems;
  17 import java.nio.file.FileVisitResult;
  18 import java.nio.file.Files;
  19 import java.nio.file.Path;
  20 import java.nio.file.PathMatcher;
  21 import java.nio.file.Paths;
  22 import java.nio.file.SimpleFileVisitor;
  23 import java.nio.file.attribute.BasicFileAttributes;
  24 import java.util.ArrayList;
  25 import java.util.HashMap;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Properties;
  29 import java.util.jar.Attributes;
  30 import java.util.jar.JarEntry;
  31 import java.util.jar.JarInputStream;
  32 import java.util.jar.Manifest;
  33
  34 import org.argeo.slc.DefaultNameVersion;
  35 import org.argeo.slc.ManifestConstants;
  36 import org.argeo.slc.NameVersion;
  37 import org.argeo.slc.build.m2.DefaultArtifact;
  38 import org.argeo.slc.build.m2.MavenConventionsUtils;
  39
  40 public class A2Factory {
  41         private final static Logger logger = System.getLogger(A2Factory.class.getName());
  42
  43         private final static String COMMON_BND = "common.bnd";
  44
  45         private Path originBase;
  46         private Path factoryBase;
  47
  48         /** key is URI prefix, value list of base URLs */
  49         private Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
  50
  51         public A2Factory(Path originBase, Path factoryBase) {
  52                 super();
  53                 this.originBase = originBase;
  54                 this.factoryBase = factoryBase;
  55
  56                 // TODO make it configurable
  57                 List<String> eclipseMirrors = new ArrayList<>();
  58                 eclipseMirrors.add("https://archive.eclipse.org/");
  59
  60                 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
  61         }
  62
  63         public void processM2BasedDistributionUnit(Path duDir) {
  64                 try {
  65                         String category = duDir.getParent().getFileName().toString();
  66                         Path targetCategoryBase = factoryBase.resolve(category);
  67                         Path commonBnd = duDir.resolve(COMMON_BND);
  68                         Properties commonProps = new Properties();
  69                         try (InputStream in = Files.newInputStream(commonBnd)) {
  70                                 commonProps.load(in);
  71                         }
  72
  73                         String m2Version = commonProps.getProperty(SLC_ORIGIN_M2.toString());
  74                         if (!m2Version.startsWith(":")) {
  75                                 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
  76                         }
  77                         m2Version = m2Version.substring(1);
  78
  79                         // String license = commonProps.getProperty(BUNDLE_LICENSE.toString());
  80
  81                         DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
  82                                         (p) -> p.getFileName().toString().endsWith(".bnd")
  83                                                         && !p.getFileName().toString().equals(COMMON_BND));
  84                         for (Path p : ds) {
  85                                 Properties fileProps = new Properties();
  86                                 try (InputStream in = Files.newInputStream(p)) {
  87                                         fileProps.load(in);
  88                                 }
  89                                 String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
  90                                 DefaultArtifact artifact = new DefaultArtifact(m2Coordinates);
  91
  92                                 // temporary rewrite, for migration
  93                                 String localLicense = fileProps.getProperty(BUNDLE_LICENSE.toString());
  94                                 if (localLicense != null || artifact.getVersion() != null) {
  95                                         fileProps.remove(BUNDLE_LICENSE.toString());
  96                                         fileProps.put(SLC_ORIGIN_M2.toString(), artifact.getGroupId() + ":" + artifact.getArtifactId());
  97                                         try (Writer writer = Files.newBufferedWriter(p)) {
  98                                                 for (Object key : fileProps.keySet()) {
  99                                                         String value = fileProps.getProperty(key.toString());
 100                                                         writer.write(key + ": " + value + '\n');
 101                                                 }
 102                                                 logger.log(Level.DEBUG, () -> "Migrated  " + p);
 103                                         }
 104                                 }
 105
 106                                 artifact.setVersion(m2Version);
 107                                 URL url = MavenConventionsUtils.mavenCentralUrl(artifact);
 108                                 Path downloaded = download(url, originBase, artifact.toM2Coordinates() + ".jar");
 109
 110                                 // prepare manifest entries
 111                                 Map<String, String> entries = new HashMap<>();
 112                                 for (Object key : commonProps.keySet()) {
 113                                         entries.put(key.toString(), commonProps.getProperty(key.toString()));
 114                                 }
 115                                 fileEntries: for (Object key : fileProps.keySet()) {
 116                                         if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
 117                                                 continue fileEntries;
 118                                         String value = fileProps.getProperty(key.toString());
 119                                         String previousValue = entries.put(key.toString(), value);
 120                                         if (previousValue != null) {
 121                                                 logger.log(Level.WARNING,
 122                                                                 downloaded + ": " + key + " was " + previousValue + ", overridden with " + value);
 123                                         }
 124                                 }
 125                                 entries.put(ManifestConstants.SLC_ORIGIN_M2.toString(), artifact.toM2Coordinates());
 126                                 Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, entries);
 127                                 logger.log(Level.DEBUG, () -> "Processed " + downloaded);
 128
 129                                 // sources
 130                                 DefaultArtifact sourcesArtifact = new DefaultArtifact(artifact.toM2Coordinates(), "sources");
 131                                 URL sourcesUrl = MavenConventionsUtils.mavenCentralUrl(sourcesArtifact);
 132                                 Path sourcesDownloaded = download(sourcesUrl, originBase, artifact.toM2Coordinates() + ".sources.jar");
 133                                 processM2SourceJar(sourcesDownloaded, targetBundleDir);
 134                                 logger.log(Level.DEBUG, () -> "Processed " + sourcesDownloaded);
 135                         }
 136                 } catch (IOException e) {
 137                         throw new RuntimeException("Cannot process " + duDir, e);
 138                 }
 139
 140         }
 141
 142         protected void processM2SourceJar(Path file, Path targetBundleDir) throws IOException {
 143                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 144                         Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
 145
 146                         // TODO make it less dangerous?
 147                         if (Files.exists(targetSourceDir)) {
 148                                 deleteDirectory(targetSourceDir);
 149                         } else {
 150                                 Files.createDirectories(targetSourceDir);
 151                         }
 152
 153                         // copy entries
 154                         JarEntry entry;
 155                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 156                                 if (entry.isDirectory())
 157                                         continue entries;
 158                                 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
 159                                         continue entries;
 160                                 Path target = targetSourceDir.resolve(entry.getName());
 161                                 Files.createDirectories(target.getParent());
 162                                 Files.copy(jarIn, target);
 163                                 logger.log(Level.TRACE, () -> "Copied source " + target);
 164                         }
 165                 }
 166
 167         }
 168
 169         public void processEclipseArchive(Path duDir) {
 170                 try {
 171                         String category = duDir.getParent().getFileName().toString();
 172                         Path targetCategoryBase = factoryBase.resolve(category);
 173                         Files.createDirectories(targetCategoryBase);
 174                         Files.createDirectories(originBase);
 175
 176                         Path commonBnd = duDir.resolve(COMMON_BND);
 177                         Properties commonProps = new Properties();
 178                         try (InputStream in = Files.newInputStream(commonBnd)) {
 179                                 commonProps.load(in);
 180                         }
 181                         Properties includes = new Properties();
 182                         try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
 183                                 includes.load(in);
 184                         }
 185                         String url = commonProps.getProperty(ManifestConstants.SLC_ORIGIN_URI.toString());
 186                         Path downloaded = tryDownload(url, originBase);
 187
 188                         FileSystem zipFs = FileSystems.newFileSystem(downloaded, null);
 189
 190                         List<PathMatcher> pathMatchers = new ArrayList<>();
 191                         for (Object pattern : includes.keySet()) {
 192                                 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
 193                                 pathMatchers.add(pathMatcher);
 194                         }
 195
 196                         Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
 197
 198                                 @Override
 199                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 200                                         pathMatchers: for (PathMatcher pathMatcher : pathMatchers) {
 201                                                 if (pathMatcher.matches(file)) {
 202 //                                                      Path target = targetBase.resolve(file.getFileName().toString());
 203 //                                                      if (!Files.exists(target)) {
 204 //                                                              Files.copy(file, target);
 205 //                                                              logger.log(Level.DEBUG, () -> "Copied " + target + " from " + downloaded);
 206 //                                                      } else {
 207 //                                                              logger.log(Level.DEBUG, () -> target + " already exists.");
 208 //
 209 //                                                      }
 210                                                         if (file.getFileName().toString().contains(".source_")) {
 211                                                                 processEclipseSourceJar(file, targetCategoryBase);
 212                                                                 logger.log(Level.DEBUG, () -> "Processed source " + file);
 213
 214                                                         } else {
 215                                                                 processBundleJar(file, targetCategoryBase, new HashMap<>());
 216                                                                 logger.log(Level.DEBUG, () -> "Processed " + file);
 217                                                         }
 218                                                         continue pathMatchers;
 219                                                 }
 220                                         }
 221                                         return super.visitFile(file, attrs);
 222                                 }
 223                         });
 224                 } catch (IOException e) {
 225                         throw new RuntimeException("Cannot process " + duDir, e);
 226                 }
 227
 228         }
 229
 230         protected Path processBundleJar(Path file, Path targetBase, Map<String, String> additionalManifestEntries)
 231                         throws IOException {
 232                 NameVersion nameVersion;
 233                 Path targetBundleDir;
 234                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 235                         Manifest manifest = jarIn.getManifest();
 236                         nameVersion = nameVersionFromManifest(manifest);
 237                         targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 238
 239                         // TODO make it less dangerous?
 240                         if (Files.exists(targetBundleDir)) {
 241                                 deleteDirectory(targetBundleDir);
 242                         }
 243
 244                         // copy entries
 245                         JarEntry entry;
 246                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 247                                 if (entry.isDirectory())
 248                                         continue entries;
 249                                 Path target = targetBundleDir.resolve(entry.getName());
 250                                 Files.createDirectories(target.getParent());
 251                                 Files.copy(jarIn, target);
 252                                 logger.log(Level.TRACE, () -> "Copied " + target);
 253                         }
 254
 255                         // copy MANIFEST
 256                         Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 257                         Files.createDirectories(manifestPath.getParent());
 258                         for (String key : additionalManifestEntries.keySet()) {
 259                                 String value = additionalManifestEntries.get(key);
 260                                 Object previousValue = manifest.getMainAttributes().putValue(key, value);
 261                                 if (previousValue != null && !previousValue.equals(value)) {
 262                                         logger.log(Level.WARNING,
 263                                                         file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value);
 264                                 }
 265                         }
 266                         try (OutputStream out = Files.newOutputStream(manifestPath)) {
 267                                 manifest.write(out);
 268                         }
 269                 }
 270                 return targetBundleDir;
 271         }
 272
 273         protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
 274                 // NameVersion nameVersion;
 275                 Path targetBundleDir;
 276                 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
 277                         Manifest manifest = jarIn.getManifest();
 278                         // nameVersion = nameVersionFromManifest(manifest);
 279
 280                         String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
 281                         String version = relatedBundle[1].substring("version=\"".length());
 282                         version = version.substring(0, version.length() - 1);
 283                         NameVersion nameVersion = new DefaultNameVersion(relatedBundle[0], version);
 284                         targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 285
 286                         Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
 287
 288                         // TODO make it less dangerous?
 289                         if (Files.exists(targetSourceDir)) {
 290                                 deleteDirectory(targetSourceDir);
 291                         } else {
 292                                 Files.createDirectories(targetSourceDir);
 293                         }
 294
 295                         // copy entries
 296                         JarEntry entry;
 297                         entries: while ((entry = jarIn.getNextJarEntry()) != null) {
 298                                 if (entry.isDirectory())
 299                                         continue entries;
 300                                 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
 301                                         continue entries;
 302                                 Path target = targetSourceDir.resolve(entry.getName());
 303                                 Files.createDirectories(target.getParent());
 304                                 Files.copy(jarIn, target);
 305                                 logger.log(Level.TRACE, () -> "Copied source " + target);
 306                         }
 307
 308                         // copy MANIFEST
 309 //                      Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
 310 //                      Files.createDirectories(manifestPath.getParent());
 311 //                      try (OutputStream out = Files.newOutputStream(manifestPath)) {
 312 //                              manifest.write(out);
 313 //                      }
 314                 }
 315
 316         }
 317
 318         static void deleteDirectory(Path path) throws IOException {
 319                 if (!Files.exists(path))
 320                         return;
 321                 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
 322                         @Override
 323                         public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
 324                                 if (e != null)
 325                                         throw e;
 326                                 Files.delete(directory);
 327                                 return FileVisitResult.CONTINUE;
 328                         }
 329
 330                         @Override
 331                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 332                                 Files.delete(file);
 333                                 return FileVisitResult.CONTINUE;
 334                         }
 335                 });
 336         }
 337
 338         protected NameVersion nameVersionFromManifest(Manifest manifest) {
 339                 Attributes attrs = manifest.getMainAttributes();
 340                 // symbolic name
 341                 String symbolicName = attrs.getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
 342                 // make sure there is no directive
 343                 symbolicName = symbolicName.split(";")[0];
 344
 345                 String version = attrs.getValue(ManifestConstants.BUNDLE_VERSION.toString());
 346                 return new DefaultNameVersion(symbolicName, version);
 347         }
 348
 349         protected Path tryDownload(String uri, Path dir) throws IOException {
 350                 // find mirror
 351                 List<String> urlBases = null;
 352                 String uriPrefix = null;
 353                 uriPrefixes: for (String uriPref : mirrors.keySet()) {
 354                         if (uri.startsWith(uriPref)) {
 355                                 if (mirrors.get(uriPref).size() > 0) {
 356                                         urlBases = mirrors.get(uriPref);
 357                                         uriPrefix = uriPref;
 358                                         break uriPrefixes;
 359                                 }
 360                         }
 361                 }
 362                 if (urlBases == null)
 363                         try {
 364                                 return download(new URL(uri), dir, null);
 365                         } catch (FileNotFoundException e) {
 366                                 throw new FileNotFoundException("Cannot find " + uri);
 367                         }
 368
 369                 // try to download
 370                 for (String urlBase : urlBases) {
 371                         String relativePath = uri.substring(uriPrefix.length());
 372                         URL url = new URL(urlBase + relativePath);
 373                         try {
 374                                 return download(url, dir, null);
 375                         } catch (FileNotFoundException e) {
 376                                 logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror");
 377                         }
 378                 }
 379                 throw new FileNotFoundException("Cannot find " + uri);
 380         }
 381
 382 //      protected String simplifyName(URL u) {
 383 //      String  name = u.getPath().substring(u.getPath().lastIndexOf('/') + 1);
 384 //
 385 //      }
 386
 387         protected Path download(URL url, Path dir, String name) throws IOException {
 388
 389                 Path dest;
 390                 if (name == null) {
 391                         name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1);
 392                 }
 393
 394                 dest = dir.resolve(name);
 395                 if (Files.exists(dest)) {
 396                         logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
 397                         return dest;
 398                 }
 399
 400                 try (InputStream in = url.openStream()) {
 401                         Files.copy(in, dest);
 402                         logger.log(Level.DEBUG, () -> "Downloaded " + dest + " from " + url);
 403                 }
 404                 return dest;
 405         }
 406
 407         public static void main(String[] args) {
 408                 Path originBase = Paths.get("../output/origin").toAbsolutePath().normalize();
 409                 Path factoryBase = Paths.get("../output/a2").toAbsolutePath().normalize();
 410                 A2Factory factory = new A2Factory(originBase, factoryBase);
 411
 412                 Path descriptorsBase = Paths.get("../tp").toAbsolutePath().normalize();
 413
 414 //              factory.processEclipseArchive(
 415 //                              descriptorsBase.resolve("org.argeo.tp.eclipse.equinox").resolve("eclipse-equinox"));
 416 //              factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rap").resolve("eclipse-rap"));
 417 //              factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp").resolve("eclipse-rcp"));
 418
 419                 factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp").resolve("jetty"));
 420         }
 421 }