X-Git-Url: http://git.argeo.org/?a=blobdiff_plain;ds=inline;f=org.argeo.slc.factory%2Fsrc%2Forg%2Fargeo%2Fslc%2Ffactory%2FA2Factory.java;h=6b61493bcc15675177b947f6cec8f91b6d396c77;hb=ab325b797629aac81fe729ba182daa48a78c045b;hp=7a6e6a21547ebfd6dca71df4fbc0b1f898acaa00;hpb=c27f5e8acaed38656756d9c4cd557cf7548647d0;p=gpl%2Fargeo-slc.git diff --git a/org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java b/org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java index 7a6e6a215..6b61493bc 100644 --- a/org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java +++ b/org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java @@ -1,17 +1,16 @@ package org.argeo.slc.factory; import static java.lang.System.Logger.Level.DEBUG; -import static org.argeo.slc.ManifestConstants.BUNDLE_LICENSE; import static org.argeo.slc.ManifestConstants.BUNDLE_SYMBOLICNAME; import static org.argeo.slc.ManifestConstants.BUNDLE_VERSION; import static org.argeo.slc.ManifestConstants.EXPORT_PACKAGE; import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2; +import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2_REPO; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.Writer; import java.lang.System.Logger; import java.lang.System.Logger.Level; import java.net.URL; @@ -24,11 +23,13 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.TreeMap; import java.util.jar.Attributes; @@ -36,10 +37,13 @@ import java.util.jar.JarEntry; import java.util.jar.JarInputStream; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; +import java.util.zip.Deflater; +import org.argeo.slc.DefaultCategoryNameVersion; import org.argeo.slc.DefaultNameVersion; import org.argeo.slc.ManifestConstants; import org.argeo.slc.NameVersion; +import org.argeo.slc.factory.m2.Artifact; import org.argeo.slc.factory.m2.DefaultArtifact; import org.argeo.slc.factory.m2.MavenConventionsUtils; @@ -51,17 +55,39 @@ public class A2Factory { private final static Logger logger = System.getLogger(A2Factory.class.getName()); private final static String COMMON_BND = "common.bnd"; + private final static String MERGE_BND = "merge.bnd"; private Path originBase; - private Path factoryBase; + private Path a2Base; + private Path a2LibBase; + private Path descriptorsBase; + + private Properties uris = new Properties(); + + private boolean includeSources = true; /** key is URI prefix, value list of base URLs */ private Map> mirrors = new HashMap>(); - public A2Factory(Path originBase, Path factoryBase) { - super(); - this.originBase = originBase; - this.factoryBase = factoryBase; + public A2Factory(Path a2Base, Path descriptorsBase, boolean includeSources) { + Objects.requireNonNull(a2Base); + Objects.requireNonNull(descriptorsBase); + this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/slc/origin"); + this.a2Base = a2Base; + this.a2LibBase = a2Base.resolve("lib"); + this.descriptorsBase = descriptorsBase; + if (!Files.exists(this.descriptorsBase)) + throw new IllegalArgumentException(this.descriptorsBase + " does not exist"); + this.includeSources = includeSources; + + Path urisPath = this.descriptorsBase.resolve("uris.properties"); + if (Files.exists(urisPath)) { + try (InputStream in = Files.newInputStream(urisPath)) { + uris.load(in); + } catch (IOException e) { + throw new IllegalStateException("Cannot load " + urisPath, e); + } + } // TODO make it configurable List eclipseMirrors = new ArrayList<>(); @@ -72,11 +98,17 @@ public class A2Factory { mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors); } - public void processCategory(Path targetCategoryBase) { + /* + * MAVEN ORIGIN + */ + + /** Process a whole category/group id. */ + public void processCategory(Path categoryRelativePath) { try { + Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath); DirectoryStream bnds = Files.newDirectoryStream(targetCategoryBase, - (p) -> p.getFileName().toString().endsWith(".bnd") - && !p.getFileName().toString().equals(COMMON_BND)); + (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND) + && !p.getFileName().toString().equals(MERGE_BND)); for (Path p : bnds) { processSingleM2ArtifactDistributionUnit(p); } @@ -86,29 +118,42 @@ public class A2Factory { processM2BasedDistributionUnit(duDir); } } catch (IOException e) { - throw new RuntimeException("Cannot process category " + targetCategoryBase, e); + throw new RuntimeException("Cannot process category " + categoryRelativePath, e); } } + /** Process a standalone Maven artifact. */ public void processSingleM2ArtifactDistributionUnit(Path bndFile) { try { - String category = bndFile.getParent().getFileName().toString(); - Path targetCategoryBase = factoryBase.resolve(category); +// String category = bndFile.getParent().getFileName().toString(); + Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent()); + Path targetCategoryBase = a2Base.resolve(categoryRelativePath); + Properties fileProps = new Properties(); try (InputStream in = Files.newInputStream(bndFile)) { fileProps.load(in); } + String repoStr = fileProps.containsKey(SLC_ORIGIN_M2_REPO.toString()) + ? fileProps.getProperty(SLC_ORIGIN_M2_REPO.toString()) + : null; + + if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) { + // use file name as symbolic name + String symbolicName = bndFile.getFileName().toString(); + symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length()); + fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName); + } String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString()); if (m2Coordinates == null) throw new IllegalArgumentException("No M2 coordinates available for " + bndFile); DefaultArtifact artifact = new DefaultArtifact(m2Coordinates); - URL url = MavenConventionsUtils.mavenCentralUrl(artifact); - Path downloaded = download(url, originBase, artifact.toM2Coordinates() + ".jar"); + URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact); + Path downloaded = download(url, originBase, artifact); Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact); - downloadAndProcessM2Sources(artifact, targetBundleDir); + downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir); createJar(targetBundleDir); } catch (Exception e) { @@ -116,27 +161,42 @@ public class A2Factory { } } + /** Process multiple Maven artifacts. */ public void processM2BasedDistributionUnit(Path duDir) { try { - String category = duDir.getParent().getFileName().toString(); - Path targetCategoryBase = factoryBase.resolve(category); + // String category = duDir.getParent().getFileName().toString(); + Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent()); + Path targetCategoryBase = a2Base.resolve(categoryRelativePath); + + // merge + Path mergeBnd = duDir.resolve(MERGE_BND); + if (Files.exists(mergeBnd)) { + mergeM2Artifacts(mergeBnd); +// return; + } + Path commonBnd = duDir.resolve(COMMON_BND); + if (!Files.exists(commonBnd)) { + return; + } Properties commonProps = new Properties(); try (InputStream in = Files.newInputStream(commonBnd)) { commonProps.load(in); } String m2Version = commonProps.getProperty(SLC_ORIGIN_M2.toString()); + if (m2Version == null) { + logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); + return;// ignore, this is probably an Eclipse archive + } if (!m2Version.startsWith(":")) { throw new IllegalStateException("Only the M2 version can be specified: " + m2Version); } m2Version = m2Version.substring(1); - // String license = commonProps.getProperty(BUNDLE_LICENSE.toString()); - DirectoryStream ds = Files.newDirectoryStream(duDir, - (p) -> p.getFileName().toString().endsWith(".bnd") - && !p.getFileName().toString().equals(COMMON_BND)); + (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND) + && !p.getFileName().toString().equals(MERGE_BND)); for (Path p : ds) { Properties fileProps = new Properties(); try (InputStream in = Files.newInputStream(p)) { @@ -146,31 +206,25 @@ public class A2Factory { DefaultArtifact artifact = new DefaultArtifact(m2Coordinates); // temporary rewrite, for migration - String localLicense = fileProps.getProperty(BUNDLE_LICENSE.toString()); - if (localLicense != null || artifact.getVersion() != null) { - fileProps.remove(BUNDLE_LICENSE.toString()); - fileProps.put(SLC_ORIGIN_M2.toString(), artifact.getGroupId() + ":" + artifact.getArtifactId()); - try (Writer writer = Files.newBufferedWriter(p)) { - for (Object key : fileProps.keySet()) { - String value = fileProps.getProperty(key.toString()); - writer.write(key + ": " + value + '\n'); - } - logger.log(DEBUG, () -> "Migrated " + p); - } - } +// String localLicense = fileProps.getProperty(BUNDLE_LICENSE.toString()); +// if (localLicense != null || artifact.getVersion() != null) { +// fileProps.remove(BUNDLE_LICENSE.toString()); +// fileProps.put(SLC_ORIGIN_M2.toString(), artifact.getGroupId() + ":" + artifact.getArtifactId()); +// try (Writer writer = Files.newBufferedWriter(p)) { +// for (Object key : fileProps.keySet()) { +// String value = fileProps.getProperty(key.toString()); +// writer.write(key + ": " + value + '\n'); +// } +// logger.log(DEBUG, () -> "Migrated " + p); +// } +// } artifact.setVersion(m2Version); - URL url = MavenConventionsUtils.mavenCentralUrl(artifact); - Path downloaded = download(url, originBase, artifact.toM2Coordinates() + ".jar"); // prepare manifest entries Properties mergeProps = new Properties(); mergeProps.putAll(commonProps); - // Map entries = new HashMap<>(); -// for (Object key : commonProps.keySet()) { -// entries.put(key.toString(), commonProps.getProperty(key.toString())); -// } fileEntries: for (Object key : fileProps.keySet()) { if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key)) continue fileEntries; @@ -178,15 +232,30 @@ public class A2Factory { Object previousValue = mergeProps.put(key.toString(), value); if (previousValue != null) { logger.log(Level.WARNING, - downloaded + ": " + key + " was " + previousValue + ", overridden with " + value); + commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value); } } mergeProps.put(ManifestConstants.SLC_ORIGIN_M2.toString(), artifact.toM2Coordinates()); + if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) { + // use file name as symbolic name + String symbolicName = p.getFileName().toString(); + symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length()); + mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName); + } + + String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString()) + ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString()) + : null; + + // download + URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact); + Path downloaded = download(url, originBase, artifact); + Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact); // logger.log(Level.DEBUG, () -> "Processed " + downloaded); // sources - downloadAndProcessM2Sources(artifact, targetBundleDir); + downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir); createJar(targetBundleDir); } @@ -196,15 +265,165 @@ public class A2Factory { } - protected void downloadAndProcessM2Sources(DefaultArtifact artifact, Path targetBundleDir) throws IOException { - DefaultArtifact sourcesArtifact = new DefaultArtifact(artifact.toM2Coordinates(), "sources"); - URL sourcesUrl = MavenConventionsUtils.mavenCentralUrl(sourcesArtifact); - Path sourcesDownloaded = download(sourcesUrl, originBase, artifact.toM2Coordinates() + ".sources.jar"); - processM2SourceJar(sourcesDownloaded, targetBundleDir); - logger.log(Level.DEBUG, () -> "Processed source " + sourcesDownloaded); + /** Merge multiple Maven artifacts. */ + protected void mergeM2Artifacts(Path mergeBnd) throws IOException { + Path duDir = mergeBnd.getParent(); + String category = duDir.getParent().getFileName().toString(); + Path targetCategoryBase = a2Base.resolve(category); + + Properties mergeProps = new Properties(); + try (InputStream in = Files.newInputStream(mergeBnd)) { + mergeProps.load(in); + } + + // Version + String m2Version = mergeProps.getProperty(SLC_ORIGIN_M2.toString()); + if (m2Version == null) { + logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); + return;// ignore, this is probably an Eclipse archive + } + if (!m2Version.startsWith(":")) { + throw new IllegalStateException("Only the M2 version can be specified: " + m2Version); + } + m2Version = m2Version.substring(1); + mergeProps.put(ManifestConstants.BUNDLE_VERSION.toString(), m2Version); + + String artifactsStr = mergeProps.getProperty(ManifestConstants.SLC_ORIGIN_M2_MERGE.toString()); + String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString()) + ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString()) + : null; + + String bundleSymbolicName = mergeProps.getProperty(ManifestConstants.BUNDLE_SYMBOLICNAME.toString()); + if (bundleSymbolicName == null) + throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd); + DefaultCategoryNameVersion nameVersion = new DefaultArtifact( + category + ":" + bundleSymbolicName + ":" + m2Version); + Path targetBundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch()); + + String[] artifacts = artifactsStr.split(","); + artifacts: for (String str : artifacts) { + String m2Coordinates = str.trim(); + if ("".equals(m2Coordinates)) + continue artifacts; + DefaultArtifact artifact = new DefaultArtifact(m2Coordinates.trim()); + if (artifact.getVersion() == null) + artifact.setVersion(m2Version); + URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact); + Path downloaded = download(url, originBase, artifact); + JarEntry entry; + try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) { + entries: while ((entry = jarIn.getNextJarEntry()) != null) { + if (entry.isDirectory()) + continue entries; + else if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF")) + continue entries; + else if (entry.getName().startsWith("META-INF/versions/")) + continue entries; + else if (entry.getName().startsWith("META-INF/maven/")) + continue entries; + else if (entry.getName().equals("module-info.class")) + continue entries; + else if (entry.getName().equals("META-INF/NOTICE")) + continue entries; + else if (entry.getName().equals("META-INF/NOTICE.txt")) + continue entries; + else if (entry.getName().equals("META-INF/LICENSE")) + continue entries; + else if (entry.getName().equals("META-INF/LICENSE.md")) + continue entries; + else if (entry.getName().equals("META-INF/LICENSE-notice.md")) + continue entries; + Path target = targetBundleDir.resolve(entry.getName()); + Files.createDirectories(target.getParent()); + if (!Files.exists(target)) { + Files.copy(jarIn, target); + } else { + if (entry.getName().startsWith("META-INF/services/")) { + try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) { + out.write("\n".getBytes()); + jarIn.transferTo(out); + if (logger.isLoggable(DEBUG)) + logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName()); + } + } else if (entry.getName().startsWith("org/apache/batik/")) { + logger.log(Level.WARNING, "Skip " + entry.getName()); + continue entries; + } else { + throw new IllegalStateException("File " + target + " from " + artifact + " already exists"); + } + } + logger.log(Level.TRACE, () -> "Copied " + target); + } + + } + downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir); + } + + // additional service files + Path servicesDir = duDir.resolve("services"); + if (Files.exists(servicesDir)) { + for (Path p : Files.newDirectoryStream(servicesDir)) { + Path target = targetBundleDir.resolve("META-INF/services/").resolve(p.getFileName()); + try (InputStream in = Files.newInputStream(p); + OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) { + out.write("\n".getBytes()); + in.transferTo(out); + if (logger.isLoggable(DEBUG)) + logger.log(DEBUG, "Appended " + p); + } + } + } + + Map entries = new TreeMap<>(); + try (Analyzer bndAnalyzer = new Analyzer()) { + bndAnalyzer.setProperties(mergeProps); + Jar jar = new Jar(targetBundleDir.toFile()); + bndAnalyzer.setJar(jar); + Manifest manifest = bndAnalyzer.calcManifest(); + + keys: for (Object key : manifest.getMainAttributes().keySet()) { + Object value = manifest.getMainAttributes().get(key); + + switch (key.toString()) { + case "Tool": + case "Bnd-LastModified": + case "Created-By": + continue keys; + } + if ("Require-Capability".equals(key.toString()) + && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) + continue keys;// hack for very old classes + entries.put(key.toString(), value.toString()); + // logger.log(DEBUG, () -> key + "=" + value); + + } + } catch (Exception e) { + throw new RuntimeException("Cannot process " + mergeBnd, e); + } + + Manifest manifest = new Manifest(); + Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF"); + Files.createDirectories(manifestPath.getParent()); + for (String key : entries.keySet()) { + String value = entries.get(key); + manifest.getMainAttributes().putValue(key, value); + } + +// // Use Maven version as Bundle-Version +// String bundleVersion = manifest.getMainAttributes().getValue(ManifestConstants.BUNDLE_VERSION.toString()); +// if (bundleVersion == null || bundleVersion.trim().equals("0")) { +// // TODO check why it is sometimes set to "0" +// manifest.getMainAttributes().putValue(ManifestConstants.BUNDLE_VERSION.toString(), m2Version); +// } + try (OutputStream out = Files.newOutputStream(manifestPath)) { + manifest.write(out); + } + + createJar(targetBundleDir); } + /** Generate MANIFEST using BND. */ protected Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, DefaultArtifact artifact) { @@ -214,15 +433,7 @@ public class A2Factory { .getOrDefault(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString()); // we always force the symbolic name - if (artifact != null) { - if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) { - fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName()); - } - if (!fileProps.containsKey(BUNDLE_VERSION.toString())) { - fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion()); - } - } - + if (doNotModify) { fileEntries: for (Object key : fileProps.keySet()) { if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key)) @@ -231,6 +442,14 @@ public class A2Factory { additionalEntries.put(key.toString(), value); } } else { + if (artifact != null) { + if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) { + fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName()); + } + if (!fileProps.containsKey(BUNDLE_VERSION.toString())) { + fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion()); + } + } if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) { fileProps.put(EXPORT_PACKAGE.toString(), @@ -259,7 +478,7 @@ public class A2Factory { && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) continue keys;// hack for very old classes additionalEntries.put(key.toString(), value.toString()); - logger.log(DEBUG, () -> key + "=" + value); + // logger.log(DEBUG, () -> key + "=" + value); } } @@ -283,13 +502,27 @@ public class A2Factory { } + /** Download and integrates sources for a single Maven artifact. */ + protected void downloadAndProcessM2Sources(String repoStr, DefaultArtifact artifact, Path targetBundleDir) + throws IOException { + if (!includeSources) + return; + DefaultArtifact sourcesArtifact = new DefaultArtifact(artifact.toM2Coordinates(), "sources"); + URL sourcesUrl = MavenConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact); + Path sourcesDownloaded = download(sourcesUrl, originBase, artifact, true); + processM2SourceJar(sourcesDownloaded, targetBundleDir); + logger.log(Level.TRACE, () -> "Processed source " + sourcesDownloaded); + + } + + /** Integrate sources from a downloaded jar file. */ protected void processM2SourceJar(Path file, Path targetBundleDir) throws IOException { try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src"); // TODO make it less dangerous? if (Files.exists(targetSourceDir)) { - deleteDirectory(targetSourceDir); +// deleteDirectory(targetSourceDir); } else { Files.createDirectories(targetSourceDir); } @@ -301,68 +534,117 @@ public class A2Factory { continue entries; if (entry.getName().startsWith("META-INF"))// skip META-INF entries continue entries; + if (entry.getName().startsWith("module-info.java"))// skip META-INF entries + continue entries; + if (entry.getName().startsWith("/")) // absolute paths + continue entries; Path target = targetSourceDir.resolve(entry.getName()); Files.createDirectories(target.getParent()); - Files.copy(jarIn, target); - logger.log(Level.TRACE, () -> "Copied source " + target); + if (!Files.exists(target)) { + Files.copy(jarIn, target); + logger.log(Level.TRACE, () -> "Copied source " + target); + } else { + logger.log(Level.WARNING, () -> target + " already exists, skipping..."); + } } } } - public void processEclipseArchive(Path duDir) { + /** Download a Maven artifact. */ + protected Path download(URL url, Path dir, Artifact artifact) throws IOException { + return download(url, dir, artifact, false); + } + + /** Download a Maven artifact. */ + protected Path download(URL url, Path dir, Artifact artifact, boolean sources) throws IOException { + return download(url, dir, artifact.getGroupId() + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() + + (sources ? "-sources" : "") + ".jar"); + } + + /* + * ECLIPSE ORIGIN + */ + + /** Process an archive in Eclipse format. */ + public void processEclipseArchive(Path relativePath) { try { - String category = duDir.getParent().getFileName().toString(); - Path targetCategoryBase = factoryBase.resolve(category); + Path categoryRelativePath = relativePath.getParent(); + // String category = categoryRelativePath.getFileName().toString(); + Path targetCategoryBase = a2Base.resolve(categoryRelativePath); Files.createDirectories(targetCategoryBase); + // first delete all directories from previous builds + for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) { + deleteDirectory(dir); + } + Files.createDirectories(originBase); + Path duDir = descriptorsBase.resolve(relativePath); Path commonBnd = duDir.resolve(COMMON_BND); Properties commonProps = new Properties(); try (InputStream in = Files.newInputStream(commonBnd)) { commonProps.load(in); } - Properties includes = new Properties(); - try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) { - includes.load(in); - } String url = commonProps.getProperty(ManifestConstants.SLC_ORIGIN_URI.toString()); + if (url == null) { + url = uris.getProperty(relativePath.getFileName().toString()); + if (url == null) + throw new IllegalStateException("No url available for " + duDir); + commonProps.put(ManifestConstants.SLC_ORIGIN_URI.toString(), url); + } Path downloaded = tryDownload(url, originBase); FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null); - List pathMatchers = new ArrayList<>(); + // filters + List includeMatchers = new ArrayList<>(); + Properties includes = new Properties(); + try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) { + includes.load(in); + } for (Object pattern : includes.keySet()) { PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern); - pathMatchers.add(pathMatcher); + includeMatchers.add(pathMatcher); + } + + List excludeMatchers = new ArrayList<>(); + Path excludeFile = duDir.resolve("excludes.properties"); + if (Files.exists(excludeFile)) { + Properties excludes = new Properties(); + try (InputStream in = Files.newInputStream(excludeFile)) { + excludes.load(in); + } + for (Object pattern : excludes.keySet()) { + PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern); + excludeMatchers.add(pathMatcher); + } } Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - pathMatchers: for (PathMatcher pathMatcher : pathMatchers) { - if (pathMatcher.matches(file)) { -// Path target = targetBase.resolve(file.getFileName().toString()); -// if (!Files.exists(target)) { -// Files.copy(file, target); -// logger.log(Level.DEBUG, () -> "Copied " + target + " from " + downloaded); -// } else { -// logger.log(Level.DEBUG, () -> target + " already exists."); -// -// } - if (file.getFileName().toString().contains(".source_")) { + includeMatchers: for (PathMatcher includeMatcher : includeMatchers) { + if (includeMatcher.matches(file)) { + for (PathMatcher excludeMatcher : excludeMatchers) { + if (excludeMatcher.matches(file)) { + logger.log(Level.WARNING, "Skipping excluded " + file); + return FileVisitResult.CONTINUE; + } + } + if (includeSources && file.getFileName().toString().contains(".source_")) { processEclipseSourceJar(file, targetCategoryBase); logger.log(Level.DEBUG, () -> "Processed source " + file); } else { - processBundleJar(file, targetCategoryBase, new HashMap<>()); + processBundleJar(file, targetCategoryBase, new HashMap(commonProps)); logger.log(Level.DEBUG, () -> "Processed " + file); } - continue pathMatchers; + break includeMatchers; } } - return super.visitFile(file, attrs); + return FileVisitResult.CONTINUE; } }); @@ -371,16 +653,69 @@ public class A2Factory { createJar(dir); } } catch (IOException e) { - throw new RuntimeException("Cannot process " + duDir, e); + throw new RuntimeException("Cannot process " + relativePath, e); + } + + } + + /** Process sources in Eclipse format. */ + protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException { + try { + Path targetBundleDir; + try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { + Manifest manifest = jarIn.getManifest(); + + String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";"); + String version = relatedBundle[1].substring("version=\"".length()); + version = version.substring(0, version.length() - 1); + NameVersion nameVersion = new DefaultNameVersion(relatedBundle[0], version); + targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch()); + + Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src"); + + // TODO make it less dangerous? + if (Files.exists(targetSourceDir)) { +// deleteDirectory(targetSourceDir); + } else { + Files.createDirectories(targetSourceDir); + } + + // copy entries + JarEntry entry; + entries: while ((entry = jarIn.getNextJarEntry()) != null) { + if (entry.isDirectory()) + continue entries; + if (entry.getName().startsWith("META-INF"))// skip META-INF entries + continue entries; + Path target = targetSourceDir.resolve(entry.getName()); + Files.createDirectories(target.getParent()); + Files.copy(jarIn, target); + logger.log(Level.TRACE, () -> "Copied source " + target); + } + + // copy MANIFEST +// Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF"); +// Files.createDirectories(manifestPath.getParent()); +// try (OutputStream out = Files.newOutputStream(manifestPath)) { +// manifest.write(out); +// } + } + } catch (IOException e) { + throw new IllegalStateException("Cannot process " + file, e); } } + /* + * COMMON PROCESSING + */ + /** Normalise a bundle. */ protected Path processBundleJar(Path file, Path targetBase, Map entries) throws IOException { - NameVersion nameVersion; + DefaultNameVersion nameVersion; Path targetBundleDir; try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { - Manifest manifest = new Manifest(jarIn.getManifest()); + Manifest sourceManifest = jarIn.getManifest(); + Manifest manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest(); // remove problematic entries in MANIFEST manifest.getEntries().clear(); @@ -401,23 +736,41 @@ public class A2Factory { // manifest.getEntries().remove(key); // } - String symbolicNameFromEntries = entries.get(BUNDLE_SYMBOLICNAME.toString()); - String versionFromEntries = entries.get(BUNDLE_VERSION.toString()); + String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString()); + String ourVersion = entries.get(BUNDLE_VERSION.toString()); - if (symbolicNameFromEntries != null && versionFromEntries != null) { - nameVersion = new DefaultNameVersion(symbolicNameFromEntries, versionFromEntries); + if (ourSymbolicName != null && ourVersion != null) { + nameVersion = new DefaultNameVersion(ourSymbolicName, ourVersion); } else { nameVersion = nameVersionFromManifest(manifest); - if (versionFromEntries != null && !nameVersion.getVersion().equals(versionFromEntries)) { - logger.log(Level.WARNING, "Original version is " + nameVersion.getVersion() - + " while new version is " + versionFromEntries); + if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) { + logger.log(Level.WARNING, + "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion); + entries.put(BUNDLE_VERSION.toString(), ourVersion); + } + if (ourSymbolicName != null) { + // we always force our symbolic name + nameVersion.setName(ourSymbolicName); } } targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch()); + // force Java 9 module name + entries.put(ManifestConstants.AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName()); + // TODO make it less dangerous? - if (Files.exists(targetBundleDir)) { - deleteDirectory(targetBundleDir); +// if (Files.exists(targetBundleDir)) { +// deleteDirectory(targetBundleDir); +// } + + boolean isNative = false; + String os = null; + String arch = null; + if (targetBundleDir.startsWith(a2LibBase)) { + isNative = true; + Path libRelativePath = a2LibBase.relativize(targetBundleDir); + os = libRelativePath.getName(0).toString(); + arch = libRelativePath.getName(1).toString(); } // copy entries @@ -427,9 +780,43 @@ public class A2Factory { continue entries; if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF")) continue entries; + if (entry.getName().endsWith("module-info.class")) // skip Java 9 module info + continue entries; + if (entry.getName().startsWith("META-INF/versions/")) // skip multi-version + continue entries; + if (entry.getName().startsWith("OSGI-OPT/src/")) // skip embedded sources + continue entries; Path target = targetBundleDir.resolve(entry.getName()); Files.createDirectories(target.getParent()); Files.copy(jarIn, target); + + // native libraries + if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll") + || entry.getName().endsWith(".jnilib"))) { + Path categoryDir = targetBundleDir.getParent(); +// String[] segments = categoryDir.getFileName().toString().split("\\."); +// String arch = segments[segments.length - 1]; +// String os = segments[segments.length - 2]; + boolean copyDll = false; + Path targetDll = categoryDir.resolve(target.getFileName()); + if (nameVersion.getName().equals("com.sun.jna")) { + if (arch.equals("x86_64")) + arch = "x86-64"; + if (os.equals("macosx")) + os = "darwin"; + if (target.getParent().getFileName().toString().equals(os + "-" + arch)) { + copyDll = true; + } + } else { + copyDll = true; + } + if (copyDll) { + if (Files.exists(targetDll)) + Files.delete(targetDll); + Files.copy(target, targetDll); + } + Files.delete(target); + } logger.log(Level.TRACE, () -> "Copied " + target); } @@ -440,9 +827,19 @@ public class A2Factory { String value = entries.get(key); Object previousValue = manifest.getMainAttributes().putValue(key, value); if (previousValue != null && !previousValue.equals(value)) { - logger.log(Level.WARNING, - file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value); + if (ManifestConstants.IMPORT_PACKAGE.toString().equals(key) + || ManifestConstants.EXPORT_PACKAGE.toString().equals(key)) + logger.log(Level.TRACE, file.getFileName() + ": " + key + " was modified"); + else + logger.log(Level.WARNING, file.getFileName() + ": " + key + " was " + previousValue + + ", overridden with " + value); } + + // hack to remove unresolvable + if (key.equals("Provide-Capability") || key.equals("Require-Capability")) + if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) { + manifest.getMainAttributes().remove(key); + } } try (OutputStream out = Files.newOutputStream(manifestPath)) { manifest.write(out); @@ -451,52 +848,12 @@ public class A2Factory { return targetBundleDir; } - protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException { - // NameVersion nameVersion; - Path targetBundleDir; - try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { - Manifest manifest = jarIn.getManifest(); - // nameVersion = nameVersionFromManifest(manifest); + /* + * UTILITIES + */ - String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";"); - String version = relatedBundle[1].substring("version=\"".length()); - version = version.substring(0, version.length() - 1); - NameVersion nameVersion = new DefaultNameVersion(relatedBundle[0], version); - targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch()); - - Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src"); - - // TODO make it less dangerous? - if (Files.exists(targetSourceDir)) { - deleteDirectory(targetSourceDir); - } else { - Files.createDirectories(targetSourceDir); - } - - // copy entries - JarEntry entry; - entries: while ((entry = jarIn.getNextJarEntry()) != null) { - if (entry.isDirectory()) - continue entries; - if (entry.getName().startsWith("META-INF"))// skip META-INF entries - continue entries; - Path target = targetSourceDir.resolve(entry.getName()); - Files.createDirectories(target.getParent()); - Files.copy(jarIn, target); - logger.log(Level.TRACE, () -> "Copied source " + target); - } - - // copy MANIFEST -// Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF"); -// Files.createDirectories(manifestPath.getParent()); -// try (OutputStream out = Files.newOutputStream(manifestPath)) { -// manifest.write(out); -// } - } - - } - - static void deleteDirectory(Path path) throws IOException { + /** Recursively deletes a directory. */ + private static void deleteDirectory(Path path) throws IOException { if (!Files.exists(path)) return; Files.walkFileTree(path, new SimpleFileVisitor() { @@ -516,7 +873,8 @@ public class A2Factory { }); } - protected NameVersion nameVersionFromManifest(Manifest manifest) { + /** Extract name/version from a MANIFEST. */ + protected DefaultNameVersion nameVersionFromManifest(Manifest manifest) { Attributes attrs = manifest.getMainAttributes(); // symbolic name String symbolicName = attrs.getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString()); @@ -529,6 +887,7 @@ public class A2Factory { return new DefaultNameVersion(symbolicName, version); } + /** Try to download from an URI. */ protected Path tryDownload(String uri, Path dir) throws IOException { // find mirror List urlBases = null; @@ -544,7 +903,7 @@ public class A2Factory { } if (urlBases == null) try { - return download(new URL(uri), dir, null); + return download(new URL(uri), dir); } catch (FileNotFoundException e) { throw new FileNotFoundException("Cannot find " + uri); } @@ -554,7 +913,7 @@ public class A2Factory { String relativePath = uri.substring(uriPrefix.length()); URL url = new URL(urlBase + relativePath); try { - return download(url, dir, null); + return download(url, dir); } catch (FileNotFoundException e) { logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror"); } @@ -567,6 +926,12 @@ public class A2Factory { // // } + /** Effectively download. */ + protected Path download(URL url, Path dir) throws IOException { + return download(url, dir, (String) null); + } + + /** Effectively download. */ protected Path download(URL url, Path dir, String name) throws IOException { Path dest; @@ -578,6 +943,8 @@ public class A2Factory { if (Files.exists(dest)) { logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again"); return dest; + } else { + Files.createDirectories(dest.getParent()); } try (InputStream in = url.openStream()) { @@ -587,7 +954,30 @@ public class A2Factory { return dest; } + /** Create a JAR file from a directory. */ protected Path createJar(Path bundleDir) throws IOException { + // Remove from source the resources already in the jar file +// Path srcDir = bundleDir.resolve("OSGI-OPT/src"); +// Files.walkFileTree(srcDir, new SimpleFileVisitor() { +// +// @Override +// public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { +// // always keep .java file +// if (file.getFileName().toString().endsWith(".java")) +// return super.visitFile(file, attrs); +// +// Path relPath = srcDir.relativize(file); +// Path bundlePath = bundleDir.resolve(relPath); +// if (Files.exists(bundlePath)) { +// Files.delete(file); +// logger.log(DEBUG, () -> "Removed " + file + " from sources."); +// } +// return super.visitFile(file, attrs); +// } +// +// }); + + // Create the jar Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar"); Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF"); Manifest manifest; @@ -595,6 +985,7 @@ public class A2Factory { manifest = new Manifest(in); } try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) { + jarOut.setLevel(Deflater.DEFAULT_COMPRESSION); Files.walkFileTree(bundleDir, new SimpleFileVisitor() { @Override @@ -613,32 +1004,55 @@ public class A2Factory { return jarPath; } + /** For development purproses. */ public static void main(String[] args) { - Path originBase = Paths.get("../output/origin").toAbsolutePath().normalize(); - Path factoryBase = Paths.get("../output/a2").toAbsolutePath().normalize(); - A2Factory factory = new A2Factory(originBase, factoryBase); - + Path factoryBase = Paths.get("../../output/a2").toAbsolutePath().normalize(); Path descriptorsBase = Paths.get("../tp").toAbsolutePath().normalize(); + A2Factory factory = new A2Factory(factoryBase, descriptorsBase, true); // factory.processSingleM2ArtifactDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache").resolve("org.apache.xml.resolver.bnd")); // factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache/apache-sshd")); - factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty")); - factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty-websocket")); - System.exit(0); +// factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty")); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.osgi")); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp")); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp")); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache")); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk")); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats")); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis")); + + // SDK + factory.processCategory(Paths.get("org.argeo.tp.sdk")); // Eclipse - factory.processEclipseArchive( - descriptorsBase.resolve("org.argeo.tp.eclipse.equinox").resolve("eclipse-equinox")); - factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rap").resolve("eclipse-rap")); - factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp").resolve("eclipse-rcp")); + factory.processCategory(Paths.get("osgi/api/org.argeo.tp.osgi")); + factory.processEclipseArchive(Paths.get("osgi/equinox/org.argeo.tp.osgi", "eclipse-equinox")); + + factory.processEclipseArchive(Paths.get("org.argeo.tp.eclipse", "eclipse-equinox")); + + factory.processEclipseArchive(Paths.get("swt/rap/org.argeo.tp.swt", "eclipse-rap")); + factory.processEclipseArchive(Paths.get("swt/rap/org.argeo.tp.swt.workbench", "eclipse-rap")); + + factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); - System.exit(0); + factory.processEclipseArchive(Paths.get("lib/linux/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); + factory.processEclipseArchive(Paths.get("lib/linux/aarch64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); + factory.processEclipseArchive(Paths.get("lib/win32/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); + factory.processEclipseArchive(Paths.get("lib/macosx/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); + System.exit(1); + factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt", "eclipse-nebula")); + // factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt", + // "eclipse-equinox")); + factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt.workbench", "eclipse-rcp")); + factory.processCategory(Paths.get("swt/rcp/org.argeo.tp.swt.workbench")); // Maven - factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk")); - factory.processCategory(descriptorsBase.resolve("org.argeo.tp")); - factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache")); - factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jetty")); - factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jcr")); + factory.processCategory(Paths.get("org.argeo.tp")); + factory.processCategory(Paths.get("org.argeo.tp.apache")); + factory.processCategory(Paths.get("org.argeo.tp.jetty")); + factory.processCategory(Paths.get("org.argeo.tp.jcr")); + factory.processCategory(Paths.get("org.argeo.tp.formats")); + factory.processCategory(Paths.get("org.argeo.tp.poi")); + factory.processCategory(Paths.get("org.argeo.tp.gis")); } }