X-Git-Url: https://git.argeo.org/?a=blobdiff_plain;f=src%2Forg%2Fargeo%2Fbuild%2FRepackage.java;h=90202952607eb15b9cab2a53e597250b7b0372ad;hb=56823b2fa7daf14ccff45e329f9653cdc3273ec4;hp=df630055aec75973df0dbbf5046d57324bdcdabb;hpb=8a5bc1878eb62e28f2cecbf610c61c456d1f1f89;p=cc0%2Fargeo-build.git diff --git a/src/org/argeo/build/Repackage.java b/src/org/argeo/build/Repackage.java index df63005..9020295 100644 --- a/src/org/argeo/build/Repackage.java +++ b/src/org/argeo/build/Repackage.java @@ -1,20 +1,31 @@ package org.argeo.build; import static java.lang.System.Logger.Level.DEBUG; +import static java.lang.System.Logger.Level.ERROR; +import static java.lang.System.Logger.Level.INFO; +import static java.lang.System.Logger.Level.TRACE; +import static java.lang.System.Logger.Level.WARNING; +import static java.nio.file.FileVisitResult.CONTINUE; +import static java.util.jar.Attributes.Name.MANIFEST_VERSION; +import static org.argeo.build.Repackage.ManifestConstants.ARGEO_ORIGIN_M2; +import static org.argeo.build.Repackage.ManifestConstants.ARGEO_ORIGIN_M2_REPO; +import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_LICENSE; import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_SYMBOLICNAME; import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_VERSION; +import static org.argeo.build.Repackage.ManifestConstants.ECLIPSE_SOURCE_BUNDLE; import static org.argeo.build.Repackage.ManifestConstants.EXPORT_PACKAGE; -import static org.argeo.build.Repackage.ManifestConstants.SLC_ORIGIN_M2; -import static org.argeo.build.Repackage.ManifestConstants.SLC_ORIGIN_M2_REPO; +import static org.argeo.build.Repackage.ManifestConstants.IMPORT_PACKAGE; +import static org.argeo.build.Repackage.ManifestConstants.SPDX_LICENSE_IDENTIFIER; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.lang.System.Logger; -import java.lang.System.Logger.Level; import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.FileSystem; import java.nio.file.FileSystems; @@ -28,11 +39,15 @@ import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Properties; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; import java.util.jar.Attributes; import java.util.jar.JarEntry; import java.util.jar.JarInputStream; @@ -43,9 +58,25 @@ import java.util.zip.Deflater; import aQute.bnd.osgi.Analyzer; import aQute.bnd.osgi.Jar; -/** The central class for A2 packaging. */ +/** + * Simple tool repackaging existing jar files into OSGi bundles in an A2 + * repository. + */ public class Repackage { - private final static Logger logger = System.getLogger(Repackage.class.getName()); + final static Logger logger = System.getLogger(Repackage.class.getName()); + + /** + * Environment variable on whether sources should be packaged separately or + * integrated in the bundles. + */ + final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES"; + + /** Whethere repackaging should run in parallel or sequentially. */ + final static boolean parallel = true; + + // cache + /** Summary of all license seen during the repackaging. */ + final static Map> licensesUsed = new TreeMap<>(); /** Main entry point. */ public static void main(String[] args) { @@ -55,41 +86,70 @@ public class Repackage { } Path a2Base = Paths.get(args[0]).toAbsolutePath().normalize(); Path descriptorsBase = Paths.get(".").toAbsolutePath().normalize(); - Repackage factory = new Repackage(a2Base, descriptorsBase, true); + Repackage factory = new Repackage(a2Base, descriptorsBase); + List> toDos = new ArrayList<>(); for (int i = 1; i < args.length; i++) { Path p = Paths.get(args[i]); - factory.processCategory(p); + if (parallel) + toDos.add(CompletableFuture.runAsync(() -> factory.processCategory(p))); + else + factory.processCategory(p); } + CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join(); + + // Summary + StringBuilder sb = new StringBuilder(); + for (String licenseId : licensesUsed.keySet()) + for (String name : licensesUsed.get(licenseId)) + sb.append((licenseId.equals("") ? "Proprietary" : licenseId) + "\t\t" + name + "\n"); + logger.log(INFO, "# License summary:\n" + sb); } - private final static String COMMON_BND = "common.bnd"; - private final static String MERGE_BND = "merge.bnd"; - - private Path originBase; - private Path a2Base; - private Path a2LibBase; - private Path descriptorsBase; + final static String COMMON_BND = "common.bnd"; + final static String MERGE_BND = "merge.bnd"; + + /** Directory where to download archives */ + Path originBase; + /** Directory where to download Maven artifacts */ + Path mavenBase; + + /** A2 repository base for binary bundles */ + Path a2Base; + /** A2 repository base for source bundles */ + Path a2SrcBase; + /** A2 base for native components */ + Path a2LibBase; + /** Location of the descriptors driving the packaging */ + Path descriptorsBase; + /** URIs of archives to download */ + Properties uris = new Properties(); + /** Mirrors for archive download. Key is URI prefix, value list of base URLs */ + Map> mirrors = new HashMap>(); + + /** Whether sources should be packaged separately */ + final boolean sourceBundles; + + /** Constructor initialises the various variables */ + public Repackage(Path a2Base, Path descriptorsBase) { + sourceBundles = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES)); + if (sourceBundles) + logger.log(INFO, "Sources will be packaged separately"); - private Properties uris = new Properties(); - - private boolean includeSources = true; - - /** key is URI prefix, value list of base URLs */ - private Map> mirrors = new HashMap>(); - - public Repackage(Path a2Base, Path descriptorsBase, boolean includeSources) { Objects.requireNonNull(a2Base); Objects.requireNonNull(descriptorsBase); this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/build/origin"); + this.mavenBase = Paths.get(System.getProperty("user.home"), ".m2", "repository"); + // TODO define and use a build base this.a2Base = a2Base; + this.a2SrcBase = a2Base.getParent().resolve(a2Base.getFileName() + ".src"); this.a2LibBase = a2Base.resolve("lib"); this.descriptorsBase = descriptorsBase; if (!Files.exists(this.descriptorsBase)) throw new IllegalArgumentException(this.descriptorsBase + " does not exist"); - this.includeSources = includeSources; + // URIs mapping Path urisPath = this.descriptorsBase.resolve("uris.properties"); if (Files.exists(urisPath)) { try (InputStream in = Files.newInputStream(urisPath)) { @@ -99,21 +159,29 @@ public class Repackage { } } - // TODO make it configurable + // Eclipse mirrors + Path eclipseMirrorsPath = this.descriptorsBase.resolve("eclipse.mirrors.txt"); List eclipseMirrors = new ArrayList<>(); - eclipseMirrors.add("https://archive.eclipse.org/"); - eclipseMirrors.add("http://ftp-stud.hs-esslingen.de/Mirrors/eclipse/"); - eclipseMirrors.add("http://ftp.fau.de/eclipse/"); - + if (Files.exists(eclipseMirrorsPath)) { + try { + eclipseMirrors = Files.readAllLines(eclipseMirrorsPath, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new IllegalStateException("Cannot load " + eclipseMirrorsPath, e); + } + for (Iterator it = eclipseMirrors.iterator(); it.hasNext();) { + String value = it.next(); + if (value.strip().equals("")) + it.remove(); + } + } mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors); } /* * MAVEN ORIGIN */ - /** Process a whole category/group id. */ - public void processCategory(Path categoryRelativePath) { + void processCategory(Path categoryRelativePath) { try { Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath); DirectoryStream bnds = Files.newDirectoryStream(targetCategoryBase, @@ -137,9 +205,8 @@ public class Repackage { } /** Process a standalone Maven artifact. */ - public void processSingleM2ArtifactDistributionUnit(Path bndFile) { + void processSingleM2ArtifactDistributionUnit(Path bndFile) { try { -// String category = bndFile.getParent().getFileName().toString(); Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent()); Path targetCategoryBase = a2Base.resolve(categoryRelativePath); @@ -147,8 +214,8 @@ public class Repackage { try (InputStream in = Files.newInputStream(bndFile)) { fileProps.load(in); } - String repoStr = fileProps.containsKey(SLC_ORIGIN_M2_REPO.toString()) - ? fileProps.getProperty(SLC_ORIGIN_M2_REPO.toString()) + String repoStr = fileProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString()) + ? fileProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString()) : null; if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) { @@ -158,15 +225,14 @@ public class Repackage { fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName); } - String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString()); + String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString()); if (m2Coordinates == null) throw new IllegalArgumentException("No M2 coordinates available for " + bndFile); M2Artifact artifact = new M2Artifact(m2Coordinates); URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact); - Path downloaded = download(url, originBase, artifact); + Path downloaded = downloadMaven(url, artifact); Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact); - downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir); createJar(targetBundleDir); @@ -176,31 +242,27 @@ public class Repackage { } /** Process multiple Maven artifacts. */ - public void processM2BasedDistributionUnit(Path duDir) { + void processM2BasedDistributionUnit(Path duDir) { try { - // String category = duDir.getParent().getFileName().toString(); Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent()); Path targetCategoryBase = a2Base.resolve(categoryRelativePath); - // merge Path mergeBnd = duDir.resolve(MERGE_BND); - if (Files.exists(mergeBnd)) { + if (Files.exists(mergeBnd)) // merge mergeM2Artifacts(mergeBnd); -// return; - } Path commonBnd = duDir.resolve(COMMON_BND); - if (!Files.exists(commonBnd)) { + if (!Files.exists(commonBnd)) return; - } + Properties commonProps = new Properties(); try (InputStream in = Files.newInputStream(commonBnd)) { commonProps.load(in); } - String m2Version = commonProps.getProperty(SLC_ORIGIN_M2.toString()); + String m2Version = commonProps.getProperty(ARGEO_ORIGIN_M2.toString()); if (m2Version == null) { - logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); + logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); return;// ignore, this is probably an Eclipse archive } if (!m2Version.startsWith(":")) { @@ -216,9 +278,8 @@ public class Repackage { try (InputStream in = Files.newInputStream(p)) { fileProps.load(in); } - String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString()); + String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString()); M2Artifact artifact = new M2Artifact(m2Coordinates); - artifact.setVersion(m2Version); // prepare manifest entries @@ -226,16 +287,16 @@ public class Repackage { mergeProps.putAll(commonProps); fileEntries: for (Object key : fileProps.keySet()) { - if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key)) + if (ManifestConstants.ARGEO_ORIGIN_M2.toString().equals(key)) continue fileEntries; String value = fileProps.getProperty(key.toString()); Object previousValue = mergeProps.put(key.toString(), value); if (previousValue != null) { - logger.log(Level.WARNING, + logger.log(WARNING, commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value); } } - mergeProps.put(ManifestConstants.SLC_ORIGIN_M2.toString(), artifact.toM2Coordinates()); + mergeProps.put(ManifestConstants.ARGEO_ORIGIN_M2.toString(), artifact.toM2Coordinates()); if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) { // use file name as symbolic name String symbolicName = p.getFileName().toString(); @@ -243,30 +304,25 @@ public class Repackage { mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName); } - String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString()) - ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString()) + String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString()) + ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString()) : null; // download URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact); - Path downloaded = download(url, originBase, artifact); + Path downloaded = downloadMaven(url, artifact); Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact); -// logger.log(Level.DEBUG, () -> "Processed " + downloaded); - - // sources downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir); - createJar(targetBundleDir); } } catch (IOException e) { throw new RuntimeException("Cannot process " + duDir, e); } - } /** Merge multiple Maven artifacts. */ - protected void mergeM2Artifacts(Path mergeBnd) throws IOException { + void mergeM2Artifacts(Path mergeBnd) throws IOException { Path duDir = mergeBnd.getParent(); String category = duDir.getParent().getFileName().toString(); Path targetCategoryBase = a2Base.resolve(category); @@ -276,10 +332,9 @@ public class Repackage { mergeProps.load(in); } - // Version - String m2Version = mergeProps.getProperty(SLC_ORIGIN_M2.toString()); + String m2Version = mergeProps.getProperty(ARGEO_ORIGIN_M2.toString()); if (m2Version == null) { - logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); + logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); return;// ignore, this is probably an Eclipse archive } if (!m2Version.startsWith(":")) { @@ -288,15 +343,20 @@ public class Repackage { m2Version = m2Version.substring(1); mergeProps.put(ManifestConstants.BUNDLE_VERSION.toString(), m2Version); - String artifactsStr = mergeProps.getProperty(ManifestConstants.SLC_ORIGIN_M2_MERGE.toString()); - String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString()) - ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString()) + String artifactsStr = mergeProps.getProperty(ManifestConstants.ARGEO_ORIGIN_M2_MERGE.toString()); + if (artifactsStr == null) + throw new IllegalArgumentException( + mergeBnd + ": " + ManifestConstants.ARGEO_ORIGIN_M2_MERGE + " must be set"); + + String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString()) + ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString()) : null; String bundleSymbolicName = mergeProps.getProperty(ManifestConstants.BUNDLE_SYMBOLICNAME.toString()); if (bundleSymbolicName == null) throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd); CategoryNameVersion nameVersion = new M2Artifact(category + ":" + bundleSymbolicName + ":" + m2Version); + Path targetBundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch()); String[] artifacts = artifactsStr.split(","); @@ -308,7 +368,7 @@ public class Repackage { if (artifact.getVersion() == null) artifact.setVersion(m2Version); URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact); - Path downloaded = download(url, originBase, artifact); + Path downloaded = downloadMaven(url, artifact); JarEntry entry; try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) { entries: while ((entry = jarIn.getNextJarEntry()) != null) { @@ -345,17 +405,16 @@ public class Repackage { try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) { out.write("\n".getBytes()); jarIn.transferTo(out); - if (logger.isLoggable(DEBUG)) - logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName()); + logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName()); } } else if (entry.getName().startsWith("org/apache/batik/")) { - logger.log(Level.WARNING, "Skip " + entry.getName()); + logger.log(TRACE, "Skip " + entry.getName()); continue entries; } else { throw new IllegalStateException("File " + target + " from " + artifact + " already exists"); } } - logger.log(Level.TRACE, () -> "Copied " + target); + logger.log(TRACE, () -> "Copied " + target); } } @@ -371,8 +430,7 @@ public class Repackage { OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) { out.write("\n".getBytes()); in.transferTo(out); - if (logger.isLoggable(DEBUG)) - logger.log(DEBUG, "Appended " + p); + logger.log(DEBUG, "Appended " + p); } } } @@ -397,8 +455,6 @@ public class Repackage { && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) continue keys;// hack for very old classes entries.put(key.toString(), value.toString()); - // logger.log(DEBUG, () -> key + "=" + value); - } } catch (Exception e) { throw new RuntimeException("Cannot process " + mergeBnd, e); @@ -419,18 +475,18 @@ public class Repackage { } /** Generate MANIFEST using BND. */ - protected Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, M2Artifact artifact) { + Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, M2Artifact artifact) { try { Map additionalEntries = new TreeMap<>(); boolean doNotModify = Boolean.parseBoolean(fileProps - .getOrDefault(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString()); + .getOrDefault(ManifestConstants.ARGEO_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString()); // we always force the symbolic name if (doNotModify) { fileEntries: for (Object key : fileProps.keySet()) { - if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key)) + if (ManifestConstants.ARGEO_ORIGIN_M2.toString().equals(key)) continue fileEntries; String value = fileProps.getProperty(key.toString()); additionalEntries.put(key.toString(), value); @@ -467,15 +523,13 @@ public class Repackage { } if ("Require-Capability".equals(key.toString()) && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) - continue keys;// hack for very old classes + continue keys;// !! hack for very old classes additionalEntries.put(key.toString(), value.toString()); - // logger.log(DEBUG, () -> key + "=" + value); - } } } Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries); - logger.log(Level.DEBUG, () -> "Processed " + downloaded); + logger.log(DEBUG, () -> "Processed " + downloaded); return targetBundleDir; } catch (Exception e) { throw new RuntimeException("Cannot BND process " + downloaded, e); @@ -484,31 +538,27 @@ public class Repackage { } /** Download and integrates sources for a single Maven artifact. */ - protected void downloadAndProcessM2Sources(String repoStr, M2Artifact artifact, Path targetBundleDir) - throws IOException { - if (!includeSources) - return; - M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources"); - URL sourcesUrl = M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact); - Path sourcesDownloaded = download(sourcesUrl, originBase, artifact, true); - processM2SourceJar(sourcesDownloaded, targetBundleDir); - logger.log(Level.TRACE, () -> "Processed source " + sourcesDownloaded); + void downloadAndProcessM2Sources(String repoStr, M2Artifact artifact, Path targetBundleDir) throws IOException { + try { + M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources"); + URL sourcesUrl = M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact); + Path sourcesDownloaded = downloadMaven(sourcesUrl, artifact, true); + processM2SourceJar(sourcesDownloaded, targetBundleDir); + logger.log(TRACE, () -> "Processed source " + sourcesDownloaded); + } catch (Exception e) { + logger.log(ERROR, () -> "Cannot download source for " + artifact); + } } /** Integrate sources from a downloaded jar file. */ - protected void processM2SourceJar(Path file, Path targetBundleDir) throws IOException { + void processM2SourceJar(Path file, Path targetBundleDir) throws IOException { try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { - Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src"); - - // TODO make it less dangerous? - if (Files.exists(targetSourceDir)) { -// deleteDirectory(targetSourceDir); - } else { - Files.createDirectories(targetSourceDir); - } + Path targetSourceDir = sourceBundles + ? targetBundleDir.getParent().resolve(targetBundleDir.toString() + ".src") + : targetBundleDir.resolve("OSGI-OPT/src"); - // copy entries + Files.createDirectories(targetSourceDir); JarEntry entry; entries: while ((entry = jarIn.getNextJarEntry()) != null) { if (entry.isDirectory()) @@ -523,9 +573,9 @@ public class Repackage { Files.createDirectories(target.getParent()); if (!Files.exists(target)) { Files.copy(jarIn, target); - logger.log(Level.TRACE, () -> "Copied source " + target); + logger.log(TRACE, () -> "Copied source " + target); } else { - logger.log(Level.WARNING, () -> target + " already exists, skipping..."); + logger.log(TRACE, () -> target + " already exists, skipping..."); } } } @@ -533,31 +583,29 @@ public class Repackage { } /** Download a Maven artifact. */ - protected Path download(URL url, Path dir, M2Artifact artifact) throws IOException { - return download(url, dir, artifact, false); + Path downloadMaven(URL url, M2Artifact artifact) throws IOException { + return downloadMaven(url, artifact, false); } /** Download a Maven artifact. */ - protected Path download(URL url, Path dir, M2Artifact artifact, boolean sources) throws IOException { - return download(url, dir, artifact.getGroupId() + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() - + (sources ? "-sources" : "") + ".jar"); + Path downloadMaven(URL url, M2Artifact artifact, boolean sources) throws IOException { + return download(url, mavenBase, artifact.getGroupId().replace(".", "/") // + + '/' + artifact.getArtifactId() + '/' + artifact.getVersion() // + + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() + (sources ? "-sources" : "") + ".jar"); } /* * ECLIPSE ORIGIN */ - /** Process an archive in Eclipse format. */ - public void processEclipseArchive(Path duDir) { + void processEclipseArchive(Path duDir) { try { Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent()); - // String category = categoryRelativePath.getFileName().toString(); Path targetCategoryBase = a2Base.resolve(categoryRelativePath); Files.createDirectories(targetCategoryBase); // first delete all directories from previous builds - for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) { + for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) deleteDirectory(dir); - } Files.createDirectories(originBase); @@ -566,14 +614,14 @@ public class Repackage { try (InputStream in = Files.newInputStream(commonBnd)) { commonProps.load(in); } - String url = commonProps.getProperty(ManifestConstants.SLC_ORIGIN_URI.toString()); + String url = commonProps.getProperty(ManifestConstants.ARGEO_ORIGIN_URI.toString()); if (url == null) { url = uris.getProperty(duDir.getFileName().toString()); if (url == null) throw new IllegalStateException("No url available for " + duDir); - commonProps.put(ManifestConstants.SLC_ORIGIN_URI.toString(), url); + commonProps.put(ManifestConstants.ARGEO_ORIGIN_URI.toString(), url); } - Path downloaded = tryDownload(url, originBase); + Path downloaded = tryDownloadArchive(url, originBase); FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null); @@ -609,20 +657,19 @@ public class Repackage { if (includeMatcher.matches(file)) { for (PathMatcher excludeMatcher : excludeMatchers) { if (excludeMatcher.matches(file)) { - logger.log(Level.WARNING, "Skipping excluded " + file); + logger.log(TRACE, "Skipping excluded " + file); return FileVisitResult.CONTINUE; } } - if (includeSources && file.getFileName().toString().contains(".source_")) { + if (file.getFileName().toString().contains(".source_")) { processEclipseSourceJar(file, targetCategoryBase); - logger.log(Level.DEBUG, () -> "Processed source " + file); - + logger.log(DEBUG, () -> "Processed source " + file); } else { Map map = new HashMap<>(); for (Object key : commonProps.keySet()) map.put(key.toString(), commonProps.getProperty(key.toString())); processBundleJar(file, targetCategoryBase, map); - logger.log(Level.DEBUG, () -> "Processed " + file); + logger.log(DEBUG, () -> "Processed " + file); } break includeMatchers; } @@ -631,8 +678,8 @@ public class Repackage { } }); - DirectoryStream dirs = Files.newDirectoryStream(targetCategoryBase, - (p) -> Files.isDirectory(p) && p.getFileName().toString().indexOf('.') >= 0); + DirectoryStream dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p) + && p.getFileName().toString().indexOf('.') >= 0 && !p.getFileName().toString().endsWith(".src")); for (Path dir : dirs) { createJar(dir); } @@ -643,28 +690,24 @@ public class Repackage { } /** Process sources in Eclipse format. */ - protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException { + void processEclipseSourceJar(Path file, Path targetBase) throws IOException { try { Path targetBundleDir; try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { Manifest manifest = jarIn.getManifest(); - String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";"); + String[] relatedBundle = manifest.getMainAttributes().getValue(ECLIPSE_SOURCE_BUNDLE.toString()) + .split(";"); String version = relatedBundle[1].substring("version=\"".length()); version = version.substring(0, version.length() - 1); NameVersion nameVersion = new NameVersion(relatedBundle[0], version); targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch()); - Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src"); + Path targetSourceDir = sourceBundles + ? targetBundleDir.getParent().resolve(targetBundleDir.toString() + ".src") + : targetBundleDir.resolve("OSGI-OPT/src"); - // TODO make it less dangerous? - if (Files.exists(targetSourceDir)) { -// deleteDirectory(targetSourceDir); - } else { - Files.createDirectories(targetSourceDir); - } - - // copy entries + Files.createDirectories(targetSourceDir); JarEntry entry; entries: while ((entry = jarIn.getNextJarEntry()) != null) { if (entry.isDirectory()) @@ -674,22 +717,19 @@ public class Repackage { Path target = targetSourceDir.resolve(entry.getName()); Files.createDirectories(target.getParent()); Files.copy(jarIn, target); - logger.log(Level.TRACE, () -> "Copied source " + target); + logger.log(TRACE, () -> "Copied source " + target); } - - // copy MANIFEST } } catch (IOException e) { throw new IllegalStateException("Cannot process " + file, e); } - } /* * COMMON PROCESSING */ /** Normalise a bundle. */ - protected Path processBundleJar(Path file, Path targetBase, Map entries) throws IOException { + Path processBundleJar(Path file, Path targetBase, Map entries) throws IOException { NameVersion nameVersion; Path targetBundleDir; try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { @@ -701,7 +741,6 @@ public class Repackage { String rawSourceSymbolicName = manifest.getMainAttributes() .getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString()); if (rawSourceSymbolicName != null) { - // make sure there is no directive String[] arr = rawSourceSymbolicName.split(";"); for (int i = 1; i < arr.length; i++) { @@ -710,7 +749,6 @@ public class Repackage { logger.log(DEBUG, file.getFileName() + " is a singleton"); } } - // remove problematic entries in MANIFEST manifest.getEntries().clear(); @@ -722,7 +760,7 @@ public class Repackage { } else { nameVersion = nameVersionFromManifest(manifest); if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) { - logger.log(Level.WARNING, + logger.log(WARNING, "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion); entries.put(BUNDLE_VERSION.toString(), ourVersion); } @@ -770,9 +808,6 @@ public class Repackage { if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll") || entry.getName().endsWith(".jnilib"))) { Path categoryDir = targetBundleDir.getParent(); -// String[] segments = categoryDir.getFileName().toString().split("\\."); -// String arch = segments[segments.length - 1]; -// String os = segments[segments.length - 2]; boolean copyDll = false; Path targetDll = categoryDir.resolve(targetBundleDir.relativize(target)); if (nameVersion.getName().equals("com.sun.jna")) { @@ -795,7 +830,7 @@ public class Repackage { } Files.delete(target); } - logger.log(Level.TRACE, () -> "Copied " + target); + logger.log(TRACE, () -> "Copied " + target); } // copy MANIFEST @@ -807,24 +842,57 @@ public class Repackage { entries.get(BUNDLE_SYMBOLICNAME.toString()) + ";singleton:=true"); } + // Final MANIFEST decisions + // This also where we check the original OSGi metadata and compare with our + // changes for (String key : entries.keySet()) { String value = entries.get(key); - Object previousValue = manifest.getMainAttributes().putValue(key, value); - if (previousValue != null && !previousValue.equals(value)) { - if (ManifestConstants.IMPORT_PACKAGE.toString().equals(key) - || ManifestConstants.EXPORT_PACKAGE.toString().equals(key)) - logger.log(Level.TRACE, file.getFileName() + ": " + key + " was modified"); + String previousValue = manifest.getMainAttributes().getValue(key); + boolean wasDifferent = previousValue != null && !previousValue.equals(value); + boolean keepPrevious = false; + if (wasDifferent) { + if (SPDX_LICENSE_IDENTIFIER.toString().equals(key) && previousValue != null) + keepPrevious = true; + else if (BUNDLE_VERSION.toString().equals(key) && wasDifferent) + if (previousValue.equals(value + ".0")) // typically a Maven first release + keepPrevious = true; + + if (keepPrevious) { + if (logger.isLoggable(TRACE)) + logger.log(TRACE, file.getFileName() + ": " + key + " was NOT modified, value kept is " + + previousValue + ", not overriden with " + value); + value = previousValue; + } + } + + manifest.getMainAttributes().putValue(key, value); + if (wasDifferent && !keepPrevious) { + if (IMPORT_PACKAGE.toString().equals(key) || EXPORT_PACKAGE.toString().equals(key)) + logger.log(TRACE, () -> file.getFileName() + ": " + key + " was modified"); else - logger.log(Level.WARNING, file.getFileName() + ": " + key + " was " + previousValue + logger.log(WARNING, file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value); } - // hack to remove unresolvable + // !! hack to remove unresolvable if (key.equals("Provide-Capability") || key.equals("Require-Capability")) if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) { manifest.getMainAttributes().remove(key); } } + + // last checks + String spdxLicenceId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString()); + String bundleLicense = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString()); + if (spdxLicenceId == null) { + logger.log(ERROR, file.getFileName() + ": " + SPDX_LICENSE_IDENTIFIER + " not available, " + + BUNDLE_LICENSE + " is " + bundleLicense); + } else { + if (!licensesUsed.containsKey(spdxLicenceId)) + licensesUsed.put(spdxLicenceId, new TreeSet<>()); + licensesUsed.get(spdxLicenceId).add(nameVersion.toString()); + } + try (OutputStream out = Files.newOutputStream(manifestPath)) { manifest.write(out); } @@ -835,9 +903,8 @@ public class Repackage { /* * UTILITIES */ - /** Recursively deletes a directory. */ - private static void deleteDirectory(Path path) throws IOException { + static void deleteDirectory(Path path) throws IOException { if (!Files.exists(path)) return; Files.walkFileTree(path, new SimpleFileVisitor() { @@ -846,19 +913,19 @@ public class Repackage { if (e != null) throw e; Files.delete(directory); - return FileVisitResult.CONTINUE; + return CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Files.delete(file); - return FileVisitResult.CONTINUE; + return CONTINUE; } }); } /** Extract name/version from a MANIFEST. */ - protected NameVersion nameVersionFromManifest(Manifest manifest) { + NameVersion nameVersionFromManifest(Manifest manifest) { Attributes attrs = manifest.getMainAttributes(); // symbolic name String symbolicName = attrs.getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString()); @@ -872,7 +939,7 @@ public class Repackage { } /** Try to download from an URI. */ - protected Path tryDownload(String uri, Path dir) throws IOException { + Path tryDownloadArchive(String uri, Path dir) throws IOException { // find mirror List urlBases = null; String uriPrefix = null; @@ -887,7 +954,7 @@ public class Repackage { } if (urlBases == null) try { - return download(new URL(uri), dir); + return downloadArchive(new URL(uri), dir); } catch (FileNotFoundException e) { throw new FileNotFoundException("Cannot find " + uri); } @@ -897,30 +964,36 @@ public class Repackage { String relativePath = uri.substring(uriPrefix.length()); URL url = new URL(urlBase + relativePath); try { - return download(url, dir); + return downloadArchive(url, dir); } catch (FileNotFoundException e) { - logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror"); + logger.log(WARNING, "Cannot download " + url + ", trying another mirror"); } } throw new FileNotFoundException("Cannot find " + uri); } - /** Effectively download. */ - protected Path download(URL url, Path dir) throws IOException { + /** + * Effectively download. Synchronised in order to avoid downloading twice in + * parallel. + */ + synchronized Path downloadArchive(URL url, Path dir) throws IOException { return download(url, dir, (String) null); } /** Effectively download. */ - protected Path download(URL url, Path dir, String name) throws IOException { + Path download(URL url, Path dir, String name) throws IOException { Path dest; if (name == null) { - name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1); + // We use also use parent directory in case the archive itself has a fixed name + String[] segments = url.getPath().split("/"); + name = segments.length > 1 ? segments[segments.length - 2] + '-' + segments[segments.length - 1] + : segments[segments.length - 1]; } dest = dir.resolve(name); if (Files.exists(dest)) { - logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again"); + logger.log(TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again"); return dest; } else { Files.createDirectories(dest.getParent()); @@ -928,13 +1001,13 @@ public class Repackage { try (InputStream in = url.openStream()) { Files.copy(in, dest); - logger.log(Level.DEBUG, () -> "Downloaded " + dest + " from " + url); + logger.log(DEBUG, () -> "Downloaded " + dest + " from " + url); } return dest; } /** Create a JAR file from a directory. */ - protected Path createJar(Path bundleDir) throws IOException { + Path createJar(Path bundleDir) throws IOException { // Create the jar Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar"); Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF"); @@ -950,7 +1023,8 @@ public class Repackage { public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (file.getFileName().toString().equals("MANIFEST.MF")) return super.visitFile(file, attrs); - JarEntry entry = new JarEntry(bundleDir.relativize(file).toString()); + JarEntry entry = new JarEntry( + bundleDir.relativize(file).toString().replace(File.separatorChar, '/')); jarOut.putNextEntry(entry); Files.copy(file, jarOut); return super.visitFile(file, attrs); @@ -959,25 +1033,101 @@ public class Repackage { }); } deleteDirectory(bundleDir); + + if (sourceBundles) { + Path bundleCategoryDir = bundleDir.getParent(); + Path sourceDir = bundleCategoryDir.resolve(bundleDir.toString() + ".src"); + if (!Files.exists(sourceDir)) { + logger.log(WARNING, sourceDir + " does not exist, skipping..."); + return jarPath; + + } + + Path relPath = a2Base.relativize(bundleCategoryDir); + Path srcCategoryDir = a2SrcBase.resolve(relPath); + Path srcJarP = srcCategoryDir.resolve(sourceDir.getFileName() + ".jar"); + Files.createDirectories(srcJarP.getParent()); + + String bundleSymbolicName = manifest.getMainAttributes().getValue("Bundle-SymbolicName").toString(); + // in case there are additional directives + bundleSymbolicName = bundleSymbolicName.split(";")[0]; + Manifest srcManifest = new Manifest(); + srcManifest.getMainAttributes().put(MANIFEST_VERSION, "1.0"); + srcManifest.getMainAttributes().putValue(BUNDLE_SYMBOLICNAME.toString(), bundleSymbolicName + ".src"); + srcManifest.getMainAttributes().putValue(BUNDLE_VERSION.toString(), + manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()).toString()); + srcManifest.getMainAttributes().putValue(ECLIPSE_SOURCE_BUNDLE.toString(), bundleSymbolicName + + ";version=\"" + manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString())); + + try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) { + srcJarOut.setLevel(Deflater.BEST_COMPRESSION); + Files.walkFileTree(sourceDir, new SimpleFileVisitor() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.getFileName().toString().equals("MANIFEST.MF")) + return super.visitFile(file, attrs); + JarEntry entry = new JarEntry( + sourceDir.relativize(file).toString().replace(File.separatorChar, '/')); + srcJarOut.putNextEntry(entry); + Files.copy(file, srcJarOut); + return super.visitFile(file, attrs); + } + + }); + } + deleteDirectory(sourceDir); + } + return jarPath; } + /** MANIFEST headers. */ enum ManifestConstants { // OSGi + /** OSGi bundle symbolic name. */ BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), // + /** OSGi bundle version. */ BUNDLE_VERSION("Bundle-Version"), // + /** OSGi bundle license. */ BUNDLE_LICENSE("Bundle-License"), // + /** OSGi exported packages list. */ EXPORT_PACKAGE("Export-Package"), // + /** OSGi imported packages list. */ IMPORT_PACKAGE("Import-Package"), // - // JAVA + // Java + /** Java module name. */ AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), // - // SLC - SLC_CATEGORY("SLC-Category"), // - SLC_ORIGIN_M2("SLC-Origin-M2"), // - SLC_ORIGIN_M2_MERGE("SLC-Origin-M2-Merge"), // - SLC_ORIGIN_M2_REPO("SLC-Origin-M2-Repo"), // - SLC_ORIGIN_MANIFEST_NOT_MODIFIED("SLC-Origin-ManifestNotModified"), // - SLC_ORIGIN_URI("SLC-Origin-URI"),// + // Eclipse + /** Eclipse source bundle. */ + ECLIPSE_SOURCE_BUNDLE("Eclipse-SourceBundle"), // + // SPDX + /** + * SPDX license identifier. + * + * @see https://spdx.org/licenses/ + */ + SPDX_LICENSE_IDENTIFIER("SPDX-License-Identifier"), // + // Argeo Origin + /** + * Maven coordinates of the origin, possibly partial when using common.bnd or + * merge.bnd. + */ + ARGEO_ORIGIN_M2("Argeo-Origin-M2"), // + /** List of Maven coordinates to merge. */ + ARGEO_ORIGIN_M2_MERGE("Argeo-Origin-M2-Merge"), // + /** Maven repository if not the default one. */ + ARGEO_ORIGIN_M2_REPO("Argeo-Origin-M2-Repo"), // + /** + * Do not perform BND analysis of the origin component. Typically IMport_package + * and Export-Package will be kept untouched. + */ + ARGEO_ORIGIN_MANIFEST_NOT_MODIFIED("Argeo-Origin-ManifestNotModified"), // + /** + * Origin (non-Maven) URI of the component. It may be anything (jar, archive, + * etc.). + */ + ARGEO_ORIGIN_URI("Argeo-Origin-URI"), // ; final String value; @@ -990,9 +1140,7 @@ public class Repackage { public String toString() { return value; } - } - } /** Simple representation of an M2 artifact. */ @@ -1084,6 +1232,7 @@ class M2ConventionsUtils { } } +/** Combination of a category, a name and a version. */ class CategoryNameVersion extends NameVersion { private String category; @@ -1115,6 +1264,7 @@ class CategoryNameVersion extends NameVersion { } +/** Combination of a name and a version. */ class NameVersion implements Comparable { private String name; private String version;