X-Git-Url: https://git.argeo.org/?a=blobdiff_plain;ds=sidebyside;f=src%2Forg%2Fargeo%2Fbuild%2FRepackage.java;h=05ff7b177616cd4e01118d3c32e15aea9a717b4a;hb=1d38b1c8155f35271cf20217167eb6901df25272;hp=35413fe336df06dee5008f9fba2b827eba98e8a0;hpb=3714839e09d67535491b15ef560b2cb3fdf482db;p=cc0%2Fargeo-build.git diff --git a/src/org/argeo/build/Repackage.java b/src/org/argeo/build/Repackage.java index 35413fe..05ff7b1 100644 --- a/src/org/argeo/build/Repackage.java +++ b/src/org/argeo/build/Repackage.java @@ -1,7 +1,11 @@ package org.argeo.build; import static java.lang.System.Logger.Level.DEBUG; +import static java.lang.System.Logger.Level.ERROR; +import static java.lang.System.Logger.Level.INFO; +import static java.lang.System.Logger.Level.TRACE; import static java.lang.System.Logger.Level.WARNING; +import static java.nio.file.FileVisitResult.CONTINUE; import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_SYMBOLICNAME; import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_VERSION; import static org.argeo.build.Repackage.ManifestConstants.EXPORT_PACKAGE; @@ -14,7 +18,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.lang.System.Logger; -import java.lang.System.Logger.Level; import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -48,12 +51,20 @@ import java.util.zip.Deflater; import aQute.bnd.osgi.Analyzer; import aQute.bnd.osgi.Jar; -/** The central class for A2 packaging. */ +/** + * Simple tool repackaging existing jar files into OSGi bundles in an A2 + * repository. + */ public class Repackage { private final static Logger logger = System.getLogger(Repackage.class.getName()); - private final static String ENV_BUILD_SOURCE_BUNDLES = "BUILD_SOURCE_BUNDLES"; + /** + * Environment variable on whether sources should be packaged separately or + * integrated in the bundles. + */ + private final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES"; + /** Whethere repackaging should run in parallel or sequentially. */ private final static boolean parallel = true; /** Main entry point. */ @@ -80,28 +91,41 @@ public class Repackage { private final static String COMMON_BND = "common.bnd"; private final static String MERGE_BND = "merge.bnd"; + /** Directory where to download archives */ private Path originBase; + /** Directory where to download Maven artifacts */ + private Path mavenBase; + + /** A2 repository base for binary bundles */ private Path a2Base; + /** A2 repository base for source bundles */ + private Path a2SrcBase; + /** A2 base for native components */ private Path a2LibBase; + /** Location of the descriptors driving the packaging */ private Path descriptorsBase; - + /** URIs of archives to download */ private Properties uris = new Properties(); - - /** key is URI prefix, value list of base URLs */ + /** Mirrors for archive download. Key is URI prefix, value list of base URLs */ private Map> mirrors = new HashMap>(); + /** Whether sources should be packaged separately */ private final boolean sourceBundles; + /** Constructor initialises the various variables */ public Repackage(Path a2Base, Path descriptorsBase) { - sourceBundles = Boolean.parseBoolean(System.getenv(ENV_BUILD_SOURCE_BUNDLES)); + sourceBundles = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES)); if (sourceBundles) - logger.log(Level.INFO, "Sources will be packaged separately"); + logger.log(INFO, "Sources will be packaged separately"); Objects.requireNonNull(a2Base); Objects.requireNonNull(descriptorsBase); this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/build/origin"); + this.mavenBase = Paths.get(System.getProperty("user.home"), ".m2", "repository"); + // TODO define and use a build base this.a2Base = a2Base; + this.a2SrcBase = a2Base.getParent().resolve(a2Base.getFileName() + ".src"); this.a2LibBase = a2Base.resolve("lib"); this.descriptorsBase = descriptorsBase; if (!Files.exists(this.descriptorsBase)) @@ -132,14 +156,12 @@ public class Repackage { it.remove(); } } - mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors); } /* * MAVEN ORIGIN */ - /** Process a whole category/group id. */ public void processCategory(Path categoryRelativePath) { try { @@ -167,7 +189,6 @@ public class Repackage { /** Process a standalone Maven artifact. */ public void processSingleM2ArtifactDistributionUnit(Path bndFile) { try { -// String category = bndFile.getParent().getFileName().toString(); Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent()); Path targetCategoryBase = a2Base.resolve(categoryRelativePath); @@ -191,10 +212,9 @@ public class Repackage { throw new IllegalArgumentException("No M2 coordinates available for " + bndFile); M2Artifact artifact = new M2Artifact(m2Coordinates); URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact); - Path downloaded = downloadMaven(url, originBase, artifact); + Path downloaded = downloadMaven(url, artifact); Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact); - downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir); createJar(targetBundleDir); @@ -206,21 +226,17 @@ public class Repackage { /** Process multiple Maven artifacts. */ public void processM2BasedDistributionUnit(Path duDir) { try { - // String category = duDir.getParent().getFileName().toString(); Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent()); Path targetCategoryBase = a2Base.resolve(categoryRelativePath); - // merge Path mergeBnd = duDir.resolve(MERGE_BND); - if (Files.exists(mergeBnd)) { + if (Files.exists(mergeBnd)) // merge mergeM2Artifacts(mergeBnd); -// return; - } Path commonBnd = duDir.resolve(COMMON_BND); - if (!Files.exists(commonBnd)) { + if (!Files.exists(commonBnd)) return; - } + Properties commonProps = new Properties(); try (InputStream in = Files.newInputStream(commonBnd)) { commonProps.load(in); @@ -228,7 +244,7 @@ public class Repackage { String m2Version = commonProps.getProperty(SLC_ORIGIN_M2.toString()); if (m2Version == null) { - logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); + logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); return;// ignore, this is probably an Eclipse archive } if (!m2Version.startsWith(":")) { @@ -246,7 +262,6 @@ public class Repackage { } String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString()); M2Artifact artifact = new M2Artifact(m2Coordinates); - artifact.setVersion(m2Version); // prepare manifest entries @@ -259,7 +274,7 @@ public class Repackage { String value = fileProps.getProperty(key.toString()); Object previousValue = mergeProps.put(key.toString(), value); if (previousValue != null) { - logger.log(Level.WARNING, + logger.log(WARNING, commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value); } } @@ -277,20 +292,15 @@ public class Repackage { // download URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact); - Path downloaded = downloadMaven(url, originBase, artifact); + Path downloaded = downloadMaven(url, artifact); Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact); -// logger.log(Level.DEBUG, () -> "Processed " + downloaded); - - // sources downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir); - createJar(targetBundleDir); } } catch (IOException e) { throw new RuntimeException("Cannot process " + duDir, e); } - } /** Merge multiple Maven artifacts. */ @@ -304,10 +314,9 @@ public class Repackage { mergeProps.load(in); } - // Version String m2Version = mergeProps.getProperty(SLC_ORIGIN_M2.toString()); if (m2Version == null) { - logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); + logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); return;// ignore, this is probably an Eclipse archive } if (!m2Version.startsWith(":")) { @@ -329,6 +338,7 @@ public class Repackage { if (bundleSymbolicName == null) throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd); CategoryNameVersion nameVersion = new M2Artifact(category + ":" + bundleSymbolicName + ":" + m2Version); + Path targetBundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch()); String[] artifacts = artifactsStr.split(","); @@ -340,7 +350,7 @@ public class Repackage { if (artifact.getVersion() == null) artifact.setVersion(m2Version); URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact); - Path downloaded = downloadMaven(url, originBase, artifact); + Path downloaded = downloadMaven(url, artifact); JarEntry entry; try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) { entries: while ((entry = jarIn.getNextJarEntry()) != null) { @@ -377,16 +387,16 @@ public class Repackage { try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) { out.write("\n".getBytes()); jarIn.transferTo(out); - logger.log(Level.WARNING, artifact.getArtifactId() + " - Appended " + entry.getName()); + logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName()); } } else if (entry.getName().startsWith("org/apache/batik/")) { - logger.log(Level.WARNING, "Skip " + entry.getName()); + logger.log(TRACE, "Skip " + entry.getName()); continue entries; } else { throw new IllegalStateException("File " + target + " from " + artifact + " already exists"); } } - logger.log(Level.TRACE, () -> "Copied " + target); + logger.log(TRACE, () -> "Copied " + target); } } @@ -402,7 +412,7 @@ public class Repackage { OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) { out.write("\n".getBytes()); in.transferTo(out); - logger.log(Level.WARNING, "Appended " + p); + logger.log(DEBUG, "Appended " + p); } } } @@ -427,8 +437,6 @@ public class Repackage { && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) continue keys;// hack for very old classes entries.put(key.toString(), value.toString()); - // logger.log(DEBUG, () -> key + "=" + value); - } } catch (Exception e) { throw new RuntimeException("Cannot process " + mergeBnd, e); @@ -505,7 +513,7 @@ public class Repackage { } } Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries); - logger.log(Level.DEBUG, () -> "Processed " + downloaded); + logger.log(DEBUG, () -> "Processed " + downloaded); return targetBundleDir; } catch (Exception e) { throw new RuntimeException("Cannot BND process " + downloaded, e); @@ -516,16 +524,14 @@ public class Repackage { /** Download and integrates sources for a single Maven artifact. */ protected void downloadAndProcessM2Sources(String repoStr, M2Artifact artifact, Path targetBundleDir) throws IOException { -// if (sourceBundles) -// return; try { M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources"); URL sourcesUrl = M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact); - Path sourcesDownloaded = downloadMaven(sourcesUrl, originBase, artifact, true); + Path sourcesDownloaded = downloadMaven(sourcesUrl, artifact, true); processM2SourceJar(sourcesDownloaded, targetBundleDir); - logger.log(Level.TRACE, () -> "Processed source " + sourcesDownloaded); + logger.log(TRACE, () -> "Processed source " + sourcesDownloaded); } catch (Exception e) { - logger.log(Level.ERROR, () -> "Cannot download source for " + artifact); + logger.log(ERROR, () -> "Cannot download source for " + artifact); } } @@ -537,14 +543,7 @@ public class Repackage { ? targetBundleDir.getParent().resolve(targetBundleDir.toString() + ".src") : targetBundleDir.resolve("OSGI-OPT/src"); - // TODO make it less dangerous? - if (Files.exists(targetSourceDir)) { -// deleteDirectory(targetSourceDir); - } else { - Files.createDirectories(targetSourceDir); - } - - // copy entries + Files.createDirectories(targetSourceDir); JarEntry entry; entries: while ((entry = jarIn.getNextJarEntry()) != null) { if (entry.isDirectory()) @@ -559,9 +558,9 @@ public class Repackage { Files.createDirectories(target.getParent()); if (!Files.exists(target)) { Files.copy(jarIn, target); - logger.log(Level.TRACE, () -> "Copied source " + target); + logger.log(TRACE, () -> "Copied source " + target); } else { - logger.log(Level.WARNING, () -> target + " already exists, skipping..."); + logger.log(TRACE, () -> target + " already exists, skipping..."); } } } @@ -569,20 +568,20 @@ public class Repackage { } /** Download a Maven artifact. */ - protected Path downloadMaven(URL url, Path dir, M2Artifact artifact) throws IOException { - return downloadMaven(url, dir, artifact, false); + protected Path downloadMaven(URL url, M2Artifact artifact) throws IOException { + return downloadMaven(url, artifact, false); } /** Download a Maven artifact. */ - protected Path downloadMaven(URL url, Path dir, M2Artifact artifact, boolean sources) throws IOException { - return download(url, dir, artifact.getGroupId() + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() - + (sources ? "-sources" : "") + ".jar"); + protected Path downloadMaven(URL url, M2Artifact artifact, boolean sources) throws IOException { + return download(url, mavenBase, artifact.getGroupId().replace(".", "/") // + + '/' + artifact.getArtifactId() + '/' + artifact.getVersion() // + + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() + (sources ? "-sources" : "") + ".jar"); } /* * ECLIPSE ORIGIN */ - /** Process an archive in Eclipse format. */ public void processEclipseArchive(Path duDir) { try { @@ -590,9 +589,8 @@ public class Repackage { Path targetCategoryBase = a2Base.resolve(categoryRelativePath); Files.createDirectories(targetCategoryBase); // first delete all directories from previous builds - for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) { + for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) deleteDirectory(dir); - } Files.createDirectories(originBase); @@ -644,22 +642,19 @@ public class Repackage { if (includeMatcher.matches(file)) { for (PathMatcher excludeMatcher : excludeMatchers) { if (excludeMatcher.matches(file)) { - logger.log(Level.TRACE, "Skipping excluded " + file); + logger.log(TRACE, "Skipping excluded " + file); return FileVisitResult.CONTINUE; } } if (file.getFileName().toString().contains(".source_")) { -// if (!sourceBundles) { processEclipseSourceJar(file, targetCategoryBase); - logger.log(Level.DEBUG, () -> "Processed source " + file); -// } - + logger.log(DEBUG, () -> "Processed source " + file); } else { Map map = new HashMap<>(); for (Object key : commonProps.keySet()) map.put(key.toString(), commonProps.getProperty(key.toString())); processBundleJar(file, targetCategoryBase, map); - logger.log(Level.DEBUG, () -> "Processed " + file); + logger.log(DEBUG, () -> "Processed " + file); } break includeMatchers; } @@ -696,14 +691,7 @@ public class Repackage { ? targetBundleDir.getParent().resolve(targetBundleDir.toString() + ".src") : targetBundleDir.resolve("OSGI-OPT/src"); - // TODO make it less dangerous? - if (Files.exists(targetSourceDir)) { -// deleteDirectory(targetSourceDir); - } else { - Files.createDirectories(targetSourceDir); - } - - // copy entries + Files.createDirectories(targetSourceDir); JarEntry entry; entries: while ((entry = jarIn.getNextJarEntry()) != null) { if (entry.isDirectory()) @@ -713,15 +701,12 @@ public class Repackage { Path target = targetSourceDir.resolve(entry.getName()); Files.createDirectories(target.getParent()); Files.copy(jarIn, target); - logger.log(Level.TRACE, () -> "Copied source " + target); + logger.log(TRACE, () -> "Copied source " + target); } - - // copy MANIFEST } } catch (IOException e) { throw new IllegalStateException("Cannot process " + file, e); } - } /* @@ -740,7 +725,6 @@ public class Repackage { String rawSourceSymbolicName = manifest.getMainAttributes() .getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString()); if (rawSourceSymbolicName != null) { - // make sure there is no directive String[] arr = rawSourceSymbolicName.split(";"); for (int i = 1; i < arr.length; i++) { @@ -749,7 +733,6 @@ public class Repackage { logger.log(DEBUG, file.getFileName() + " is a singleton"); } } - // remove problematic entries in MANIFEST manifest.getEntries().clear(); @@ -761,7 +744,7 @@ public class Repackage { } else { nameVersion = nameVersionFromManifest(manifest); if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) { - logger.log(Level.WARNING, + logger.log(WARNING, "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion); entries.put(BUNDLE_VERSION.toString(), ourVersion); } @@ -809,9 +792,6 @@ public class Repackage { if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll") || entry.getName().endsWith(".jnilib"))) { Path categoryDir = targetBundleDir.getParent(); -// String[] segments = categoryDir.getFileName().toString().split("\\."); -// String arch = segments[segments.length - 1]; -// String os = segments[segments.length - 2]; boolean copyDll = false; Path targetDll = categoryDir.resolve(targetBundleDir.relativize(target)); if (nameVersion.getName().equals("com.sun.jna")) { @@ -834,7 +814,7 @@ public class Repackage { } Files.delete(target); } - logger.log(Level.TRACE, () -> "Copied " + target); + logger.log(TRACE, () -> "Copied " + target); } // copy MANIFEST @@ -853,9 +833,9 @@ public class Repackage { if (ManifestConstants.IMPORT_PACKAGE.toString().equals(key) || ManifestConstants.EXPORT_PACKAGE.toString().equals(key) || ManifestConstants.BUNDLE_LICENSE.toString().equals(key)) - logger.log(Level.TRACE, file.getFileName() + ": " + key + " was modified"); + logger.log(TRACE, file.getFileName() + ": " + key + " was modified"); else - logger.log(Level.WARNING, file.getFileName() + ": " + key + " was " + previousValue + logger.log(WARNING, file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value); } @@ -875,7 +855,6 @@ public class Repackage { /* * UTILITIES */ - /** Recursively deletes a directory. */ private static void deleteDirectory(Path path) throws IOException { if (!Files.exists(path)) @@ -886,13 +865,13 @@ public class Repackage { if (e != null) throw e; Files.delete(directory); - return FileVisitResult.CONTINUE; + return CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Files.delete(file); - return FileVisitResult.CONTINUE; + return CONTINUE; } }); } @@ -939,7 +918,7 @@ public class Repackage { try { return downloadArchive(url, dir); } catch (FileNotFoundException e) { - logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror"); + logger.log(WARNING, "Cannot download " + url + ", trying another mirror"); } } throw new FileNotFoundException("Cannot find " + uri); @@ -966,7 +945,7 @@ public class Repackage { dest = dir.resolve(name); if (Files.exists(dest)) { - logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again"); + logger.log(TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again"); return dest; } else { Files.createDirectories(dest.getParent()); @@ -974,7 +953,7 @@ public class Repackage { try (InputStream in = url.openStream()) { Files.copy(in, dest); - logger.log(Level.DEBUG, () -> "Downloaded " + dest + " from " + url); + logger.log(DEBUG, () -> "Downloaded " + dest + " from " + url); } return dest; } @@ -1008,12 +987,19 @@ public class Repackage { deleteDirectory(bundleDir); if (sourceBundles) { - Path sourceDir = bundleDir.getParent().resolve(bundleDir.toString() + ".src"); + Path bundleCategoryDir = bundleDir.getParent(); + Path sourceDir = bundleCategoryDir.resolve(bundleDir.toString() + ".src"); if (!Files.exists(sourceDir)) { logger.log(WARNING, sourceDir + " does not exist, skipping..."); return jarPath; + } - Path srcJarP = sourceDir.getParent().resolve(sourceDir.getFileName() + ".jar"); + + Path relPath = a2Base.relativize(bundleCategoryDir); + Path srcCategoryDir = a2SrcBase.resolve(relPath); + Path srcJarP = srcCategoryDir.resolve(sourceDir.getFileName() + ".jar"); + Files.createDirectories(srcJarP.getParent()); + String bundleSymbolicName = manifest.getMainAttributes().getValue("Bundle-SymbolicName").toString(); // in case there are additional directives bundleSymbolicName = bundleSymbolicName.split(";")[0]; @@ -1048,6 +1034,7 @@ public class Repackage { return jarPath; } + /** MANIFEST headers. */ enum ManifestConstants { // OSGi BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), // @@ -1058,12 +1045,12 @@ public class Repackage { // JAVA AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), // // SLC - SLC_CATEGORY("SLC-Category"), // +// SLC_CATEGORY("SLC-Category"), // SLC_ORIGIN_M2("SLC-Origin-M2"), // SLC_ORIGIN_M2_MERGE("SLC-Origin-M2-Merge"), // SLC_ORIGIN_M2_REPO("SLC-Origin-M2-Repo"), // SLC_ORIGIN_MANIFEST_NOT_MODIFIED("SLC-Origin-ManifestNotModified"), // - SLC_ORIGIN_URI("SLC-Origin-URI"),// + SLC_ORIGIN_URI("SLC-Origin-URI"), // ; final String value; @@ -1076,9 +1063,7 @@ public class Repackage { public String toString() { return value; } - } - } /** Simple representation of an M2 artifact. */ @@ -1170,6 +1155,7 @@ class M2ConventionsUtils { } } +/** Combination of a category, a name and a version. */ class CategoryNameVersion extends NameVersion { private String category; @@ -1201,6 +1187,7 @@ class CategoryNameVersion extends NameVersion { } +/** Combination of a name and a version. */ class NameVersion implements Comparable { private String name; private String version;