X-Git-Url: http://git.argeo.org/?a=blobdiff_plain;f=org.argeo.slc.factory%2Fsrc%2Forg%2Fargeo%2Fslc%2Ffactory%2FA2Factory.java;h=b0bfefac632fc6fe0efea9c04c89908b3635797d;hb=ff6b85864271f75b795f88333f329a52db3b901f;hp=e1310c1c441e6c7a0eca0de2e0309efddf0773c4;hpb=3b274da1118e8f941f93676f28aa9991d12dac56;p=gpl%2Fargeo-slc.git diff --git a/org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java b/org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java index e1310c1c4..b0bfefac6 100644 --- a/org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java +++ b/org.argeo.slc.factory/src/org/argeo/slc/factory/A2Factory.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.TreeMap; import java.util.jar.Attributes; @@ -36,11 +37,13 @@ import java.util.jar.JarEntry; import java.util.jar.JarInputStream; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; +import java.util.zip.Deflater; import org.argeo.slc.DefaultCategoryNameVersion; import org.argeo.slc.DefaultNameVersion; import org.argeo.slc.ManifestConstants; import org.argeo.slc.NameVersion; +import org.argeo.slc.factory.m2.Artifact; import org.argeo.slc.factory.m2.DefaultArtifact; import org.argeo.slc.factory.m2.MavenConventionsUtils; @@ -56,13 +59,35 @@ public class A2Factory { private Path originBase; private Path a2Base; + private Path a2LibBase; + private Path descriptorsBase; + + private Properties uris = new Properties(); + + private boolean includeSources = true; /** key is URI prefix, value list of base URLs */ private Map> mirrors = new HashMap>(); - public A2Factory(Path a2Base) { + public A2Factory(Path a2Base, Path descriptorsBase, boolean includeSources) { + Objects.requireNonNull(a2Base); + Objects.requireNonNull(descriptorsBase); this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/slc/origin"); this.a2Base = a2Base; + this.a2LibBase = a2Base.resolve("lib"); + this.descriptorsBase = descriptorsBase; + if (!Files.exists(this.descriptorsBase)) + throw new IllegalArgumentException(this.descriptorsBase + " does not exist"); + this.includeSources = includeSources; + + Path urisPath = this.descriptorsBase.resolve("uris.properties"); + if (Files.exists(urisPath)) { + try (InputStream in = Files.newInputStream(urisPath)) { + uris.load(in); + } catch (IOException e) { + throw new IllegalStateException("Cannot load " + urisPath, e); + } + } // TODO make it configurable List eclipseMirrors = new ArrayList<>(); @@ -73,8 +98,14 @@ public class A2Factory { mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors); } - public void processCategory(Path targetCategoryBase) { + /* + * MAVEN ORIGIN + */ + + /** Process a whole category/group id. */ + public void processCategory(Path categoryRelativePath) { try { + Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath); DirectoryStream bnds = Files.newDirectoryStream(targetCategoryBase, (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND) && !p.getFileName().toString().equals(MERGE_BND)); @@ -87,14 +118,17 @@ public class A2Factory { processM2BasedDistributionUnit(duDir); } } catch (IOException e) { - throw new RuntimeException("Cannot process category " + targetCategoryBase, e); + throw new RuntimeException("Cannot process category " + categoryRelativePath, e); } } + /** Process a standalone Maven artifact. */ public void processSingleM2ArtifactDistributionUnit(Path bndFile) { try { - String category = bndFile.getParent().getFileName().toString(); - Path targetCategoryBase = a2Base.resolve(category); +// String category = bndFile.getParent().getFileName().toString(); + Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent()); + Path targetCategoryBase = a2Base.resolve(categoryRelativePath); + Properties fileProps = new Properties(); try (InputStream in = Files.newInputStream(bndFile)) { fileProps.load(in); @@ -103,8 +137,7 @@ public class A2Factory { ? fileProps.getProperty(SLC_ORIGIN_M2_REPO.toString()) : null; - if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString()) - && !fileProps.containsKey(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString())) { + if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) { // use file name as symbolic name String symbolicName = bndFile.getFileName().toString(); symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length()); @@ -116,7 +149,7 @@ public class A2Factory { throw new IllegalArgumentException("No M2 coordinates available for " + bndFile); DefaultArtifact artifact = new DefaultArtifact(m2Coordinates); URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact); - Path downloaded = download(url, originBase, artifact.toM2Coordinates() + ".jar"); + Path downloaded = download(url, originBase, artifact); Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact); @@ -128,19 +161,24 @@ public class A2Factory { } } + /** Process multiple Maven artifacts. */ public void processM2BasedDistributionUnit(Path duDir) { try { - String category = duDir.getParent().getFileName().toString(); - Path targetCategoryBase = a2Base.resolve(category); + // String category = duDir.getParent().getFileName().toString(); + Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent()); + Path targetCategoryBase = a2Base.resolve(categoryRelativePath); // merge Path mergeBnd = duDir.resolve(MERGE_BND); if (Files.exists(mergeBnd)) { mergeM2Artifacts(mergeBnd); - return; +// return; } Path commonBnd = duDir.resolve(COMMON_BND); + if (!Files.exists(commonBnd)) { + return; + } Properties commonProps = new Properties(); try (InputStream in = Files.newInputStream(commonBnd)) { commonProps.load(in); @@ -157,8 +195,8 @@ public class A2Factory { m2Version = m2Version.substring(1); DirectoryStream ds = Files.newDirectoryStream(duDir, - (p) -> p.getFileName().toString().endsWith(".bnd") - && !p.getFileName().toString().equals(COMMON_BND)); + (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND) + && !p.getFileName().toString().equals(MERGE_BND)); for (Path p : ds) { Properties fileProps = new Properties(); try (InputStream in = Files.newInputStream(p)) { @@ -198,8 +236,7 @@ public class A2Factory { } } mergeProps.put(ManifestConstants.SLC_ORIGIN_M2.toString(), artifact.toM2Coordinates()); - if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString()) - && !mergeProps.containsKey(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString())) { + if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) { // use file name as symbolic name String symbolicName = p.getFileName().toString(); symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length()); @@ -212,7 +249,7 @@ public class A2Factory { // download URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact); - Path downloaded = download(url, originBase, artifact.toM2Coordinates() + ".jar"); + Path downloaded = download(url, originBase, artifact); Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact); // logger.log(Level.DEBUG, () -> "Processed " + downloaded); @@ -228,6 +265,7 @@ public class A2Factory { } + /** Merge multiple Maven artifacts. */ protected void mergeM2Artifacts(Path mergeBnd) throws IOException { Path duDir = mergeBnd.getParent(); String category = duDir.getParent().getFileName().toString(); @@ -237,6 +275,8 @@ public class A2Factory { try (InputStream in = Files.newInputStream(mergeBnd)) { mergeProps.load(in); } + + // Version String m2Version = mergeProps.getProperty(SLC_ORIGIN_M2.toString()); if (m2Version == null) { logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit"); @@ -246,6 +286,7 @@ public class A2Factory { throw new IllegalStateException("Only the M2 version can be specified: " + m2Version); } m2Version = m2Version.substring(1); + mergeProps.put(ManifestConstants.BUNDLE_VERSION.toString(), m2Version); String artifactsStr = mergeProps.getProperty(ManifestConstants.SLC_ORIGIN_M2_MERGE.toString()); String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString()) @@ -253,6 +294,8 @@ public class A2Factory { : null; String bundleSymbolicName = mergeProps.getProperty(ManifestConstants.BUNDLE_SYMBOLICNAME.toString()); + if (bundleSymbolicName == null) + throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd); DefaultCategoryNameVersion nameVersion = new DefaultArtifact( category + ":" + bundleSymbolicName + ":" + m2Version); Path targetBundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch()); @@ -263,15 +306,32 @@ public class A2Factory { if ("".equals(m2Coordinates)) continue artifacts; DefaultArtifact artifact = new DefaultArtifact(m2Coordinates.trim()); - artifact.setVersion(m2Version); + if (artifact.getVersion() == null) + artifact.setVersion(m2Version); URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact); - Path downloaded = download(url, originBase, artifact.toM2Coordinates() + ".jar"); + Path downloaded = download(url, originBase, artifact); JarEntry entry; try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) { entries: while ((entry = jarIn.getNextJarEntry()) != null) { if (entry.isDirectory()) continue entries; - if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF")) + else if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF")) + continue entries; + else if (entry.getName().startsWith("META-INF/versions/")) + continue entries; + else if (entry.getName().startsWith("META-INF/maven/")) + continue entries; + else if (entry.getName().equals("module-info.class")) + continue entries; + else if (entry.getName().equals("META-INF/NOTICE")) + continue entries; + else if (entry.getName().equals("META-INF/NOTICE.txt")) + continue entries; + else if (entry.getName().equals("META-INF/LICENSE")) + continue entries; + else if (entry.getName().equals("META-INF/LICENSE.md")) + continue entries; + else if (entry.getName().equals("META-INF/LICENSE-notice.md")) continue entries; Path target = targetBundleDir.resolve(entry.getName()); Files.createDirectories(target.getParent()); @@ -280,10 +340,16 @@ public class A2Factory { } else { if (entry.getName().startsWith("META-INF/services/")) { try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) { + out.write("\n".getBytes()); jarIn.transferTo(out); + if (logger.isLoggable(DEBUG)) + logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName()); } + } else if (entry.getName().startsWith("org/apache/batik/")) { + logger.log(Level.WARNING, "Skip " + entry.getName()); + continue entries; } else { - throw new IllegalStateException("File " + target + " already exists"); + throw new IllegalStateException("File " + target + " from " + artifact + " already exists"); } } logger.log(Level.TRACE, () -> "Copied " + target); @@ -293,6 +359,21 @@ public class A2Factory { downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir); } + // additional service files + Path servicesDir = duDir.resolve("services"); + if (Files.exists(servicesDir)) { + for (Path p : Files.newDirectoryStream(servicesDir)) { + Path target = targetBundleDir.resolve("META-INF/services/").resolve(p.getFileName()); + try (InputStream in = Files.newInputStream(p); + OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) { + out.write("\n".getBytes()); + in.transferTo(out); + if (logger.isLoggable(DEBUG)) + logger.log(DEBUG, "Appended " + p); + } + } + } + Map entries = new TreeMap<>(); try (Analyzer bndAnalyzer = new Analyzer()) { bndAnalyzer.setProperties(mergeProps); @@ -313,7 +394,7 @@ public class A2Factory { && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) continue keys;// hack for very old classes entries.put(key.toString(), value.toString()); - logger.log(DEBUG, () -> key + "=" + value); + // logger.log(DEBUG, () -> key + "=" + value); } } catch (Exception e) { @@ -327,6 +408,13 @@ public class A2Factory { String value = entries.get(key); manifest.getMainAttributes().putValue(key, value); } + +// // Use Maven version as Bundle-Version +// String bundleVersion = manifest.getMainAttributes().getValue(ManifestConstants.BUNDLE_VERSION.toString()); +// if (bundleVersion == null || bundleVersion.trim().equals("0")) { +// // TODO check why it is sometimes set to "0" +// manifest.getMainAttributes().putValue(ManifestConstants.BUNDLE_VERSION.toString(), m2Version); +// } try (OutputStream out = Files.newOutputStream(manifestPath)) { manifest.write(out); } @@ -335,16 +423,7 @@ public class A2Factory { } - protected void downloadAndProcessM2Sources(String repoStr, DefaultArtifact artifact, Path targetBundleDir) - throws IOException { - DefaultArtifact sourcesArtifact = new DefaultArtifact(artifact.toM2Coordinates(), "sources"); - URL sourcesUrl = MavenConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact); - Path sourcesDownloaded = download(sourcesUrl, originBase, artifact.toM2Coordinates() + ".sources.jar"); - processM2SourceJar(sourcesDownloaded, targetBundleDir); - logger.log(Level.DEBUG, () -> "Processed source " + sourcesDownloaded); - - } - + /** Generate MANIFEST using BND. */ protected Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, DefaultArtifact artifact) { @@ -399,7 +478,7 @@ public class A2Factory { && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) continue keys;// hack for very old classes additionalEntries.put(key.toString(), value.toString()); - logger.log(DEBUG, () -> key + "=" + value); + // logger.log(DEBUG, () -> key + "=" + value); } } @@ -423,13 +502,27 @@ public class A2Factory { } + /** Download and integrates sources for a single Maven artifact. */ + protected void downloadAndProcessM2Sources(String repoStr, DefaultArtifact artifact, Path targetBundleDir) + throws IOException { + if (!includeSources) + return; + DefaultArtifact sourcesArtifact = new DefaultArtifact(artifact.toM2Coordinates(), "sources"); + URL sourcesUrl = MavenConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact); + Path sourcesDownloaded = download(sourcesUrl, originBase, artifact, true); + processM2SourceJar(sourcesDownloaded, targetBundleDir); + logger.log(Level.TRACE, () -> "Processed source " + sourcesDownloaded); + + } + + /** Integrate sources from a downloaded jar file. */ protected void processM2SourceJar(Path file, Path targetBundleDir) throws IOException { try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src"); // TODO make it less dangerous? if (Files.exists(targetSourceDir)) { - deleteDirectory(targetSourceDir); +// deleteDirectory(targetSourceDir); } else { Files.createDirectories(targetSourceDir); } @@ -441,19 +534,44 @@ public class A2Factory { continue entries; if (entry.getName().startsWith("META-INF"))// skip META-INF entries continue entries; + if (entry.getName().startsWith("module-info.java"))// skip META-INF entries + continue entries; + if (entry.getName().startsWith("/")) // absolute paths + continue entries; Path target = targetSourceDir.resolve(entry.getName()); Files.createDirectories(target.getParent()); - Files.copy(jarIn, target); - logger.log(Level.TRACE, () -> "Copied source " + target); + if (!Files.exists(target)) { + Files.copy(jarIn, target); + logger.log(Level.TRACE, () -> "Copied source " + target); + } else { + logger.log(Level.WARNING, () -> target + " already exists, skipping..."); + } } } } - public void processEclipseArchive(Path duDir) { + /** Download a Maven artifact. */ + protected Path download(URL url, Path dir, Artifact artifact) throws IOException { + return download(url, dir, artifact, false); + } + + /** Download a Maven artifact. */ + protected Path download(URL url, Path dir, Artifact artifact, boolean sources) throws IOException { + return download(url, dir, artifact.getGroupId() + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() + + (sources ? "-sources" : "") + ".jar"); + } + + /* + * ECLIPSE ORIGIN + */ + + /** Process an archive in Eclipse format. */ + public void processEclipseArchive(Path relativePath) { try { - String category = duDir.getParent().getFileName().toString(); - Path targetCategoryBase = a2Base.resolve(category); + Path categoryRelativePath = relativePath.getParent(); + // String category = categoryRelativePath.getFileName().toString(); + Path targetCategoryBase = a2Base.resolve(categoryRelativePath); Files.createDirectories(targetCategoryBase); // first delete all directories from previous builds for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) { @@ -462,41 +580,68 @@ public class A2Factory { Files.createDirectories(originBase); + Path duDir = descriptorsBase.resolve(relativePath); Path commonBnd = duDir.resolve(COMMON_BND); Properties commonProps = new Properties(); try (InputStream in = Files.newInputStream(commonBnd)) { commonProps.load(in); } - Properties includes = new Properties(); - try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) { - includes.load(in); - } String url = commonProps.getProperty(ManifestConstants.SLC_ORIGIN_URI.toString()); + if (url == null) { + url = uris.getProperty(relativePath.getFileName().toString()); + if (url == null) + throw new IllegalStateException("No url available for " + duDir); + commonProps.put(ManifestConstants.SLC_ORIGIN_URI.toString(), url); + } Path downloaded = tryDownload(url, originBase); FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null); - List pathMatchers = new ArrayList<>(); + // filters + List includeMatchers = new ArrayList<>(); + Properties includes = new Properties(); + try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) { + includes.load(in); + } for (Object pattern : includes.keySet()) { PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern); - pathMatchers.add(pathMatcher); + includeMatchers.add(pathMatcher); + } + + List excludeMatchers = new ArrayList<>(); + Path excludeFile = duDir.resolve("excludes.properties"); + if (Files.exists(excludeFile)) { + Properties excludes = new Properties(); + try (InputStream in = Files.newInputStream(excludeFile)) { + excludes.load(in); + } + for (Object pattern : excludes.keySet()) { + PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern); + excludeMatchers.add(pathMatcher); + } } Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - pathMatchers: for (PathMatcher pathMatcher : pathMatchers) { - if (pathMatcher.matches(file)) { - if (file.getFileName().toString().contains(".source_")) { + includeMatchers: for (PathMatcher includeMatcher : includeMatchers) { + if (includeMatcher.matches(file)) { + for (PathMatcher excludeMatcher : excludeMatchers) { + if (excludeMatcher.matches(file)) { + logger.log(Level.WARNING, "Skipping excluded " + file); + return FileVisitResult.CONTINUE; + } + } + if (includeSources && file.getFileName().toString().contains(".source_")) { processEclipseSourceJar(file, targetCategoryBase); logger.log(Level.DEBUG, () -> "Processed source " + file); } else { - processBundleJar(file, targetCategoryBase, new HashMap<>()); + processBundleJar(file, targetCategoryBase, new HashMap(commonProps)); logger.log(Level.DEBUG, () -> "Processed " + file); } - break pathMatchers; + break includeMatchers; } } return FileVisitResult.CONTINUE; @@ -508,16 +653,69 @@ public class A2Factory { createJar(dir); } } catch (IOException e) { - throw new RuntimeException("Cannot process " + duDir, e); + throw new RuntimeException("Cannot process " + relativePath, e); + } + + } + + /** Process sources in Eclipse format. */ + protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException { + try { + Path targetBundleDir; + try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { + Manifest manifest = jarIn.getManifest(); + + String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";"); + String version = relatedBundle[1].substring("version=\"".length()); + version = version.substring(0, version.length() - 1); + NameVersion nameVersion = new DefaultNameVersion(relatedBundle[0], version); + targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch()); + + Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src"); + + // TODO make it less dangerous? + if (Files.exists(targetSourceDir)) { +// deleteDirectory(targetSourceDir); + } else { + Files.createDirectories(targetSourceDir); + } + + // copy entries + JarEntry entry; + entries: while ((entry = jarIn.getNextJarEntry()) != null) { + if (entry.isDirectory()) + continue entries; + if (entry.getName().startsWith("META-INF"))// skip META-INF entries + continue entries; + Path target = targetSourceDir.resolve(entry.getName()); + Files.createDirectories(target.getParent()); + Files.copy(jarIn, target); + logger.log(Level.TRACE, () -> "Copied source " + target); + } + + // copy MANIFEST +// Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF"); +// Files.createDirectories(manifestPath.getParent()); +// try (OutputStream out = Files.newOutputStream(manifestPath)) { +// manifest.write(out); +// } + } + } catch (IOException e) { + throw new IllegalStateException("Cannot process " + file, e); } } + /* + * COMMON PROCESSING + */ + /** Normalise a bundle. */ protected Path processBundleJar(Path file, Path targetBase, Map entries) throws IOException { DefaultNameVersion nameVersion; Path targetBundleDir; try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { - Manifest manifest = new Manifest(jarIn.getManifest()); + Manifest sourceManifest = jarIn.getManifest(); + Manifest manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest(); // remove problematic entries in MANIFEST manifest.getEntries().clear(); @@ -538,29 +736,43 @@ public class A2Factory { // manifest.getEntries().remove(key); // } - String symbolicNameFromEntries = entries.get(BUNDLE_SYMBOLICNAME.toString()); - String versionFromEntries = entries.get(BUNDLE_VERSION.toString()); + String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString()); + String ourVersion = entries.get(BUNDLE_VERSION.toString()); - if (symbolicNameFromEntries != null && versionFromEntries != null) { - nameVersion = new DefaultNameVersion(symbolicNameFromEntries, versionFromEntries); + if (ourSymbolicName != null && ourVersion != null) { + nameVersion = new DefaultNameVersion(ourSymbolicName, ourVersion); } else { nameVersion = nameVersionFromManifest(manifest); - if (versionFromEntries != null && !nameVersion.getVersion().equals(versionFromEntries)) { - logger.log(Level.WARNING, "Original version is " + nameVersion.getVersion() - + " while new version is " + versionFromEntries); + if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) { + logger.log(Level.WARNING, + "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion); + entries.put(BUNDLE_VERSION.toString(), ourVersion); } - if (symbolicNameFromEntries != null) { + if (ourSymbolicName != null) { // we always force our symbolic name - nameVersion.setName(symbolicNameFromEntries); + nameVersion.setName(ourSymbolicName); } } targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch()); + // force Java 9 module name + entries.put(ManifestConstants.AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName()); + // TODO make it less dangerous? // if (Files.exists(targetBundleDir)) { // deleteDirectory(targetBundleDir); // } + boolean isNative = false; + String os = null; + String arch = null; + if (targetBundleDir.startsWith(a2LibBase)) { + isNative = true; + Path libRelativePath = a2LibBase.relativize(targetBundleDir); + os = libRelativePath.getName(0).toString(); + arch = libRelativePath.getName(1).toString(); + } + // copy entries JarEntry entry; entries: while ((entry = jarIn.getNextJarEntry()) != null) { @@ -568,9 +780,46 @@ public class A2Factory { continue entries; if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF")) continue entries; + if (entry.getName().endsWith("module-info.class")) // skip Java 9 module info + continue entries; + if (entry.getName().startsWith("META-INF/versions/")) // skip multi-version + continue entries; + // skip file system providers as they cause issues with native image + if (entry.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider")) + continue entries; + if (entry.getName().startsWith("OSGI-OPT/src/")) // skip embedded sources + continue entries; Path target = targetBundleDir.resolve(entry.getName()); Files.createDirectories(target.getParent()); Files.copy(jarIn, target); + + // native libraries + if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll") + || entry.getName().endsWith(".jnilib"))) { + Path categoryDir = targetBundleDir.getParent(); +// String[] segments = categoryDir.getFileName().toString().split("\\."); +// String arch = segments[segments.length - 1]; +// String os = segments[segments.length - 2]; + boolean copyDll = false; + Path targetDll = categoryDir.resolve(target.getFileName()); + if (nameVersion.getName().equals("com.sun.jna")) { + if (arch.equals("x86_64")) + arch = "x86-64"; + if (os.equals("macosx")) + os = "darwin"; + if (target.getParent().getFileName().toString().equals(os + "-" + arch)) { + copyDll = true; + } + } else { + copyDll = true; + } + if (copyDll) { + if (Files.exists(targetDll)) + Files.delete(targetDll); + Files.copy(target, targetDll); + } + Files.delete(target); + } logger.log(Level.TRACE, () -> "Copied " + target); } @@ -583,12 +832,17 @@ public class A2Factory { if (previousValue != null && !previousValue.equals(value)) { if (ManifestConstants.IMPORT_PACKAGE.toString().equals(key) || ManifestConstants.EXPORT_PACKAGE.toString().equals(key)) - logger.log(Level.WARNING, file.getFileName() + ": " + key + " was modified"); - + logger.log(Level.TRACE, file.getFileName() + ": " + key + " was modified"); else logger.log(Level.WARNING, file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value); } + + // hack to remove unresolvable + if (key.equals("Provide-Capability") || key.equals("Require-Capability")) + if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) { + manifest.getMainAttributes().remove(key); + } } try (OutputStream out = Files.newOutputStream(manifestPath)) { manifest.write(out); @@ -597,54 +851,12 @@ public class A2Factory { return targetBundleDir; } - protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException { - try { - Path targetBundleDir; - try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) { - Manifest manifest = jarIn.getManifest(); - - String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";"); - String version = relatedBundle[1].substring("version=\"".length()); - version = version.substring(0, version.length() - 1); - NameVersion nameVersion = new DefaultNameVersion(relatedBundle[0], version); - targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch()); - - Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src"); - - // TODO make it less dangerous? - if (Files.exists(targetSourceDir)) { -// deleteDirectory(targetSourceDir); - } else { - Files.createDirectories(targetSourceDir); - } + /* + * UTILITIES + */ - // copy entries - JarEntry entry; - entries: while ((entry = jarIn.getNextJarEntry()) != null) { - if (entry.isDirectory()) - continue entries; - if (entry.getName().startsWith("META-INF"))// skip META-INF entries - continue entries; - Path target = targetSourceDir.resolve(entry.getName()); - Files.createDirectories(target.getParent()); - Files.copy(jarIn, target); - logger.log(Level.TRACE, () -> "Copied source " + target); - } - - // copy MANIFEST -// Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF"); -// Files.createDirectories(manifestPath.getParent()); -// try (OutputStream out = Files.newOutputStream(manifestPath)) { -// manifest.write(out); -// } - } - } catch (IOException e) { - throw new IllegalStateException("Cannot process " + file, e); - } - - } - - static void deleteDirectory(Path path) throws IOException { + /** Recursively deletes a directory. */ + private static void deleteDirectory(Path path) throws IOException { if (!Files.exists(path)) return; Files.walkFileTree(path, new SimpleFileVisitor() { @@ -664,6 +876,7 @@ public class A2Factory { }); } + /** Extract name/version from a MANIFEST. */ protected DefaultNameVersion nameVersionFromManifest(Manifest manifest) { Attributes attrs = manifest.getMainAttributes(); // symbolic name @@ -677,6 +890,7 @@ public class A2Factory { return new DefaultNameVersion(symbolicName, version); } + /** Try to download from an URI. */ protected Path tryDownload(String uri, Path dir) throws IOException { // find mirror List urlBases = null; @@ -692,7 +906,7 @@ public class A2Factory { } if (urlBases == null) try { - return download(new URL(uri), dir, null); + return download(new URL(uri), dir); } catch (FileNotFoundException e) { throw new FileNotFoundException("Cannot find " + uri); } @@ -702,7 +916,7 @@ public class A2Factory { String relativePath = uri.substring(uriPrefix.length()); URL url = new URL(urlBase + relativePath); try { - return download(url, dir, null); + return download(url, dir); } catch (FileNotFoundException e) { logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror"); } @@ -715,6 +929,12 @@ public class A2Factory { // // } + /** Effectively download. */ + protected Path download(URL url, Path dir) throws IOException { + return download(url, dir, (String) null); + } + + /** Effectively download. */ protected Path download(URL url, Path dir, String name) throws IOException { Path dest; @@ -726,6 +946,8 @@ public class A2Factory { if (Files.exists(dest)) { logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again"); return dest; + } else { + Files.createDirectories(dest.getParent()); } try (InputStream in = url.openStream()) { @@ -735,7 +957,30 @@ public class A2Factory { return dest; } + /** Create a JAR file from a directory. */ protected Path createJar(Path bundleDir) throws IOException { + // Remove from source the resources already in the jar file +// Path srcDir = bundleDir.resolve("OSGI-OPT/src"); +// Files.walkFileTree(srcDir, new SimpleFileVisitor() { +// +// @Override +// public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { +// // always keep .java file +// if (file.getFileName().toString().endsWith(".java")) +// return super.visitFile(file, attrs); +// +// Path relPath = srcDir.relativize(file); +// Path bundlePath = bundleDir.resolve(relPath); +// if (Files.exists(bundlePath)) { +// Files.delete(file); +// logger.log(DEBUG, () -> "Removed " + file + " from sources."); +// } +// return super.visitFile(file, attrs); +// } +// +// }); + + // Create the jar Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar"); Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF"); Manifest manifest; @@ -743,6 +988,7 @@ public class A2Factory { manifest = new Manifest(in); } try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) { + jarOut.setLevel(Deflater.DEFAULT_COMPRESSION); Files.walkFileTree(bundleDir, new SimpleFileVisitor() { @Override @@ -761,36 +1007,56 @@ public class A2Factory { return jarPath; } + /** For development purproses. */ public static void main(String[] args) { Path factoryBase = Paths.get("../../output/a2").toAbsolutePath().normalize(); - A2Factory factory = new A2Factory(factoryBase); - Path descriptorsBase = Paths.get("../tp").toAbsolutePath().normalize(); + A2Factory factory = new A2Factory(factoryBase, descriptorsBase, true); // factory.processSingleM2ArtifactDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache").resolve("org.apache.xml.resolver.bnd")); // factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache/apache-sshd")); // factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty")); -// factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty-websocket")); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.osgi")); // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp")); // factory.processCategory(descriptorsBase.resolve("org.argeo.tp")); -// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache")); + factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache")); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk")); // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats")); - factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis")); - System.exit(0); +// factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis")); + System.exit(1); + + // SDK + factory.processCategory(Paths.get("org.argeo.tp.sdk")); // Eclipse - factory.processEclipseArchive( - descriptorsBase.resolve("org.argeo.tp.eclipse.equinox").resolve("eclipse-equinox")); - factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rap").resolve("eclipse-rap")); - factory.processEclipseArchive(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp").resolve("eclipse-rcp")); + factory.processCategory(Paths.get("osgi/api/org.argeo.tp.osgi")); + factory.processEclipseArchive(Paths.get("osgi/equinox/org.argeo.tp.osgi", "eclipse-equinox")); + + factory.processEclipseArchive(Paths.get("org.argeo.tp.eclipse", "eclipse-equinox")); + + factory.processEclipseArchive(Paths.get("swt/rap/org.argeo.tp.swt", "eclipse-rap")); + factory.processEclipseArchive(Paths.get("swt/rap/org.argeo.tp.swt.workbench", "eclipse-rap")); - System.exit(0); + factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); + + factory.processEclipseArchive(Paths.get("lib/linux/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); + factory.processEclipseArchive(Paths.get("lib/linux/aarch64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); + factory.processEclipseArchive(Paths.get("lib/win32/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); + factory.processEclipseArchive(Paths.get("lib/macosx/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp")); + + factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt", "eclipse-nebula")); + // factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt", + // "eclipse-equinox")); + factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt.workbench", "eclipse-rcp")); + factory.processCategory(Paths.get("swt/rcp/org.argeo.tp.swt.workbench")); // Maven - factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk")); - factory.processCategory(descriptorsBase.resolve("org.argeo.tp")); - factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache")); - factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jetty")); - factory.processCategory(descriptorsBase.resolve("org.argeo.tp.jcr")); + factory.processCategory(Paths.get("org.argeo.tp")); + factory.processCategory(Paths.get("org.argeo.tp.apache")); + factory.processCategory(Paths.get("org.argeo.tp.jetty")); + factory.processCategory(Paths.get("org.argeo.tp.jcr")); + factory.processCategory(Paths.get("org.argeo.tp.formats")); + factory.processCategory(Paths.get("org.argeo.tp.poi")); + factory.processCategory(Paths.get("org.argeo.tp.gis")); } }