Merge tag 'v2.3.8' into testing
authorMathieu Baudier <mbaudier@argeo.org>
Thu, 9 Nov 2023 10:13:58 +0000 (11:13 +0100)
committerMathieu Baudier <mbaudier@argeo.org>
Thu, 9 Nov 2023 10:13:58 +0000 (11:13 +0100)
common.mk
excludes.txt
osgi.mk
sdk/branches/unstable.bnd
src/org/argeo/build/Make.java
src/org/argeo/build/Repackage.java

index bc1fa82b0cc89a950f146a52116154050d119a02..46b03984697b9a2ce6049b2f36a5af5a1fb7e778 100644 (file)
--- a/common.mk
+++ b/common.mk
@@ -2,7 +2,7 @@ build-major=2
 build-minor=3
 
 # Required third party libraries
-ECJ_BRANCH=3.32
+ECJ_MAJOR=3
 BNDLIB_BRANCH=5.3
 SYSLOGGER_BRANCH=$(build-major).$(build-minor)
 
index 0b155d9e36eab0f3e93a82e2b81bd12a7372aff8..06eaa255eb722c140d4a948ac84b177714ce0232 100644 (file)
@@ -7,3 +7,12 @@ build.properties
 bin
 generated
 META-INF/MANIFEST.MF
+configure
+Makefile
+*.mk
+sdk
+output
+node_modules
+package.json
+package-lock.json
+webpack.*.js
diff --git a/osgi.mk b/osgi.mk
index 52bdba6443406c039693cb49fc232f7ac77d3ef5..ca17cbe820b8fb7b7f7150da2f3d7f4085cab2e3 100644 (file)
--- a/osgi.mk
+++ b/osgi.mk
@@ -27,9 +27,10 @@ DEP_CATEGORIES ?=
 JAVADOC_PACKAGES ?=
 A2_BASE ?=/usr/share/a2 /usr/local/share/a2 $(A2_OUTPUT)
 
+# We always use the latest version of the ECJ compiler
+ECJ_JAR ?= $(lastword $(foreach base, $(A2_BASE), $(sort $(wildcard $(base)/org.argeo.tp.build/org.eclipse.jdt.core.compiler.batch.$(ECJ_MAJOR).*.jar))))
 # Third-party libraries
 LOGGER_JAR ?= $(lastword $(foreach base, $(A2_BASE), $(wildcard $(base)/log/syslogger/org.argeo.tp/org.argeo.tp.syslogger.$(SYSLOGGER_BRANCH).jar)))
-ECJ_JAR ?= $(lastword $(foreach base, $(A2_BASE), $(wildcard $(base)/org.argeo.tp.build/org.eclipse.jdt.core.compiler.batch.$(ECJ_BRANCH).jar)))
 BNDLIB_JAR ?= $(lastword $(foreach base, $(A2_BASE), $(wildcard $(base)/org.argeo.tp.build/biz.aQute.bndlib.$(BNDLIB_BRANCH).jar)))
 
 # Internal variables
@@ -49,9 +50,13 @@ TODOS = $(foreach bundle, $(BUNDLES),$(BUILD_BASE)/$(bundle)/to-build)
 osgi: $(BUILD_BASE)/built $(MANIFESTS)
 
 # Actual build (compilation + bundle packaging)
-$(BUILD_BASE)/built : BUNDLES_TO_BUILD = $(subst $(abspath $(BUILD_BASE))/,, $(subst to-build,, $?))
+$(BUILD_BASE)/built : BUNDLES_TO_BUILD = $(strip $(subst $(abspath $(BUILD_BASE))/,, $(subst to-build,, $?)))
 $(BUILD_BASE)/built : $(TODOS)
-       $(ARGEO_MAKE) \
+       @echo "| A2 category  : $(A2_CATEGORY)"
+       @echo "| Bundles      : $(BUNDLES_TO_BUILD)"
+       @echo "| Dependencies : $(DEP_CATEGORIES)"
+       @echo "| Compiler     : $(notdir $(ECJ_JAR))"
+       @$(ARGEO_MAKE) \
         all --a2-bases $(A2_BASE) --dep-categories $(DEP_CATEGORIES) \
         --category $(A2_CATEGORY) --bundles $(BUNDLES_TO_BUILD)
        @touch $(BUILD_BASE)/built 
index 41547c2d3974de70ccbdfc6156037f9b08550d87..5ec42c40b2878e7769e36b4fe24fa52d90ec08ec 100644 (file)
@@ -1,6 +1,6 @@
 major=2
 minor=3
-micro=6
+micro=8
 qualifier=
 
 SPDX-License-Identifier=CC0-1.0
\ No newline at end of file
index 798e1ea8627758a5a6ba1ff08ed74e14f405e255..9ebbfd462f10c8f0052c065b0c19e871ba1cbdb8 100644 (file)
@@ -226,7 +226,8 @@ public class Make {
                }
 
                // sources
-               for (String bundle : bundles) {
+               boolean atLeastOneBundleToCompile = false;
+               bundles: for (String bundle : bundles) {
                        StringBuilder sb = new StringBuilder();
                        Path bundlePath = execDirectory.resolve(bundle);
                        if (!Files.exists(bundlePath)) {
@@ -237,15 +238,24 @@ public class Make {
                                } else
                                        throw new IllegalArgumentException("Bundle " + bundle + " not found in " + execDirectory);
                        }
-                       sb.append(bundlePath.resolve("src"));
+                       Path bundleSrc = bundlePath.resolve("src");
+                       if (!Files.exists(bundleSrc)) {
+                               logger.log(WARNING, bundleSrc + " does not exist, skipping it, as this is not a Java bundle");
+                               continue bundles;
+                       }
+                       sb.append(bundleSrc);
                        sb.append("[-d");
                        compilerArgs.add(sb.toString());
                        sb = new StringBuilder();
                        sb.append(buildBase.resolve(bundle).resolve("bin"));
                        sb.append("]");
                        compilerArgs.add(sb.toString());
+                       atLeastOneBundleToCompile = true;
                }
 
+               if (!atLeastOneBundleToCompile)
+                       return;
+
                if (logger.isLoggable(INFO))
                        compilerArgs.add("-time");
 
@@ -386,18 +396,18 @@ public class Make {
 
        /** Package a single bundle. */
        void createBundle(String branch, String bundle, String category) throws IOException {
-               final Path source;
+               final Path bundleSourceBase;
                if (!Files.exists(execDirectory.resolve(bundle))) {
                        logger.log(WARNING,
                                        "Bundle " + bundle + " not found in " + execDirectory + ", assuming this is this directory.");
-                       source = execDirectory;
+                       bundleSourceBase = execDirectory;
                } else {
-                       source = execDirectory.resolve(bundle);
+                       bundleSourceBase = execDirectory.resolve(bundle);
                }
-               Path srcP = source.resolve("src");
+               Path srcP = bundleSourceBase.resolve("src");
 
                Path compiled = buildBase.resolve(bundle);
-               String bundleSymbolicName = source.getFileName().toString();
+               String bundleSymbolicName = bundleSourceBase.getFileName().toString();
 
                // Metadata
                Properties properties = new Properties();
@@ -414,7 +424,7 @@ public class Make {
                                }
                }
 
-               Path bndBnd = source.resolve("bnd.bnd");
+               Path bndBnd = bundleSourceBase.resolve("bnd.bnd");
                if (Files.exists(bndBnd))
                        try (InputStream in = Files.newInputStream(bndBnd)) {
                                properties.load(in);
@@ -477,7 +487,7 @@ public class Make {
                        });
 
                        // add resources
-                       Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
+                       Files.walkFileTree(bundleSourceBase, new SimpleFileVisitor<Path>() {
                                @Override
                                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                                        // skip output directory if it happens to be within the sources
@@ -485,7 +495,7 @@ public class Make {
                                                return FileVisitResult.SKIP_SUBTREE;
 
                                        // skip excluded patterns
-                                       Path relativeP = source.relativize(dir);
+                                       Path relativeP = bundleSourceBase.relativize(dir);
                                        for (PathMatcher exclude : excludes)
                                                if (exclude.matches(relativeP))
                                                        return FileVisitResult.SKIP_SUBTREE;
@@ -495,10 +505,14 @@ public class Make {
 
                                @Override
                                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                                       Path relativeP = source.relativize(file);
+                                       Path relativeP = bundleSourceBase.relativize(file);
                                        for (PathMatcher exclude : excludes)
                                                if (exclude.matches(relativeP))
                                                        return FileVisitResult.CONTINUE;
+                                       // skip JavaScript source maps
+                                       if (sourceBundles && file.getFileName().toString().endsWith(".map"))
+                                               return FileVisitResult.CONTINUE;
+
                                        JarEntry entry = new JarEntry(relativeP.toString());
                                        jarOut.putNextEntry(entry);
                                        Files.copy(file, jarOut);
@@ -506,32 +520,43 @@ public class Make {
                                }
                        });
 
-                       // Add all resources from src/
-                       Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
-                               @Override
-                               public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                                       if (file.getFileName().toString().endsWith(".java")
-                                                       || file.getFileName().toString().endsWith(".class"))
+                       if (Files.exists(srcP)) {
+                               // Add all resources from src/
+                               Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
+                                       @Override
+                                       public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                                               // skip directories ending with .js
+                                               // TODO find something more robust?
+                                               if (dir.getFileName().toString().endsWith(".js"))
+                                                       return FileVisitResult.SKIP_SUBTREE;
+                                               return super.preVisitDirectory(dir, attrs);
+                                       }
+
+                                       @Override
+                                       public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                                               if (file.getFileName().toString().endsWith(".java")
+                                                               || file.getFileName().toString().endsWith(".class"))
+                                                       return FileVisitResult.CONTINUE;
+                                               jarOut.putNextEntry(new JarEntry(srcP.relativize(file).toString()));
+                                               if (!Files.isDirectory(file))
+                                                       Files.copy(file, jarOut);
                                                return FileVisitResult.CONTINUE;
-                                       jarOut.putNextEntry(new JarEntry(srcP.relativize(file).toString()));
-                                       if (!Files.isDirectory(file))
-                                               Files.copy(file, jarOut);
-                                       return FileVisitResult.CONTINUE;
+                                       }
+                               });
+
+                               // add sources
+                               // TODO add effective BND, Eclipse project file, etc., in order to be able to
+                               // repackage
+                               if (!sourceBundles) {
+                                       copySourcesToJar(srcP, jarOut, "OSGI-OPT/src/");
                                }
-                       });
+                       }
 
                        // add legal notices and licenses
-                       for (Path p : listLegalFilesToInclude(source).values()) {
+                       for (Path p : listLegalFilesToInclude(bundleSourceBase).values()) {
                                jarOut.putNextEntry(new JarEntry(p.getFileName().toString()));
                                Files.copy(p, jarOut);
                        }
-
-                       // add sources
-                       // TODO add effective BND, Eclipse project file, etc., in order to be able to
-                       // repackage
-                       if (!sourceBundles) {
-                               copySourcesToJar(srcP, jarOut, "OSGI-OPT/src/");
-                       }
                }
 
                if (sourceBundles) {// create separate sources jar
@@ -539,22 +564,49 @@ public class Make {
                                        : a2srcOutput.resolve(category);
                        Files.createDirectories(a2srcJarDirectory);
                        Path srcJarP = a2srcJarDirectory.resolve(compiled.getFileName() + "." + major + "." + minor + ".src.jar");
-                       Manifest srcManifest = new Manifest();
-                       srcManifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
-                       srcManifest.getMainAttributes().putValue("Bundle-SymbolicName", bundleSymbolicName + ".src");
-                       srcManifest.getMainAttributes().putValue("Bundle-Version",
-                                       manifest.getMainAttributes().getValue("Bundle-Version").toString());
+                       createSourceBundle(bundleSymbolicName, manifest, bundleSourceBase, srcP, srcJarP);
+               }
+       }
+
+       /** Create a separate bundle containing the sources. */
+       void createSourceBundle(String bundleSymbolicName, Manifest manifest, Path bundleSourceBase, Path srcP,
+                       Path srcJarP) throws IOException {
+               Manifest srcManifest = new Manifest();
+               srcManifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+               srcManifest.getMainAttributes().putValue("Bundle-SymbolicName", bundleSymbolicName + ".src");
+               srcManifest.getMainAttributes().putValue("Bundle-Version",
+                               manifest.getMainAttributes().getValue("Bundle-Version").toString());
+
+               boolean isJsBundle = bundleSymbolicName.endsWith(".js");
+               if (!isJsBundle) {
                        srcManifest.getMainAttributes().putValue("Eclipse-SourceBundle",
                                        bundleSymbolicName + ";version=\"" + manifest.getMainAttributes().getValue("Bundle-Version"));
 
                        try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
                                copySourcesToJar(srcP, srcJarOut, "");
                                // add legal notices and licenses
-                               for (Path p : listLegalFilesToInclude(source).values()) {
+                               for (Path p : listLegalFilesToInclude(bundleSourceBase).values()) {
                                        srcJarOut.putNextEntry(new JarEntry(p.getFileName().toString()));
                                        Files.copy(p, srcJarOut);
                                }
                        }
+               } else {// JavaScript source maps
+                       srcManifest.getMainAttributes().putValue("Fragment-Host", bundleSymbolicName + ";bundle-version=\""
+                                       + manifest.getMainAttributes().getValue("Bundle-Version"));
+                       try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
+                               Files.walkFileTree(bundleSourceBase, new SimpleFileVisitor<Path>() {
+                                       @Override
+                                       public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                                               Path relativeP = bundleSourceBase.relativize(file);
+                                               if (!file.getFileName().toString().endsWith(".map"))
+                                                       return FileVisitResult.CONTINUE;
+                                               JarEntry entry = new JarEntry(relativeP.toString());
+                                               srcJarOut.putNextEntry(entry);
+                                               Files.copy(file, srcJarOut);
+                                               return FileVisitResult.CONTINUE;
+                                       }
+                               });
+                       }
                }
        }
 
index a6c84edd81f627e86410707cfbf81b364078dcdf..6f5fbf83e9083b250c793fea6adabaa023bc3401 100644 (file)
@@ -98,13 +98,14 @@ public class Repackage {
 
                List<CompletableFuture<Void>> toDos = new ArrayList<>();
                for (int i = 1; i < args.length; i++) {
-                       Path p = Paths.get(args[i]);
-                       if (sequential)
-                               factory.processCategory(p);
+                       Path categoryPath = Paths.get(args[i]);
+                       factory.cleanPreviousFailedBuild(categoryPath);
+                       if (sequential) // sequential processing happens here
+                               factory.processCategory(categoryPath);
                        else
-                               toDos.add(CompletableFuture.runAsync(() -> factory.processCategory(p)));
+                               toDos.add(CompletableFuture.runAsync(() -> factory.processCategory(categoryPath)));
                }
-               if (!sequential)
+               if (!sequential)// parallel processing
                        CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
 
                // Summary
@@ -115,6 +116,25 @@ public class Repackage {
                logger.log(INFO, "# License summary:\n" + sb);
        }
 
+       /** Deletes remaining sub directories. */
+       void cleanPreviousFailedBuild(Path categoryPath) {
+               Path outputCategoryPath = a2Base.resolve(categoryPath);
+               if (!Files.exists(outputCategoryPath))
+                       return;
+               // clean previous failed build
+               try {
+                       for (Path subDir : Files.newDirectoryStream(outputCategoryPath, (d) -> Files.isDirectory(d))) {
+                               if (Files.exists(subDir)) {
+                                       logger.log(WARNING, "Bundle dir " + subDir
+                                                       + " already exists, probably from a previous failed build, deleting it...");
+                                       deleteDirectory(subDir);
+                               }
+                       }
+               } catch (IOException e) {
+                       logger.log(ERROR, "Cannot clean previous build", e);
+               }
+       }
+
        /** MANIFEST headers. */
        enum ManifestHeader {
                // OSGi
@@ -584,6 +604,10 @@ public class Repackage {
                                                } else if (entry.getName().startsWith("org/apache/batik/")) {
                                                        logger.log(TRACE, "Skip " + entry.getName());
                                                        continue entries;
+                                               } else if (entry.getName().startsWith("META-INF/NOTICE")) {
+                                                       logger.log(WARNING, "Skip " + entry.getName() + " from " + artifact);
+                                                       // TODO merge them?
+                                                       continue entries;
                                                } else {
                                                        throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
                                                }
@@ -910,6 +934,10 @@ public class Repackage {
                                                                        map.put(key.toString(), commonProps.getProperty(key.toString()));
                                                                A2Origin origin = new A2Origin();
                                                                Path bundleDir = processBundleJar(file, targetCategoryBase, map, origin);
+                                                               if (bundleDir == null) {
+                                                                       logger.log(WARNING, "No bundle dir created for " + file + ", skipping...");
+                                                                       return FileVisitResult.CONTINUE;
+                                                               }
                                                                origins.put(bundleDir, origin);
                                                                logger.log(DEBUG, () -> "Processed " + file);
                                                        }
@@ -993,6 +1021,8 @@ public class Repackage {
                Manifest sourceManifest;
                try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
                        sourceManifest = jarIn.getManifest();
+                       if (sourceManifest == null)
+                               logger.log(WARNING, file + " has no manifest");
                        manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
 
                        String rawSourceSymbolicName = manifest.getMainAttributes().getValue(BUNDLE_SYMBOLICNAME.toString());
@@ -1015,6 +1045,8 @@ public class Repackage {
                                nameVersion = new NameVersion(ourSymbolicName, ourVersion);
                        } else {
                                nameVersion = nameVersionFromManifest(manifest);
+                               if (nameVersion == null)
+                                       throw new IllegalStateException("Could not compute name/version from Manifest");
                                if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) {
                                        logger.log(WARNING,
                                                        "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion);
@@ -1025,6 +1057,7 @@ public class Repackage {
                                        nameVersion.setName(ourSymbolicName);
                                }
                        }
+
                        bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
 
                        // copy original MANIFEST
@@ -1050,7 +1083,6 @@ public class Repackage {
                                arch = libRelativePath.getName(1).toString();
                        }
 
-//                     if (!embed) {
                        // copy entries
                        JarEntry entry;
                        entries: while ((entry = jarIn.getNextJarEntry()) != null) {
@@ -1117,7 +1149,6 @@ public class Repackage {
                                        origin.deleted.add(bundleDir.relativize(target).toString());
                                }
                                logger.log(TRACE, () -> "Copied " + target);
-//                             }
                        }
                }
 
@@ -1130,11 +1161,6 @@ public class Repackage {
                                        entries.get(BUNDLE_SYMBOLICNAME.toString()) + ";singleton:=true");
                }
 
-//             if (embed) {// copy embedded jar
-//                     Files.copy(file, bundleDir.resolve(file.getFileName()));
-//                     entries.put(ManifestHeader.BUNDLE_CLASSPATH.toString(), file.getFileName().toString());
-//             }
-
                // Final MANIFEST decisions
                // We also check the original OSGi metadata and compare with our changes
                for (String key : entries.keySet()) {