]> git.argeo.org Git - cc0/argeo-build.git/blob - src/org/argeo/build/Make.java
List all modifications of the origin artifacts when repackaging
[cc0/argeo-build.git] / src / org / argeo / build / Make.java
1 package org.argeo.build;
2
3 import static java.lang.System.Logger.Level.DEBUG;
4 import static java.lang.System.Logger.Level.ERROR;
5 import static java.lang.System.Logger.Level.INFO;
6 import static java.lang.System.Logger.Level.WARNING;
7
8 import java.io.File;
9 import java.io.IOException;
10 import java.io.InputStream;
11 import java.io.OutputStream;
12 import java.io.PrintWriter;
13 import java.lang.System.Logger;
14 import java.lang.System.Logger.Level;
15 import java.lang.management.ManagementFactory;
16 import java.nio.file.FileVisitResult;
17 import java.nio.file.Files;
18 import java.nio.file.Path;
19 import java.nio.file.PathMatcher;
20 import java.nio.file.Paths;
21 import java.nio.file.SimpleFileVisitor;
22 import java.nio.file.attribute.BasicFileAttributes;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Objects;
30 import java.util.Properties;
31 import java.util.StringJoiner;
32 import java.util.StringTokenizer;
33 import java.util.TreeMap;
34 import java.util.concurrent.CompletableFuture;
35 import java.util.jar.Attributes;
36 import java.util.jar.JarEntry;
37 import java.util.jar.JarOutputStream;
38 import java.util.jar.Manifest;
39 import java.util.zip.Deflater;
40
41 import org.eclipse.jdt.core.compiler.CompilationProgress;
42
43 import aQute.bnd.osgi.Analyzer;
44 import aQute.bnd.osgi.Jar;
45
46 /**
47 * Minimalistic OSGi compiler and packager, meant to be used as a single file
48 * without being itself compiled first. It depends on the Eclipse batch compiler
49 * (aka. ECJ) and the BND Libs library for OSGi metadata generation (which
50 * itselfs depends on slf4j).<br/>
51 * <br/>
52 * For example, a typical system call would be:<br/>
53 * <code>java -cp "/path/to/ECJ jar:/path/to/bndlib jar:/path/to/SLF4J jar" /path/to/cloned/argeo-build/src/org/argeo/build/Make.java action --option1 argument1 argument2 --option2 argument3 </code>
54 */
55 public class Make {
56 private final static Logger logger = System.getLogger(Make.class.getName());
57
58 /**
59 * Environment variable on whether sources should be packaged separately or
60 * integrated in the bundles.
61 */
62 private final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES";
63
64 /**
65 * Environment variable to override the default location for the Argeo Build
66 * configuration files. Typically used if Argeo Build has been compiled and
67 * packaged separately.
68 */
69 private final static String ENV_ARGEO_BUILD_CONFIG = "ARGEO_BUILD_CONFIG";
70
71 /** Name of the local-specific Makefile (sdk.mk). */
72 final static String SDK_MK = "sdk.mk";
73 /** Name of the branch definition Makefile (branch.mk). */
74 final static String BRANCH_MK = "branch.mk";
75
76 /** The execution directory (${user.dir}). */
77 final Path execDirectory;
78 /** Base of the source code, typically the cloned git repository. */
79 final Path sdkSrcBase;
80 /**
81 * The base of the builder, typically a submodule pointing to the public
82 * argeo-build directory.
83 */
84 final Path argeoBuildBase;
85 /** The base of the build for all layers. */
86 final Path sdkBuildBase;
87 /** The base of the build for this layer. */
88 final Path buildBase;
89 /** The base of the a2 output for all layers. */
90 final Path a2Output;
91 /** The base of the a2 sources when packages separately. */
92 final Path a2srcOutput;
93
94 /** Whether sources should be packaged separately */
95 final boolean sourceBundles;
96
97 /** Constructor initialises the base directories. */
98 public Make() throws IOException {
99 sourceBundles = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES));
100 if (sourceBundles)
101 logger.log(Level.INFO, "Sources will be packaged separately");
102
103 execDirectory = Paths.get(System.getProperty("user.dir"));
104 Path sdkMkP = findSdkMk(execDirectory);
105 Objects.requireNonNull(sdkMkP, "No " + SDK_MK + " found under " + execDirectory);
106
107 Map<String, String> context = readeMakefileVariables(sdkMkP);
108 sdkSrcBase = Paths.get(context.computeIfAbsent("SDK_SRC_BASE", (key) -> {
109 throw new IllegalStateException(key + " not found");
110 })).toAbsolutePath();
111
112 Path argeoBuildBaseT = sdkSrcBase.resolve("sdk/argeo-build");
113 if (!Files.exists(argeoBuildBaseT)) {
114 String fromEnv = System.getenv(ENV_ARGEO_BUILD_CONFIG);
115 if (fromEnv != null)
116 argeoBuildBaseT = Paths.get(fromEnv);
117 if (fromEnv == null || !Files.exists(argeoBuildBaseT)) {
118 throw new IllegalStateException(
119 "Argeo Build not found. Did you initialise the git submodules or set the "
120 + ENV_ARGEO_BUILD_CONFIG + " environment variable?");
121 }
122 }
123 argeoBuildBase = argeoBuildBaseT;
124
125 sdkBuildBase = Paths.get(context.computeIfAbsent("SDK_BUILD_BASE", (key) -> {
126 throw new IllegalStateException(key + " not found");
127 })).toAbsolutePath();
128 buildBase = sdkBuildBase.resolve(sdkSrcBase.getFileName());
129 a2Output = sdkBuildBase.resolve("a2");
130 a2srcOutput = sdkBuildBase.resolve("a2.src");
131 }
132
133 /*
134 * ACTIONS
135 */
136 /** Compile and create the bundles in one go. */
137 void all(Map<String, List<String>> options) throws IOException {
138 compile(options);
139 bundle(options);
140 }
141
142 /** Compile all the bundles which have been passed via the --bundle argument. */
143 void compile(Map<String, List<String>> options) throws IOException {
144 List<String> bundles = options.get("--bundles");
145 Objects.requireNonNull(bundles, "--bundles argument must be set");
146 if (bundles.isEmpty())
147 return;
148
149 List<String> a2Categories = options.getOrDefault("--dep-categories", new ArrayList<>());
150 List<String> a2Bases = options.getOrDefault("--a2-bases", new ArrayList<>());
151 if (a2Bases.isEmpty() || !a2Bases.contains(a2Output.toString())) {
152 a2Bases.add(a2Output.toString());
153 }
154
155 List<String> compilerArgs = new ArrayList<>();
156
157 Path ecjArgs = argeoBuildBase.resolve("ecj.args");
158 compilerArgs.add("@" + ecjArgs);
159
160 // classpath
161 if (!a2Categories.isEmpty()) {
162 // We will keep only the highest major.minor
163 // and order by bundle name, for predictability
164 Map<String, A2Jar> a2Jars = new TreeMap<>();
165
166 // StringJoiner modulePath = new StringJoiner(File.pathSeparator);
167 for (String a2Base : a2Bases) {
168 categories: for (String a2Category : a2Categories) {
169 Path a2Dir = Paths.get(a2Base).resolve(a2Category);
170 if (!Files.exists(a2Dir))
171 continue categories;
172 // modulePath.add(a2Dir.toString());
173 for (Path jarP : Files.newDirectoryStream(a2Dir,
174 (p) -> p.getFileName().toString().endsWith(".jar"))) {
175 A2Jar a2Jar = new A2Jar(jarP);
176 if (a2Jars.containsKey(a2Jar.name)) {
177 A2Jar current = a2Jars.get(a2Jar.name);
178 if (a2Jar.major > current.major)
179 a2Jars.put(a2Jar.name, a2Jar);
180 else if (a2Jar.major == current.major //
181 // if minor equals, we take the last one
182 && a2Jar.minor >= current.minor)
183 a2Jars.put(a2Jar.name, a2Jar);
184 } else {
185 a2Jars.put(a2Jar.name, a2Jar);
186 }
187 }
188 }
189 }
190
191 StringJoiner classPath = new StringJoiner(File.pathSeparator);
192 for (Iterator<A2Jar> it = a2Jars.values().iterator(); it.hasNext();)
193 classPath.add(it.next().path.toString());
194
195 compilerArgs.add("-cp");
196 compilerArgs.add(classPath.toString());
197 // compilerArgs.add("--module-path");
198 // compilerArgs.add(modulePath.toString());
199 }
200
201 // sources
202 for (String bundle : bundles) {
203 StringBuilder sb = new StringBuilder();
204 Path bundlePath = execDirectory.resolve(bundle);
205 if (!Files.exists(bundlePath)) {
206 if (bundles.size() == 1) {
207 logger.log(WARNING, "Bundle " + bundle + " not found in " + execDirectory
208 + ", assuming this is this directory, as only one bundle was requested.");
209 bundlePath = execDirectory;
210 } else
211 throw new IllegalArgumentException("Bundle " + bundle + " not found in " + execDirectory);
212 }
213 sb.append(bundlePath.resolve("src"));
214 sb.append("[-d");
215 compilerArgs.add(sb.toString());
216 sb = new StringBuilder();
217 sb.append(buildBase.resolve(bundle).resolve("bin"));
218 sb.append("]");
219 compilerArgs.add(sb.toString());
220 }
221
222 if (logger.isLoggable(INFO))
223 compilerArgs.add("-time");
224
225 if (logger.isLoggable(DEBUG)) {
226 logger.log(DEBUG, "Compiler arguments:");
227 for (String arg : compilerArgs)
228 logger.log(DEBUG, arg);
229 }
230
231 boolean success = org.eclipse.jdt.core.compiler.batch.BatchCompiler.compile(
232 compilerArgs.toArray(new String[compilerArgs.size()]), new PrintWriter(System.out),
233 new PrintWriter(System.err), new MakeCompilationProgress());
234 if (!success) // kill the process if compilation failed
235 throw new IllegalStateException("Compilation failed");
236 }
237
238 /** Package the bundles. */
239 void bundle(Map<String, List<String>> options) throws IOException {
240 // check arguments
241 List<String> bundles = options.get("--bundles");
242 Objects.requireNonNull(bundles, "--bundles argument must be set");
243 if (bundles.isEmpty())
244 return;
245
246 List<String> categories = options.get("--category");
247 Objects.requireNonNull(bundles, "--category argument must be set");
248 if (categories.size() != 1)
249 throw new IllegalArgumentException("One and only one --category must be specified");
250 String category = categories.get(0);
251
252 final String branch;
253 Path branchMk = sdkSrcBase.resolve(BRANCH_MK);
254 if (Files.exists(branchMk)) {
255 Map<String, String> branchVariables = readeMakefileVariables(branchMk);
256 branch = branchVariables.get("BRANCH");
257 } else {
258 branch = null;
259 }
260
261 long begin = System.currentTimeMillis();
262 // create jars in parallel
263 List<CompletableFuture<Void>> toDos = new ArrayList<>();
264 for (String bundle : bundles) {
265 toDos.add(CompletableFuture.runAsync(() -> {
266 try {
267 createBundle(branch, bundle, category);
268 } catch (IOException e) {
269 throw new RuntimeException("Packaging of " + bundle + " failed", e);
270 }
271 }));
272 }
273 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
274 long duration = System.currentTimeMillis() - begin;
275 logger.log(INFO, "Packaging took " + duration + " ms");
276 }
277
278 /** Package a single bundle. */
279 void createBundle(String branch, String bundle, String category) throws IOException {
280 final Path source;
281 if (!Files.exists(execDirectory.resolve(bundle))) {
282 logger.log(WARNING,
283 "Bundle " + bundle + " not found in " + execDirectory + ", assuming this is this directory.");
284 source = execDirectory;
285 } else {
286 source = execDirectory.resolve(bundle);
287 }
288
289 Path compiled = buildBase.resolve(bundle);
290 String bundleSymbolicName = source.getFileName().toString();
291
292 // Metadata
293 Properties properties = new Properties();
294 Path argeoBnd = argeoBuildBase.resolve("argeo.bnd");
295 try (InputStream in = Files.newInputStream(argeoBnd)) {
296 properties.load(in);
297 }
298
299 if (branch != null) {
300 Path branchBnd = sdkSrcBase.resolve("sdk/branches/" + branch + ".bnd");
301 if (Files.exists(branchBnd))
302 try (InputStream in = Files.newInputStream(branchBnd)) {
303 properties.load(in);
304 }
305 }
306
307 Path bndBnd = source.resolve("bnd.bnd");
308 if (Files.exists(bndBnd))
309 try (InputStream in = Files.newInputStream(bndBnd)) {
310 properties.load(in);
311 }
312
313 // Normalise
314 if (!properties.containsKey("Bundle-SymbolicName"))
315 properties.put("Bundle-SymbolicName", bundleSymbolicName);
316
317 // Calculate MANIFEST
318 Path binP = compiled.resolve("bin");
319 if (!Files.exists(binP))
320 Files.createDirectories(binP);
321 Manifest manifest;
322 try (Analyzer bndAnalyzer = new Analyzer()) {
323 bndAnalyzer.setProperties(properties);
324 Jar jar = new Jar(bundleSymbolicName, binP.toFile());
325 bndAnalyzer.setJar(jar);
326 manifest = bndAnalyzer.calcManifest();
327 } catch (Exception e) {
328 throw new RuntimeException("Bnd analysis of " + compiled + " failed", e);
329 }
330
331 String major = properties.getProperty("major");
332 Objects.requireNonNull(major, "'major' must be set");
333 String minor = properties.getProperty("minor");
334 Objects.requireNonNull(minor, "'minor' must be set");
335
336 // Write manifest
337 Path manifestP = compiled.resolve("META-INF/MANIFEST.MF");
338 Files.createDirectories(manifestP.getParent());
339 try (OutputStream out = Files.newOutputStream(manifestP)) {
340 manifest.write(out);
341 }
342
343 // Load excludes
344 List<PathMatcher> excludes = new ArrayList<>();
345 Path excludesP = argeoBuildBase.resolve("excludes.txt");
346 for (String line : Files.readAllLines(excludesP)) {
347 PathMatcher pathMatcher = excludesP.getFileSystem().getPathMatcher("glob:" + line);
348 excludes.add(pathMatcher);
349 }
350
351 Path bundleParent = Paths.get(bundle).getParent();
352 Path a2JarDirectory = bundleParent != null ? a2Output.resolve(bundleParent).resolve(category)
353 : a2Output.resolve(category);
354 Path jarP = a2JarDirectory.resolve(compiled.getFileName() + "." + major + "." + minor + ".jar");
355 Files.createDirectories(jarP.getParent());
356
357 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarP), manifest)) {
358 jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
359 // add all classes first
360 Files.walkFileTree(binP, new SimpleFileVisitor<Path>() {
361 @Override
362 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
363 jarOut.putNextEntry(new JarEntry(binP.relativize(file).toString()));
364 Files.copy(file, jarOut);
365 return FileVisitResult.CONTINUE;
366 }
367 });
368
369 // add resources
370 Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
371 @Override
372 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
373 Path relativeP = source.relativize(dir);
374 for (PathMatcher exclude : excludes)
375 if (exclude.matches(relativeP))
376 return FileVisitResult.SKIP_SUBTREE;
377
378 return FileVisitResult.CONTINUE;
379 }
380
381 @Override
382 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
383 Path relativeP = source.relativize(file);
384 for (PathMatcher exclude : excludes)
385 if (exclude.matches(relativeP))
386 return FileVisitResult.CONTINUE;
387 JarEntry entry = new JarEntry(relativeP.toString());
388 jarOut.putNextEntry(entry);
389 Files.copy(file, jarOut);
390 return FileVisitResult.CONTINUE;
391 }
392 });
393
394 Path srcP = source.resolve("src");
395 // Add all resources from src/
396 Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
397 @Override
398 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
399 if (file.getFileName().toString().endsWith(".java")
400 || file.getFileName().toString().endsWith(".class"))
401 return FileVisitResult.CONTINUE;
402 jarOut.putNextEntry(new JarEntry(srcP.relativize(file).toString()));
403 if (!Files.isDirectory(file))
404 Files.copy(file, jarOut);
405 return FileVisitResult.CONTINUE;
406 }
407 });
408
409 // add sources
410 // TODO add effective BND, Eclipse project file, etc., in order to be able to
411 // repackage
412 if (sourceBundles) {
413 Path a2srcJarDirectory = bundleParent != null ? a2srcOutput.resolve(bundleParent).resolve(category)
414 : a2srcOutput.resolve(category);
415 Files.createDirectories(a2srcJarDirectory);
416 Path srcJarP = a2srcJarDirectory
417 .resolve(compiled.getFileName() + "." + major + "." + minor + ".src.jar");
418 Manifest srcManifest = new Manifest();
419 srcManifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
420 srcManifest.getMainAttributes().putValue("Bundle-SymbolicName", bundleSymbolicName + ".src");
421 srcManifest.getMainAttributes().putValue("Bundle-Version",
422 manifest.getMainAttributes().getValue("Bundle-Version").toString());
423 srcManifest.getMainAttributes().putValue("Eclipse-SourceBundle",
424 bundleSymbolicName + ";version=\"" + manifest.getMainAttributes().getValue("Bundle-Version"));
425
426 try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
427 copySourcesToJar(srcP, srcJarOut, "");
428 }
429 } else {
430 copySourcesToJar(srcP, jarOut, "OSGI-OPT/src/");
431 }
432 }
433 }
434
435 /*
436 * UTILITIES
437 */
438 /** Add sources to a jar file */
439 void copySourcesToJar(Path srcP, JarOutputStream srcJarOut, String prefix) throws IOException {
440 Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
441 @Override
442 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
443 srcJarOut.putNextEntry(new JarEntry(prefix + srcP.relativize(file).toString()));
444 if (!Files.isDirectory(file))
445 Files.copy(file, srcJarOut);
446 return FileVisitResult.CONTINUE;
447 }
448 });
449 }
450
451 /**
452 * Recursively find the base source directory (which contains the
453 * <code>{@value #SDK_MK}</code> file).
454 */
455 Path findSdkMk(Path directory) {
456 Path sdkMkP = directory.resolve(SDK_MK);
457 if (Files.exists(sdkMkP)) {
458 return sdkMkP.toAbsolutePath();
459 }
460 if (directory.getParent() == null)
461 return null;
462 return findSdkMk(directory.getParent());
463 }
464
465 /**
466 * Reads Makefile variable assignments of the form =, :=, or ?=, ignoring white
467 * spaces. To be used with very simple included Makefiles only.
468 */
469 Map<String, String> readeMakefileVariables(Path path) throws IOException {
470 Map<String, String> context = new HashMap<>();
471 List<String> sdkMkLines = Files.readAllLines(path);
472 lines: for (String line : sdkMkLines) {
473 StringTokenizer st = new StringTokenizer(line, " :=?");
474 if (!st.hasMoreTokens())
475 continue lines;
476 String key = st.nextToken();
477 if (!st.hasMoreTokens())
478 continue lines;
479 String value = st.nextToken();
480 if (st.hasMoreTokens()) // probably not a simple variable assignment
481 continue lines;
482 context.put(key, value);
483 }
484 return context;
485 }
486
487 /** Main entry point, interpreting actions and arguments. */
488 public static void main(String... args) {
489 if (args.length == 0)
490 throw new IllegalArgumentException("At least an action must be provided");
491 int actionIndex = 0;
492 String action = args[actionIndex];
493 if (args.length > actionIndex + 1 && !args[actionIndex + 1].startsWith("-"))
494 throw new IllegalArgumentException(
495 "Action " + action + " must be followed by an option: " + Arrays.asList(args));
496
497 Map<String, List<String>> options = new HashMap<>();
498 String currentOption = null;
499 for (int i = actionIndex + 1; i < args.length; i++) {
500 if (args[i].startsWith("-")) {
501 currentOption = args[i];
502 if (!options.containsKey(currentOption))
503 options.put(currentOption, new ArrayList<>());
504
505 } else {
506 options.get(currentOption).add(args[i]);
507 }
508 }
509
510 try {
511 Make argeoMake = new Make();
512 switch (action) {
513 case "compile" -> argeoMake.compile(options);
514 case "bundle" -> argeoMake.bundle(options);
515 case "all" -> argeoMake.all(options);
516
517 default -> throw new IllegalArgumentException("Unkown action: " + action);
518 }
519
520 long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
521 logger.log(INFO, "Make.java action '" + action + "' succesfully completed after " + (jvmUptime / 1000) + "."
522 + (jvmUptime % 1000) + " s");
523 } catch (Exception e) {
524 long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
525 logger.log(ERROR, "Make.java action '" + action + "' failed after " + (jvmUptime / 1000) + "."
526 + (jvmUptime % 1000) + " s", e);
527 System.exit(1);
528 }
529 }
530
531 static class A2Jar {
532 final Path path;
533 final String name;
534 final int major;
535 final int minor;
536
537 A2Jar(Path path) {
538 this.path = path;
539 String fileName = path.getFileName().toString();
540 fileName = fileName.substring(0, fileName.lastIndexOf('.'));
541 minor = Integer.parseInt(fileName.substring(fileName.lastIndexOf('.') + 1));
542 fileName = fileName.substring(0, fileName.lastIndexOf('.'));
543 major = Integer.parseInt(fileName.substring(fileName.lastIndexOf('.') + 1));
544 name = fileName.substring(0, fileName.lastIndexOf('.'));
545 }
546 }
547
548 /**
549 * An ECJ {@link CompilationProgress} printing a progress bar while compiling.
550 */
551 static class MakeCompilationProgress extends CompilationProgress {
552 private int totalWork;
553 private long currentChunk = 0;
554 private long chunksCount = 80;
555
556 @Override
557 public void worked(int workIncrement, int remainingWork) {
558 if (!logger.isLoggable(Level.INFO)) // progress bar only at INFO level
559 return;
560 long chunk = ((totalWork - remainingWork) * chunksCount) / totalWork;
561 if (chunk != currentChunk) {
562 currentChunk = chunk;
563 for (long i = 0; i < currentChunk; i++) {
564 System.out.print("#");
565 }
566 for (long i = currentChunk; i < chunksCount; i++) {
567 System.out.print("-");
568 }
569 System.out.print("\r");
570 }
571 if (remainingWork == 0)
572 System.out.print("\n");
573 }
574
575 @Override
576 public void setTaskName(String name) {
577 }
578
579 @Override
580 public boolean isCanceled() {
581 return false;
582 }
583
584 @Override
585 public void done() {
586 }
587
588 @Override
589 public void begin(int remainingWork) {
590 this.totalWork = remainingWork;
591 }
592 }
593 }