build/Make.java

   1 package org.argeo.build;
   2
   3 import static java.lang.System.Logger.Level.ERROR;
   4 import static java.lang.System.Logger.Level.INFO;
   5
   6 import java.io.File;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.io.OutputStream;
  10 import java.io.PrintWriter;
  11 import java.lang.System.Logger;
  12 import java.lang.System.Logger.Level;
  13 import java.lang.management.ManagementFactory;
  14 import java.nio.file.FileVisitResult;
  15 import java.nio.file.Files;
  16 import java.nio.file.Path;
  17 import java.nio.file.PathMatcher;
  18 import java.nio.file.Paths;
  19 import java.nio.file.SimpleFileVisitor;
  20 import java.nio.file.attribute.BasicFileAttributes;
  21 import java.util.ArrayList;
  22 import java.util.Arrays;
  23 import java.util.HashMap;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Properties;
  28 import java.util.StringJoiner;
  29 import java.util.StringTokenizer;
  30 import java.util.concurrent.CompletableFuture;
  31 import java.util.jar.JarEntry;
  32 import java.util.jar.JarOutputStream;
  33 import java.util.jar.Manifest;
  34 import java.util.zip.Deflater;
  35
  36 import org.eclipse.jdt.core.compiler.CompilationProgress;
  37
  38 import aQute.bnd.osgi.Analyzer;
  39 import aQute.bnd.osgi.Jar;
  40
  41 /**
  42  * Minimalistic OSGi compiler and packager, meant to be used as a single file
  43  * without being itself compiled first. It depends on the Eclipse batch compiler
  44  * (aka. ECJ) and the BND Libs library for OSGi metadata generation (which
  45  * itselfs depends on slf4j).<br/>
  46  * <br/>
  47  * For example, a typical system call would be:<br/>
  48  * <code>java -cp "/path/to/ECJ jar:/path/to/bndlib jar:/path/to/SLF4J jar" /path/to/cloned/argeo-build/src/org/argeo/build/Make.java action --option1 argument1 argument2 --option2 argument3 </code>
  49  */
  50 public class Make {
  51         private final static Logger logger = System.getLogger(Make.class.getName());
  52
  53         /** Name of the local-specific Makefile (sdk.mk). */
  54         final static String SDK_MK = "sdk.mk";
  55         /** Name of the branch definition Makefile (branch.mk). */
  56         final static String BRANCH_MK = "branch.mk";
  57
  58         /** The execution directory (${user.dir}). */
  59         final Path execDirectory;
  60         /** Base of the source code, typically the cloned git repository. */
  61         final Path sdkSrcBase;
  62         /**
  63          * The base of the builder, typically a submodule pointing to the public
  64          * argeo-build directory.
  65          */
  66         final Path argeoBuildBase;
  67         /** The base of the build for all layers. */
  68         final Path sdkBuildBase;
  69         /** The base of the build for this layer. */
  70         final Path buildBase;
  71         /** The base of the a2 output for all layers. */
  72         final Path a2Output;
  73
  74         /** Constructor initialises the base directories. */
  75         public Make() throws IOException {
  76                 execDirectory = Paths.get(System.getProperty("user.dir"));
  77                 Path sdkMkP = findSdkMk(execDirectory);
  78                 Objects.requireNonNull(sdkMkP, "No " + SDK_MK + " found under " + execDirectory);
  79
  80                 Map<String, String> context = readeMakefileVariables(sdkMkP);
  81                 sdkSrcBase = Paths.get(context.computeIfAbsent("SDK_SRC_BASE", (key) -> {
  82                         throw new IllegalStateException(key + " not found");
  83                 })).toAbsolutePath();
  84                 argeoBuildBase = sdkSrcBase.resolve("sdk/argeo-build");
  85
  86                 sdkBuildBase = Paths.get(context.computeIfAbsent("SDK_BUILD_BASE", (key) -> {
  87                         throw new IllegalStateException(key + " not found");
  88                 })).toAbsolutePath();
  89                 buildBase = sdkBuildBase.resolve(sdkSrcBase.getFileName());
  90                 a2Output = sdkBuildBase.resolve("a2");
  91         }
  92
  93         /*
  94          * ACTIONS
  95          */
  96         /** Compile and create the bundles in one go. */
  97         void all(Map<String, List<String>> options) throws IOException {
  98                 compile(options);
  99                 bundle(options);
 100         }
 101
 102         /** Compile all the bundles which have been passed via the --bundle argument. */
 103         @SuppressWarnings("restriction")
 104         void compile(Map<String, List<String>> options) throws IOException {
 105                 List<String> bundles = options.get("--bundles");
 106                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 107                 if (bundles.isEmpty())
 108                         return;
 109
 110                 List<String> a2Categories = options.getOrDefault("--dep-categories", new ArrayList<>());
 111                 List<String> a2Bases = options.getOrDefault("--a2-bases", new ArrayList<>());
 112                 if (a2Bases.isEmpty()) {
 113                         a2Bases.add(a2Output.toString());
 114                 }
 115
 116                 List<String> compilerArgs = new ArrayList<>();
 117
 118                 Path ecjArgs = argeoBuildBase.resolve("ecj.args");
 119                 compilerArgs.add("@" + ecjArgs);
 120
 121                 // classpath
 122                 if (!a2Categories.isEmpty()) {
 123                         StringJoiner classPath = new StringJoiner(File.pathSeparator);
 124                         StringJoiner modulePath = new StringJoiner(File.pathSeparator);
 125                         for (String a2Base : a2Bases) {
 126                                 for (String a2Category : a2Categories) {
 127                                         Path a2Dir = Paths.get(a2Base).resolve(a2Category);
 128                                         if (!Files.exists(a2Dir))
 129                                                 Files.createDirectories(a2Dir);
 130                                         modulePath.add(a2Dir.toString());
 131                                         for (Path jarP : Files.newDirectoryStream(a2Dir,
 132                                                         (p) -> p.getFileName().toString().endsWith(".jar"))) {
 133                                                 classPath.add(jarP.toString());
 134                                         }
 135                                 }
 136                         }
 137                         compilerArgs.add("-cp");
 138                         compilerArgs.add(classPath.toString());
 139 //                      compilerArgs.add("--module-path");
 140 //                      compilerArgs.add(modulePath.toString());
 141                 }
 142
 143                 // sources
 144                 for (String bundle : bundles) {
 145                         StringBuilder sb = new StringBuilder();
 146                         Path bundlePath = execDirectory.resolve(bundle);
 147                         if (!Files.exists(bundlePath))
 148                                 throw new IllegalArgumentException("Bundle " + bundle + " not found in " + execDirectory);
 149                         sb.append(bundlePath.resolve("src"));
 150                         sb.append("[-d");
 151                         compilerArgs.add(sb.toString());
 152                         sb = new StringBuilder();
 153                         sb.append(buildBase.resolve(bundle).resolve("bin"));
 154                         sb.append("]");
 155                         compilerArgs.add(sb.toString());
 156                 }
 157
 158                 if (logger.isLoggable(INFO))
 159                         compilerArgs.add("-time");
 160
 161 //              for (String arg : compilerArgs)
 162 //                      System.out.println(arg);
 163
 164                 boolean success = org.eclipse.jdt.core.compiler.batch.BatchCompiler.compile(
 165                                 compilerArgs.toArray(new String[compilerArgs.size()]), new PrintWriter(System.out),
 166                                 new PrintWriter(System.err), new MakeCompilationProgress());
 167                 if (!success) // kill the process if compilation failed
 168                         throw new IllegalStateException("Compilation failed");
 169         }
 170
 171         /** Package the bundles. */
 172         void bundle(Map<String, List<String>> options) throws IOException {
 173                 // check arguments
 174                 List<String> bundles = options.get("--bundles");
 175                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 176                 if (bundles.isEmpty())
 177                         return;
 178
 179                 List<String> categories = options.get("--category");
 180                 Objects.requireNonNull(bundles, "--category argument must be set");
 181                 if (categories.size() != 1)
 182                         throw new IllegalArgumentException("One and only one --category must be specified");
 183                 String category = categories.get(0);
 184
 185                 Path branchMk = sdkSrcBase.resolve(BRANCH_MK);
 186                 if (!Files.exists(branchMk))
 187                         throw new IllegalStateException("No " + branchMk + " file available");
 188                 Map<String, String> branchVariables = readeMakefileVariables(branchMk);
 189
 190                 String branch = branchVariables.get("BRANCH");
 191
 192                 long begin = System.currentTimeMillis();
 193                 // create jars in parallel
 194                 List<CompletableFuture<Void>> toDos = new ArrayList<>();
 195                 for (String bundle : bundles) {
 196                         toDos.add(CompletableFuture.runAsync(() -> {
 197                                 try {
 198                                         createBundle(branch, bundle, category);
 199                                 } catch (IOException e) {
 200                                         throw new RuntimeException("Packaging of " + bundle + " failed", e);
 201                                 }
 202                         }));
 203                 }
 204                 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
 205                 long duration = System.currentTimeMillis() - begin;
 206                 logger.log(INFO, "Packaging took " + duration + " ms");
 207         }
 208
 209         /*
 210          * UTILITIES
 211          */
 212         /** Package a single bundle. */
 213         void createBundle(String branch, String bundle, String category) throws IOException {
 214                 Path source = execDirectory.resolve(bundle);
 215                 Path compiled = buildBase.resolve(bundle);
 216                 String bundleSymbolicName = source.getFileName().toString();
 217
 218                 // Metadata
 219                 Properties properties = new Properties();
 220                 Path argeoBnd = argeoBuildBase.resolve("argeo.bnd");
 221                 try (InputStream in = Files.newInputStream(argeoBnd)) {
 222                         properties.load(in);
 223                 }
 224
 225                 Path branchBnd = sdkSrcBase.resolve("sdk/branches/" + branch + ".bnd");
 226                 try (InputStream in = Files.newInputStream(branchBnd)) {
 227                         properties.load(in);
 228                 }
 229
 230                 Path bndBnd = source.resolve("bnd.bnd");
 231                 try (InputStream in = Files.newInputStream(bndBnd)) {
 232                         properties.load(in);
 233                 }
 234
 235                 // Normalise
 236                 properties.put("Bundle-SymbolicName", bundleSymbolicName);
 237
 238                 // Calculate MANIFEST
 239                 Path binP = compiled.resolve("bin");
 240                 if (!Files.exists(binP))
 241                         Files.createDirectories(binP);
 242                 Manifest manifest;
 243                 try (Analyzer bndAnalyzer = new Analyzer()) {
 244                         bndAnalyzer.setProperties(properties);
 245                         Jar jar = new Jar(bundleSymbolicName, binP.toFile());
 246                         bndAnalyzer.setJar(jar);
 247                         manifest = bndAnalyzer.calcManifest();
 248                 } catch (Exception e) {
 249                         throw new RuntimeException("Bnd analysis of " + compiled + " failed", e);
 250                 }
 251
 252                 String major = properties.getProperty("major");
 253                 Objects.requireNonNull(major, "'major' must be set");
 254                 String minor = properties.getProperty("minor");
 255                 Objects.requireNonNull(minor, "'minor' must be set");
 256
 257                 // Write manifest
 258                 Path manifestP = compiled.resolve("META-INF/MANIFEST.MF");
 259                 Files.createDirectories(manifestP.getParent());
 260                 try (OutputStream out = Files.newOutputStream(manifestP)) {
 261                         manifest.write(out);
 262                 }
 263
 264                 // Load excludes
 265                 List<PathMatcher> excludes = new ArrayList<>();
 266                 Path excludesP = argeoBuildBase.resolve("excludes.txt");
 267                 for (String line : Files.readAllLines(excludesP)) {
 268                         PathMatcher pathMatcher = excludesP.getFileSystem().getPathMatcher("glob:" + line);
 269                         excludes.add(pathMatcher);
 270                 }
 271
 272                 Path bundleParent = Paths.get(bundle).getParent();
 273                 Path a2JarDirectory = bundleParent != null ? a2Output.resolve(bundleParent).resolve(category)
 274                                 : a2Output.resolve(category);
 275                 Path jarP = a2JarDirectory.resolve(compiled.getFileName() + "." + major + "." + minor + ".jar");
 276                 Files.createDirectories(jarP.getParent());
 277
 278                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarP), manifest)) {
 279                         jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
 280                         // add all classes first
 281                         Files.walkFileTree(binP, new SimpleFileVisitor<Path>() {
 282                                 @Override
 283                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 284                                         jarOut.putNextEntry(new JarEntry(binP.relativize(file).toString()));
 285                                         Files.copy(file, jarOut);
 286                                         return FileVisitResult.CONTINUE;
 287                                 }
 288                         });
 289
 290                         // add resources
 291                         Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
 292                                 @Override
 293                                 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 294                                         Path relativeP = source.relativize(dir);
 295                                         for (PathMatcher exclude : excludes)
 296                                                 if (exclude.matches(relativeP))
 297                                                         return FileVisitResult.SKIP_SUBTREE;
 298
 299                                         return FileVisitResult.CONTINUE;
 300                                 }
 301
 302                                 @Override
 303                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 304                                         Path relativeP = source.relativize(file);
 305                                         for (PathMatcher exclude : excludes)
 306                                                 if (exclude.matches(relativeP))
 307                                                         return FileVisitResult.CONTINUE;
 308                                         JarEntry entry = new JarEntry(relativeP.toString());
 309                                         jarOut.putNextEntry(entry);
 310                                         Files.copy(file, jarOut);
 311                                         return FileVisitResult.CONTINUE;
 312                                 }
 313                         });
 314
 315                         Path srcP = source.resolve("src");
 316                         // Add all resources from src/
 317                         Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
 318                                 @Override
 319                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 320                                         if (file.getFileName().toString().endsWith(".java")
 321                                                         || file.getFileName().toString().endsWith(".class"))
 322                                                 return FileVisitResult.CONTINUE;
 323                                         jarOut.putNextEntry(new JarEntry(srcP.relativize(file).toString()));
 324                                         if (!Files.isDirectory(file))
 325                                                 Files.copy(file, jarOut);
 326                                         return FileVisitResult.CONTINUE;
 327                                 }
 328                         });
 329
 330                         // add sources
 331                         // TODO add effective BND, Eclipse project file, etc., in order to be able to
 332                         // repackage
 333                         Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
 334                                 @Override
 335                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 336                                         jarOut.putNextEntry(new JarEntry("OSGI-OPT/src/" + srcP.relativize(file).toString()));
 337                                         if (!Files.isDirectory(file))
 338                                                 Files.copy(file, jarOut);
 339                                         return FileVisitResult.CONTINUE;
 340                                 }
 341                         });
 342
 343                 }
 344         }
 345
 346         /**
 347          * Recursively find the base source directory (which contains the
 348          * <code>{@value #SDK_MK}</code> file).
 349          */
 350         Path findSdkMk(Path directory) {
 351                 Path sdkMkP = directory.resolve(SDK_MK);
 352                 if (Files.exists(sdkMkP)) {
 353                         return sdkMkP.toAbsolutePath();
 354                 }
 355                 if (directory.getParent() == null)
 356                         return null;
 357                 return findSdkMk(directory.getParent());
 358         }
 359
 360         /**
 361          * Reads Makefile variable assignments of the form =, :=, or ?=, ignoring white
 362          * spaces. To be used with very simple included Makefiles only.
 363          */
 364         Map<String, String> readeMakefileVariables(Path path) throws IOException {
 365                 Map<String, String> context = new HashMap<>();
 366                 List<String> sdkMkLines = Files.readAllLines(path);
 367                 lines: for (String line : sdkMkLines) {
 368                         StringTokenizer st = new StringTokenizer(line, " :=?");
 369                         if (!st.hasMoreTokens())
 370                                 continue lines;
 371                         String key = st.nextToken();
 372                         if (!st.hasMoreTokens())
 373                                 continue lines;
 374                         String value = st.nextToken();
 375                         if (st.hasMoreTokens()) // probably not a simple variable assignment
 376                                 continue lines;
 377                         context.put(key, value);
 378                 }
 379                 return context;
 380         }
 381
 382         /** Main entry point, interpreting actions and arguments. */
 383         public static void main(String... args) {
 384                 if (args.length == 0)
 385                         throw new IllegalArgumentException("At least an action must be provided");
 386                 int actionIndex = 0;
 387                 String action = args[actionIndex];
 388                 if (args.length > actionIndex + 1 && !args[actionIndex + 1].startsWith("-"))
 389                         throw new IllegalArgumentException(
 390                                         "Action " + action + " must be followed by an option: " + Arrays.asList(args));
 391
 392                 Map<String, List<String>> options = new HashMap<>();
 393                 String currentOption = null;
 394                 for (int i = actionIndex + 1; i < args.length; i++) {
 395                         if (args[i].startsWith("-")) {
 396                                 currentOption = args[i];
 397                                 if (!options.containsKey(currentOption))
 398                                         options.put(currentOption, new ArrayList<>());
 399
 400                         } else {
 401                                 options.get(currentOption).add(args[i]);
 402                         }
 403                 }
 404
 405                 try {
 406                         Make argeoMake = new Make();
 407                         switch (action) {
 408                         case "compile" -> argeoMake.compile(options);
 409                         case "bundle" -> argeoMake.bundle(options);
 410                         case "all" -> argeoMake.all(options);
 411
 412                         default -> throw new IllegalArgumentException("Unkown action: " + action);
 413                         }
 414
 415                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 416                         logger.log(INFO, "Make.java action '" + action + "' succesfully completed after " + (jvmUptime / 1000) + "."
 417                                         + (jvmUptime % 1000) + " s");
 418                 } catch (Exception e) {
 419                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 420                         logger.log(ERROR, "Make.java action '" + action + "' failed after " + (jvmUptime / 1000) + "."
 421                                         + (jvmUptime % 1000) + " s", e);
 422                         System.exit(1);
 423                 }
 424         }
 425
 426         /**
 427          * An ECJ {@link CompilationProgress} printing a progress bar while compiling.
 428          */
 429         static class MakeCompilationProgress extends CompilationProgress {
 430                 private int totalWork;
 431                 private long currentChunk = 0;
 432                 private long chunksCount = 80;
 433
 434                 @Override
 435                 public void worked(int workIncrement, int remainingWork) {
 436                         if (!logger.isLoggable(Level.INFO)) // progress bar only at INFO level
 437                                 return;
 438                         long chunk = ((totalWork - remainingWork) * chunksCount) / totalWork;
 439                         if (chunk != currentChunk) {
 440                                 currentChunk = chunk;
 441                                 for (long i = 0; i < currentChunk; i++) {
 442                                         System.out.print("#");
 443                                 }
 444                                 for (long i = currentChunk; i < chunksCount; i++) {
 445                                         System.out.print("-");
 446                                 }
 447                                 System.out.print("\r");
 448                         }
 449                         if (remainingWork == 0)
 450                                 System.out.print("\n");
 451                 }
 452
 453                 @Override
 454                 public void setTaskName(String name) {
 455                 }
 456
 457                 @Override
 458                 public boolean isCanceled() {
 459                         return false;
 460                 }
 461
 462                 @Override
 463                 public void done() {
 464                 }
 465
 466                 @Override
 467                 public void begin(int remainingWork) {
 468                         this.totalWork = remainingWork;
 469                 }
 470         }
 471 }