Make.java

   1 package org.argeo.build;
   2
   3 import static java.lang.System.Logger.Level.ERROR;
   4 import static java.lang.System.Logger.Level.INFO;
   5
   6 import java.io.File;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.io.OutputStream;
  10 import java.io.PrintWriter;
  11 import java.lang.System.Logger;
  12 import java.lang.System.Logger.Level;
  13 import java.lang.management.ManagementFactory;
  14 import java.nio.file.FileVisitResult;
  15 import java.nio.file.Files;
  16 import java.nio.file.Path;
  17 import java.nio.file.PathMatcher;
  18 import java.nio.file.Paths;
  19 import java.nio.file.SimpleFileVisitor;
  20 import java.nio.file.attribute.BasicFileAttributes;
  21 import java.util.ArrayList;
  22 import java.util.Arrays;
  23 import java.util.HashMap;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Properties;
  28 import java.util.StringJoiner;
  29 import java.util.StringTokenizer;
  30 import java.util.concurrent.CompletableFuture;
  31 import java.util.jar.Attributes;
  32 import java.util.jar.JarEntry;
  33 import java.util.jar.JarOutputStream;
  34 import java.util.jar.Manifest;
  35 import java.util.zip.Deflater;
  36
  37 import org.eclipse.jdt.core.compiler.CompilationProgress;
  38
  39 import aQute.bnd.osgi.Analyzer;
  40 import aQute.bnd.osgi.Jar;
  41
  42 /**
  43  * Minimalistic OSGi compiler and packager, meant to be used as a single file
  44  * without being itself compiled first. It depends on the Eclipse batch compiler
  45  * (aka. ECJ) and the BND Libs library for OSGi metadata generation (which
  46  * itselfs depends on slf4j).<br/>
  47  * <br/>
  48  * For example, a typical system call would be:<br/>
  49  * <code>java -cp "/path/to/ECJ jar:/path/to/bndlib jar:/path/to/SLF4J jar" /path/to/cloned/argeo-build/src/org/argeo/build/Make.java action --option1 argument1 argument2 --option2 argument3 </code>
  50  */
  51 public class Make {
  52         private final static Logger logger = System.getLogger(Make.class.getName());
  53
  54         /**
  55          * Environment properties on whether sources should be packaged separately or
  56          * integrated in the bundles.
  57          */
  58         private final static String ENV_BUILD_SOURCE_BUNDLES = "BUILD_SOURCE_BUNDLES";
  59
  60         /** Name of the local-specific Makefile (sdk.mk). */
  61         final static String SDK_MK = "sdk.mk";
  62         /** Name of the branch definition Makefile (branch.mk). */
  63         final static String BRANCH_MK = "branch.mk";
  64
  65         /** The execution directory (${user.dir}). */
  66         final Path execDirectory;
  67         /** Base of the source code, typically the cloned git repository. */
  68         final Path sdkSrcBase;
  69         /**
  70          * The base of the builder, typically a submodule pointing to the public
  71          * argeo-build directory.
  72          */
  73         final Path argeoBuildBase;
  74         /** The base of the build for all layers. */
  75         final Path sdkBuildBase;
  76         /** The base of the build for this layer. */
  77         final Path buildBase;
  78         /** The base of the a2 output for all layers. */
  79         final Path a2Output;
  80
  81         /** Whether sources should be packaged separately */
  82         final boolean sourceBundles;
  83
  84         /** Constructor initialises the base directories. */
  85         public Make() throws IOException {
  86                 sourceBundles = Boolean.parseBoolean(System.getenv(ENV_BUILD_SOURCE_BUNDLES));
  87                 if (sourceBundles)
  88                         logger.log(Level.INFO, "Sources will be packaged separately");
  89
  90                 execDirectory = Paths.get(System.getProperty("user.dir"));
  91                 Path sdkMkP = findSdkMk(execDirectory);
  92                 Objects.requireNonNull(sdkMkP, "No " + SDK_MK + " found under " + execDirectory);
  93
  94                 Map<String, String> context = readeMakefileVariables(sdkMkP);
  95                 sdkSrcBase = Paths.get(context.computeIfAbsent("SDK_SRC_BASE", (key) -> {
  96                         throw new IllegalStateException(key + " not found");
  97                 })).toAbsolutePath();
  98                 argeoBuildBase = sdkSrcBase.resolve("sdk/argeo-build");
  99
 100                 sdkBuildBase = Paths.get(context.computeIfAbsent("SDK_BUILD_BASE", (key) -> {
 101                         throw new IllegalStateException(key + " not found");
 102                 })).toAbsolutePath();
 103                 buildBase = sdkBuildBase.resolve(sdkSrcBase.getFileName());
 104                 a2Output = sdkBuildBase.resolve("a2");
 105         }
 106
 107         /*
 108          * ACTIONS
 109          */
 110         /** Compile and create the bundles in one go. */
 111         void all(Map<String, List<String>> options) throws IOException {
 112                 compile(options);
 113                 bundle(options);
 114         }
 115
 116         /** Compile all the bundles which have been passed via the --bundle argument. */
 117         @SuppressWarnings("restriction")
 118         void compile(Map<String, List<String>> options) throws IOException {
 119                 List<String> bundles = options.get("--bundles");
 120                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 121                 if (bundles.isEmpty())
 122                         return;
 123
 124                 List<String> a2Categories = options.getOrDefault("--dep-categories", new ArrayList<>());
 125                 List<String> a2Bases = options.getOrDefault("--a2-bases", new ArrayList<>());
 126                 if (a2Bases.isEmpty()) {
 127                         a2Bases.add(a2Output.toString());
 128                 }
 129
 130                 List<String> compilerArgs = new ArrayList<>();
 131
 132                 Path ecjArgs = argeoBuildBase.resolve("ecj.args");
 133                 compilerArgs.add("@" + ecjArgs);
 134
 135                 // classpath
 136                 if (!a2Categories.isEmpty()) {
 137                         StringJoiner classPath = new StringJoiner(File.pathSeparator);
 138                         StringJoiner modulePath = new StringJoiner(File.pathSeparator);
 139                         for (String a2Base : a2Bases) {
 140                                 for (String a2Category : a2Categories) {
 141                                         Path a2Dir = Paths.get(a2Base).resolve(a2Category);
 142                                         if (!Files.exists(a2Dir))
 143                                                 Files.createDirectories(a2Dir);
 144                                         modulePath.add(a2Dir.toString());
 145                                         for (Path jarP : Files.newDirectoryStream(a2Dir,
 146                                                         (p) -> p.getFileName().toString().endsWith(".jar"))) {
 147                                                 classPath.add(jarP.toString());
 148                                         }
 149                                 }
 150                         }
 151                         compilerArgs.add("-cp");
 152                         compilerArgs.add(classPath.toString());
 153 //                      compilerArgs.add("--module-path");
 154 //                      compilerArgs.add(modulePath.toString());
 155                 }
 156
 157                 // sources
 158                 for (String bundle : bundles) {
 159                         StringBuilder sb = new StringBuilder();
 160                         Path bundlePath = execDirectory.resolve(bundle);
 161                         if (!Files.exists(bundlePath))
 162                                 throw new IllegalArgumentException("Bundle " + bundle + " not found in " + execDirectory);
 163                         sb.append(bundlePath.resolve("src"));
 164                         sb.append("[-d");
 165                         compilerArgs.add(sb.toString());
 166                         sb = new StringBuilder();
 167                         sb.append(buildBase.resolve(bundle).resolve("bin"));
 168                         sb.append("]");
 169                         compilerArgs.add(sb.toString());
 170                 }
 171
 172                 if (logger.isLoggable(INFO))
 173                         compilerArgs.add("-time");
 174
 175 //              for (String arg : compilerArgs)
 176 //                      System.out.println(arg);
 177
 178                 boolean success = org.eclipse.jdt.core.compiler.batch.BatchCompiler.compile(
 179                                 compilerArgs.toArray(new String[compilerArgs.size()]), new PrintWriter(System.out),
 180                                 new PrintWriter(System.err), new MakeCompilationProgress());
 181                 if (!success) // kill the process if compilation failed
 182                         throw new IllegalStateException("Compilation failed");
 183         }
 184
 185         /** Package the bundles. */
 186         void bundle(Map<String, List<String>> options) throws IOException {
 187                 // check arguments
 188                 List<String> bundles = options.get("--bundles");
 189                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 190                 if (bundles.isEmpty())
 191                         return;
 192
 193                 List<String> categories = options.get("--category");
 194                 Objects.requireNonNull(bundles, "--category argument must be set");
 195                 if (categories.size() != 1)
 196                         throw new IllegalArgumentException("One and only one --category must be specified");
 197                 String category = categories.get(0);
 198
 199                 final String branch;
 200                 Path branchMk = sdkSrcBase.resolve(BRANCH_MK);
 201                 if (Files.exists(branchMk)) {
 202                         Map<String, String> branchVariables = readeMakefileVariables(branchMk);
 203                         branch = branchVariables.get("BRANCH");
 204                 } else {
 205                         branch = null;
 206                 }
 207
 208                 long begin = System.currentTimeMillis();
 209                 // create jars in parallel
 210                 List<CompletableFuture<Void>> toDos = new ArrayList<>();
 211                 for (String bundle : bundles) {
 212                         toDos.add(CompletableFuture.runAsync(() -> {
 213                                 try {
 214                                         createBundle(branch, bundle, category);
 215                                 } catch (IOException e) {
 216                                         throw new RuntimeException("Packaging of " + bundle + " failed", e);
 217                                 }
 218                         }));
 219                 }
 220                 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
 221                 long duration = System.currentTimeMillis() - begin;
 222                 logger.log(INFO, "Packaging took " + duration + " ms");
 223         }
 224
 225         /*
 226          * UTILITIES
 227          */
 228         /** Package a single bundle. */
 229         void createBundle(String branch, String bundle, String category) throws IOException {
 230                 Path source = execDirectory.resolve(bundle);
 231                 Path compiled = buildBase.resolve(bundle);
 232                 String bundleSymbolicName = source.getFileName().toString();
 233
 234                 // Metadata
 235                 Properties properties = new Properties();
 236                 Path argeoBnd = argeoBuildBase.resolve("argeo.bnd");
 237                 try (InputStream in = Files.newInputStream(argeoBnd)) {
 238                         properties.load(in);
 239                 }
 240
 241                 if (branch != null) {
 242                         Path branchBnd = sdkSrcBase.resolve("sdk/branches/" + branch + ".bnd");
 243                         if (Files.exists(branchBnd))
 244                                 try (InputStream in = Files.newInputStream(branchBnd)) {
 245                                         properties.load(in);
 246                                 }
 247                 }
 248
 249                 Path bndBnd = source.resolve("bnd.bnd");
 250                 if (Files.exists(bndBnd))
 251                         try (InputStream in = Files.newInputStream(bndBnd)) {
 252                                 properties.load(in);
 253                         }
 254
 255                 // Normalise
 256                 if (!properties.containsKey("Bundle-SymbolicName"))
 257                         properties.put("Bundle-SymbolicName", bundleSymbolicName);
 258
 259                 // Calculate MANIFEST
 260                 Path binP = compiled.resolve("bin");
 261                 if (!Files.exists(binP))
 262                         Files.createDirectories(binP);
 263                 Manifest manifest;
 264                 try (Analyzer bndAnalyzer = new Analyzer()) {
 265                         bndAnalyzer.setProperties(properties);
 266                         Jar jar = new Jar(bundleSymbolicName, binP.toFile());
 267                         bndAnalyzer.setJar(jar);
 268                         manifest = bndAnalyzer.calcManifest();
 269                 } catch (Exception e) {
 270                         throw new RuntimeException("Bnd analysis of " + compiled + " failed", e);
 271                 }
 272
 273                 String major = properties.getProperty("major");
 274                 Objects.requireNonNull(major, "'major' must be set");
 275                 String minor = properties.getProperty("minor");
 276                 Objects.requireNonNull(minor, "'minor' must be set");
 277
 278                 // Write manifest
 279                 Path manifestP = compiled.resolve("META-INF/MANIFEST.MF");
 280                 Files.createDirectories(manifestP.getParent());
 281                 try (OutputStream out = Files.newOutputStream(manifestP)) {
 282                         manifest.write(out);
 283                 }
 284
 285                 // Load excludes
 286                 List<PathMatcher> excludes = new ArrayList<>();
 287                 Path excludesP = argeoBuildBase.resolve("excludes.txt");
 288                 for (String line : Files.readAllLines(excludesP)) {
 289                         PathMatcher pathMatcher = excludesP.getFileSystem().getPathMatcher("glob:" + line);
 290                         excludes.add(pathMatcher);
 291                 }
 292
 293                 Path bundleParent = Paths.get(bundle).getParent();
 294                 Path a2JarDirectory = bundleParent != null ? a2Output.resolve(bundleParent).resolve(category)
 295                                 : a2Output.resolve(category);
 296                 Path jarP = a2JarDirectory.resolve(compiled.getFileName() + "." + major + "." + minor + ".jar");
 297                 Files.createDirectories(jarP.getParent());
 298
 299                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarP), manifest)) {
 300                         jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
 301                         // add all classes first
 302                         Files.walkFileTree(binP, new SimpleFileVisitor<Path>() {
 303                                 @Override
 304                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 305                                         jarOut.putNextEntry(new JarEntry(binP.relativize(file).toString()));
 306                                         Files.copy(file, jarOut);
 307                                         return FileVisitResult.CONTINUE;
 308                                 }
 309                         });
 310
 311                         // add resources
 312                         Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
 313                                 @Override
 314                                 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 315                                         Path relativeP = source.relativize(dir);
 316                                         for (PathMatcher exclude : excludes)
 317                                                 if (exclude.matches(relativeP))
 318                                                         return FileVisitResult.SKIP_SUBTREE;
 319
 320                                         return FileVisitResult.CONTINUE;
 321                                 }
 322
 323                                 @Override
 324                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 325                                         Path relativeP = source.relativize(file);
 326                                         for (PathMatcher exclude : excludes)
 327                                                 if (exclude.matches(relativeP))
 328                                                         return FileVisitResult.CONTINUE;
 329                                         JarEntry entry = new JarEntry(relativeP.toString());
 330                                         jarOut.putNextEntry(entry);
 331                                         Files.copy(file, jarOut);
 332                                         return FileVisitResult.CONTINUE;
 333                                 }
 334                         });
 335
 336                         Path srcP = source.resolve("src");
 337                         // Add all resources from src/
 338                         Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
 339                                 @Override
 340                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 341                                         if (file.getFileName().toString().endsWith(".java")
 342                                                         || file.getFileName().toString().endsWith(".class"))
 343                                                 return FileVisitResult.CONTINUE;
 344                                         jarOut.putNextEntry(new JarEntry(srcP.relativize(file).toString()));
 345                                         if (!Files.isDirectory(file))
 346                                                 Files.copy(file, jarOut);
 347                                         return FileVisitResult.CONTINUE;
 348                                 }
 349                         });
 350
 351                         // add sources
 352                         // TODO add effective BND, Eclipse project file, etc., in order to be able to
 353                         // repackage
 354                         if (sourceBundles) {
 355                                 Path srcJarP = a2JarDirectory.resolve(compiled.getFileName() + "." + major + "." + minor + ".src.jar");
 356                                 Manifest srcManifest = new Manifest();
 357                                 srcManifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
 358                                 srcManifest.getMainAttributes().putValue("Bundle-SymbolicName", bundleSymbolicName + ".src");
 359                                 srcManifest.getMainAttributes().putValue("Bundle-Version",
 360                                                 manifest.getMainAttributes().getValue("Bundle-Version").toString());
 361                                 srcManifest.getMainAttributes().putValue("Eclipse-SourceBundle",
 362                                                 bundleSymbolicName + ";version=\"" + manifest.getMainAttributes().getValue("Bundle-Version"));
 363
 364                                 try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
 365                                         copySourcesToJar(srcP, srcJarOut, "");
 366                                 }
 367                         } else {
 368                                 copySourcesToJar(srcP, jarOut, "OSGI-OPT/src/");
 369                         }
 370                 }
 371         }
 372
 373         void copySourcesToJar(Path srcP, JarOutputStream srcJarOut, String prefix) throws IOException {
 374                 Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
 375                         @Override
 376                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 377                                 srcJarOut.putNextEntry(new JarEntry(prefix + srcP.relativize(file).toString()));
 378                                 if (!Files.isDirectory(file))
 379                                         Files.copy(file, srcJarOut);
 380                                 return FileVisitResult.CONTINUE;
 381                         }
 382                 });
 383         }
 384
 385         /**
 386          * Recursively find the base source directory (which contains the
 387          * <code>{@value #SDK_MK}</code> file).
 388          */
 389         Path findSdkMk(Path directory) {
 390                 Path sdkMkP = directory.resolve(SDK_MK);
 391                 if (Files.exists(sdkMkP)) {
 392                         return sdkMkP.toAbsolutePath();
 393                 }
 394                 if (directory.getParent() == null)
 395                         return null;
 396                 return findSdkMk(directory.getParent());
 397         }
 398
 399         /**
 400          * Reads Makefile variable assignments of the form =, :=, or ?=, ignoring white
 401          * spaces. To be used with very simple included Makefiles only.
 402          */
 403         Map<String, String> readeMakefileVariables(Path path) throws IOException {
 404                 Map<String, String> context = new HashMap<>();
 405                 List<String> sdkMkLines = Files.readAllLines(path);
 406                 lines: for (String line : sdkMkLines) {
 407                         StringTokenizer st = new StringTokenizer(line, " :=?");
 408                         if (!st.hasMoreTokens())
 409                                 continue lines;
 410                         String key = st.nextToken();
 411                         if (!st.hasMoreTokens())
 412                                 continue lines;
 413                         String value = st.nextToken();
 414                         if (st.hasMoreTokens()) // probably not a simple variable assignment
 415                                 continue lines;
 416                         context.put(key, value);
 417                 }
 418                 return context;
 419         }
 420
 421         /** Main entry point, interpreting actions and arguments. */
 422         public static void main(String... args) {
 423                 if (args.length == 0)
 424                         throw new IllegalArgumentException("At least an action must be provided");
 425                 int actionIndex = 0;
 426                 String action = args[actionIndex];
 427                 if (args.length > actionIndex + 1 && !args[actionIndex + 1].startsWith("-"))
 428                         throw new IllegalArgumentException(
 429                                         "Action " + action + " must be followed by an option: " + Arrays.asList(args));
 430
 431                 Map<String, List<String>> options = new HashMap<>();
 432                 String currentOption = null;
 433                 for (int i = actionIndex + 1; i < args.length; i++) {
 434                         if (args[i].startsWith("-")) {
 435                                 currentOption = args[i];
 436                                 if (!options.containsKey(currentOption))
 437                                         options.put(currentOption, new ArrayList<>());
 438
 439                         } else {
 440                                 options.get(currentOption).add(args[i]);
 441                         }
 442                 }
 443
 444                 try {
 445                         Make argeoMake = new Make();
 446                         switch (action) {
 447                         case "compile" -> argeoMake.compile(options);
 448                         case "bundle" -> argeoMake.bundle(options);
 449                         case "all" -> argeoMake.all(options);
 450
 451                         default -> throw new IllegalArgumentException("Unkown action: " + action);
 452                         }
 453
 454                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 455                         logger.log(INFO, "Make.java action '" + action + "' succesfully completed after " + (jvmUptime / 1000) + "."
 456                                         + (jvmUptime % 1000) + " s");
 457                 } catch (Exception e) {
 458                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 459                         logger.log(ERROR, "Make.java action '" + action + "' failed after " + (jvmUptime / 1000) + "."
 460                                         + (jvmUptime % 1000) + " s", e);
 461                         System.exit(1);
 462                 }
 463         }
 464
 465         /**
 466          * An ECJ {@link CompilationProgress} printing a progress bar while compiling.
 467          */
 468         static class MakeCompilationProgress extends CompilationProgress {
 469                 private int totalWork;
 470                 private long currentChunk = 0;
 471                 private long chunksCount = 80;
 472
 473                 @Override
 474                 public void worked(int workIncrement, int remainingWork) {
 475                         if (!logger.isLoggable(Level.INFO)) // progress bar only at INFO level
 476                                 return;
 477                         long chunk = ((totalWork - remainingWork) * chunksCount) / totalWork;
 478                         if (chunk != currentChunk) {
 479                                 currentChunk = chunk;
 480                                 for (long i = 0; i < currentChunk; i++) {
 481                                         System.out.print("#");
 482                                 }
 483                                 for (long i = currentChunk; i < chunksCount; i++) {
 484                                         System.out.print("-");
 485                                 }
 486                                 System.out.print("\r");
 487                         }
 488                         if (remainingWork == 0)
 489                                 System.out.print("\n");
 490                 }
 491
 492                 @Override
 493                 public void setTaskName(String name) {
 494                 }
 495
 496                 @Override
 497                 public boolean isCanceled() {
 498                         return false;
 499                 }
 500
 501                 @Override
 502                 public void done() {
 503                 }
 504
 505                 @Override
 506                 public void begin(int remainingWork) {
 507                         this.totalWork = remainingWork;
 508                 }
 509         }
 510 }