src/org/argeo/build/Make.java

   1 package org.argeo.build;
   2
   3 import static java.lang.System.Logger.Level.ERROR;
   4 import static java.lang.System.Logger.Level.INFO;
   5
   6 import java.io.File;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.io.OutputStream;
  10 import java.io.PrintWriter;
  11 import java.lang.System.Logger;
  12 import java.lang.System.Logger.Level;
  13 import java.lang.management.ManagementFactory;
  14 import java.nio.file.FileVisitResult;
  15 import java.nio.file.Files;
  16 import java.nio.file.Path;
  17 import java.nio.file.PathMatcher;
  18 import java.nio.file.Paths;
  19 import java.nio.file.SimpleFileVisitor;
  20 import java.nio.file.attribute.BasicFileAttributes;
  21 import java.util.ArrayList;
  22 import java.util.Arrays;
  23 import java.util.HashMap;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Properties;
  28 import java.util.StringJoiner;
  29 import java.util.StringTokenizer;
  30 import java.util.concurrent.CompletableFuture;
  31 import java.util.jar.Attributes;
  32 import java.util.jar.JarEntry;
  33 import java.util.jar.JarOutputStream;
  34 import java.util.jar.Manifest;
  35 import java.util.zip.Deflater;
  36
  37 import org.eclipse.jdt.core.compiler.CompilationProgress;
  38
  39 import aQute.bnd.osgi.Analyzer;
  40 import aQute.bnd.osgi.Jar;
  41
  42 /**
  43  * Minimalistic OSGi compiler and packager, meant to be used as a single file
  44  * without being itself compiled first. It depends on the Eclipse batch compiler
  45  * (aka. ECJ) and the BND Libs library for OSGi metadata generation (which
  46  * itselfs depends on slf4j).<br/>
  47  * <br/>
  48  * For example, a typical system call would be:<br/>
  49  * <code>java -cp "/path/to/ECJ jar:/path/to/bndlib jar:/path/to/SLF4J jar" /path/to/cloned/argeo-build/src/org/argeo/build/Make.java action --option1 argument1 argument2 --option2 argument3 </code>
  50  */
  51 public class Make {
  52         private final static Logger logger = System.getLogger(Make.class.getName());
  53
  54         /**
  55          * Environment properties on whether sources should be packaged separately or
  56          * integrated in the bundles.
  57          */
  58         private final static String ENV_BUILD_SOURCE_BUNDLES = "BUILD_SOURCE_BUNDLES";
  59
  60         /** Name of the local-specific Makefile (sdk.mk). */
  61         final static String SDK_MK = "sdk.mk";
  62         /** Name of the branch definition Makefile (branch.mk). */
  63         final static String BRANCH_MK = "branch.mk";
  64
  65         /** The execution directory (${user.dir}). */
  66         final Path execDirectory;
  67         /** Base of the source code, typically the cloned git repository. */
  68         final Path sdkSrcBase;
  69         /**
  70          * The base of the builder, typically a submodule pointing to the public
  71          * argeo-build directory.
  72          */
  73         final Path argeoBuildBase;
  74         /** The base of the build for all layers. */
  75         final Path sdkBuildBase;
  76         /** The base of the build for this layer. */
  77         final Path buildBase;
  78         /** The base of the a2 output for all layers. */
  79         final Path a2Output;
  80         /** The base of the a2 sources when packages separately. */
  81         final Path a2srcOutput;
  82
  83         /** Whether sources should be packaged separately */
  84         final boolean sourceBundles;
  85
  86         /** Constructor initialises the base directories. */
  87         public Make() throws IOException {
  88                 sourceBundles = Boolean.parseBoolean(System.getenv(ENV_BUILD_SOURCE_BUNDLES));
  89                 if (sourceBundles)
  90                         logger.log(Level.INFO, "Sources will be packaged separately");
  91
  92                 execDirectory = Paths.get(System.getProperty("user.dir"));
  93                 Path sdkMkP = findSdkMk(execDirectory);
  94                 Objects.requireNonNull(sdkMkP, "No " + SDK_MK + " found under " + execDirectory);
  95
  96                 Map<String, String> context = readeMakefileVariables(sdkMkP);
  97                 sdkSrcBase = Paths.get(context.computeIfAbsent("SDK_SRC_BASE", (key) -> {
  98                         throw new IllegalStateException(key + " not found");
  99                 })).toAbsolutePath();
 100                 argeoBuildBase = sdkSrcBase.resolve("sdk/argeo-build");
 101
 102                 sdkBuildBase = Paths.get(context.computeIfAbsent("SDK_BUILD_BASE", (key) -> {
 103                         throw new IllegalStateException(key + " not found");
 104                 })).toAbsolutePath();
 105                 buildBase = sdkBuildBase.resolve(sdkSrcBase.getFileName());
 106                 a2Output = sdkBuildBase.resolve("a2");
 107                 a2srcOutput = sdkBuildBase.resolve("a2.src");
 108         }
 109
 110         /*
 111          * ACTIONS
 112          */
 113         /** Compile and create the bundles in one go. */
 114         void all(Map<String, List<String>> options) throws IOException {
 115                 compile(options);
 116                 bundle(options);
 117         }
 118
 119         /** Compile all the bundles which have been passed via the --bundle argument. */
 120         @SuppressWarnings("restriction")
 121         void compile(Map<String, List<String>> options) throws IOException {
 122                 List<String> bundles = options.get("--bundles");
 123                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 124                 if (bundles.isEmpty())
 125                         return;
 126
 127                 List<String> a2Categories = options.getOrDefault("--dep-categories", new ArrayList<>());
 128                 List<String> a2Bases = options.getOrDefault("--a2-bases", new ArrayList<>());
 129                 if (a2Bases.isEmpty()) {
 130                         a2Bases.add(a2Output.toString());
 131                 }
 132
 133                 List<String> compilerArgs = new ArrayList<>();
 134
 135                 Path ecjArgs = argeoBuildBase.resolve("ecj.args");
 136                 compilerArgs.add("@" + ecjArgs);
 137
 138                 // classpath
 139                 if (!a2Categories.isEmpty()) {
 140                         StringJoiner classPath = new StringJoiner(File.pathSeparator);
 141                         StringJoiner modulePath = new StringJoiner(File.pathSeparator);
 142                         for (String a2Base : a2Bases) {
 143                                 for (String a2Category : a2Categories) {
 144                                         Path a2Dir = Paths.get(a2Base).resolve(a2Category);
 145                                         if (!Files.exists(a2Dir))
 146                                                 Files.createDirectories(a2Dir);
 147                                         modulePath.add(a2Dir.toString());
 148                                         for (Path jarP : Files.newDirectoryStream(a2Dir,
 149                                                         (p) -> p.getFileName().toString().endsWith(".jar"))) {
 150                                                 classPath.add(jarP.toString());
 151                                         }
 152                                 }
 153                         }
 154                         compilerArgs.add("-cp");
 155                         compilerArgs.add(classPath.toString());
 156 //                      compilerArgs.add("--module-path");
 157 //                      compilerArgs.add(modulePath.toString());
 158                 }
 159
 160                 // sources
 161                 for (String bundle : bundles) {
 162                         StringBuilder sb = new StringBuilder();
 163                         Path bundlePath = execDirectory.resolve(bundle);
 164                         if (!Files.exists(bundlePath))
 165                                 throw new IllegalArgumentException("Bundle " + bundle + " not found in " + execDirectory);
 166                         sb.append(bundlePath.resolve("src"));
 167                         sb.append("[-d");
 168                         compilerArgs.add(sb.toString());
 169                         sb = new StringBuilder();
 170                         sb.append(buildBase.resolve(bundle).resolve("bin"));
 171                         sb.append("]");
 172                         compilerArgs.add(sb.toString());
 173                 }
 174
 175                 if (logger.isLoggable(INFO))
 176                         compilerArgs.add("-time");
 177
 178 //              for (String arg : compilerArgs)
 179 //                      System.out.println(arg);
 180
 181                 boolean success = org.eclipse.jdt.core.compiler.batch.BatchCompiler.compile(
 182                                 compilerArgs.toArray(new String[compilerArgs.size()]), new PrintWriter(System.out),
 183                                 new PrintWriter(System.err), new MakeCompilationProgress());
 184                 if (!success) // kill the process if compilation failed
 185                         throw new IllegalStateException("Compilation failed");
 186         }
 187
 188         /** Package the bundles. */
 189         void bundle(Map<String, List<String>> options) throws IOException {
 190                 // check arguments
 191                 List<String> bundles = options.get("--bundles");
 192                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 193                 if (bundles.isEmpty())
 194                         return;
 195
 196                 List<String> categories = options.get("--category");
 197                 Objects.requireNonNull(bundles, "--category argument must be set");
 198                 if (categories.size() != 1)
 199                         throw new IllegalArgumentException("One and only one --category must be specified");
 200                 String category = categories.get(0);
 201
 202                 final String branch;
 203                 Path branchMk = sdkSrcBase.resolve(BRANCH_MK);
 204                 if (Files.exists(branchMk)) {
 205                         Map<String, String> branchVariables = readeMakefileVariables(branchMk);
 206                         branch = branchVariables.get("BRANCH");
 207                 } else {
 208                         branch = null;
 209                 }
 210
 211                 long begin = System.currentTimeMillis();
 212                 // create jars in parallel
 213                 List<CompletableFuture<Void>> toDos = new ArrayList<>();
 214                 for (String bundle : bundles) {
 215                         toDos.add(CompletableFuture.runAsync(() -> {
 216                                 try {
 217                                         createBundle(branch, bundle, category);
 218                                 } catch (IOException e) {
 219                                         throw new RuntimeException("Packaging of " + bundle + " failed", e);
 220                                 }
 221                         }));
 222                 }
 223                 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
 224                 long duration = System.currentTimeMillis() - begin;
 225                 logger.log(INFO, "Packaging took " + duration + " ms");
 226         }
 227
 228         /*
 229          * UTILITIES
 230          */
 231         /** Package a single bundle. */
 232         void createBundle(String branch, String bundle, String category) throws IOException {
 233                 Path source = execDirectory.resolve(bundle);
 234                 Path compiled = buildBase.resolve(bundle);
 235                 String bundleSymbolicName = source.getFileName().toString();
 236
 237                 // Metadata
 238                 Properties properties = new Properties();
 239                 Path argeoBnd = argeoBuildBase.resolve("argeo.bnd");
 240                 try (InputStream in = Files.newInputStream(argeoBnd)) {
 241                         properties.load(in);
 242                 }
 243
 244                 if (branch != null) {
 245                         Path branchBnd = sdkSrcBase.resolve("sdk/branches/" + branch + ".bnd");
 246                         if (Files.exists(branchBnd))
 247                                 try (InputStream in = Files.newInputStream(branchBnd)) {
 248                                         properties.load(in);
 249                                 }
 250                 }
 251
 252                 Path bndBnd = source.resolve("bnd.bnd");
 253                 if (Files.exists(bndBnd))
 254                         try (InputStream in = Files.newInputStream(bndBnd)) {
 255                                 properties.load(in);
 256                         }
 257
 258                 // Normalise
 259                 if (!properties.containsKey("Bundle-SymbolicName"))
 260                         properties.put("Bundle-SymbolicName", bundleSymbolicName);
 261
 262                 // Calculate MANIFEST
 263                 Path binP = compiled.resolve("bin");
 264                 if (!Files.exists(binP))
 265                         Files.createDirectories(binP);
 266                 Manifest manifest;
 267                 try (Analyzer bndAnalyzer = new Analyzer()) {
 268                         bndAnalyzer.setProperties(properties);
 269                         Jar jar = new Jar(bundleSymbolicName, binP.toFile());
 270                         bndAnalyzer.setJar(jar);
 271                         manifest = bndAnalyzer.calcManifest();
 272                 } catch (Exception e) {
 273                         throw new RuntimeException("Bnd analysis of " + compiled + " failed", e);
 274                 }
 275
 276                 String major = properties.getProperty("major");
 277                 Objects.requireNonNull(major, "'major' must be set");
 278                 String minor = properties.getProperty("minor");
 279                 Objects.requireNonNull(minor, "'minor' must be set");
 280
 281                 // Write manifest
 282                 Path manifestP = compiled.resolve("META-INF/MANIFEST.MF");
 283                 Files.createDirectories(manifestP.getParent());
 284                 try (OutputStream out = Files.newOutputStream(manifestP)) {
 285                         manifest.write(out);
 286                 }
 287
 288                 // Load excludes
 289                 List<PathMatcher> excludes = new ArrayList<>();
 290                 Path excludesP = argeoBuildBase.resolve("excludes.txt");
 291                 for (String line : Files.readAllLines(excludesP)) {
 292                         PathMatcher pathMatcher = excludesP.getFileSystem().getPathMatcher("glob:" + line);
 293                         excludes.add(pathMatcher);
 294                 }
 295
 296                 Path bundleParent = Paths.get(bundle).getParent();
 297                 Path a2JarDirectory = bundleParent != null ? a2Output.resolve(bundleParent).resolve(category)
 298                                 : a2Output.resolve(category);
 299                 Path jarP = a2JarDirectory.resolve(compiled.getFileName() + "." + major + "." + minor + ".jar");
 300                 Files.createDirectories(jarP.getParent());
 301
 302                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarP), manifest)) {
 303                         jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
 304                         // add all classes first
 305                         Files.walkFileTree(binP, new SimpleFileVisitor<Path>() {
 306                                 @Override
 307                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 308                                         jarOut.putNextEntry(new JarEntry(binP.relativize(file).toString()));
 309                                         Files.copy(file, jarOut);
 310                                         return FileVisitResult.CONTINUE;
 311                                 }
 312                         });
 313
 314                         // add resources
 315                         Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
 316                                 @Override
 317                                 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 318                                         Path relativeP = source.relativize(dir);
 319                                         for (PathMatcher exclude : excludes)
 320                                                 if (exclude.matches(relativeP))
 321                                                         return FileVisitResult.SKIP_SUBTREE;
 322
 323                                         return FileVisitResult.CONTINUE;
 324                                 }
 325
 326                                 @Override
 327                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 328                                         Path relativeP = source.relativize(file);
 329                                         for (PathMatcher exclude : excludes)
 330                                                 if (exclude.matches(relativeP))
 331                                                         return FileVisitResult.CONTINUE;
 332                                         JarEntry entry = new JarEntry(relativeP.toString());
 333                                         jarOut.putNextEntry(entry);
 334                                         Files.copy(file, jarOut);
 335                                         return FileVisitResult.CONTINUE;
 336                                 }
 337                         });
 338
 339                         Path srcP = source.resolve("src");
 340                         // Add all resources from src/
 341                         Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
 342                                 @Override
 343                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 344                                         if (file.getFileName().toString().endsWith(".java")
 345                                                         || file.getFileName().toString().endsWith(".class"))
 346                                                 return FileVisitResult.CONTINUE;
 347                                         jarOut.putNextEntry(new JarEntry(srcP.relativize(file).toString()));
 348                                         if (!Files.isDirectory(file))
 349                                                 Files.copy(file, jarOut);
 350                                         return FileVisitResult.CONTINUE;
 351                                 }
 352                         });
 353
 354                         // add sources
 355                         // TODO add effective BND, Eclipse project file, etc., in order to be able to
 356                         // repackage
 357                         if (sourceBundles) {
 358                                 Path a2srcJarDirectory = bundleParent != null ? a2srcOutput.resolve(bundleParent).resolve(category)
 359                                                 : a2srcOutput.resolve(category);
 360                                 Files.createDirectories(a2srcJarDirectory);
 361                                 Path srcJarP = a2srcJarDirectory
 362                                                 .resolve(compiled.getFileName() + "." + major + "." + minor + ".src.jar");
 363                                 Manifest srcManifest = new Manifest();
 364                                 srcManifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
 365                                 srcManifest.getMainAttributes().putValue("Bundle-SymbolicName", bundleSymbolicName + ".src");
 366                                 srcManifest.getMainAttributes().putValue("Bundle-Version",
 367                                                 manifest.getMainAttributes().getValue("Bundle-Version").toString());
 368                                 srcManifest.getMainAttributes().putValue("Eclipse-SourceBundle",
 369                                                 bundleSymbolicName + ";version=\"" + manifest.getMainAttributes().getValue("Bundle-Version"));
 370
 371                                 try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
 372                                         copySourcesToJar(srcP, srcJarOut, "");
 373                                 }
 374                         } else {
 375                                 copySourcesToJar(srcP, jarOut, "OSGI-OPT/src/");
 376                         }
 377                 }
 378         }
 379
 380         void copySourcesToJar(Path srcP, JarOutputStream srcJarOut, String prefix) throws IOException {
 381                 Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
 382                         @Override
 383                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 384                                 srcJarOut.putNextEntry(new JarEntry(prefix + srcP.relativize(file).toString()));
 385                                 if (!Files.isDirectory(file))
 386                                         Files.copy(file, srcJarOut);
 387                                 return FileVisitResult.CONTINUE;
 388                         }
 389                 });
 390         }
 391
 392         /**
 393          * Recursively find the base source directory (which contains the
 394          * <code>{@value #SDK_MK}</code> file).
 395          */
 396         Path findSdkMk(Path directory) {
 397                 Path sdkMkP = directory.resolve(SDK_MK);
 398                 if (Files.exists(sdkMkP)) {
 399                         return sdkMkP.toAbsolutePath();
 400                 }
 401                 if (directory.getParent() == null)
 402                         return null;
 403                 return findSdkMk(directory.getParent());
 404         }
 405
 406         /**
 407          * Reads Makefile variable assignments of the form =, :=, or ?=, ignoring white
 408          * spaces. To be used with very simple included Makefiles only.
 409          */
 410         Map<String, String> readeMakefileVariables(Path path) throws IOException {
 411                 Map<String, String> context = new HashMap<>();
 412                 List<String> sdkMkLines = Files.readAllLines(path);
 413                 lines: for (String line : sdkMkLines) {
 414                         StringTokenizer st = new StringTokenizer(line, " :=?");
 415                         if (!st.hasMoreTokens())
 416                                 continue lines;
 417                         String key = st.nextToken();
 418                         if (!st.hasMoreTokens())
 419                                 continue lines;
 420                         String value = st.nextToken();
 421                         if (st.hasMoreTokens()) // probably not a simple variable assignment
 422                                 continue lines;
 423                         context.put(key, value);
 424                 }
 425                 return context;
 426         }
 427
 428         /** Main entry point, interpreting actions and arguments. */
 429         public static void main(String... args) {
 430                 if (args.length == 0)
 431                         throw new IllegalArgumentException("At least an action must be provided");
 432                 int actionIndex = 0;
 433                 String action = args[actionIndex];
 434                 if (args.length > actionIndex + 1 && !args[actionIndex + 1].startsWith("-"))
 435                         throw new IllegalArgumentException(
 436                                         "Action " + action + " must be followed by an option: " + Arrays.asList(args));
 437
 438                 Map<String, List<String>> options = new HashMap<>();
 439                 String currentOption = null;
 440                 for (int i = actionIndex + 1; i < args.length; i++) {
 441                         if (args[i].startsWith("-")) {
 442                                 currentOption = args[i];
 443                                 if (!options.containsKey(currentOption))
 444                                         options.put(currentOption, new ArrayList<>());
 445
 446                         } else {
 447                                 options.get(currentOption).add(args[i]);
 448                         }
 449                 }
 450
 451                 try {
 452                         Make argeoMake = new Make();
 453                         switch (action) {
 454                         case "compile" -> argeoMake.compile(options);
 455                         case "bundle" -> argeoMake.bundle(options);
 456                         case "all" -> argeoMake.all(options);
 457
 458                         default -> throw new IllegalArgumentException("Unkown action: " + action);
 459                         }
 460
 461                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 462                         logger.log(INFO, "Make.java action '" + action + "' succesfully completed after " + (jvmUptime / 1000) + "."
 463                                         + (jvmUptime % 1000) + " s");
 464                 } catch (Exception e) {
 465                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 466                         logger.log(ERROR, "Make.java action '" + action + "' failed after " + (jvmUptime / 1000) + "."
 467                                         + (jvmUptime % 1000) + " s", e);
 468                         System.exit(1);
 469                 }
 470         }
 471
 472         /**
 473          * An ECJ {@link CompilationProgress} printing a progress bar while compiling.
 474          */
 475         static class MakeCompilationProgress extends CompilationProgress {
 476                 private int totalWork;
 477                 private long currentChunk = 0;
 478                 private long chunksCount = 80;
 479
 480                 @Override
 481                 public void worked(int workIncrement, int remainingWork) {
 482                         if (!logger.isLoggable(Level.INFO)) // progress bar only at INFO level
 483                                 return;
 484                         long chunk = ((totalWork - remainingWork) * chunksCount) / totalWork;
 485                         if (chunk != currentChunk) {
 486                                 currentChunk = chunk;
 487                                 for (long i = 0; i < currentChunk; i++) {
 488                                         System.out.print("#");
 489                                 }
 490                                 for (long i = currentChunk; i < chunksCount; i++) {
 491                                         System.out.print("-");
 492                                 }
 493                                 System.out.print("\r");
 494                         }
 495                         if (remainingWork == 0)
 496                                 System.out.print("\n");
 497                 }
 498
 499                 @Override
 500                 public void setTaskName(String name) {
 501                 }
 502
 503                 @Override
 504                 public boolean isCanceled() {
 505                         return false;
 506                 }
 507
 508                 @Override
 509                 public void done() {
 510                 }
 511
 512                 @Override
 513                 public void begin(int remainingWork) {
 514                         this.totalWork = remainingWork;
 515                 }
 516         }
 517 }