Make.java

   1 package org.argeo.build;
   2
   3 import static java.lang.System.Logger.Level.ERROR;
   4 import static java.lang.System.Logger.Level.INFO;
   5
   6 import java.io.File;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.io.OutputStream;
  10 import java.io.PrintWriter;
  11 import java.lang.System.Logger;
  12 import java.lang.System.Logger.Level;
  13 import java.lang.management.ManagementFactory;
  14 import java.nio.file.FileVisitResult;
  15 import java.nio.file.Files;
  16 import java.nio.file.Path;
  17 import java.nio.file.PathMatcher;
  18 import java.nio.file.Paths;
  19 import java.nio.file.SimpleFileVisitor;
  20 import java.nio.file.attribute.BasicFileAttributes;
  21 import java.util.ArrayList;
  22 import java.util.Arrays;
  23 import java.util.HashMap;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Properties;
  28 import java.util.StringJoiner;
  29 import java.util.StringTokenizer;
  30 import java.util.concurrent.CompletableFuture;
  31 import java.util.jar.JarEntry;
  32 import java.util.jar.JarOutputStream;
  33 import java.util.jar.Manifest;
  34
  35 import org.eclipse.jdt.core.compiler.CompilationProgress;
  36
  37 import aQute.bnd.osgi.Analyzer;
  38 import aQute.bnd.osgi.Jar;
  39
  40 /**
  41  * Minimalistic OSGi compiler and packager, meant to be used as a single file
  42  * without being itself compiled first. It depends on the Eclipse batch compiler
  43  * (aka. ECJ) and the BND Libs library for OSGi metadata generation (which
  44  * itselfs depends on slf4j).<br/>
  45  * <br/>
  46  * For example, a typical system call would be:<br/>
  47  * <code>java -cp "/path/to/ECJ jar:/path/to/bndlib jar:/path/to/SLF4J jar" /path/to/cloned/argeo-build/src/org/argeo/build/Make.java action --option1 argument1 argument2 --option2 argument3 </code>
  48  */
  49 public class Make {
  50         private final static Logger logger = System.getLogger(Make.class.getName());
  51
  52         /** Name of the local-specific Makefile (sdk.mk). */
  53         final static String SDK_MK = "sdk.mk";
  54
  55         /** The execution directory (${user.dir}). */
  56         final Path execDirectory;
  57         /** Base of the source code, typically the cloned git repository. */
  58         final Path sdkSrcBase;
  59         /**
  60          * The base of the builder, typically a submodule pointing to the public
  61          * argeo-build directory.
  62          */
  63         final Path argeoBuildBase;
  64         /** The base of the build for all layers. */
  65         final Path sdkBuildBase;
  66         /** The base of the build for this layer. */
  67         final Path buildBase;
  68         /** The base of the a2 output for all layers. */
  69         final Path a2Output;
  70
  71         /** Constructor initialises the base directories. */
  72         public Make() throws IOException {
  73                 execDirectory = Paths.get(System.getProperty("user.dir"));
  74                 Path sdkMkP = findSdkMk(execDirectory);
  75                 Objects.requireNonNull(sdkMkP, "No " + SDK_MK + " found under " + execDirectory);
  76
  77                 Map<String, String> context = new HashMap<>();
  78                 List<String> sdkMkLines = Files.readAllLines(sdkMkP);
  79                 lines: for (String line : sdkMkLines) {
  80                         StringTokenizer st = new StringTokenizer(line, " :=");
  81                         if (!st.hasMoreTokens())
  82                                 continue lines;
  83                         String key = st.nextToken();
  84                         if (!st.hasMoreTokens())
  85                                 continue lines;
  86                         String value = st.nextToken();
  87                         context.put(key, value);
  88                 }
  89
  90                 sdkSrcBase = Paths.get(context.computeIfAbsent("SDK_SRC_BASE", (key) -> {
  91                         throw new IllegalStateException(key + " not found");
  92                 })).toAbsolutePath();
  93                 argeoBuildBase = sdkSrcBase.resolve("sdk/argeo-build");
  94
  95                 sdkBuildBase = Paths.get(context.computeIfAbsent("SDK_BUILD_BASE", (key) -> {
  96                         throw new IllegalStateException(key + " not found");
  97                 })).toAbsolutePath();
  98                 buildBase = sdkBuildBase.resolve(sdkSrcBase.getFileName());
  99                 a2Output = sdkBuildBase.resolve("a2");
 100         }
 101
 102         /*
 103          * ACTIONS
 104          */
 105         /** Compile and create the bundles in one go. */
 106         void all(Map<String, List<String>> options) throws IOException {
 107                 compile(options);
 108                 bundle(options);
 109         }
 110
 111         /** Compile all the bundles which have been passed via the --bundle argument. */
 112         @SuppressWarnings("restriction")
 113         void compile(Map<String, List<String>> options) throws IOException {
 114                 List<String> bundles = options.get("--bundles");
 115                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 116                 if (bundles.isEmpty())
 117                         return;
 118
 119                 List<String> a2Categories = options.getOrDefault("--dep-categories", new ArrayList<>());
 120                 List<String> a2Bases = options.getOrDefault("--a2-bases", new ArrayList<>());
 121                 if (a2Bases.isEmpty()) {
 122                         a2Bases.add(a2Output.toString());
 123                 }
 124
 125                 List<String> compilerArgs = new ArrayList<>();
 126
 127                 Path ecjArgs = argeoBuildBase.resolve("ecj.args");
 128                 compilerArgs.add("@" + ecjArgs);
 129
 130                 // classpath
 131                 if (!a2Categories.isEmpty()) {
 132                         compilerArgs.add("-cp");
 133                         StringJoiner classPath = new StringJoiner(File.pathSeparator);
 134                         for (String a2Base : a2Bases) {
 135                                 for (String a2Category : a2Categories) {
 136                                         Path a2Dir = Paths.get(a2Base).resolve(a2Category);
 137                                         if (!Files.exists(a2Dir))
 138                                                 Files.createDirectories(a2Dir);
 139                                         for (Path jarP : Files.newDirectoryStream(a2Dir,
 140                                                         (p) -> p.getFileName().toString().endsWith(".jar"))) {
 141                                                 classPath.add(jarP.toString());
 142                                         }
 143                                 }
 144                         }
 145                         compilerArgs.add(classPath.toString());
 146                 }
 147
 148                 // sources
 149                 for (String bundle : bundles) {
 150                         StringBuilder sb = new StringBuilder();
 151                         sb.append(execDirectory.resolve(bundle).resolve("src"));
 152                         sb.append("[-d");
 153                         compilerArgs.add(sb.toString());
 154                         sb = new StringBuilder();
 155                         sb.append(buildBase.resolve(bundle).resolve("bin"));
 156                         sb.append("]");
 157                         compilerArgs.add(sb.toString());
 158                 }
 159
 160                 if (logger.isLoggable(INFO))
 161                         compilerArgs.add("-time");
 162
 163                 boolean success = org.eclipse.jdt.core.compiler.batch.BatchCompiler.compile(
 164                                 compilerArgs.toArray(new String[compilerArgs.size()]), new PrintWriter(System.out),
 165                                 new PrintWriter(System.err), new MakeCompilationProgress());
 166                 if (!success) // kill the process if compilation failed
 167                         throw new IllegalStateException("Compilation failed");
 168         }
 169
 170         /** Package the bundles. */
 171         void bundle(Map<String, List<String>> options) {
 172                 // check arguments
 173                 List<String> bundles = options.get("--bundles");
 174                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 175                 if (bundles.isEmpty())
 176                         return;
 177
 178                 List<String> categories = options.get("--category");
 179                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 180                 if (categories.size() != 1)
 181                         throw new IllegalArgumentException("One and only one category must be specified");
 182                 String category = categories.get(0);
 183
 184                 long begin = System.currentTimeMillis();
 185                 // create jars in parallel
 186                 List<CompletableFuture<Void>> toDos = new ArrayList<>();
 187                 for (String bundle : bundles) {
 188                         toDos.add(CompletableFuture.runAsync(() -> {
 189                                 try {
 190                                         createBundle(bundle, category);
 191                                 } catch (IOException e) {
 192                                         throw new RuntimeException("Packaging of " + bundle + " failed", e);
 193                                 }
 194                         }));
 195                 }
 196                 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
 197                 long duration = System.currentTimeMillis() - begin;
 198                 logger.log(INFO, "Packaging took " + duration + " ms");
 199         }
 200
 201         /*
 202          * UTILITIES
 203          */
 204         /** Package a single bundle. */
 205         void createBundle(String bundle, String category) throws IOException {
 206                 Path source = execDirectory.resolve(bundle);
 207                 Path compiled = buildBase.resolve(bundle);
 208                 String bundleSymbolicName = source.getFileName().toString();
 209
 210                 // Metadata
 211                 Properties properties = new Properties();
 212                 Path argeoBnd = argeoBuildBase.resolve("argeo.bnd");
 213                 try (InputStream in = Files.newInputStream(argeoBnd)) {
 214                         properties.load(in);
 215                 }
 216                 // FIXME make it configurable
 217                 Path branchBnd = sdkSrcBase.resolve("cnf/unstable.bnd");
 218                 try (InputStream in = Files.newInputStream(branchBnd)) {
 219                         properties.load(in);
 220                 }
 221
 222                 Path bndBnd = source.resolve("bnd.bnd");
 223                 try (InputStream in = Files.newInputStream(bndBnd)) {
 224                         properties.load(in);
 225                 }
 226
 227                 // Normalise
 228                 properties.put("Bundle-SymbolicName", bundleSymbolicName);
 229
 230                 // Calculate MANIFEST
 231                 Path binP = compiled.resolve("bin");
 232                 if (!Files.exists(binP))
 233                         Files.createDirectories(binP);
 234                 Manifest manifest;
 235                 try (Analyzer bndAnalyzer = new Analyzer()) {
 236                         bndAnalyzer.setProperties(properties);
 237                         Jar jar = new Jar(bundleSymbolicName, binP.toFile());
 238                         bndAnalyzer.setJar(jar);
 239                         manifest = bndAnalyzer.calcManifest();
 240                 } catch (Exception e) {
 241                         throw new RuntimeException("Bnd analysis of " + compiled + " failed", e);
 242                 }
 243
 244                 String major = properties.getProperty("MAJOR");
 245                 Objects.requireNonNull(major, "MAJOR must be set");
 246                 String minor = properties.getProperty("MINOR");
 247                 Objects.requireNonNull(minor, "MINOR must be set");
 248
 249                 // Write manifest
 250                 Path manifestP = compiled.resolve("META-INF/MANIFEST.MF");
 251                 Files.createDirectories(manifestP.getParent());
 252                 try (OutputStream out = Files.newOutputStream(manifestP)) {
 253                         manifest.write(out);
 254                 }
 255
 256                 // Load excludes
 257                 List<PathMatcher> excludes = new ArrayList<>();
 258                 Path excludesP = argeoBuildBase.resolve("excludes.txt");
 259                 for (String line : Files.readAllLines(excludesP)) {
 260                         PathMatcher pathMatcher = excludesP.getFileSystem().getPathMatcher("glob:" + line);
 261                         excludes.add(pathMatcher);
 262                 }
 263
 264                 Path bundleParent = Paths.get(bundle).getParent();
 265                 Path a2JarDirectory = bundleParent != null ? a2Output.resolve(bundleParent).resolve(category)
 266                                 : a2Output.resolve(category);
 267                 Path jarP = a2JarDirectory.resolve(compiled.getFileName() + "." + major + "." + minor + ".jar");
 268                 Files.createDirectories(jarP.getParent());
 269
 270                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarP), manifest)) {
 271                         // add all classes first
 272                         Files.walkFileTree(binP, new SimpleFileVisitor<Path>() {
 273                                 @Override
 274                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 275                                         jarOut.putNextEntry(new JarEntry(binP.relativize(file).toString()));
 276                                         Files.copy(file, jarOut);
 277                                         return FileVisitResult.CONTINUE;
 278                                 }
 279                         });
 280
 281                         // add resources
 282                         Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
 283                                 @Override
 284                                 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 285                                         Path relativeP = source.relativize(dir);
 286                                         for (PathMatcher exclude : excludes)
 287                                                 if (exclude.matches(relativeP))
 288                                                         return FileVisitResult.SKIP_SUBTREE;
 289
 290                                         return FileVisitResult.CONTINUE;
 291                                 }
 292
 293                                 @Override
 294                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 295                                         Path relativeP = source.relativize(file);
 296                                         for (PathMatcher exclude : excludes)
 297                                                 if (exclude.matches(relativeP))
 298                                                         return FileVisitResult.CONTINUE;
 299                                         JarEntry entry = new JarEntry(relativeP.toString());
 300                                         jarOut.putNextEntry(entry);
 301                                         Files.copy(file, jarOut);
 302                                         return FileVisitResult.CONTINUE;
 303                                 }
 304                         });
 305
 306                         // add sources
 307                         // TODO add effective BND, Eclipse project file, etc., in order to be able to
 308                         // repackage
 309                         Path srcP = source.resolve("src");
 310                         Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
 311                                 @Override
 312                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 313                                         jarOut.putNextEntry(new JarEntry("OSGI-OPT/src/" + srcP.relativize(file).toString()));
 314                                         Files.copy(file, jarOut);
 315                                         return FileVisitResult.CONTINUE;
 316                                 }
 317                         });
 318                 }
 319         }
 320
 321         /**
 322          * Recursively find the base source directory (which contains the
 323          * <code>{@value #SDK_MK}</code> file).
 324          */
 325         private Path findSdkMk(Path directory) {
 326                 Path sdkMkP = directory.resolve(SDK_MK);
 327                 if (Files.exists(sdkMkP)) {
 328                         return sdkMkP.toAbsolutePath();
 329                 }
 330                 if (directory.getParent() == null)
 331                         return null;
 332                 return findSdkMk(directory.getParent());
 333         }
 334
 335         /** Main entry point, interpreting actions and arguments. */
 336         public static void main(String... args) {
 337                 if (args.length == 0)
 338                         throw new IllegalArgumentException("At least an action must be provided");
 339                 int actionIndex = 0;
 340                 String action = args[actionIndex];
 341                 if (args.length > actionIndex + 1 && !args[actionIndex + 1].startsWith("-"))
 342                         throw new IllegalArgumentException(
 343                                         "Action " + action + " must be followed by an option: " + Arrays.asList(args));
 344
 345                 Map<String, List<String>> options = new HashMap<>();
 346                 String currentOption = null;
 347                 for (int i = actionIndex + 1; i < args.length; i++) {
 348                         if (args[i].startsWith("-")) {
 349                                 currentOption = args[i];
 350                                 if (!options.containsKey(currentOption))
 351                                         options.put(currentOption, new ArrayList<>());
 352
 353                         } else {
 354                                 options.get(currentOption).add(args[i]);
 355                         }
 356                 }
 357
 358                 try {
 359                         Make argeoMake = new Make();
 360                         switch (action) {
 361                         case "compile" -> argeoMake.compile(options);
 362                         case "bundle" -> argeoMake.bundle(options);
 363                         case "all" -> argeoMake.all(options);
 364
 365                         default -> throw new IllegalArgumentException("Unkown action: " + action);
 366                         }
 367
 368                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 369                         logger.log(INFO, "Make.java action '" + action + "' succesfully completed after " + (jvmUptime / 1000) + "."
 370                                         + (jvmUptime % 1000) + " s");
 371                 } catch (Exception e) {
 372                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 373                         logger.log(ERROR,
 374                                         "Make.java action '" + action + "' failed after " + (jvmUptime / 1000) + "." + (jvmUptime % 1000) + " s",
 375                                         e);
 376                         System.exit(1);
 377                 }
 378         }
 379
 380         /**
 381          * An ECJ {@link CompilationProgress} printing a progress bar while compiling.
 382          */
 383         static class MakeCompilationProgress extends CompilationProgress {
 384                 private int totalWork;
 385                 private long currentChunk = 0;
 386                 private long chunksCount = 80;
 387
 388                 @Override
 389                 public void worked(int workIncrement, int remainingWork) {
 390                         if (!logger.isLoggable(Level.INFO)) // progress bar only at INFO level
 391                                 return;
 392                         long chunk = ((totalWork - remainingWork) * chunksCount) / totalWork;
 393                         if (chunk != currentChunk) {
 394                                 currentChunk = chunk;
 395                                 for (long i = 0; i < currentChunk; i++) {
 396                                         System.out.print("#");
 397                                 }
 398                                 for (long i = currentChunk; i < chunksCount; i++) {
 399                                         System.out.print("-");
 400                                 }
 401                                 System.out.print("\r");
 402                         }
 403                         if (remainingWork == 0)
 404                                 System.out.print("\n");
 405                 }
 406
 407                 @Override
 408                 public void setTaskName(String name) {
 409                 }
 410
 411                 @Override
 412                 public boolean isCanceled() {
 413                         return false;
 414                 }
 415
 416                 @Override
 417                 public void done() {
 418                 }
 419
 420                 @Override
 421                 public void begin(int remainingWork) {
 422                         this.totalWork = remainingWork;
 423                 }
 424         }
 425 }