src/org/argeo/build/Make.java

   1 package org.argeo.build;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static java.lang.System.Logger.Level.ERROR;
   5 import static java.lang.System.Logger.Level.INFO;
   6 import static java.lang.System.Logger.Level.WARNING;
   7
   8 import java.io.File;
   9 import java.io.IOException;
  10 import java.io.InputStream;
  11 import java.io.OutputStream;
  12 import java.io.PrintWriter;
  13 import java.lang.System.Logger;
  14 import java.lang.System.Logger.Level;
  15 import java.lang.management.ManagementFactory;
  16 import java.nio.file.DirectoryStream;
  17 import java.nio.file.FileVisitResult;
  18 import java.nio.file.Files;
  19 import java.nio.file.Path;
  20 import java.nio.file.PathMatcher;
  21 import java.nio.file.Paths;
  22 import java.nio.file.SimpleFileVisitor;
  23 import java.nio.file.StandardCopyOption;
  24 import java.nio.file.attribute.BasicFileAttributes;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.HashMap;
  28 import java.util.Iterator;
  29 import java.util.List;
  30 import java.util.Map;
  31 import java.util.Objects;
  32 import java.util.Properties;
  33 import java.util.StringJoiner;
  34 import java.util.StringTokenizer;
  35 import java.util.TreeMap;
  36 import java.util.concurrent.CompletableFuture;
  37 import java.util.jar.Attributes;
  38 import java.util.jar.JarEntry;
  39 import java.util.jar.JarOutputStream;
  40 import java.util.jar.Manifest;
  41 import java.util.stream.Collectors;
  42 import java.util.zip.Deflater;
  43
  44 import org.eclipse.jdt.core.compiler.CompilationProgress;
  45
  46 import aQute.bnd.osgi.Analyzer;
  47 import aQute.bnd.osgi.Jar;
  48 import aQute.bnd.osgi.Resource;
  49 import aQute.bnd.plugin.jpms.JPMSModuleInfoPlugin;
  50
  51 /**
  52  * Minimalistic OSGi compiler and packager, meant to be used as a single file
  53  * without being itself compiled first. It depends on the Eclipse batch compiler
  54  * (aka. ECJ) and the BND Libs library for OSGi metadata generation (which
  55  * itselfs depends on slf4j).<br/>
  56  * <br/>
  57  * For example, a typical system call would be:<br/>
  58  * <code>java -cp "/path/to/ECJ jar:/path/to/bndlib jar:/path/to/SLF4J jar" /path/to/cloned/argeo-build/src/org/argeo/build/Make.java action --option1 argument1 argument2 --option2 argument3 </code>
  59  */
  60 public class Make {
  61         private final static Logger logger = System.getLogger(Make.class.getName());
  62
  63         /**
  64          * Environment variable on whether sources should be packaged separately or
  65          * integrated in the bundles.
  66          */
  67         private final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES";
  68
  69         /**
  70          * Environment variable on whether legal files at the root of the sources should
  71          * be included in the generated bundles. Should be set to true when building
  72          * third-party software in order no to include the build harness license into
  73          * the generated bundles.
  74          */
  75         private final static String ENV_NO_SDK_LEGAL = "NO_SDK_LEGAL";
  76
  77         /**
  78          * Environment variable to override the default location for the Argeo Build
  79          * configuration files. Typically used if Argeo Build has been compiled and
  80          * packaged separately.
  81          */
  82         private final static String ENV_ARGEO_BUILD_CONFIG = "ARGEO_BUILD_CONFIG";
  83
  84         /** Make file variable (in {@link #SDK_MK}) with a path to the sources base. */
  85         private final static String VAR_SDK_SRC_BASE = "SDK_SRC_BASE";
  86
  87         /**
  88          * Make file variable (in {@link #SDK_MK}) with a path to the build output base.
  89          */
  90         private final static String VAR_SDK_BUILD_BASE = "SDK_BUILD_BASE";
  91         /**
  92          * Make file variable (in {@link #BRANCH_MK}) with the branch.
  93          */
  94         private final static String VAR_BRANCH = "BRANCH";
  95
  96         /** Name of the local-specific Makefile (sdk.mk). */
  97         final static String SDK_MK = "sdk.mk";
  98         /** Name of the branch definition Makefile (branch.mk). */
  99         final static String BRANCH_MK = "branch.mk";
 100
 101         /** The execution directory (${user.dir}). */
 102         final Path execDirectory;
 103         /** Base of the source code, typically the cloned git repository. */
 104         final Path sdkSrcBase;
 105         /**
 106          * The base of the builder, typically a submodule pointing to the INCLUDEpublic
 107          * argeo-build directory.
 108          */
 109         final Path argeoBuildBase;
 110         /** The base of the build for all layers. */
 111         final Path sdkBuildBase;
 112         /** The base of the build for this layer. */
 113         final Path buildBase;
 114         /** The base of the a2 output for all layers. */
 115         final Path a2Output;
 116         /** The base of the a2 sources when packaged separately. */
 117         final Path a2srcOutput;
 118
 119         /** Whether sources should be packaged separately. */
 120         final boolean sourceBundles;
 121         /** Whether common legal files should be included. */
 122         final boolean noSdkLegal;
 123
 124         /** Constructor initialises the base directories. */
 125         public Make() throws IOException {
 126                 sourceBundles = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES));
 127                 if (sourceBundles)
 128                         logger.log(Level.INFO, "Sources will be packaged separately");
 129                 noSdkLegal = Boolean.parseBoolean(System.getenv(ENV_NO_SDK_LEGAL));
 130                 if (noSdkLegal)
 131                         logger.log(Level.INFO, "SDK legal files will NOT be included");
 132
 133                 execDirectory = Paths.get(System.getProperty("user.dir"));
 134                 Path sdkMkP = findSdkMk(execDirectory);
 135                 Objects.requireNonNull(sdkMkP, "No " + SDK_MK + " found under " + execDirectory);
 136
 137                 Map<String, String> context = readMakefileVariables(sdkMkP);
 138                 sdkSrcBase = Paths.get(context.computeIfAbsent(VAR_SDK_SRC_BASE, (key) -> {
 139                         throw new IllegalStateException(key + " not found");
 140                 })).toAbsolutePath();
 141
 142                 Path argeoBuildBaseT = sdkSrcBase.resolve("sdk/argeo-build");
 143                 if (!Files.exists(argeoBuildBaseT)) {
 144                         String fromEnv = System.getenv(ENV_ARGEO_BUILD_CONFIG);
 145                         if (fromEnv != null)
 146                                 argeoBuildBaseT = Paths.get(fromEnv);
 147                         if (fromEnv == null || !Files.exists(argeoBuildBaseT)) {
 148                                 throw new IllegalStateException(
 149                                                 "Argeo Build not found. Did you initialise the git submodules or set the "
 150                                                                 + ENV_ARGEO_BUILD_CONFIG + " environment variable?");
 151                         }
 152                 }
 153                 argeoBuildBase = argeoBuildBaseT;
 154
 155                 sdkBuildBase = Paths.get(context.computeIfAbsent(VAR_SDK_BUILD_BASE, (key) -> {
 156                         throw new IllegalStateException(key + " not found");
 157                 })).toAbsolutePath();
 158                 buildBase = sdkBuildBase.resolve(sdkSrcBase.getFileName());
 159                 a2Output = sdkBuildBase.resolve("a2");
 160                 a2srcOutput = sdkBuildBase.resolve("a2.src");
 161         }
 162
 163         /*
 164          * ACTIONS
 165          */
 166         /** Compile and create the bundles in one go. */
 167         void all(Map<String, List<String>> options) throws IOException {
 168                 compile(options);
 169                 bundle(options);
 170         }
 171
 172         /** Compile all the bundles which have been passed via the --bundle argument. */
 173         void compile(Map<String, List<String>> options) throws IOException {
 174                 List<String> bundles = options.get("--bundles");
 175                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 176                 if (bundles.isEmpty())
 177                         return;
 178
 179                 List<String> a2Categories = options.getOrDefault("--dep-categories", new ArrayList<>());
 180                 List<String> a2Bases = options.getOrDefault("--a2-bases", new ArrayList<>());
 181                 a2Bases = a2Bases.stream().distinct().collect(Collectors.toList());// remove duplicates
 182                 if (a2Bases.isEmpty() || !a2Bases.contains(a2Output.toString())) {// make sure a2 output is available
 183                         a2Bases.add(a2Output.toString());
 184                 }
 185
 186                 List<String> compilerArgs = new ArrayList<>();
 187
 188                 Path ecjArgs = argeoBuildBase.resolve("ecj.args");
 189                 compilerArgs.add("@" + ecjArgs);
 190
 191                 // classpath
 192                 if (!a2Categories.isEmpty()) {
 193                         // We will keep only the highest major.minor
 194                         // and order by bundle name, for predictability
 195                         Map<String, A2Jar> a2Jars = new TreeMap<>();
 196
 197 //                      StringJoiner modulePath = new StringJoiner(File.pathSeparator);
 198                         for (String a2Base : a2Bases) {
 199                                 categories: for (String a2Category : a2Categories) {
 200                                         Path a2Dir = Paths.get(a2Base).resolve(a2Category);
 201                                         if (!Files.exists(a2Dir))
 202                                                 continue categories;
 203 //                                      modulePath.add(a2Dir.toString());
 204                                         for (Path jarP : Files.newDirectoryStream(a2Dir, (p) -> p.getFileName().toString().endsWith(".jar")
 205                                                         && !p.getFileName().toString().endsWith(".src.jar"))) {
 206                                                 A2Jar a2Jar = new A2Jar(jarP);
 207                                                 if (a2Jars.containsKey(a2Jar.name)) {
 208                                                         A2Jar current = a2Jars.get(a2Jar.name);
 209                                                         if (a2Jar.major > current.major)
 210                                                                 a2Jars.put(a2Jar.name, a2Jar);
 211                                                         else if (a2Jar.major == current.major && a2Jar.minor > current.minor)
 212                                                                 a2Jars.put(a2Jar.name, a2Jar);
 213                                                         // keep if minor equals
 214                                                 } else {
 215                                                         a2Jars.put(a2Jar.name, a2Jar);
 216                                                 }
 217                                         }
 218                                 }
 219                         }
 220
 221                         StringJoiner classPath = new StringJoiner(File.pathSeparator);
 222                         for (Iterator<A2Jar> it = a2Jars.values().iterator(); it.hasNext();)
 223                                 classPath.add(it.next().path.toString());
 224
 225                         compilerArgs.add("-cp");
 226                         compilerArgs.add(classPath.toString());
 227 //                      compilerArgs.add("--module-path");
 228 //                      compilerArgs.add(modulePath.toString());
 229                 }
 230
 231                 // sources
 232                 boolean atLeastOneBundleToCompile = false;
 233                 bundles: for (String bundle : bundles) {
 234                         StringBuilder sb = new StringBuilder();
 235                         Path bundlePath = execDirectory.resolve(bundle);
 236                         if (!Files.exists(bundlePath)) {
 237                                 if (bundles.size() == 1) {
 238                                         logger.log(WARNING, "Bundle " + bundle + " not found in " + execDirectory
 239                                                         + ", assuming this is this directory, as only one bundle was requested.");
 240                                         bundlePath = execDirectory;
 241                                 } else
 242                                         throw new IllegalArgumentException("Bundle " + bundle + " not found in " + execDirectory);
 243                         }
 244                         Path bundleSrc = bundlePath.resolve("src");
 245                         if (!Files.exists(bundleSrc)) {
 246                                 logger.log(WARNING, bundleSrc + " does not exist, skipping it, as this is not a Java bundle");
 247                                 continue bundles;
 248                         }
 249                         sb.append(bundleSrc);
 250                         sb.append("[-d");
 251                         compilerArgs.add(sb.toString());
 252                         sb = new StringBuilder();
 253                         sb.append(buildBase.resolve(bundle).resolve("bin"));
 254                         sb.append("]");
 255                         compilerArgs.add(sb.toString());
 256                         atLeastOneBundleToCompile = true;
 257                 }
 258
 259                 if (!atLeastOneBundleToCompile)
 260                         return;
 261
 262                 if (logger.isLoggable(INFO))
 263                         compilerArgs.add("-time");
 264
 265                 if (logger.isLoggable(DEBUG)) {
 266                         logger.log(DEBUG, "Compiler arguments:");
 267                         for (String arg : compilerArgs)
 268                                 logger.log(DEBUG, arg);
 269                 }
 270
 271                 boolean success = org.eclipse.jdt.core.compiler.batch.BatchCompiler.compile(
 272                                 compilerArgs.toArray(new String[compilerArgs.size()]), new PrintWriter(System.out),
 273                                 new PrintWriter(System.err), new MakeCompilationProgress());
 274                 if (!success) // kill the process if compilation failed
 275                         throw new IllegalStateException("Compilation failed");
 276         }
 277
 278         /** Package the bundles. */
 279         void bundle(Map<String, List<String>> options) throws IOException {
 280                 // check arguments
 281                 List<String> bundles = options.get("--bundles");
 282                 Objects.requireNonNull(bundles, "--bundles argument must be set");
 283                 if (bundles.isEmpty())
 284                         return;
 285
 286                 List<String> categories = options.get("--category");
 287                 Objects.requireNonNull(categories, "--category argument must be set");
 288                 if (categories.size() != 1)
 289                         throw new IllegalArgumentException("One and only one --category must be specified");
 290                 String category = categories.get(0);
 291
 292                 final String branch;
 293                 Path branchMk = sdkSrcBase.resolve(BRANCH_MK);
 294                 if (Files.exists(branchMk)) {
 295                         Map<String, String> branchVariables = readMakefileVariables(branchMk);
 296                         branch = branchVariables.get(VAR_BRANCH);
 297                 } else {
 298                         branch = null;
 299                 }
 300
 301                 long begin = System.currentTimeMillis();
 302                 // create jars in parallel
 303                 List<CompletableFuture<Void>> toDos = new ArrayList<>();
 304                 for (String bundle : bundles) {
 305                         toDos.add(CompletableFuture.runAsync(() -> {
 306                                 try {
 307                                         createBundle(branch, bundle, category);
 308                                 } catch (IOException e) {
 309                                         throw new RuntimeException("Packaging of " + bundle + " failed", e);
 310                                 }
 311                         }));
 312                 }
 313                 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
 314                 long duration = System.currentTimeMillis() - begin;
 315                 logger.log(DEBUG, "Packaging took " + duration + " ms");
 316         }
 317
 318         /** Install or uninstall bundles and native output. */
 319         void install(Map<String, List<String>> options, boolean uninstall) throws IOException {
 320                 final String LIB_ = "lib/";
 321                 final String NATIVE_ = "native/";
 322
 323                 // check arguments
 324                 List<String> bundles = multiArg(options, "--bundles", true);
 325                 if (bundles.isEmpty())
 326                         return;
 327                 String category = singleArg(options, "--category", true);
 328                 Path targetA2 = Paths.get(singleArg(options, "--target", true));
 329                 String nativeTargetArg = singleArg(options, "--target-native", false);
 330                 Path nativeTargetA2 = nativeTargetArg != null ? Paths.get(nativeTargetArg) : null;
 331                 String targetOs = singleArg(options, "--os", nativeTargetArg != null);
 332                 logger.log(INFO, (uninstall ? "Uninstalling bundles from " : "Installing bundles to ") + targetA2);
 333
 334                 final String branch;
 335                 Path branchMk = sdkSrcBase.resolve(BRANCH_MK);
 336                 if (Files.exists(branchMk)) {
 337                         Map<String, String> branchVariables = readMakefileVariables(branchMk);
 338                         branch = branchVariables.get(VAR_BRANCH);
 339                 } else {
 340                         throw new IllegalArgumentException(VAR_BRANCH + " variable must be set.");
 341                 }
 342
 343                 Properties properties = new Properties();
 344                 Path branchBnd = sdkSrcBase.resolve("sdk/branches/" + branch + ".bnd");
 345                 if (Files.exists(branchBnd))
 346                         try (InputStream in = Files.newInputStream(branchBnd)) {
 347                                 properties.load(in);
 348                         }
 349                 String major = properties.getProperty("major");
 350                 Objects.requireNonNull(major, "'major' must be set");
 351                 String minor = properties.getProperty("minor");
 352                 Objects.requireNonNull(minor, "'minor' must be set");
 353
 354                 int count = 0;
 355                 bundles: for (String bundle : bundles) {
 356                         Path bundlePath = Paths.get(bundle);
 357                         Path bundleParent = bundlePath.getParent();
 358                         Path a2JarDirectory = bundleParent != null ? a2Output.resolve(bundleParent).resolve(category)
 359                                         : a2Output.resolve(category);
 360                         Path jarP = a2JarDirectory.resolve(bundlePath.getFileName() + "." + major + "." + minor + ".jar");
 361
 362                         Path targetJarP;
 363                         if (bundle.startsWith(LIB_)) {// OS-specific
 364                                 Objects.requireNonNull(nativeTargetA2);
 365                                 if (bundle.startsWith(LIB_ + NATIVE_) // portable native
 366                                                 || bundle.startsWith(LIB_ + targetOs + "/" + NATIVE_)) {// OS-specific native
 367                                         targetJarP = nativeTargetA2.resolve(category).resolve(jarP.getFileName());
 368                                 } else if (bundle.startsWith(LIB_ + targetOs)) {// OS-specific portable
 369                                         targetJarP = targetA2.resolve(category).resolve(jarP.getFileName());
 370                                 } else { // ignore other OS
 371                                         continue bundles;
 372                                 }
 373                         } else {
 374                                 targetJarP = targetA2.resolve(a2Output.relativize(jarP));
 375                         }
 376
 377                         if (uninstall) { // uninstall
 378                                 if (Files.exists(targetJarP)) {
 379                                         Files.delete(targetJarP);
 380                                         logger.log(DEBUG, "Removed " + targetJarP);
 381                                         count++;
 382                                 }
 383                                 Path targetParent = targetJarP.getParent();
 384                                 if (targetParent.startsWith(targetA2))
 385                                         deleteEmptyParents(targetA2, targetParent);
 386                                 if (nativeTargetA2 != null && targetParent.startsWith(nativeTargetA2))
 387                                         deleteEmptyParents(nativeTargetA2, targetParent);
 388                         } else { // install
 389                                 Files.createDirectories(targetJarP.getParent());
 390                                 boolean update = Files.exists(targetJarP);
 391                                 Files.copy(jarP, targetJarP, StandardCopyOption.REPLACE_EXISTING);
 392                                 logger.log(DEBUG, (update ? "Updated " : "Installed ") + targetJarP);
 393                                 count++;
 394                         }
 395                 }
 396                 logger.log(INFO, uninstall ? count + " bundles removed" : count + " bundles installed or updated");
 397         }
 398
 399         /** Extracts an argument which must be unique. */
 400         String singleArg(Map<String, List<String>> options, String arg, boolean mandatory) {
 401                 List<String> values = options.get(arg);
 402                 if (values == null || values.size() == 0)
 403                         if (mandatory)
 404                                 throw new IllegalArgumentException(arg + " argument must be set");
 405                         else
 406                                 return null;
 407                 if (values.size() != 1)
 408                         throw new IllegalArgumentException("One and only one " + arg + " arguments must be specified");
 409                 return values.get(0);
 410         }
 411
 412         /** Extracts an argument which can have multiple values. */
 413         List<String> multiArg(Map<String, List<String>> options, String arg, boolean mandatory) {
 414                 List<String> values = options.get(arg);
 415                 if (mandatory && values == null)
 416                         throw new IllegalArgumentException(arg + " argument must be set");
 417                 return values != null ? values : new ArrayList<>();
 418         }
 419
 420         /** Delete empty parent directory up to the base directory (included). */
 421         void deleteEmptyParents(Path baseDir, Path targetParent) throws IOException {
 422                 if (!targetParent.startsWith(baseDir))
 423                         throw new IllegalArgumentException(targetParent + " does not start with " + baseDir);
 424                 if (!Files.exists(baseDir))
 425                         return;
 426                 if (!Files.exists(targetParent)) {
 427                         deleteEmptyParents(baseDir, targetParent.getParent());
 428                         return;
 429                 }
 430                 if (!Files.isDirectory(targetParent))
 431                         throw new IllegalArgumentException(targetParent + " must be a directory");
 432                 boolean isA2target = Files.isSameFile(baseDir, targetParent);
 433                 if (!Files.list(targetParent).iterator().hasNext()) {
 434                         Files.delete(targetParent);
 435                         if (isA2target)
 436                                 return;// stop after deleting A2 base
 437                         deleteEmptyParents(baseDir, targetParent.getParent());
 438                 }
 439         }
 440
 441         /** Package a single bundle. */
 442         void createBundle(String branch, String bundle, String category) throws IOException {
 443                 final Path bundleSourceBase;
 444                 if (!Files.exists(execDirectory.resolve(bundle))) {
 445                         logger.log(WARNING,
 446                                         "Bundle " + bundle + " not found in " + execDirectory + ", assuming this is this directory.");
 447                         bundleSourceBase = execDirectory;
 448                 } else {
 449                         bundleSourceBase = execDirectory.resolve(bundle);
 450                 }
 451                 Path srcP = bundleSourceBase.resolve("src");
 452
 453                 Path compiled = buildBase.resolve(bundle);
 454                 String bundleSymbolicName = bundleSourceBase.getFileName().toString();
 455
 456                 // Metadata
 457                 Properties properties = new Properties();
 458                 Path argeoBnd = argeoBuildBase.resolve("argeo.bnd");
 459                 try (InputStream in = Files.newInputStream(argeoBnd)) {
 460                         properties.load(in);
 461                 }
 462
 463                 if (branch != null) {
 464                         Path branchBnd = sdkSrcBase.resolve("sdk/branches/" + branch + ".bnd");
 465                         if (Files.exists(branchBnd))
 466                                 try (InputStream in = Files.newInputStream(branchBnd)) {
 467                                         properties.load(in);
 468                                 }
 469                 }
 470
 471                 Path bndBnd = bundleSourceBase.resolve("bnd.bnd");
 472                 if (Files.exists(bndBnd))
 473                         try (InputStream in = Files.newInputStream(bndBnd)) {
 474                                 properties.load(in);
 475                         }
 476
 477                 // Normalise
 478                 if (!properties.containsKey("Bundle-SymbolicName"))
 479                         properties.put("Bundle-SymbolicName", bundleSymbolicName);
 480
 481                 // Calculate MANIFEST
 482                 Path binP = compiled.resolve("bin");
 483                 if (!Files.exists(binP))
 484                         Files.createDirectories(binP);
 485                 Manifest manifest;
 486                 Resource moduleInfoClass = null;
 487                 try (Analyzer bndAnalyzer = new Analyzer()) {
 488                         bndAnalyzer.setProperties(properties);
 489                         Jar jar = new Jar(bundleSymbolicName, binP.toFile());
 490                         bndAnalyzer.setJar(jar);
 491                         manifest = bndAnalyzer.calcManifest();
 492
 493                         // JPMS module
 494                         jar.setManifest(manifest);
 495                         JPMSModuleInfoPlugin jpmsModuleInfoPlugin = new JPMSModuleInfoPlugin();
 496                         jpmsModuleInfoPlugin.verify(bndAnalyzer);
 497                         moduleInfoClass = bndAnalyzer.getJar().getResource("module-info.class");
 498                 } catch (Exception e) {
 499                         throw new RuntimeException("Bnd analysis of " + compiled + " failed", e);
 500                 }
 501
 502                 String major = properties.getProperty("major");
 503                 Objects.requireNonNull(major, "'major' must be set");
 504                 String minor = properties.getProperty("minor");
 505                 Objects.requireNonNull(minor, "'minor' must be set");
 506
 507                 // Write manifest
 508                 Path manifestP = compiled.resolve("META-INF/MANIFEST.MF");
 509                 Files.createDirectories(manifestP.getParent());
 510                 try (OutputStream out = Files.newOutputStream(manifestP)) {
 511                         manifest.write(out);
 512                 }
 513
 514                 // Write module-info.class
 515                 if (moduleInfoClass != null) {
 516                         Path moduleInfoClassP = binP.resolve("module-info.class");
 517                         Files.createDirectories(moduleInfoClassP.getParent());
 518                         try (OutputStream out = Files.newOutputStream(moduleInfoClassP)) {
 519                                 moduleInfoClass.write(out);
 520 //                              logger.log(INFO, "Wrote " + moduleInfoClassP);
 521                         } catch (Exception e) {
 522                                 throw new RuntimeException("Cannot write module-info.class");
 523                         }
 524                 }
 525
 526                 // Load excludes
 527                 List<PathMatcher> excludes = new ArrayList<>();
 528                 Path excludesP = argeoBuildBase.resolve("excludes.txt");
 529                 for (String line : Files.readAllLines(excludesP)) {
 530                         PathMatcher pathMatcher = excludesP.getFileSystem().getPathMatcher("glob:" + line);
 531                         excludes.add(pathMatcher);
 532                 }
 533
 534                 Path bundleParent = Paths.get(bundle).getParent();
 535                 Path a2JarDirectory = bundleParent != null ? a2Output.resolve(bundleParent).resolve(category)
 536                                 : a2Output.resolve(category);
 537                 Path jarP = a2JarDirectory.resolve(compiled.getFileName() + "." + major + "." + minor + ".jar");
 538                 Files.createDirectories(jarP.getParent());
 539
 540                 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarP), manifest)) {
 541                         jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
 542                         // add all classes first
 543                         Files.walkFileTree(binP, new SimpleFileVisitor<Path>() {
 544                                 @Override
 545                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 546                                         jarOut.putNextEntry(new JarEntry(binP.relativize(file).toString()));
 547                                         Files.copy(file, jarOut);
 548                                         return FileVisitResult.CONTINUE;
 549                                 }
 550                         });
 551
 552                         // add resources
 553                         Files.walkFileTree(bundleSourceBase, new SimpleFileVisitor<Path>() {
 554                                 @Override
 555                                 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 556                                         // skip output directory if it happens to be within the sources
 557                                         if (Files.isSameFile(sdkBuildBase, dir))
 558                                                 return FileVisitResult.SKIP_SUBTREE;
 559
 560                                         // skip excluded patterns
 561                                         Path relativeP = bundleSourceBase.relativize(dir);
 562                                         for (PathMatcher exclude : excludes)
 563                                                 if (exclude.matches(relativeP))
 564                                                         return FileVisitResult.SKIP_SUBTREE;
 565
 566                                         return FileVisitResult.CONTINUE;
 567                                 }
 568
 569                                 @Override
 570                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 571                                         Path relativeP = bundleSourceBase.relativize(file);
 572                                         for (PathMatcher exclude : excludes)
 573                                                 if (exclude.matches(relativeP))
 574                                                         return FileVisitResult.CONTINUE;
 575                                         // skip JavaScript source maps
 576                                         if (sourceBundles && file.getFileName().toString().endsWith(".map"))
 577                                                 return FileVisitResult.CONTINUE;
 578
 579                                         JarEntry entry = new JarEntry(relativeP.toString());
 580                                         jarOut.putNextEntry(entry);
 581                                         Files.copy(file, jarOut);
 582                                         return FileVisitResult.CONTINUE;
 583                                 }
 584                         });
 585
 586                         if (Files.exists(srcP)) {
 587                                 // Add all resources from src/
 588                                 Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
 589                                         @Override
 590                                         public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 591                                                 // skip directories ending with .js
 592                                                 // TODO find something more robust?
 593                                                 if (dir.getFileName().toString().endsWith(".js"))
 594                                                         return FileVisitResult.SKIP_SUBTREE;
 595                                                 return super.preVisitDirectory(dir, attrs);
 596                                         }
 597
 598                                         @Override
 599                                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 600                                                 if (file.getFileName().toString().endsWith(".java")
 601                                                                 || file.getFileName().toString().endsWith(".class"))
 602                                                         return FileVisitResult.CONTINUE;
 603                                                 jarOut.putNextEntry(new JarEntry(srcP.relativize(file).toString()));
 604                                                 if (!Files.isDirectory(file))
 605                                                         Files.copy(file, jarOut);
 606                                                 return FileVisitResult.CONTINUE;
 607                                         }
 608                                 });
 609
 610                                 // add sources
 611                                 // TODO add effective BND, Eclipse project file, etc., in order to be able to
 612                                 // repackage
 613                                 if (!sourceBundles) {
 614                                         copySourcesToJar(srcP, jarOut, "OSGI-OPT/src/");
 615                                 }
 616                         }
 617
 618                         // add legal notices and licenses
 619                         for (Path p : listLegalFilesToInclude(bundleSourceBase).values()) {
 620                                 jarOut.putNextEntry(new JarEntry(p.getFileName().toString()));
 621                                 Files.copy(p, jarOut);
 622                         }
 623                 }
 624
 625                 if (sourceBundles) {// create separate sources jar
 626                         Path a2srcJarDirectory = bundleParent != null ? a2srcOutput.resolve(bundleParent).resolve(category)
 627                                         : a2srcOutput.resolve(category);
 628                         Files.createDirectories(a2srcJarDirectory);
 629                         Path srcJarP = a2srcJarDirectory.resolve(compiled.getFileName() + "." + major + "." + minor + ".src.jar");
 630                         createSourceBundle(bundleSymbolicName, manifest, bundleSourceBase, srcP, srcJarP);
 631                 }
 632         }
 633
 634         /** Create a separate bundle containing the sources. */
 635         void createSourceBundle(String bundleSymbolicName, Manifest manifest, Path bundleSourceBase, Path srcP,
 636                         Path srcJarP) throws IOException {
 637                 Manifest srcManifest = new Manifest();
 638                 srcManifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
 639                 srcManifest.getMainAttributes().putValue("Bundle-SymbolicName", bundleSymbolicName + ".src");
 640                 srcManifest.getMainAttributes().putValue("Bundle-Version",
 641                                 manifest.getMainAttributes().getValue("Bundle-Version").toString());
 642
 643                 boolean isJsBundle = bundleSymbolicName.endsWith(".js");
 644                 if (!isJsBundle) {
 645                         srcManifest.getMainAttributes().putValue("Eclipse-SourceBundle",
 646                                         bundleSymbolicName + ";version=\"" + manifest.getMainAttributes().getValue("Bundle-Version"));
 647
 648                         try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
 649                                 copySourcesToJar(srcP, srcJarOut, "");
 650                                 // add legal notices and licenses
 651                                 for (Path p : listLegalFilesToInclude(bundleSourceBase).values()) {
 652                                         srcJarOut.putNextEntry(new JarEntry(p.getFileName().toString()));
 653                                         Files.copy(p, srcJarOut);
 654                                 }
 655                         }
 656                 } else {// JavaScript source maps
 657                         srcManifest.getMainAttributes().putValue("Fragment-Host", bundleSymbolicName + ";bundle-version=\""
 658                                         + manifest.getMainAttributes().getValue("Bundle-Version"));
 659                         try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
 660                                 Files.walkFileTree(bundleSourceBase, new SimpleFileVisitor<Path>() {
 661                                         @Override
 662                                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 663                                                 Path relativeP = bundleSourceBase.relativize(file);
 664                                                 if (!file.getFileName().toString().endsWith(".map"))
 665                                                         return FileVisitResult.CONTINUE;
 666                                                 JarEntry entry = new JarEntry(relativeP.toString());
 667                                                 srcJarOut.putNextEntry(entry);
 668                                                 Files.copy(file, srcJarOut);
 669                                                 return FileVisitResult.CONTINUE;
 670                                         }
 671                                 });
 672                         }
 673                 }
 674         }
 675
 676         /** List the relevant legal files to include, from the SDK source base. */
 677         Map<String, Path> listLegalFilesToInclude(Path bundleBase) throws IOException {
 678                 Map<String, Path> toInclude = new HashMap<>();
 679                 if (!noSdkLegal) {
 680                         try (DirectoryStream<Path> sdkSrcLegal = Files.newDirectoryStream(sdkSrcBase, (p) -> {
 681                                 String fileName = p.getFileName().toString();
 682                                 return switch (fileName) {
 683                                 case "NOTICE":
 684                                 case "LICENSE":
 685                                 case "COPYING":
 686                                 case "COPYING.LESSER":
 687                                         yield true;
 688                                 default:
 689                                         yield false;
 690                                 };
 691                         });) {
 692                                 for (Path p : sdkSrcLegal)
 693                                         toInclude.put(p.getFileName().toString(), p);
 694                         }
 695                 }
 696                 for (Iterator<Map.Entry<String, Path>> entries = toInclude.entrySet().iterator(); entries.hasNext();) {
 697                         Map.Entry<String, Path> entry = entries.next();
 698                         Path inBundle = bundleBase.resolve(entry.getValue().getFileName());
 699                         // remove file if it is also defined at bundle level
 700                         // since it has already been copied
 701                         // and has priority
 702                         if (Files.exists(inBundle))
 703                                 entries.remove();
 704                 }
 705                 return toInclude;
 706         }
 707
 708         /*
 709          * UTILITIES
 710          */
 711         /** Add sources to a jar file */
 712         void copySourcesToJar(Path srcP, JarOutputStream srcJarOut, String prefix) throws IOException {
 713                 Files.walkFileTree(srcP, new SimpleFileVisitor<Path>() {
 714                         @Override
 715                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 716                                 srcJarOut.putNextEntry(new JarEntry(prefix + srcP.relativize(file).toString()));
 717                                 if (!Files.isDirectory(file))
 718                                         Files.copy(file, srcJarOut);
 719                                 return FileVisitResult.CONTINUE;
 720                         }
 721                 });
 722         }
 723
 724         /**
 725          * Recursively find the base source directory (which contains the
 726          * <code>{@value #SDK_MK}</code> file).
 727          */
 728         Path findSdkMk(Path directory) {
 729                 Path sdkMkP = directory.resolve(SDK_MK);
 730                 if (Files.exists(sdkMkP)) {
 731                         return sdkMkP.toAbsolutePath();
 732                 }
 733                 if (directory.getParent() == null)
 734                         return null;
 735                 return findSdkMk(directory.getParent());
 736         }
 737
 738         /**
 739          * Reads Makefile variable assignments of the form =, :=, or ?=, ignoring white
 740          * spaces. To be used with very simple included Makefiles only.
 741          */
 742         Map<String, String> readMakefileVariables(Path path) throws IOException {
 743                 Map<String, String> context = new HashMap<>();
 744                 List<String> sdkMkLines = Files.readAllLines(path);
 745                 lines: for (String line : sdkMkLines) {
 746                         StringTokenizer st = new StringTokenizer(line, " :=?");
 747                         if (!st.hasMoreTokens())
 748                                 continue lines;
 749                         String key = st.nextToken();
 750                         if (!st.hasMoreTokens())
 751                                 continue lines;
 752                         String value = st.nextToken();
 753                         if (st.hasMoreTokens()) // probably not a simple variable assignment
 754                                 continue lines;
 755                         context.put(key, value);
 756                 }
 757                 return context;
 758         }
 759
 760         /** Main entry point, interpreting actions and arguments. */
 761         public static void main(String... args) {
 762                 if (args.length == 0)
 763                         throw new IllegalArgumentException("At least an action must be provided");
 764                 int actionIndex = 0;
 765                 String action = args[actionIndex];
 766                 if (args.length > actionIndex + 1 && !args[actionIndex + 1].startsWith("-"))
 767                         throw new IllegalArgumentException(
 768                                         "Action " + action + " must be followed by an option: " + Arrays.asList(args));
 769
 770                 Map<String, List<String>> options = new HashMap<>();
 771                 String currentOption = null;
 772                 for (int i = actionIndex + 1; i < args.length; i++) {
 773                         if (args[i].startsWith("-")) {
 774                                 currentOption = args[i];
 775                                 if (!options.containsKey(currentOption))
 776                                         options.put(currentOption, new ArrayList<>());
 777
 778                         } else {
 779                                 options.get(currentOption).add(args[i]);
 780                         }
 781                 }
 782
 783                 try {
 784                         Make argeoMake = new Make();
 785                         switch (action) {
 786                         case "compile" -> argeoMake.compile(options);
 787                         case "bundle" -> argeoMake.bundle(options);
 788                         case "all" -> argeoMake.all(options);
 789                         case "install" -> argeoMake.install(options, false);
 790                         case "uninstall" -> argeoMake.install(options, true);
 791
 792                         default -> throw new IllegalArgumentException("Unkown action: " + action);
 793                         }
 794
 795                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 796                         logger.log(INFO, "Make.java action '" + action + "' successfully completed after " + (jvmUptime / 1000)
 797                                         + "." + (jvmUptime % 1000) + " s");
 798                 } catch (Exception e) {
 799                         long jvmUptime = ManagementFactory.getRuntimeMXBean().getUptime();
 800                         logger.log(ERROR, "Make.java action '" + action + "' failed after " + (jvmUptime / 1000) + "."
 801                                         + (jvmUptime % 1000) + " s", e);
 802                         System.exit(1);
 803                 }
 804         }
 805
 806         /** A jar file in A2 format */
 807         static class A2Jar {
 808                 final Path path;
 809                 final String name;
 810                 final int major;
 811                 final int minor;
 812
 813                 A2Jar(Path path) {
 814                         try {
 815                                 this.path = path;
 816                                 String fileName = path.getFileName().toString();
 817                                 fileName = fileName.substring(0, fileName.lastIndexOf('.'));
 818                                 minor = Integer.parseInt(fileName.substring(fileName.lastIndexOf('.') + 1));
 819                                 fileName = fileName.substring(0, fileName.lastIndexOf('.'));
 820                                 major = Integer.parseInt(fileName.substring(fileName.lastIndexOf('.') + 1));
 821                                 name = fileName.substring(0, fileName.lastIndexOf('.'));
 822                         } catch (Exception e) {
 823                                 throw new IllegalArgumentException("Badly formatted A2 jar " + path, e);
 824                         }
 825                 }
 826         }
 827
 828         /**
 829          * An ECJ {@link CompilationProgress} printing a progress bar while compiling.
 830          */
 831         static class MakeCompilationProgress extends CompilationProgress {
 832                 private int totalWork;
 833                 private long currentChunk = 0;
 834                 private long chunksCount = 80;
 835
 836                 @Override
 837                 public void worked(int workIncrement, int remainingWork) {
 838                         if (!logger.isLoggable(Level.INFO)) // progress bar only at INFO level
 839                                 return;
 840                         long chunk = ((totalWork - remainingWork) * chunksCount) / totalWork;
 841                         if (chunk != currentChunk) {
 842                                 currentChunk = chunk;
 843                                 for (long i = 0; i < currentChunk; i++) {
 844                                         System.out.print("#");
 845                                 }
 846                                 for (long i = currentChunk; i < chunksCount; i++) {
 847                                         System.out.print("-");
 848                                 }
 849                                 System.out.print("\r");
 850                         }
 851                         if (remainingWork == 0)
 852                                 System.out.print("\n");
 853                 }
 854
 855                 @Override
 856                 public void setTaskName(String name) {
 857                 }
 858
 859                 @Override
 860                 public boolean isCanceled() {
 861                         return false;
 862                 }
 863
 864                 @Override
 865                 public void done() {
 866                 }
 867
 868                 @Override
 869                 public void begin(int remainingWork) {
 870                         this.totalWork = remainingWork;
 871                 }
 872         }
 873 }