1 package org
.argeo
.build
;
3 import static java
.lang
.System
.Logger
.Level
.DEBUG
;
4 import static java
.lang
.System
.Logger
.Level
.ERROR
;
5 import static java
.lang
.System
.Logger
.Level
.INFO
;
6 import static java
.lang
.System
.Logger
.Level
.WARNING
;
9 import java
.io
.IOException
;
10 import java
.io
.InputStream
;
11 import java
.io
.OutputStream
;
12 import java
.io
.PrintWriter
;
13 import java
.lang
.System
.Logger
;
14 import java
.lang
.System
.Logger
.Level
;
15 import java
.lang
.management
.ManagementFactory
;
16 import java
.nio
.file
.FileVisitResult
;
17 import java
.nio
.file
.Files
;
18 import java
.nio
.file
.Path
;
19 import java
.nio
.file
.PathMatcher
;
20 import java
.nio
.file
.Paths
;
21 import java
.nio
.file
.SimpleFileVisitor
;
22 import java
.nio
.file
.attribute
.BasicFileAttributes
;
23 import java
.util
.ArrayList
;
24 import java
.util
.Arrays
;
25 import java
.util
.HashMap
;
26 import java
.util
.Iterator
;
27 import java
.util
.List
;
29 import java
.util
.Objects
;
30 import java
.util
.Properties
;
31 import java
.util
.StringJoiner
;
32 import java
.util
.StringTokenizer
;
33 import java
.util
.TreeMap
;
34 import java
.util
.concurrent
.CompletableFuture
;
35 import java
.util
.jar
.Attributes
;
36 import java
.util
.jar
.JarEntry
;
37 import java
.util
.jar
.JarOutputStream
;
38 import java
.util
.jar
.Manifest
;
39 import java
.util
.zip
.Deflater
;
41 import org
.eclipse
.jdt
.core
.compiler
.CompilationProgress
;
43 import aQute
.bnd
.osgi
.Analyzer
;
44 import aQute
.bnd
.osgi
.Jar
;
47 * Minimalistic OSGi compiler and packager, meant to be used as a single file
48 * without being itself compiled first. It depends on the Eclipse batch compiler
49 * (aka. ECJ) and the BND Libs library for OSGi metadata generation (which
50 * itselfs depends on slf4j).<br/>
52 * For example, a typical system call would be:<br/>
53 * <code>java -cp "/path/to/ECJ jar:/path/to/bndlib jar:/path/to/SLF4J jar" /path/to/cloned/argeo-build/src/org/argeo/build/Make.java action --option1 argument1 argument2 --option2 argument3 </code>
56 private final static Logger logger
= System
.getLogger(Make
.class.getName());
59 * Environment variable on whether sources should be packaged separately or
60 * integrated in the bundles.
62 private final static String ENV_SOURCE_BUNDLES
= "SOURCE_BUNDLES";
65 * Environment variable to override the default location for the Argeo Build
66 * configuration files. Typically used if Argeo Build has been compiled and
67 * packaged separately.
69 private final static String ENV_ARGEO_BUILD_CONFIG
= "ARGEO_BUILD_CONFIG";
71 /** Name of the local-specific Makefile (sdk.mk). */
72 final static String SDK_MK
= "sdk.mk";
73 /** Name of the branch definition Makefile (branch.mk). */
74 final static String BRANCH_MK
= "branch.mk";
76 /** The execution directory (${user.dir}). */
77 final Path execDirectory
;
78 /** Base of the source code, typically the cloned git repository. */
79 final Path sdkSrcBase
;
81 * The base of the builder, typically a submodule pointing to the public
82 * argeo-build directory.
84 final Path argeoBuildBase
;
85 /** The base of the build for all layers. */
86 final Path sdkBuildBase
;
87 /** The base of the build for this layer. */
89 /** The base of the a2 output for all layers. */
91 /** The base of the a2 sources when packages separately. */
92 final Path a2srcOutput
;
94 /** Whether sources should be packaged separately */
95 final boolean sourceBundles
;
97 /** Constructor initialises the base directories. */
98 public Make() throws IOException
{
99 sourceBundles
= Boolean
.parseBoolean(System
.getenv(ENV_SOURCE_BUNDLES
));
101 logger
.log(Level
.INFO
, "Sources will be packaged separately");
103 execDirectory
= Paths
.get(System
.getProperty("user.dir"));
104 Path sdkMkP
= findSdkMk(execDirectory
);
105 Objects
.requireNonNull(sdkMkP
, "No " + SDK_MK
+ " found under " + execDirectory
);
107 Map
<String
, String
> context
= readeMakefileVariables(sdkMkP
);
108 sdkSrcBase
= Paths
.get(context
.computeIfAbsent("SDK_SRC_BASE", (key
) -> {
109 throw new IllegalStateException(key
+ " not found");
110 })).toAbsolutePath();
112 Path argeoBuildBaseT
= sdkSrcBase
.resolve("sdk/argeo-build");
113 if (!Files
.exists(argeoBuildBaseT
)) {
114 String fromEnv
= System
.getenv(ENV_ARGEO_BUILD_CONFIG
);
116 argeoBuildBaseT
= Paths
.get(fromEnv
);
117 if (fromEnv
== null || !Files
.exists(argeoBuildBaseT
)) {
118 throw new IllegalStateException(
119 "Argeo Build not found. Did you initialise the git submodules or set the "
120 + ENV_ARGEO_BUILD_CONFIG
+ " environment variable?");
123 argeoBuildBase
= argeoBuildBaseT
;
125 sdkBuildBase
= Paths
.get(context
.computeIfAbsent("SDK_BUILD_BASE", (key
) -> {
126 throw new IllegalStateException(key
+ " not found");
127 })).toAbsolutePath();
128 buildBase
= sdkBuildBase
.resolve(sdkSrcBase
.getFileName());
129 a2Output
= sdkBuildBase
.resolve("a2");
130 a2srcOutput
= sdkBuildBase
.resolve("a2.src");
136 /** Compile and create the bundles in one go. */
137 void all(Map
<String
, List
<String
>> options
) throws IOException
{
142 /** Compile all the bundles which have been passed via the --bundle argument. */
143 @SuppressWarnings("restriction")
144 void compile(Map
<String
, List
<String
>> options
) throws IOException
{
145 List
<String
> bundles
= options
.get("--bundles");
146 Objects
.requireNonNull(bundles
, "--bundles argument must be set");
147 if (bundles
.isEmpty())
150 List
<String
> a2Categories
= options
.getOrDefault("--dep-categories", new ArrayList
<>());
151 List
<String
> a2Bases
= options
.getOrDefault("--a2-bases", new ArrayList
<>());
152 if (a2Bases
.isEmpty() || !a2Bases
.contains(a2Output
.toString())) {
153 a2Bases
.add(a2Output
.toString());
156 List
<String
> compilerArgs
= new ArrayList
<>();
158 Path ecjArgs
= argeoBuildBase
.resolve("ecj.args");
159 compilerArgs
.add("@" + ecjArgs
);
162 if (!a2Categories
.isEmpty()) {
163 // We will keep only the highest major.minor
164 // and order by bundle name, for predictability
165 Map
<String
, A2Jar
> a2Jars
= new TreeMap
<>();
167 // StringJoiner modulePath = new StringJoiner(File.pathSeparator);
168 for (String a2Base
: a2Bases
) {
169 categories
: for (String a2Category
: a2Categories
) {
170 Path a2Dir
= Paths
.get(a2Base
).resolve(a2Category
);
171 if (!Files
.exists(a2Dir
))
173 // modulePath.add(a2Dir.toString());
174 for (Path jarP
: Files
.newDirectoryStream(a2Dir
,
175 (p
) -> p
.getFileName().toString().endsWith(".jar"))) {
176 A2Jar a2Jar
= new A2Jar(jarP
);
177 if (a2Jars
.containsKey(a2Jar
.name
)) {
178 A2Jar current
= a2Jars
.get(a2Jar
.name
);
179 if (a2Jar
.major
> current
.major
)
180 a2Jars
.put(a2Jar
.name
, a2Jar
);
181 else if (a2Jar
.major
== current
.major
//
182 // if minor equals, we take the last one
183 && a2Jar
.minor
>= current
.minor
)
184 a2Jars
.put(a2Jar
.name
, a2Jar
);
186 a2Jars
.put(a2Jar
.name
, a2Jar
);
192 StringJoiner classPath
= new StringJoiner(File
.pathSeparator
);
193 for (Iterator
<A2Jar
> it
= a2Jars
.values().iterator(); it
.hasNext();)
194 classPath
.add(it
.next().path
.toString());
196 compilerArgs
.add("-cp");
197 compilerArgs
.add(classPath
.toString());
198 // compilerArgs.add("--module-path");
199 // compilerArgs.add(modulePath.toString());
203 for (String bundle
: bundles
) {
204 StringBuilder sb
= new StringBuilder();
205 Path bundlePath
= execDirectory
.resolve(bundle
);
206 if (!Files
.exists(bundlePath
)) {
207 if (bundles
.size() == 1) {
208 logger
.log(WARNING
, "Bundle " + bundle
+ " not found in " + execDirectory
209 + ", assuming this is this directory, as only one bundle was requested.");
210 bundlePath
= execDirectory
;
212 throw new IllegalArgumentException("Bundle " + bundle
+ " not found in " + execDirectory
);
214 sb
.append(bundlePath
.resolve("src"));
216 compilerArgs
.add(sb
.toString());
217 sb
= new StringBuilder();
218 sb
.append(buildBase
.resolve(bundle
).resolve("bin"));
220 compilerArgs
.add(sb
.toString());
223 if (logger
.isLoggable(INFO
))
224 compilerArgs
.add("-time");
226 if (logger
.isLoggable(DEBUG
)) {
227 logger
.log(DEBUG
, "Compiler arguments:");
228 for (String arg
: compilerArgs
)
229 logger
.log(DEBUG
, arg
);
232 boolean success
= org
.eclipse
.jdt
.core
.compiler
.batch
.BatchCompiler
.compile(
233 compilerArgs
.toArray(new String
[compilerArgs
.size()]), new PrintWriter(System
.out
),
234 new PrintWriter(System
.err
), new MakeCompilationProgress());
235 if (!success
) // kill the process if compilation failed
236 throw new IllegalStateException("Compilation failed");
239 /** Package the bundles. */
240 void bundle(Map
<String
, List
<String
>> options
) throws IOException
{
242 List
<String
> bundles
= options
.get("--bundles");
243 Objects
.requireNonNull(bundles
, "--bundles argument must be set");
244 if (bundles
.isEmpty())
247 List
<String
> categories
= options
.get("--category");
248 Objects
.requireNonNull(bundles
, "--category argument must be set");
249 if (categories
.size() != 1)
250 throw new IllegalArgumentException("One and only one --category must be specified");
251 String category
= categories
.get(0);
254 Path branchMk
= sdkSrcBase
.resolve(BRANCH_MK
);
255 if (Files
.exists(branchMk
)) {
256 Map
<String
, String
> branchVariables
= readeMakefileVariables(branchMk
);
257 branch
= branchVariables
.get("BRANCH");
262 long begin
= System
.currentTimeMillis();
263 // create jars in parallel
264 List
<CompletableFuture
<Void
>> toDos
= new ArrayList
<>();
265 for (String bundle
: bundles
) {
266 toDos
.add(CompletableFuture
.runAsync(() -> {
268 createBundle(branch
, bundle
, category
);
269 } catch (IOException e
) {
270 throw new RuntimeException("Packaging of " + bundle
+ " failed", e
);
274 CompletableFuture
.allOf(toDos
.toArray(new CompletableFuture
[toDos
.size()])).join();
275 long duration
= System
.currentTimeMillis() - begin
;
276 logger
.log(INFO
, "Packaging took " + duration
+ " ms");
279 /** Package a single bundle. */
280 void createBundle(String branch
, String bundle
, String category
) throws IOException
{
282 if (!Files
.exists(execDirectory
.resolve(bundle
))) {
284 "Bundle " + bundle
+ " not found in " + execDirectory
+ ", assuming this is this directory.");
285 source
= execDirectory
;
287 source
= execDirectory
.resolve(bundle
);
290 Path compiled
= buildBase
.resolve(bundle
);
291 String bundleSymbolicName
= source
.getFileName().toString();
294 Properties properties
= new Properties();
295 Path argeoBnd
= argeoBuildBase
.resolve("argeo.bnd");
296 try (InputStream in
= Files
.newInputStream(argeoBnd
)) {
300 if (branch
!= null) {
301 Path branchBnd
= sdkSrcBase
.resolve("sdk/branches/" + branch
+ ".bnd");
302 if (Files
.exists(branchBnd
))
303 try (InputStream in
= Files
.newInputStream(branchBnd
)) {
308 Path bndBnd
= source
.resolve("bnd.bnd");
309 if (Files
.exists(bndBnd
))
310 try (InputStream in
= Files
.newInputStream(bndBnd
)) {
315 if (!properties
.containsKey("Bundle-SymbolicName"))
316 properties
.put("Bundle-SymbolicName", bundleSymbolicName
);
318 // Calculate MANIFEST
319 Path binP
= compiled
.resolve("bin");
320 if (!Files
.exists(binP
))
321 Files
.createDirectories(binP
);
323 try (Analyzer bndAnalyzer
= new Analyzer()) {
324 bndAnalyzer
.setProperties(properties
);
325 Jar jar
= new Jar(bundleSymbolicName
, binP
.toFile());
326 bndAnalyzer
.setJar(jar
);
327 manifest
= bndAnalyzer
.calcManifest();
328 } catch (Exception e
) {
329 throw new RuntimeException("Bnd analysis of " + compiled
+ " failed", e
);
332 String major
= properties
.getProperty("major");
333 Objects
.requireNonNull(major
, "'major' must be set");
334 String minor
= properties
.getProperty("minor");
335 Objects
.requireNonNull(minor
, "'minor' must be set");
338 Path manifestP
= compiled
.resolve("META-INF/MANIFEST.MF");
339 Files
.createDirectories(manifestP
.getParent());
340 try (OutputStream out
= Files
.newOutputStream(manifestP
)) {
345 List
<PathMatcher
> excludes
= new ArrayList
<>();
346 Path excludesP
= argeoBuildBase
.resolve("excludes.txt");
347 for (String line
: Files
.readAllLines(excludesP
)) {
348 PathMatcher pathMatcher
= excludesP
.getFileSystem().getPathMatcher("glob:" + line
);
349 excludes
.add(pathMatcher
);
352 Path bundleParent
= Paths
.get(bundle
).getParent();
353 Path a2JarDirectory
= bundleParent
!= null ? a2Output
.resolve(bundleParent
).resolve(category
)
354 : a2Output
.resolve(category
);
355 Path jarP
= a2JarDirectory
.resolve(compiled
.getFileName() + "." + major
+ "." + minor
+ ".jar");
356 Files
.createDirectories(jarP
.getParent());
358 try (JarOutputStream jarOut
= new JarOutputStream(Files
.newOutputStream(jarP
), manifest
)) {
359 jarOut
.setLevel(Deflater
.DEFAULT_COMPRESSION
);
360 // add all classes first
361 Files
.walkFileTree(binP
, new SimpleFileVisitor
<Path
>() {
363 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
364 jarOut
.putNextEntry(new JarEntry(binP
.relativize(file
).toString()));
365 Files
.copy(file
, jarOut
);
366 return FileVisitResult
.CONTINUE
;
371 Files
.walkFileTree(source
, new SimpleFileVisitor
<Path
>() {
373 public FileVisitResult
preVisitDirectory(Path dir
, BasicFileAttributes attrs
) throws IOException
{
374 Path relativeP
= source
.relativize(dir
);
375 for (PathMatcher exclude
: excludes
)
376 if (exclude
.matches(relativeP
))
377 return FileVisitResult
.SKIP_SUBTREE
;
379 return FileVisitResult
.CONTINUE
;
383 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
384 Path relativeP
= source
.relativize(file
);
385 for (PathMatcher exclude
: excludes
)
386 if (exclude
.matches(relativeP
))
387 return FileVisitResult
.CONTINUE
;
388 JarEntry entry
= new JarEntry(relativeP
.toString());
389 jarOut
.putNextEntry(entry
);
390 Files
.copy(file
, jarOut
);
391 return FileVisitResult
.CONTINUE
;
395 Path srcP
= source
.resolve("src");
396 // Add all resources from src/
397 Files
.walkFileTree(srcP
, new SimpleFileVisitor
<Path
>() {
399 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
400 if (file
.getFileName().toString().endsWith(".java")
401 || file
.getFileName().toString().endsWith(".class"))
402 return FileVisitResult
.CONTINUE
;
403 jarOut
.putNextEntry(new JarEntry(srcP
.relativize(file
).toString()));
404 if (!Files
.isDirectory(file
))
405 Files
.copy(file
, jarOut
);
406 return FileVisitResult
.CONTINUE
;
411 // TODO add effective BND, Eclipse project file, etc., in order to be able to
414 Path a2srcJarDirectory
= bundleParent
!= null ? a2srcOutput
.resolve(bundleParent
).resolve(category
)
415 : a2srcOutput
.resolve(category
);
416 Files
.createDirectories(a2srcJarDirectory
);
417 Path srcJarP
= a2srcJarDirectory
418 .resolve(compiled
.getFileName() + "." + major
+ "." + minor
+ ".src.jar");
419 Manifest srcManifest
= new Manifest();
420 srcManifest
.getMainAttributes().put(Attributes
.Name
.MANIFEST_VERSION
, "1.0");
421 srcManifest
.getMainAttributes().putValue("Bundle-SymbolicName", bundleSymbolicName
+ ".src");
422 srcManifest
.getMainAttributes().putValue("Bundle-Version",
423 manifest
.getMainAttributes().getValue("Bundle-Version").toString());
424 srcManifest
.getMainAttributes().putValue("Eclipse-SourceBundle",
425 bundleSymbolicName
+ ";version=\"" + manifest
.getMainAttributes().getValue("Bundle-Version"));
427 try (JarOutputStream srcJarOut
= new JarOutputStream(Files
.newOutputStream(srcJarP
), srcManifest
)) {
428 copySourcesToJar(srcP
, srcJarOut
, "");
431 copySourcesToJar(srcP
, jarOut
, "OSGI-OPT/src/");
439 /** Add sources to a jar file */
440 void copySourcesToJar(Path srcP
, JarOutputStream srcJarOut
, String prefix
) throws IOException
{
441 Files
.walkFileTree(srcP
, new SimpleFileVisitor
<Path
>() {
443 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
444 srcJarOut
.putNextEntry(new JarEntry(prefix
+ srcP
.relativize(file
).toString()));
445 if (!Files
.isDirectory(file
))
446 Files
.copy(file
, srcJarOut
);
447 return FileVisitResult
.CONTINUE
;
453 * Recursively find the base source directory (which contains the
454 * <code>{@value #SDK_MK}</code> file).
456 Path
findSdkMk(Path directory
) {
457 Path sdkMkP
= directory
.resolve(SDK_MK
);
458 if (Files
.exists(sdkMkP
)) {
459 return sdkMkP
.toAbsolutePath();
461 if (directory
.getParent() == null)
463 return findSdkMk(directory
.getParent());
467 * Reads Makefile variable assignments of the form =, :=, or ?=, ignoring white
468 * spaces. To be used with very simple included Makefiles only.
470 Map
<String
, String
> readeMakefileVariables(Path path
) throws IOException
{
471 Map
<String
, String
> context
= new HashMap
<>();
472 List
<String
> sdkMkLines
= Files
.readAllLines(path
);
473 lines
: for (String line
: sdkMkLines
) {
474 StringTokenizer st
= new StringTokenizer(line
, " :=?");
475 if (!st
.hasMoreTokens())
477 String key
= st
.nextToken();
478 if (!st
.hasMoreTokens())
480 String value
= st
.nextToken();
481 if (st
.hasMoreTokens()) // probably not a simple variable assignment
483 context
.put(key
, value
);
488 /** Main entry point, interpreting actions and arguments. */
489 public static void main(String
... args
) {
490 if (args
.length
== 0)
491 throw new IllegalArgumentException("At least an action must be provided");
493 String action
= args
[actionIndex
];
494 if (args
.length
> actionIndex
+ 1 && !args
[actionIndex
+ 1].startsWith("-"))
495 throw new IllegalArgumentException(
496 "Action " + action
+ " must be followed by an option: " + Arrays
.asList(args
));
498 Map
<String
, List
<String
>> options
= new HashMap
<>();
499 String currentOption
= null;
500 for (int i
= actionIndex
+ 1; i
< args
.length
; i
++) {
501 if (args
[i
].startsWith("-")) {
502 currentOption
= args
[i
];
503 if (!options
.containsKey(currentOption
))
504 options
.put(currentOption
, new ArrayList
<>());
507 options
.get(currentOption
).add(args
[i
]);
512 Make argeoMake
= new Make();
514 case "compile" -> argeoMake
.compile(options
);
515 case "bundle" -> argeoMake
.bundle(options
);
516 case "all" -> argeoMake
.all(options
);
518 default -> throw new IllegalArgumentException("Unkown action: " + action
);
521 long jvmUptime
= ManagementFactory
.getRuntimeMXBean().getUptime();
522 logger
.log(INFO
, "Make.java action '" + action
+ "' succesfully completed after " + (jvmUptime
/ 1000) + "."
523 + (jvmUptime
% 1000) + " s");
524 } catch (Exception e
) {
525 long jvmUptime
= ManagementFactory
.getRuntimeMXBean().getUptime();
526 logger
.log(ERROR
, "Make.java action '" + action
+ "' failed after " + (jvmUptime
/ 1000) + "."
527 + (jvmUptime
% 1000) + " s", e
);
540 String fileName
= path
.getFileName().toString();
541 fileName
= fileName
.substring(0, fileName
.lastIndexOf('.'));
542 minor
= Integer
.parseInt(fileName
.substring(fileName
.lastIndexOf('.') + 1));
543 fileName
= fileName
.substring(0, fileName
.lastIndexOf('.'));
544 major
= Integer
.parseInt(fileName
.substring(fileName
.lastIndexOf('.') + 1));
545 name
= fileName
.substring(0, fileName
.lastIndexOf('.'));
550 * An ECJ {@link CompilationProgress} printing a progress bar while compiling.
552 static class MakeCompilationProgress
extends CompilationProgress
{
553 private int totalWork
;
554 private long currentChunk
= 0;
555 private long chunksCount
= 80;
558 public void worked(int workIncrement
, int remainingWork
) {
559 if (!logger
.isLoggable(Level
.INFO
)) // progress bar only at INFO level
561 long chunk
= ((totalWork
- remainingWork
) * chunksCount
) / totalWork
;
562 if (chunk
!= currentChunk
) {
563 currentChunk
= chunk
;
564 for (long i
= 0; i
< currentChunk
; i
++) {
565 System
.out
.print("#");
567 for (long i
= currentChunk
; i
< chunksCount
; i
++) {
568 System
.out
.print("-");
570 System
.out
.print("\r");
572 if (remainingWork
== 0)
573 System
.out
.print("\n");
577 public void setTaskName(String name
) {
581 public boolean isCanceled() {
590 public void begin(int remainingWork
) {
591 this.totalWork
= remainingWork
;