1 package org
.argeo
.build
;
3 import static java
.lang
.System
.Logger
.Level
.ERROR
;
4 import static java
.lang
.System
.Logger
.Level
.INFO
;
7 import java
.io
.IOException
;
8 import java
.io
.InputStream
;
9 import java
.io
.OutputStream
;
10 import java
.io
.PrintWriter
;
11 import java
.lang
.System
.Logger
;
12 import java
.lang
.System
.Logger
.Level
;
13 import java
.lang
.management
.ManagementFactory
;
14 import java
.nio
.file
.FileVisitResult
;
15 import java
.nio
.file
.Files
;
16 import java
.nio
.file
.Path
;
17 import java
.nio
.file
.PathMatcher
;
18 import java
.nio
.file
.Paths
;
19 import java
.nio
.file
.SimpleFileVisitor
;
20 import java
.nio
.file
.attribute
.BasicFileAttributes
;
21 import java
.util
.ArrayList
;
22 import java
.util
.Arrays
;
23 import java
.util
.HashMap
;
24 import java
.util
.List
;
26 import java
.util
.Objects
;
27 import java
.util
.Properties
;
28 import java
.util
.StringJoiner
;
29 import java
.util
.StringTokenizer
;
30 import java
.util
.concurrent
.CompletableFuture
;
31 import java
.util
.jar
.JarEntry
;
32 import java
.util
.jar
.JarOutputStream
;
33 import java
.util
.jar
.Manifest
;
34 import java
.util
.zip
.Deflater
;
36 import org
.eclipse
.jdt
.core
.compiler
.CompilationProgress
;
38 import aQute
.bnd
.osgi
.Analyzer
;
39 import aQute
.bnd
.osgi
.Jar
;
42 * Minimalistic OSGi compiler and packager, meant to be used as a single file
43 * without being itself compiled first. It depends on the Eclipse batch compiler
44 * (aka. ECJ) and the BND Libs library for OSGi metadata generation (which
45 * itselfs depends on slf4j).<br/>
47 * For example, a typical system call would be:<br/>
48 * <code>java -cp "/path/to/ECJ jar:/path/to/bndlib jar:/path/to/SLF4J jar" /path/to/cloned/argeo-build/src/org/argeo/build/Make.java action --option1 argument1 argument2 --option2 argument3 </code>
51 private final static Logger logger
= System
.getLogger(Make
.class.getName());
54 * Environment properties on whether sources should be packaged separately or
55 * integrated in the bundles.
57 private final static String ENV_BUILD_SOURCE_BUNDLES
= "BUILD_SOURCE_BUNDLES";
59 /** Name of the local-specific Makefile (sdk.mk). */
60 final static String SDK_MK
= "sdk.mk";
61 /** Name of the branch definition Makefile (branch.mk). */
62 final static String BRANCH_MK
= "branch.mk";
64 /** The execution directory (${user.dir}). */
65 final Path execDirectory
;
66 /** Base of the source code, typically the cloned git repository. */
67 final Path sdkSrcBase
;
69 * The base of the builder, typically a submodule pointing to the public
70 * argeo-build directory.
72 final Path argeoBuildBase
;
73 /** The base of the build for all layers. */
74 final Path sdkBuildBase
;
75 /** The base of the build for this layer. */
77 /** The base of the a2 output for all layers. */
80 /** Whether sources should be packaged separately */
81 final boolean sourceBundles
;
83 /** Constructor initialises the base directories. */
84 public Make() throws IOException
{
85 sourceBundles
= Boolean
.parseBoolean(System
.getenv(ENV_BUILD_SOURCE_BUNDLES
));
87 logger
.log(Level
.INFO
, "Sources will be packaged separately");
89 execDirectory
= Paths
.get(System
.getProperty("user.dir"));
90 Path sdkMkP
= findSdkMk(execDirectory
);
91 Objects
.requireNonNull(sdkMkP
, "No " + SDK_MK
+ " found under " + execDirectory
);
93 Map
<String
, String
> context
= readeMakefileVariables(sdkMkP
);
94 sdkSrcBase
= Paths
.get(context
.computeIfAbsent("SDK_SRC_BASE", (key
) -> {
95 throw new IllegalStateException(key
+ " not found");
97 argeoBuildBase
= sdkSrcBase
.resolve("sdk/argeo-build");
99 sdkBuildBase
= Paths
.get(context
.computeIfAbsent("SDK_BUILD_BASE", (key
) -> {
100 throw new IllegalStateException(key
+ " not found");
101 })).toAbsolutePath();
102 buildBase
= sdkBuildBase
.resolve(sdkSrcBase
.getFileName());
103 a2Output
= sdkBuildBase
.resolve("a2");
109 /** Compile and create the bundles in one go. */
110 void all(Map
<String
, List
<String
>> options
) throws IOException
{
115 /** Compile all the bundles which have been passed via the --bundle argument. */
116 @SuppressWarnings("restriction")
117 void compile(Map
<String
, List
<String
>> options
) throws IOException
{
118 List
<String
> bundles
= options
.get("--bundles");
119 Objects
.requireNonNull(bundles
, "--bundles argument must be set");
120 if (bundles
.isEmpty())
123 List
<String
> a2Categories
= options
.getOrDefault("--dep-categories", new ArrayList
<>());
124 List
<String
> a2Bases
= options
.getOrDefault("--a2-bases", new ArrayList
<>());
125 if (a2Bases
.isEmpty()) {
126 a2Bases
.add(a2Output
.toString());
129 List
<String
> compilerArgs
= new ArrayList
<>();
131 Path ecjArgs
= argeoBuildBase
.resolve("ecj.args");
132 compilerArgs
.add("@" + ecjArgs
);
135 if (!a2Categories
.isEmpty()) {
136 StringJoiner classPath
= new StringJoiner(File
.pathSeparator
);
137 StringJoiner modulePath
= new StringJoiner(File
.pathSeparator
);
138 for (String a2Base
: a2Bases
) {
139 for (String a2Category
: a2Categories
) {
140 Path a2Dir
= Paths
.get(a2Base
).resolve(a2Category
);
141 if (!Files
.exists(a2Dir
))
142 Files
.createDirectories(a2Dir
);
143 modulePath
.add(a2Dir
.toString());
144 for (Path jarP
: Files
.newDirectoryStream(a2Dir
,
145 (p
) -> p
.getFileName().toString().endsWith(".jar"))) {
146 classPath
.add(jarP
.toString());
150 compilerArgs
.add("-cp");
151 compilerArgs
.add(classPath
.toString());
152 // compilerArgs.add("--module-path");
153 // compilerArgs.add(modulePath.toString());
157 for (String bundle
: bundles
) {
158 StringBuilder sb
= new StringBuilder();
159 Path bundlePath
= execDirectory
.resolve(bundle
);
160 if (!Files
.exists(bundlePath
))
161 throw new IllegalArgumentException("Bundle " + bundle
+ " not found in " + execDirectory
);
162 sb
.append(bundlePath
.resolve("src"));
164 compilerArgs
.add(sb
.toString());
165 sb
= new StringBuilder();
166 sb
.append(buildBase
.resolve(bundle
).resolve("bin"));
168 compilerArgs
.add(sb
.toString());
171 if (logger
.isLoggable(INFO
))
172 compilerArgs
.add("-time");
174 // for (String arg : compilerArgs)
175 // System.out.println(arg);
177 boolean success
= org
.eclipse
.jdt
.core
.compiler
.batch
.BatchCompiler
.compile(
178 compilerArgs
.toArray(new String
[compilerArgs
.size()]), new PrintWriter(System
.out
),
179 new PrintWriter(System
.err
), new MakeCompilationProgress());
180 if (!success
) // kill the process if compilation failed
181 throw new IllegalStateException("Compilation failed");
184 /** Package the bundles. */
185 void bundle(Map
<String
, List
<String
>> options
) throws IOException
{
187 List
<String
> bundles
= options
.get("--bundles");
188 Objects
.requireNonNull(bundles
, "--bundles argument must be set");
189 if (bundles
.isEmpty())
192 List
<String
> categories
= options
.get("--category");
193 Objects
.requireNonNull(bundles
, "--category argument must be set");
194 if (categories
.size() != 1)
195 throw new IllegalArgumentException("One and only one --category must be specified");
196 String category
= categories
.get(0);
198 Path branchMk
= sdkSrcBase
.resolve(BRANCH_MK
);
199 if (!Files
.exists(branchMk
))
200 throw new IllegalStateException("No " + branchMk
+ " file available");
201 Map
<String
, String
> branchVariables
= readeMakefileVariables(branchMk
);
203 String branch
= branchVariables
.get("BRANCH");
205 long begin
= System
.currentTimeMillis();
206 // create jars in parallel
207 List
<CompletableFuture
<Void
>> toDos
= new ArrayList
<>();
208 for (String bundle
: bundles
) {
209 toDos
.add(CompletableFuture
.runAsync(() -> {
211 createBundle(branch
, bundle
, category
);
212 } catch (IOException e
) {
213 throw new RuntimeException("Packaging of " + bundle
+ " failed", e
);
217 CompletableFuture
.allOf(toDos
.toArray(new CompletableFuture
[toDos
.size()])).join();
218 long duration
= System
.currentTimeMillis() - begin
;
219 logger
.log(INFO
, "Packaging took " + duration
+ " ms");
225 /** Package a single bundle. */
226 void createBundle(String branch
, String bundle
, String category
) throws IOException
{
227 Path source
= execDirectory
.resolve(bundle
);
228 Path compiled
= buildBase
.resolve(bundle
);
229 String bundleSymbolicName
= source
.getFileName().toString();
232 Properties properties
= new Properties();
233 Path argeoBnd
= argeoBuildBase
.resolve("argeo.bnd");
234 try (InputStream in
= Files
.newInputStream(argeoBnd
)) {
238 Path branchBnd
= sdkSrcBase
.resolve("sdk/branches/" + branch
+ ".bnd");
239 try (InputStream in
= Files
.newInputStream(branchBnd
)) {
243 Path bndBnd
= source
.resolve("bnd.bnd");
244 try (InputStream in
= Files
.newInputStream(bndBnd
)) {
249 if (!properties
.containsKey("Bundle-SymbolicName"))
250 properties
.put("Bundle-SymbolicName", bundleSymbolicName
);
252 // Calculate MANIFEST
253 Path binP
= compiled
.resolve("bin");
254 if (!Files
.exists(binP
))
255 Files
.createDirectories(binP
);
257 try (Analyzer bndAnalyzer
= new Analyzer()) {
258 bndAnalyzer
.setProperties(properties
);
259 Jar jar
= new Jar(bundleSymbolicName
, binP
.toFile());
260 bndAnalyzer
.setJar(jar
);
261 manifest
= bndAnalyzer
.calcManifest();
262 } catch (Exception e
) {
263 throw new RuntimeException("Bnd analysis of " + compiled
+ " failed", e
);
266 String major
= properties
.getProperty("major");
267 Objects
.requireNonNull(major
, "'major' must be set");
268 String minor
= properties
.getProperty("minor");
269 Objects
.requireNonNull(minor
, "'minor' must be set");
272 Path manifestP
= compiled
.resolve("META-INF/MANIFEST.MF");
273 Files
.createDirectories(manifestP
.getParent());
274 try (OutputStream out
= Files
.newOutputStream(manifestP
)) {
279 List
<PathMatcher
> excludes
= new ArrayList
<>();
280 Path excludesP
= argeoBuildBase
.resolve("excludes.txt");
281 for (String line
: Files
.readAllLines(excludesP
)) {
282 PathMatcher pathMatcher
= excludesP
.getFileSystem().getPathMatcher("glob:" + line
);
283 excludes
.add(pathMatcher
);
286 Path bundleParent
= Paths
.get(bundle
).getParent();
287 Path a2JarDirectory
= bundleParent
!= null ? a2Output
.resolve(bundleParent
).resolve(category
)
288 : a2Output
.resolve(category
);
289 Path jarP
= a2JarDirectory
.resolve(compiled
.getFileName() + "." + major
+ "." + minor
+ ".jar");
290 Files
.createDirectories(jarP
.getParent());
292 try (JarOutputStream jarOut
= new JarOutputStream(Files
.newOutputStream(jarP
), manifest
)) {
293 jarOut
.setLevel(Deflater
.DEFAULT_COMPRESSION
);
294 // add all classes first
295 Files
.walkFileTree(binP
, new SimpleFileVisitor
<Path
>() {
297 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
298 jarOut
.putNextEntry(new JarEntry(binP
.relativize(file
).toString()));
299 Files
.copy(file
, jarOut
);
300 return FileVisitResult
.CONTINUE
;
305 Files
.walkFileTree(source
, new SimpleFileVisitor
<Path
>() {
307 public FileVisitResult
preVisitDirectory(Path dir
, BasicFileAttributes attrs
) throws IOException
{
308 Path relativeP
= source
.relativize(dir
);
309 for (PathMatcher exclude
: excludes
)
310 if (exclude
.matches(relativeP
))
311 return FileVisitResult
.SKIP_SUBTREE
;
313 return FileVisitResult
.CONTINUE
;
317 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
318 Path relativeP
= source
.relativize(file
);
319 for (PathMatcher exclude
: excludes
)
320 if (exclude
.matches(relativeP
))
321 return FileVisitResult
.CONTINUE
;
322 JarEntry entry
= new JarEntry(relativeP
.toString());
323 jarOut
.putNextEntry(entry
);
324 Files
.copy(file
, jarOut
);
325 return FileVisitResult
.CONTINUE
;
329 Path srcP
= source
.resolve("src");
330 // Add all resources from src/
331 Files
.walkFileTree(srcP
, new SimpleFileVisitor
<Path
>() {
333 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
334 if (file
.getFileName().toString().endsWith(".java")
335 || file
.getFileName().toString().endsWith(".class"))
336 return FileVisitResult
.CONTINUE
;
337 jarOut
.putNextEntry(new JarEntry(srcP
.relativize(file
).toString()));
338 if (!Files
.isDirectory(file
))
339 Files
.copy(file
, jarOut
);
340 return FileVisitResult
.CONTINUE
;
345 // TODO add effective BND, Eclipse project file, etc., in order to be able to
348 // TODO package sources separately
350 Files
.walkFileTree(srcP
, new SimpleFileVisitor
<Path
>() {
352 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
353 jarOut
.putNextEntry(new JarEntry("OSGI-OPT/src/" + srcP
.relativize(file
).toString()));
354 if (!Files
.isDirectory(file
))
355 Files
.copy(file
, jarOut
);
356 return FileVisitResult
.CONTINUE
;
364 * Recursively find the base source directory (which contains the
365 * <code>{@value #SDK_MK}</code> file).
367 Path
findSdkMk(Path directory
) {
368 Path sdkMkP
= directory
.resolve(SDK_MK
);
369 if (Files
.exists(sdkMkP
)) {
370 return sdkMkP
.toAbsolutePath();
372 if (directory
.getParent() == null)
374 return findSdkMk(directory
.getParent());
378 * Reads Makefile variable assignments of the form =, :=, or ?=, ignoring white
379 * spaces. To be used with very simple included Makefiles only.
381 Map
<String
, String
> readeMakefileVariables(Path path
) throws IOException
{
382 Map
<String
, String
> context
= new HashMap
<>();
383 List
<String
> sdkMkLines
= Files
.readAllLines(path
);
384 lines
: for (String line
: sdkMkLines
) {
385 StringTokenizer st
= new StringTokenizer(line
, " :=?");
386 if (!st
.hasMoreTokens())
388 String key
= st
.nextToken();
389 if (!st
.hasMoreTokens())
391 String value
= st
.nextToken();
392 if (st
.hasMoreTokens()) // probably not a simple variable assignment
394 context
.put(key
, value
);
399 /** Main entry point, interpreting actions and arguments. */
400 public static void main(String
... args
) {
401 if (args
.length
== 0)
402 throw new IllegalArgumentException("At least an action must be provided");
404 String action
= args
[actionIndex
];
405 if (args
.length
> actionIndex
+ 1 && !args
[actionIndex
+ 1].startsWith("-"))
406 throw new IllegalArgumentException(
407 "Action " + action
+ " must be followed by an option: " + Arrays
.asList(args
));
409 Map
<String
, List
<String
>> options
= new HashMap
<>();
410 String currentOption
= null;
411 for (int i
= actionIndex
+ 1; i
< args
.length
; i
++) {
412 if (args
[i
].startsWith("-")) {
413 currentOption
= args
[i
];
414 if (!options
.containsKey(currentOption
))
415 options
.put(currentOption
, new ArrayList
<>());
418 options
.get(currentOption
).add(args
[i
]);
423 Make argeoMake
= new Make();
425 case "compile" -> argeoMake
.compile(options
);
426 case "bundle" -> argeoMake
.bundle(options
);
427 case "all" -> argeoMake
.all(options
);
429 default -> throw new IllegalArgumentException("Unkown action: " + action
);
432 long jvmUptime
= ManagementFactory
.getRuntimeMXBean().getUptime();
433 logger
.log(INFO
, "Make.java action '" + action
+ "' succesfully completed after " + (jvmUptime
/ 1000) + "."
434 + (jvmUptime
% 1000) + " s");
435 } catch (Exception e
) {
436 long jvmUptime
= ManagementFactory
.getRuntimeMXBean().getUptime();
437 logger
.log(ERROR
, "Make.java action '" + action
+ "' failed after " + (jvmUptime
/ 1000) + "."
438 + (jvmUptime
% 1000) + " s", e
);
444 * An ECJ {@link CompilationProgress} printing a progress bar while compiling.
446 static class MakeCompilationProgress
extends CompilationProgress
{
447 private int totalWork
;
448 private long currentChunk
= 0;
449 private long chunksCount
= 80;
452 public void worked(int workIncrement
, int remainingWork
) {
453 if (!logger
.isLoggable(Level
.INFO
)) // progress bar only at INFO level
455 long chunk
= ((totalWork
- remainingWork
) * chunksCount
) / totalWork
;
456 if (chunk
!= currentChunk
) {
457 currentChunk
= chunk
;
458 for (long i
= 0; i
< currentChunk
; i
++) {
459 System
.out
.print("#");
461 for (long i
= currentChunk
; i
< chunksCount
; i
++) {
462 System
.out
.print("-");
464 System
.out
.print("\r");
466 if (remainingWork
== 0)
467 System
.out
.print("\n");
471 public void setTaskName(String name
) {
475 public boolean isCanceled() {
484 public void begin(int remainingWork
) {
485 this.totalWork
= remainingWork
;