1 package org
.argeo
.build
;
3 import static java
.lang
.System
.Logger
.Level
.DEBUG
;
4 import static org
.argeo
.build
.Repackage
.ManifestConstants
.BUNDLE_SYMBOLICNAME
;
5 import static org
.argeo
.build
.Repackage
.ManifestConstants
.BUNDLE_VERSION
;
6 import static org
.argeo
.build
.Repackage
.ManifestConstants
.EXPORT_PACKAGE
;
7 import static org
.argeo
.build
.Repackage
.ManifestConstants
.SLC_ORIGIN_M2
;
8 import static org
.argeo
.build
.Repackage
.ManifestConstants
.SLC_ORIGIN_M2_REPO
;
11 import java
.io
.FileNotFoundException
;
12 import java
.io
.IOException
;
13 import java
.io
.InputStream
;
14 import java
.io
.OutputStream
;
15 import java
.lang
.System
.Logger
;
16 import java
.lang
.System
.Logger
.Level
;
17 import java
.net
.MalformedURLException
;
19 import java
.nio
.charset
.StandardCharsets
;
20 import java
.nio
.file
.DirectoryStream
;
21 import java
.nio
.file
.FileSystem
;
22 import java
.nio
.file
.FileSystems
;
23 import java
.nio
.file
.FileVisitResult
;
24 import java
.nio
.file
.Files
;
25 import java
.nio
.file
.Path
;
26 import java
.nio
.file
.PathMatcher
;
27 import java
.nio
.file
.Paths
;
28 import java
.nio
.file
.SimpleFileVisitor
;
29 import java
.nio
.file
.StandardOpenOption
;
30 import java
.nio
.file
.attribute
.BasicFileAttributes
;
31 import java
.util
.ArrayList
;
32 import java
.util
.HashMap
;
33 import java
.util
.Iterator
;
34 import java
.util
.List
;
36 import java
.util
.Objects
;
37 import java
.util
.Properties
;
38 import java
.util
.TreeMap
;
39 import java
.util
.concurrent
.CompletableFuture
;
40 import java
.util
.jar
.Attributes
;
41 import java
.util
.jar
.JarEntry
;
42 import java
.util
.jar
.JarInputStream
;
43 import java
.util
.jar
.JarOutputStream
;
44 import java
.util
.jar
.Manifest
;
45 import java
.util
.zip
.Deflater
;
47 import aQute
.bnd
.osgi
.Analyzer
;
48 import aQute
.bnd
.osgi
.Jar
;
50 /** The central class for A2 packaging. */
51 public class Repackage
{
52 private final static Logger logger
= System
.getLogger(Repackage
.class.getName());
54 private final static String ENV_BUILD_SOURCE_BUNDLES
= "BUILD_SOURCE_BUNDLES";
56 /** Main entry point. */
57 public static void main(String
[] args
) {
58 if (args
.length
< 2) {
59 System
.err
.println("Usage: <path to a2 output dir> <category1> <category2> ...");
62 Path a2Base
= Paths
.get(args
[0]).toAbsolutePath().normalize();
63 Path descriptorsBase
= Paths
.get(".").toAbsolutePath().normalize();
64 Repackage factory
= new Repackage(a2Base
, descriptorsBase
);
66 List
<CompletableFuture
<Void
>> toDos
= new ArrayList
<>();
67 for (int i
= 1; i
< args
.length
; i
++) {
68 Path p
= Paths
.get(args
[i
]);
69 toDos
.add(CompletableFuture
.runAsync(() -> factory
.processCategory(p
)));
71 CompletableFuture
.allOf(toDos
.toArray(new CompletableFuture
[toDos
.size()])).join();
74 private final static String COMMON_BND
= "common.bnd";
75 private final static String MERGE_BND
= "merge.bnd";
77 private Path originBase
;
79 private Path a2LibBase
;
80 private Path descriptorsBase
;
82 private Properties uris
= new Properties();
84 /** key is URI prefix, value list of base URLs */
85 private Map
<String
, List
<String
>> mirrors
= new HashMap
<String
, List
<String
>>();
87 private final boolean sourceBundles
;
89 public Repackage(Path a2Base
, Path descriptorsBase
) {
90 sourceBundles
= Boolean
.parseBoolean(System
.getenv(ENV_BUILD_SOURCE_BUNDLES
));
92 logger
.log(Level
.INFO
, "Sources will be packaged separately");
94 Objects
.requireNonNull(a2Base
);
95 Objects
.requireNonNull(descriptorsBase
);
96 this.originBase
= Paths
.get(System
.getProperty("user.home"), ".cache", "argeo/build/origin");
97 // TODO define and use a build base
99 this.a2LibBase
= a2Base
.resolve("lib");
100 this.descriptorsBase
= descriptorsBase
;
101 if (!Files
.exists(this.descriptorsBase
))
102 throw new IllegalArgumentException(this.descriptorsBase
+ " does not exist");
105 Path urisPath
= this.descriptorsBase
.resolve("uris.properties");
106 if (Files
.exists(urisPath
)) {
107 try (InputStream in
= Files
.newInputStream(urisPath
)) {
109 } catch (IOException e
) {
110 throw new IllegalStateException("Cannot load " + urisPath
, e
);
115 Path eclipseMirrorsPath
= this.descriptorsBase
.resolve("eclipse.mirrors.txt");
116 List
<String
> eclipseMirrors
= new ArrayList
<>();
117 if (Files
.exists(eclipseMirrorsPath
)) {
119 eclipseMirrors
= Files
.readAllLines(eclipseMirrorsPath
, StandardCharsets
.UTF_8
);
120 } catch (IOException e
) {
121 throw new IllegalStateException("Cannot load " + eclipseMirrorsPath
, e
);
123 for (Iterator
<String
> it
= eclipseMirrors
.iterator(); it
.hasNext();) {
124 String value
= it
.next();
125 if (value
.strip().equals(""))
130 mirrors
.put("http://www.eclipse.org/downloads", eclipseMirrors
);
137 /** Process a whole category/group id. */
138 public void processCategory(Path categoryRelativePath
) {
140 Path targetCategoryBase
= descriptorsBase
.resolve(categoryRelativePath
);
141 DirectoryStream
<Path
> bnds
= Files
.newDirectoryStream(targetCategoryBase
,
142 (p
) -> p
.getFileName().toString().endsWith(".bnd") && !p
.getFileName().toString().equals(COMMON_BND
)
143 && !p
.getFileName().toString().equals(MERGE_BND
));
144 for (Path p
: bnds
) {
145 processSingleM2ArtifactDistributionUnit(p
);
148 DirectoryStream
<Path
> dus
= Files
.newDirectoryStream(targetCategoryBase
, (p
) -> Files
.isDirectory(p
));
149 for (Path duDir
: dus
) {
150 if (duDir
.getFileName().toString().startsWith("eclipse-")) {
151 processEclipseArchive(duDir
);
153 processM2BasedDistributionUnit(duDir
);
156 } catch (IOException e
) {
157 throw new RuntimeException("Cannot process category " + categoryRelativePath
, e
);
161 /** Process a standalone Maven artifact. */
162 public void processSingleM2ArtifactDistributionUnit(Path bndFile
) {
164 // String category = bndFile.getParent().getFileName().toString();
165 Path categoryRelativePath
= descriptorsBase
.relativize(bndFile
.getParent());
166 Path targetCategoryBase
= a2Base
.resolve(categoryRelativePath
);
168 Properties fileProps
= new Properties();
169 try (InputStream in
= Files
.newInputStream(bndFile
)) {
172 String repoStr
= fileProps
.containsKey(SLC_ORIGIN_M2_REPO
.toString())
173 ? fileProps
.getProperty(SLC_ORIGIN_M2_REPO
.toString())
176 if (!fileProps
.containsKey(BUNDLE_SYMBOLICNAME
.toString())) {
177 // use file name as symbolic name
178 String symbolicName
= bndFile
.getFileName().toString();
179 symbolicName
= symbolicName
.substring(0, symbolicName
.length() - ".bnd".length());
180 fileProps
.put(BUNDLE_SYMBOLICNAME
.toString(), symbolicName
);
183 String m2Coordinates
= fileProps
.getProperty(SLC_ORIGIN_M2
.toString());
184 if (m2Coordinates
== null)
185 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile
);
186 M2Artifact artifact
= new M2Artifact(m2Coordinates
);
187 URL url
= M2ConventionsUtils
.mavenRepoUrl(repoStr
, artifact
);
188 Path downloaded
= downloadMaven(url
, originBase
, artifact
);
190 Path targetBundleDir
= processBndJar(downloaded
, targetCategoryBase
, fileProps
, artifact
);
192 downloadAndProcessM2Sources(repoStr
, artifact
, targetBundleDir
);
194 createJar(targetBundleDir
);
195 } catch (Exception e
) {
196 throw new RuntimeException("Cannot process " + bndFile
, e
);
200 /** Process multiple Maven artifacts. */
201 public void processM2BasedDistributionUnit(Path duDir
) {
203 // String category = duDir.getParent().getFileName().toString();
204 Path categoryRelativePath
= descriptorsBase
.relativize(duDir
.getParent());
205 Path targetCategoryBase
= a2Base
.resolve(categoryRelativePath
);
208 Path mergeBnd
= duDir
.resolve(MERGE_BND
);
209 if (Files
.exists(mergeBnd
)) {
210 mergeM2Artifacts(mergeBnd
);
214 Path commonBnd
= duDir
.resolve(COMMON_BND
);
215 if (!Files
.exists(commonBnd
)) {
218 Properties commonProps
= new Properties();
219 try (InputStream in
= Files
.newInputStream(commonBnd
)) {
220 commonProps
.load(in
);
223 String m2Version
= commonProps
.getProperty(SLC_ORIGIN_M2
.toString());
224 if (m2Version
== null) {
225 logger
.log(Level
.WARNING
, "Ignoring " + duDir
+ " as it is not an M2-based distribution unit");
226 return;// ignore, this is probably an Eclipse archive
228 if (!m2Version
.startsWith(":")) {
229 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version
);
231 m2Version
= m2Version
.substring(1);
233 DirectoryStream
<Path
> ds
= Files
.newDirectoryStream(duDir
,
234 (p
) -> p
.getFileName().toString().endsWith(".bnd") && !p
.getFileName().toString().equals(COMMON_BND
)
235 && !p
.getFileName().toString().equals(MERGE_BND
));
237 Properties fileProps
= new Properties();
238 try (InputStream in
= Files
.newInputStream(p
)) {
241 String m2Coordinates
= fileProps
.getProperty(SLC_ORIGIN_M2
.toString());
242 M2Artifact artifact
= new M2Artifact(m2Coordinates
);
244 artifact
.setVersion(m2Version
);
246 // prepare manifest entries
247 Properties mergeProps
= new Properties();
248 mergeProps
.putAll(commonProps
);
250 fileEntries
: for (Object key
: fileProps
.keySet()) {
251 if (ManifestConstants
.SLC_ORIGIN_M2
.toString().equals(key
))
252 continue fileEntries
;
253 String value
= fileProps
.getProperty(key
.toString());
254 Object previousValue
= mergeProps
.put(key
.toString(), value
);
255 if (previousValue
!= null) {
256 logger
.log(Level
.WARNING
,
257 commonBnd
+ ": " + key
+ " was " + previousValue
+ ", overridden with " + value
);
260 mergeProps
.put(ManifestConstants
.SLC_ORIGIN_M2
.toString(), artifact
.toM2Coordinates());
261 if (!mergeProps
.containsKey(BUNDLE_SYMBOLICNAME
.toString())) {
262 // use file name as symbolic name
263 String symbolicName
= p
.getFileName().toString();
264 symbolicName
= symbolicName
.substring(0, symbolicName
.length() - ".bnd".length());
265 mergeProps
.put(BUNDLE_SYMBOLICNAME
.toString(), symbolicName
);
268 String repoStr
= mergeProps
.containsKey(SLC_ORIGIN_M2_REPO
.toString())
269 ? mergeProps
.getProperty(SLC_ORIGIN_M2_REPO
.toString())
273 URL url
= M2ConventionsUtils
.mavenRepoUrl(repoStr
, artifact
);
274 Path downloaded
= downloadMaven(url
, originBase
, artifact
);
276 Path targetBundleDir
= processBndJar(downloaded
, targetCategoryBase
, mergeProps
, artifact
);
277 // logger.log(Level.DEBUG, () -> "Processed " + downloaded);
280 downloadAndProcessM2Sources(repoStr
, artifact
, targetBundleDir
);
282 createJar(targetBundleDir
);
284 } catch (IOException e
) {
285 throw new RuntimeException("Cannot process " + duDir
, e
);
290 /** Merge multiple Maven artifacts. */
291 protected void mergeM2Artifacts(Path mergeBnd
) throws IOException
{
292 Path duDir
= mergeBnd
.getParent();
293 String category
= duDir
.getParent().getFileName().toString();
294 Path targetCategoryBase
= a2Base
.resolve(category
);
296 Properties mergeProps
= new Properties();
297 try (InputStream in
= Files
.newInputStream(mergeBnd
)) {
302 String m2Version
= mergeProps
.getProperty(SLC_ORIGIN_M2
.toString());
303 if (m2Version
== null) {
304 logger
.log(Level
.WARNING
, "Ignoring " + duDir
+ " as it is not an M2-based distribution unit");
305 return;// ignore, this is probably an Eclipse archive
307 if (!m2Version
.startsWith(":")) {
308 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version
);
310 m2Version
= m2Version
.substring(1);
311 mergeProps
.put(ManifestConstants
.BUNDLE_VERSION
.toString(), m2Version
);
313 String artifactsStr
= mergeProps
.getProperty(ManifestConstants
.SLC_ORIGIN_M2_MERGE
.toString());
314 if (artifactsStr
== null)
315 throw new IllegalArgumentException(
316 mergeBnd
+ ": " + ManifestConstants
.SLC_ORIGIN_M2_MERGE
+ " must be set");
318 String repoStr
= mergeProps
.containsKey(SLC_ORIGIN_M2_REPO
.toString())
319 ? mergeProps
.getProperty(SLC_ORIGIN_M2_REPO
.toString())
322 String bundleSymbolicName
= mergeProps
.getProperty(ManifestConstants
.BUNDLE_SYMBOLICNAME
.toString());
323 if (bundleSymbolicName
== null)
324 throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd
);
325 CategoryNameVersion nameVersion
= new M2Artifact(category
+ ":" + bundleSymbolicName
+ ":" + m2Version
);
326 Path targetBundleDir
= targetCategoryBase
.resolve(bundleSymbolicName
+ "." + nameVersion
.getBranch());
328 String
[] artifacts
= artifactsStr
.split(",");
329 artifacts
: for (String str
: artifacts
) {
330 String m2Coordinates
= str
.trim();
331 if ("".equals(m2Coordinates
))
333 M2Artifact artifact
= new M2Artifact(m2Coordinates
.trim());
334 if (artifact
.getVersion() == null)
335 artifact
.setVersion(m2Version
);
336 URL url
= M2ConventionsUtils
.mavenRepoUrl(repoStr
, artifact
);
337 Path downloaded
= downloadMaven(url
, originBase
, artifact
);
339 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(downloaded
), false)) {
340 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
341 if (entry
.isDirectory())
343 else if (entry
.getName().endsWith(".RSA") || entry
.getName().endsWith(".SF"))
345 else if (entry
.getName().startsWith("META-INF/versions/"))
347 else if (entry
.getName().startsWith("META-INF/maven/"))
349 else if (entry
.getName().equals("module-info.class"))
351 else if (entry
.getName().equals("META-INF/NOTICE"))
353 else if (entry
.getName().equals("META-INF/NOTICE.txt"))
355 else if (entry
.getName().equals("META-INF/LICENSE"))
357 else if (entry
.getName().equals("META-INF/LICENSE.md"))
359 else if (entry
.getName().equals("META-INF/LICENSE-notice.md"))
361 else if (entry
.getName().equals("META-INF/DEPENDENCIES"))
363 if (entry
.getName().startsWith(".cache/")) // Apache SSHD
365 Path target
= targetBundleDir
.resolve(entry
.getName());
366 Files
.createDirectories(target
.getParent());
367 if (!Files
.exists(target
)) {
368 Files
.copy(jarIn
, target
);
370 if (entry
.getName().startsWith("META-INF/services/")) {
371 try (OutputStream out
= Files
.newOutputStream(target
, StandardOpenOption
.APPEND
)) {
372 out
.write("\n".getBytes());
373 jarIn
.transferTo(out
);
374 logger
.log(Level
.WARNING
, artifact
.getArtifactId() + " - Appended " + entry
.getName());
376 } else if (entry
.getName().startsWith("org/apache/batik/")) {
377 logger
.log(Level
.WARNING
, "Skip " + entry
.getName());
380 throw new IllegalStateException("File " + target
+ " from " + artifact
+ " already exists");
383 logger
.log(Level
.TRACE
, () -> "Copied " + target
);
387 downloadAndProcessM2Sources(repoStr
, artifact
, targetBundleDir
);
390 // additional service files
391 Path servicesDir
= duDir
.resolve("services");
392 if (Files
.exists(servicesDir
)) {
393 for (Path p
: Files
.newDirectoryStream(servicesDir
)) {
394 Path target
= targetBundleDir
.resolve("META-INF/services/").resolve(p
.getFileName());
395 try (InputStream in
= Files
.newInputStream(p
);
396 OutputStream out
= Files
.newOutputStream(target
, StandardOpenOption
.APPEND
);) {
397 out
.write("\n".getBytes());
399 logger
.log(Level
.WARNING
, "Appended " + p
);
404 Map
<String
, String
> entries
= new TreeMap
<>();
405 try (Analyzer bndAnalyzer
= new Analyzer()) {
406 bndAnalyzer
.setProperties(mergeProps
);
407 Jar jar
= new Jar(targetBundleDir
.toFile());
408 bndAnalyzer
.setJar(jar
);
409 Manifest manifest
= bndAnalyzer
.calcManifest();
411 keys
: for (Object key
: manifest
.getMainAttributes().keySet()) {
412 Object value
= manifest
.getMainAttributes().get(key
);
414 switch (key
.toString()) {
416 case "Bnd-LastModified":
420 if ("Require-Capability".equals(key
.toString())
421 && value
.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
422 continue keys
;// hack for very old classes
423 entries
.put(key
.toString(), value
.toString());
424 // logger.log(DEBUG, () -> key + "=" + value);
427 } catch (Exception e
) {
428 throw new RuntimeException("Cannot process " + mergeBnd
, e
);
431 Manifest manifest
= new Manifest();
432 Path manifestPath
= targetBundleDir
.resolve("META-INF/MANIFEST.MF");
433 Files
.createDirectories(manifestPath
.getParent());
434 for (String key
: entries
.keySet()) {
435 String value
= entries
.get(key
);
436 manifest
.getMainAttributes().putValue(key
, value
);
439 try (OutputStream out
= Files
.newOutputStream(manifestPath
)) {
442 createJar(targetBundleDir
);
445 /** Generate MANIFEST using BND. */
446 protected Path
processBndJar(Path downloaded
, Path targetCategoryBase
, Properties fileProps
, M2Artifact artifact
) {
449 Map
<String
, String
> additionalEntries
= new TreeMap
<>();
450 boolean doNotModify
= Boolean
.parseBoolean(fileProps
451 .getOrDefault(ManifestConstants
.SLC_ORIGIN_MANIFEST_NOT_MODIFIED
.toString(), "false").toString());
453 // we always force the symbolic name
456 fileEntries
: for (Object key
: fileProps
.keySet()) {
457 if (ManifestConstants
.SLC_ORIGIN_M2
.toString().equals(key
))
458 continue fileEntries
;
459 String value
= fileProps
.getProperty(key
.toString());
460 additionalEntries
.put(key
.toString(), value
);
463 if (artifact
!= null) {
464 if (!fileProps
.containsKey(BUNDLE_SYMBOLICNAME
.toString())) {
465 fileProps
.put(BUNDLE_SYMBOLICNAME
.toString(), artifact
.getName());
467 if (!fileProps
.containsKey(BUNDLE_VERSION
.toString())) {
468 fileProps
.put(BUNDLE_VERSION
.toString(), artifact
.getVersion());
472 if (!fileProps
.containsKey(EXPORT_PACKAGE
.toString())) {
473 fileProps
.put(EXPORT_PACKAGE
.toString(),
474 "*;version=\"" + fileProps
.getProperty(BUNDLE_VERSION
.toString()) + "\"");
477 try (Analyzer bndAnalyzer
= new Analyzer()) {
478 bndAnalyzer
.setProperties(fileProps
);
479 Jar jar
= new Jar(downloaded
.toFile());
480 bndAnalyzer
.setJar(jar
);
481 Manifest manifest
= bndAnalyzer
.calcManifest();
483 keys
: for (Object key
: manifest
.getMainAttributes().keySet()) {
484 Object value
= manifest
.getMainAttributes().get(key
);
486 switch (key
.toString()) {
488 case "Bnd-LastModified":
492 if ("Require-Capability".equals(key
.toString())
493 && value
.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
494 continue keys
;// hack for very old classes
495 additionalEntries
.put(key
.toString(), value
.toString());
496 // logger.log(DEBUG, () -> key + "=" + value);
501 Path targetBundleDir
= processBundleJar(downloaded
, targetCategoryBase
, additionalEntries
);
502 logger
.log(Level
.DEBUG
, () -> "Processed " + downloaded
);
503 return targetBundleDir
;
504 } catch (Exception e
) {
505 throw new RuntimeException("Cannot BND process " + downloaded
, e
);
510 /** Download and integrates sources for a single Maven artifact. */
511 protected void downloadAndProcessM2Sources(String repoStr
, M2Artifact artifact
, Path targetBundleDir
)
516 M2Artifact sourcesArtifact
= new M2Artifact(artifact
.toM2Coordinates(), "sources");
517 URL sourcesUrl
= M2ConventionsUtils
.mavenRepoUrl(repoStr
, sourcesArtifact
);
518 Path sourcesDownloaded
= downloadMaven(sourcesUrl
, originBase
, artifact
, true);
519 processM2SourceJar(sourcesDownloaded
, targetBundleDir
);
520 logger
.log(Level
.TRACE
, () -> "Processed source " + sourcesDownloaded
);
521 } catch (Exception e
) {
522 logger
.log(Level
.ERROR
, () -> "Cannot download source for " + artifact
);
527 /** Integrate sources from a downloaded jar file. */
528 protected void processM2SourceJar(Path file
, Path targetBundleDir
) throws IOException
{
529 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
530 Path targetSourceDir
= targetBundleDir
.resolve("OSGI-OPT/src");
532 // TODO make it less dangerous?
533 if (Files
.exists(targetSourceDir
)) {
534 // deleteDirectory(targetSourceDir);
536 Files
.createDirectories(targetSourceDir
);
541 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
542 if (entry
.isDirectory())
544 if (entry
.getName().startsWith("META-INF"))// skip META-INF entries
546 if (entry
.getName().startsWith("module-info.java"))// skip META-INF entries
548 if (entry
.getName().startsWith("/")) // absolute paths
550 Path target
= targetSourceDir
.resolve(entry
.getName());
551 Files
.createDirectories(target
.getParent());
552 if (!Files
.exists(target
)) {
553 Files
.copy(jarIn
, target
);
554 logger
.log(Level
.TRACE
, () -> "Copied source " + target
);
556 logger
.log(Level
.WARNING
, () -> target
+ " already exists, skipping...");
563 /** Download a Maven artifact. */
564 protected Path
downloadMaven(URL url
, Path dir
, M2Artifact artifact
) throws IOException
{
565 return downloadMaven(url
, dir
, artifact
, false);
568 /** Download a Maven artifact. */
569 protected Path
downloadMaven(URL url
, Path dir
, M2Artifact artifact
, boolean sources
) throws IOException
{
570 return download(url
, dir
, artifact
.getGroupId() + '/' + artifact
.getArtifactId() + "-" + artifact
.getVersion()
571 + (sources ?
"-sources" : "") + ".jar");
578 /** Process an archive in Eclipse format. */
579 public void processEclipseArchive(Path duDir
) {
581 Path categoryRelativePath
= descriptorsBase
.relativize(duDir
.getParent());
582 Path targetCategoryBase
= a2Base
.resolve(categoryRelativePath
);
583 Files
.createDirectories(targetCategoryBase
);
584 // first delete all directories from previous builds
585 for (Path dir
: Files
.newDirectoryStream(targetCategoryBase
, (p
) -> Files
.isDirectory(p
))) {
586 deleteDirectory(dir
);
589 Files
.createDirectories(originBase
);
591 Path commonBnd
= duDir
.resolve(COMMON_BND
);
592 Properties commonProps
= new Properties();
593 try (InputStream in
= Files
.newInputStream(commonBnd
)) {
594 commonProps
.load(in
);
596 String url
= commonProps
.getProperty(ManifestConstants
.SLC_ORIGIN_URI
.toString());
598 url
= uris
.getProperty(duDir
.getFileName().toString());
600 throw new IllegalStateException("No url available for " + duDir
);
601 commonProps
.put(ManifestConstants
.SLC_ORIGIN_URI
.toString(), url
);
603 Path downloaded
= tryDownloadArchive(url
, originBase
);
605 FileSystem zipFs
= FileSystems
.newFileSystem(downloaded
, (ClassLoader
) null);
608 List
<PathMatcher
> includeMatchers
= new ArrayList
<>();
609 Properties includes
= new Properties();
610 try (InputStream in
= Files
.newInputStream(duDir
.resolve("includes.properties"))) {
613 for (Object pattern
: includes
.keySet()) {
614 PathMatcher pathMatcher
= zipFs
.getPathMatcher("glob:/" + pattern
);
615 includeMatchers
.add(pathMatcher
);
618 List
<PathMatcher
> excludeMatchers
= new ArrayList
<>();
619 Path excludeFile
= duDir
.resolve("excludes.properties");
620 if (Files
.exists(excludeFile
)) {
621 Properties excludes
= new Properties();
622 try (InputStream in
= Files
.newInputStream(excludeFile
)) {
625 for (Object pattern
: excludes
.keySet()) {
626 PathMatcher pathMatcher
= zipFs
.getPathMatcher("glob:/" + pattern
);
627 excludeMatchers
.add(pathMatcher
);
631 Files
.walkFileTree(zipFs
.getRootDirectories().iterator().next(), new SimpleFileVisitor
<Path
>() {
634 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
635 includeMatchers
: for (PathMatcher includeMatcher
: includeMatchers
) {
636 if (includeMatcher
.matches(file
)) {
637 for (PathMatcher excludeMatcher
: excludeMatchers
) {
638 if (excludeMatcher
.matches(file
)) {
639 logger
.log(Level
.TRACE
, "Skipping excluded " + file
);
640 return FileVisitResult
.CONTINUE
;
643 if (file
.getFileName().toString().contains(".source_")) {
644 if (!sourceBundles
) {
645 processEclipseSourceJar(file
, targetCategoryBase
);
646 logger
.log(Level
.DEBUG
, () -> "Processed source " + file
);
650 Map
<String
, String
> map
= new HashMap
<>();
651 for (Object key
: commonProps
.keySet())
652 map
.put(key
.toString(), commonProps
.getProperty(key
.toString()));
653 processBundleJar(file
, targetCategoryBase
, map
);
654 logger
.log(Level
.DEBUG
, () -> "Processed " + file
);
656 break includeMatchers
;
659 return FileVisitResult
.CONTINUE
;
663 DirectoryStream
<Path
> dirs
= Files
.newDirectoryStream(targetCategoryBase
,
664 (p
) -> Files
.isDirectory(p
) && p
.getFileName().toString().indexOf('.') >= 0);
665 for (Path dir
: dirs
) {
668 } catch (IOException e
) {
669 throw new RuntimeException("Cannot process " + duDir
, e
);
674 /** Process sources in Eclipse format. */
675 protected void processEclipseSourceJar(Path file
, Path targetBase
) throws IOException
{
677 Path targetBundleDir
;
678 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
679 Manifest manifest
= jarIn
.getManifest();
681 String
[] relatedBundle
= manifest
.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
682 String version
= relatedBundle
[1].substring("version=\"".length());
683 version
= version
.substring(0, version
.length() - 1);
684 NameVersion nameVersion
= new NameVersion(relatedBundle
[0], version
);
685 targetBundleDir
= targetBase
.resolve(nameVersion
.getName() + "." + nameVersion
.getBranch());
687 Path targetSourceDir
= targetBundleDir
.resolve("OSGI-OPT/src");
689 // TODO make it less dangerous?
690 if (Files
.exists(targetSourceDir
)) {
691 // deleteDirectory(targetSourceDir);
693 Files
.createDirectories(targetSourceDir
);
698 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
699 if (entry
.isDirectory())
701 if (entry
.getName().startsWith("META-INF"))// skip META-INF entries
703 Path target
= targetSourceDir
.resolve(entry
.getName());
704 Files
.createDirectories(target
.getParent());
705 Files
.copy(jarIn
, target
);
706 logger
.log(Level
.TRACE
, () -> "Copied source " + target
);
711 } catch (IOException e
) {
712 throw new IllegalStateException("Cannot process " + file
, e
);
720 /** Normalise a bundle. */
721 protected Path
processBundleJar(Path file
, Path targetBase
, Map
<String
, String
> entries
) throws IOException
{
722 NameVersion nameVersion
;
723 Path targetBundleDir
;
724 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
725 Manifest sourceManifest
= jarIn
.getManifest();
726 Manifest manifest
= sourceManifest
!= null ?
new Manifest(sourceManifest
) : new Manifest();
729 boolean isSingleton
= false;
730 String rawSourceSymbolicName
= manifest
.getMainAttributes()
731 .getValue(ManifestConstants
.BUNDLE_SYMBOLICNAME
.toString());
732 if (rawSourceSymbolicName
!= null) {
734 // make sure there is no directive
735 String
[] arr
= rawSourceSymbolicName
.split(";");
736 for (int i
= 1; i
< arr
.length
; i
++) {
737 if (arr
[i
].trim().equals("singleton:=true"))
739 logger
.log(DEBUG
, file
.getFileName() + " is a singleton");
743 // remove problematic entries in MANIFEST
744 manifest
.getEntries().clear();
746 String ourSymbolicName
= entries
.get(BUNDLE_SYMBOLICNAME
.toString());
747 String ourVersion
= entries
.get(BUNDLE_VERSION
.toString());
749 if (ourSymbolicName
!= null && ourVersion
!= null) {
750 nameVersion
= new NameVersion(ourSymbolicName
, ourVersion
);
752 nameVersion
= nameVersionFromManifest(manifest
);
753 if (ourVersion
!= null && !nameVersion
.getVersion().equals(ourVersion
)) {
754 logger
.log(Level
.WARNING
,
755 "Original version is " + nameVersion
.getVersion() + " while new version is " + ourVersion
);
756 entries
.put(BUNDLE_VERSION
.toString(), ourVersion
);
758 if (ourSymbolicName
!= null) {
759 // we always force our symbolic name
760 nameVersion
.setName(ourSymbolicName
);
763 targetBundleDir
= targetBase
.resolve(nameVersion
.getName() + "." + nameVersion
.getBranch());
765 // force Java 9 module name
766 entries
.put(ManifestConstants
.AUTOMATIC_MODULE_NAME
.toString(), nameVersion
.getName());
768 boolean isNative
= false;
771 if (targetBundleDir
.startsWith(a2LibBase
)) {
773 Path libRelativePath
= a2LibBase
.relativize(targetBundleDir
);
774 os
= libRelativePath
.getName(0).toString();
775 arch
= libRelativePath
.getName(1).toString();
780 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
781 if (entry
.isDirectory())
783 if (entry
.getName().endsWith(".RSA") || entry
.getName().endsWith(".SF"))
785 if (entry
.getName().endsWith("module-info.class")) // skip Java 9 module info
787 if (entry
.getName().startsWith("META-INF/versions/")) // skip multi-version
789 // skip file system providers as they cause issues with native image
790 if (entry
.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider"))
792 if (entry
.getName().startsWith("OSGI-OPT/src/")) // skip embedded sources
794 Path target
= targetBundleDir
.resolve(entry
.getName());
795 Files
.createDirectories(target
.getParent());
796 Files
.copy(jarIn
, target
);
799 if (isNative
&& (entry
.getName().endsWith(".so") || entry
.getName().endsWith(".dll")
800 || entry
.getName().endsWith(".jnilib"))) {
801 Path categoryDir
= targetBundleDir
.getParent();
802 // String[] segments = categoryDir.getFileName().toString().split("\\.");
803 // String arch = segments[segments.length - 1];
804 // String os = segments[segments.length - 2];
805 boolean copyDll
= false;
806 Path targetDll
= categoryDir
.resolve(targetBundleDir
.relativize(target
));
807 if (nameVersion
.getName().equals("com.sun.jna")) {
808 if (arch
.equals("x86_64"))
810 if (os
.equals("macosx"))
812 if (target
.getParent().getFileName().toString().equals(os
+ "-" + arch
)) {
815 targetDll
= categoryDir
.resolve(target
.getFileName());
820 Files
.createDirectories(targetDll
.getParent());
821 if (Files
.exists(targetDll
))
822 Files
.delete(targetDll
);
823 Files
.copy(target
, targetDll
);
825 Files
.delete(target
);
827 logger
.log(Level
.TRACE
, () -> "Copied " + target
);
831 Path manifestPath
= targetBundleDir
.resolve("META-INF/MANIFEST.MF");
832 Files
.createDirectories(manifestPath
.getParent());
834 if (isSingleton
&& entries
.containsKey(BUNDLE_SYMBOLICNAME
.toString())) {
835 entries
.put(BUNDLE_SYMBOLICNAME
.toString(),
836 entries
.get(BUNDLE_SYMBOLICNAME
.toString()) + ";singleton:=true");
839 for (String key
: entries
.keySet()) {
840 String value
= entries
.get(key
);
841 Object previousValue
= manifest
.getMainAttributes().putValue(key
, value
);
842 if (previousValue
!= null && !previousValue
.equals(value
)) {
843 if (ManifestConstants
.IMPORT_PACKAGE
.toString().equals(key
)
844 || ManifestConstants
.EXPORT_PACKAGE
.toString().equals(key
)
845 || ManifestConstants
.BUNDLE_LICENSE
.toString().equals(key
))
846 logger
.log(Level
.TRACE
, file
.getFileName() + ": " + key
+ " was modified");
848 logger
.log(Level
.WARNING
, file
.getFileName() + ": " + key
+ " was " + previousValue
849 + ", overridden with " + value
);
852 // hack to remove unresolvable
853 if (key
.equals("Provide-Capability") || key
.equals("Require-Capability"))
854 if (nameVersion
.getName().equals("osgi.core") || nameVersion
.getName().equals("osgi.cmpn")) {
855 manifest
.getMainAttributes().remove(key
);
858 try (OutputStream out
= Files
.newOutputStream(manifestPath
)) {
862 return targetBundleDir
;
869 /** Recursively deletes a directory. */
870 private static void deleteDirectory(Path path
) throws IOException
{
871 if (!Files
.exists(path
))
873 Files
.walkFileTree(path
, new SimpleFileVisitor
<Path
>() {
875 public FileVisitResult
postVisitDirectory(Path directory
, IOException e
) throws IOException
{
878 Files
.delete(directory
);
879 return FileVisitResult
.CONTINUE
;
883 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
885 return FileVisitResult
.CONTINUE
;
890 /** Extract name/version from a MANIFEST. */
891 protected NameVersion
nameVersionFromManifest(Manifest manifest
) {
892 Attributes attrs
= manifest
.getMainAttributes();
894 String symbolicName
= attrs
.getValue(ManifestConstants
.BUNDLE_SYMBOLICNAME
.toString());
895 if (symbolicName
== null)
897 // make sure there is no directive
898 symbolicName
= symbolicName
.split(";")[0];
900 String version
= attrs
.getValue(ManifestConstants
.BUNDLE_VERSION
.toString());
901 return new NameVersion(symbolicName
, version
);
904 /** Try to download from an URI. */
905 protected Path
tryDownloadArchive(String uri
, Path dir
) throws IOException
{
907 List
<String
> urlBases
= null;
908 String uriPrefix
= null;
909 uriPrefixes
: for (String uriPref
: mirrors
.keySet()) {
910 if (uri
.startsWith(uriPref
)) {
911 if (mirrors
.get(uriPref
).size() > 0) {
912 urlBases
= mirrors
.get(uriPref
);
918 if (urlBases
== null)
920 return downloadArchive(new URL(uri
), dir
);
921 } catch (FileNotFoundException e
) {
922 throw new FileNotFoundException("Cannot find " + uri
);
926 for (String urlBase
: urlBases
) {
927 String relativePath
= uri
.substring(uriPrefix
.length());
928 URL url
= new URL(urlBase
+ relativePath
);
930 return downloadArchive(url
, dir
);
931 } catch (FileNotFoundException e
) {
932 logger
.log(Level
.WARNING
, "Cannot download " + url
+ ", trying another mirror");
935 throw new FileNotFoundException("Cannot find " + uri
);
939 * Effectively download. Synchronised in order to avoid downloading twice in
942 protected synchronized Path
downloadArchive(URL url
, Path dir
) throws IOException
{
943 return download(url
, dir
, (String
) null);
946 /** Effectively download. */
947 protected Path
download(URL url
, Path dir
, String name
) throws IOException
{
951 // We use also use parent directory in case the archive itself has a fixed name
952 String
[] segments
= url
.getPath().split("/");
953 name
= segments
.length
> 1 ? segments
[segments
.length
- 2] + '-' + segments
[segments
.length
- 1]
954 : segments
[segments
.length
- 1];
957 dest
= dir
.resolve(name
);
958 if (Files
.exists(dest
)) {
959 logger
.log(Level
.TRACE
, () -> "File " + dest
+ " already exists for " + url
+ ", not downloading again");
962 Files
.createDirectories(dest
.getParent());
965 try (InputStream in
= url
.openStream()) {
966 Files
.copy(in
, dest
);
967 logger
.log(Level
.DEBUG
, () -> "Downloaded " + dest
+ " from " + url
);
972 /** Create a JAR file from a directory. */
973 protected Path
createJar(Path bundleDir
) throws IOException
{
975 Path jarPath
= bundleDir
.getParent().resolve(bundleDir
.getFileName() + ".jar");
976 Path manifestPath
= bundleDir
.resolve("META-INF/MANIFEST.MF");
978 try (InputStream in
= Files
.newInputStream(manifestPath
)) {
979 manifest
= new Manifest(in
);
981 try (JarOutputStream jarOut
= new JarOutputStream(Files
.newOutputStream(jarPath
), manifest
)) {
982 jarOut
.setLevel(Deflater
.DEFAULT_COMPRESSION
);
983 Files
.walkFileTree(bundleDir
, new SimpleFileVisitor
<Path
>() {
986 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
987 if (file
.getFileName().toString().equals("MANIFEST.MF"))
988 return super.visitFile(file
, attrs
);
989 JarEntry entry
= new JarEntry(
990 bundleDir
.relativize(file
).toString().replace(File
.separatorChar
, '/'));
991 jarOut
.putNextEntry(entry
);
992 Files
.copy(file
, jarOut
);
993 return super.visitFile(file
, attrs
);
998 deleteDirectory(bundleDir
);
1002 enum ManifestConstants
{
1004 BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), //
1005 BUNDLE_VERSION("Bundle-Version"), //
1006 BUNDLE_LICENSE("Bundle-License"), //
1007 EXPORT_PACKAGE("Export-Package"), //
1008 IMPORT_PACKAGE("Import-Package"), //
1010 AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), //
1012 SLC_CATEGORY("SLC-Category"), //
1013 SLC_ORIGIN_M2("SLC-Origin-M2"), //
1014 SLC_ORIGIN_M2_MERGE("SLC-Origin-M2-Merge"), //
1015 SLC_ORIGIN_M2_REPO("SLC-Origin-M2-Repo"), //
1016 SLC_ORIGIN_MANIFEST_NOT_MODIFIED("SLC-Origin-ManifestNotModified"), //
1017 SLC_ORIGIN_URI("SLC-Origin-URI"),//
1022 private ManifestConstants(String value
) {
1027 public String
toString() {
1035 /** Simple representation of an M2 artifact. */
1036 class M2Artifact
extends CategoryNameVersion
{
1037 private String classifier
;
1039 M2Artifact(String m2coordinates
) {
1040 this(m2coordinates
, null);
1043 M2Artifact(String m2coordinates
, String classifier
) {
1044 String
[] parts
= m2coordinates
.split(":");
1045 setCategory(parts
[0]);
1047 if (parts
.length
> 2) {
1048 setVersion(parts
[2]);
1050 this.classifier
= classifier
;
1053 String
getGroupId() {
1054 return super.getCategory();
1057 String
getArtifactId() {
1058 return super.getName();
1061 String
toM2Coordinates() {
1062 return getCategory() + ":" + getName() + (getVersion() != null ?
":" + getVersion() : "");
1065 String
getClassifier() {
1066 return classifier
!= null ? classifier
: "";
1069 String
getExtension() {
1074 /** Utilities around Maven (conventions based). */
1075 class M2ConventionsUtils
{
1076 final static String MAVEN_CENTRAL_BASE_URL
= "https://repo1.maven.org/maven2/";
1078 /** The file name of this artifact when stored */
1079 static String
artifactFileName(M2Artifact artifact
) {
1080 return artifact
.getArtifactId() + '-' + artifact
.getVersion()
1081 + (artifact
.getClassifier().equals("") ?
"" : '-' + artifact
.getClassifier()) + '.'
1082 + artifact
.getExtension();
1085 /** Absolute path to the file */
1086 static String
artifactPath(String artifactBasePath
, M2Artifact artifact
) {
1087 return artifactParentPath(artifactBasePath
, artifact
) + '/' + artifactFileName(artifact
);
1090 /** Absolute path to the file */
1091 static String
artifactUrl(String repoUrl
, M2Artifact artifact
) {
1092 if (repoUrl
.endsWith("/"))
1093 return repoUrl
+ artifactPath("/", artifact
).substring(1);
1095 return repoUrl
+ artifactPath("/", artifact
);
1098 /** Absolute path to the file */
1099 static URL
mavenRepoUrl(String repoBase
, M2Artifact artifact
) {
1100 String url
= artifactUrl(repoBase
== null ? MAVEN_CENTRAL_BASE_URL
: repoBase
, artifact
);
1102 return new URL(url
);
1103 } catch (MalformedURLException e
) {
1104 // it should not happen
1105 throw new IllegalStateException(e
);
1109 /** Absolute path to the directories where the files will be stored */
1110 static String
artifactParentPath(String artifactBasePath
, M2Artifact artifact
) {
1111 return artifactBasePath
+ (artifactBasePath
.endsWith("/") ?
"" : "/") + artifactParentPath(artifact
);
1114 /** Relative path to the directories where the files will be stored */
1115 static String
artifactParentPath(M2Artifact artifact
) {
1116 return artifact
.getGroupId().replace('.', '/') + '/' + artifact
.getArtifactId() + '/' + artifact
.getVersion();
1120 private M2ConventionsUtils() {
1124 class CategoryNameVersion
extends NameVersion
{
1125 private String category
;
1127 CategoryNameVersion() {
1130 CategoryNameVersion(String category
, String name
, String version
) {
1131 super(name
, version
);
1132 this.category
= category
;
1135 CategoryNameVersion(String category
, NameVersion nameVersion
) {
1137 this.category
= category
;
1140 String
getCategory() {
1144 void setCategory(String category
) {
1145 this.category
= category
;
1149 public String
toString() {
1150 return category
+ ":" + super.toString();
1155 class NameVersion
implements Comparable
<NameVersion
> {
1156 private String name
;
1157 private String version
;
1162 /** Interprets string in OSGi-like format my.module.name;version=0.0.0 */
1163 NameVersion(String nameVersion
) {
1164 int index
= nameVersion
.indexOf(";version=");
1166 setName(nameVersion
);
1169 setName(nameVersion
.substring(0, index
));
1170 setVersion(nameVersion
.substring(index
+ ";version=".length()));
1174 NameVersion(String name
, String version
) {
1176 this.version
= version
;
1179 NameVersion(NameVersion nameVersion
) {
1180 this.name
= nameVersion
.getName();
1181 this.version
= nameVersion
.getVersion();
1188 void setName(String name
) {
1192 String
getVersion() {
1196 void setVersion(String version
) {
1197 this.version
= version
;
1200 String
getBranch() {
1201 String
[] parts
= getVersion().split("\\.");
1202 if (parts
.length
< 2)
1203 throw new IllegalStateException("Version " + getVersion() + " cannot be interpreted as branch.");
1204 return parts
[0] + "." + parts
[1];
1208 public boolean equals(Object obj
) {
1209 if (obj
instanceof NameVersion
) {
1210 NameVersion nameVersion
= (NameVersion
) obj
;
1211 return name
.equals(nameVersion
.getName()) && version
.equals(nameVersion
.getVersion());
1217 public int hashCode() {
1218 return name
.hashCode();
1222 public String
toString() {
1223 return name
+ ":" + version
;
1226 public int compareTo(NameVersion o
) {
1227 if (o
.getName().equals(name
))
1228 return version
.compareTo(o
.getVersion());
1230 return name
.compareTo(o
.getName());