1 package org
.argeo
.slc
.factory
;
3 import static java
.lang
.System
.Logger
.Level
.DEBUG
;
4 import static org
.argeo
.slc
.ManifestConstants
.BUNDLE_SYMBOLICNAME
;
5 import static org
.argeo
.slc
.ManifestConstants
.BUNDLE_VERSION
;
6 import static org
.argeo
.slc
.ManifestConstants
.EXPORT_PACKAGE
;
7 import static org
.argeo
.slc
.ManifestConstants
.SLC_ORIGIN_M2
;
8 import static org
.argeo
.slc
.ManifestConstants
.SLC_ORIGIN_M2_REPO
;
10 import java
.io
.FileNotFoundException
;
11 import java
.io
.IOException
;
12 import java
.io
.InputStream
;
13 import java
.io
.OutputStream
;
14 import java
.lang
.System
.Logger
;
15 import java
.lang
.System
.Logger
.Level
;
17 import java
.nio
.file
.DirectoryStream
;
18 import java
.nio
.file
.FileSystem
;
19 import java
.nio
.file
.FileSystems
;
20 import java
.nio
.file
.FileVisitResult
;
21 import java
.nio
.file
.Files
;
22 import java
.nio
.file
.Path
;
23 import java
.nio
.file
.PathMatcher
;
24 import java
.nio
.file
.Paths
;
25 import java
.nio
.file
.SimpleFileVisitor
;
26 import java
.nio
.file
.StandardOpenOption
;
27 import java
.nio
.file
.attribute
.BasicFileAttributes
;
28 import java
.util
.ArrayList
;
29 import java
.util
.HashMap
;
30 import java
.util
.List
;
32 import java
.util
.Objects
;
33 import java
.util
.Properties
;
34 import java
.util
.TreeMap
;
35 import java
.util
.jar
.Attributes
;
36 import java
.util
.jar
.JarEntry
;
37 import java
.util
.jar
.JarInputStream
;
38 import java
.util
.jar
.JarOutputStream
;
39 import java
.util
.jar
.Manifest
;
40 import java
.util
.zip
.Deflater
;
42 import org
.argeo
.slc
.DefaultCategoryNameVersion
;
43 import org
.argeo
.slc
.DefaultNameVersion
;
44 import org
.argeo
.slc
.ManifestConstants
;
45 import org
.argeo
.slc
.NameVersion
;
46 import org
.argeo
.slc
.factory
.m2
.Artifact
;
47 import org
.argeo
.slc
.factory
.m2
.DefaultArtifact
;
48 import org
.argeo
.slc
.factory
.m2
.MavenConventionsUtils
;
50 import aQute
.bnd
.osgi
.Analyzer
;
51 import aQute
.bnd
.osgi
.Jar
;
53 /** The central class for A2 packaging. */
54 public class A2Factory
{
55 private final static Logger logger
= System
.getLogger(A2Factory
.class.getName());
57 private final static String COMMON_BND
= "common.bnd";
58 private final static String MERGE_BND
= "merge.bnd";
60 private Path originBase
;
62 private Path a2LibBase
;
63 private Path descriptorsBase
;
65 private Properties uris
= new Properties();
67 private boolean includeSources
= true;
69 /** key is URI prefix, value list of base URLs */
70 private Map
<String
, List
<String
>> mirrors
= new HashMap
<String
, List
<String
>>();
72 public A2Factory(Path a2Base
, Path descriptorsBase
, boolean includeSources
) {
73 Objects
.requireNonNull(a2Base
);
74 Objects
.requireNonNull(descriptorsBase
);
75 this.originBase
= Paths
.get(System
.getProperty("user.home"), ".cache", "argeo/slc/origin");
77 this.a2LibBase
= a2Base
.resolve("lib");
78 this.descriptorsBase
= descriptorsBase
;
79 if (!Files
.exists(this.descriptorsBase
))
80 throw new IllegalArgumentException(this.descriptorsBase
+ " does not exist");
81 this.includeSources
= includeSources
;
83 Path urisPath
= this.descriptorsBase
.resolve("uris.properties");
84 if (Files
.exists(urisPath
)) {
85 try (InputStream in
= Files
.newInputStream(urisPath
)) {
87 } catch (IOException e
) {
88 throw new IllegalStateException("Cannot load " + urisPath
, e
);
92 // TODO make it configurable
93 List
<String
> eclipseMirrors
= new ArrayList
<>();
94 eclipseMirrors
.add("https://archive.eclipse.org/");
95 eclipseMirrors
.add("http://ftp-stud.hs-esslingen.de/Mirrors/eclipse/");
96 eclipseMirrors
.add("http://ftp.fau.de/eclipse/");
98 mirrors
.put("http://www.eclipse.org/downloads", eclipseMirrors
);
105 /** Process a whole category/group id. */
106 public void processCategory(Path categoryRelativePath
) {
108 Path targetCategoryBase
= descriptorsBase
.resolve(categoryRelativePath
);
109 DirectoryStream
<Path
> bnds
= Files
.newDirectoryStream(targetCategoryBase
,
110 (p
) -> p
.getFileName().toString().endsWith(".bnd") && !p
.getFileName().toString().equals(COMMON_BND
)
111 && !p
.getFileName().toString().equals(MERGE_BND
));
112 for (Path p
: bnds
) {
113 processSingleM2ArtifactDistributionUnit(p
);
116 DirectoryStream
<Path
> dus
= Files
.newDirectoryStream(targetCategoryBase
, (p
) -> Files
.isDirectory(p
));
117 for (Path duDir
: dus
) {
118 processM2BasedDistributionUnit(duDir
);
120 } catch (IOException e
) {
121 throw new RuntimeException("Cannot process category " + categoryRelativePath
, e
);
125 /** Process a standalone Maven artifact. */
126 public void processSingleM2ArtifactDistributionUnit(Path bndFile
) {
128 // String category = bndFile.getParent().getFileName().toString();
129 Path categoryRelativePath
= descriptorsBase
.relativize(bndFile
.getParent());
130 Path targetCategoryBase
= a2Base
.resolve(categoryRelativePath
);
132 Properties fileProps
= new Properties();
133 try (InputStream in
= Files
.newInputStream(bndFile
)) {
136 String repoStr
= fileProps
.containsKey(SLC_ORIGIN_M2_REPO
.toString())
137 ? fileProps
.getProperty(SLC_ORIGIN_M2_REPO
.toString())
140 if (!fileProps
.containsKey(BUNDLE_SYMBOLICNAME
.toString())) {
141 // use file name as symbolic name
142 String symbolicName
= bndFile
.getFileName().toString();
143 symbolicName
= symbolicName
.substring(0, symbolicName
.length() - ".bnd".length());
144 fileProps
.put(BUNDLE_SYMBOLICNAME
.toString(), symbolicName
);
147 String m2Coordinates
= fileProps
.getProperty(SLC_ORIGIN_M2
.toString());
148 if (m2Coordinates
== null)
149 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile
);
150 DefaultArtifact artifact
= new DefaultArtifact(m2Coordinates
);
151 URL url
= MavenConventionsUtils
.mavenRepoUrl(repoStr
, artifact
);
152 Path downloaded
= download(url
, originBase
, artifact
);
154 Path targetBundleDir
= processBndJar(downloaded
, targetCategoryBase
, fileProps
, artifact
);
156 downloadAndProcessM2Sources(repoStr
, artifact
, targetBundleDir
);
158 createJar(targetBundleDir
);
159 } catch (Exception e
) {
160 throw new RuntimeException("Cannot process " + bndFile
, e
);
164 /** Process multiple Maven artifacts. */
165 public void processM2BasedDistributionUnit(Path duDir
) {
167 // String category = duDir.getParent().getFileName().toString();
168 Path categoryRelativePath
= descriptorsBase
.relativize(duDir
.getParent());
169 Path targetCategoryBase
= a2Base
.resolve(categoryRelativePath
);
172 Path mergeBnd
= duDir
.resolve(MERGE_BND
);
173 if (Files
.exists(mergeBnd
)) {
174 mergeM2Artifacts(mergeBnd
);
178 Path commonBnd
= duDir
.resolve(COMMON_BND
);
179 if (!Files
.exists(commonBnd
)) {
182 Properties commonProps
= new Properties();
183 try (InputStream in
= Files
.newInputStream(commonBnd
)) {
184 commonProps
.load(in
);
187 String m2Version
= commonProps
.getProperty(SLC_ORIGIN_M2
.toString());
188 if (m2Version
== null) {
189 logger
.log(Level
.WARNING
, "Ignoring " + duDir
+ " as it is not an M2-based distribution unit");
190 return;// ignore, this is probably an Eclipse archive
192 if (!m2Version
.startsWith(":")) {
193 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version
);
195 m2Version
= m2Version
.substring(1);
197 DirectoryStream
<Path
> ds
= Files
.newDirectoryStream(duDir
,
198 (p
) -> p
.getFileName().toString().endsWith(".bnd") && !p
.getFileName().toString().equals(COMMON_BND
)
199 && !p
.getFileName().toString().equals(MERGE_BND
));
201 Properties fileProps
= new Properties();
202 try (InputStream in
= Files
.newInputStream(p
)) {
205 String m2Coordinates
= fileProps
.getProperty(SLC_ORIGIN_M2
.toString());
206 DefaultArtifact artifact
= new DefaultArtifact(m2Coordinates
);
208 // temporary rewrite, for migration
209 // String localLicense = fileProps.getProperty(BUNDLE_LICENSE.toString());
210 // if (localLicense != null || artifact.getVersion() != null) {
211 // fileProps.remove(BUNDLE_LICENSE.toString());
212 // fileProps.put(SLC_ORIGIN_M2.toString(), artifact.getGroupId() + ":" + artifact.getArtifactId());
213 // try (Writer writer = Files.newBufferedWriter(p)) {
214 // for (Object key : fileProps.keySet()) {
215 // String value = fileProps.getProperty(key.toString());
216 // writer.write(key + ": " + value + '\n');
218 // logger.log(DEBUG, () -> "Migrated " + p);
222 artifact
.setVersion(m2Version
);
224 // prepare manifest entries
225 Properties mergeProps
= new Properties();
226 mergeProps
.putAll(commonProps
);
228 fileEntries
: for (Object key
: fileProps
.keySet()) {
229 if (ManifestConstants
.SLC_ORIGIN_M2
.toString().equals(key
))
230 continue fileEntries
;
231 String value
= fileProps
.getProperty(key
.toString());
232 Object previousValue
= mergeProps
.put(key
.toString(), value
);
233 if (previousValue
!= null) {
234 logger
.log(Level
.WARNING
,
235 commonBnd
+ ": " + key
+ " was " + previousValue
+ ", overridden with " + value
);
238 mergeProps
.put(ManifestConstants
.SLC_ORIGIN_M2
.toString(), artifact
.toM2Coordinates());
239 if (!mergeProps
.containsKey(BUNDLE_SYMBOLICNAME
.toString())) {
240 // use file name as symbolic name
241 String symbolicName
= p
.getFileName().toString();
242 symbolicName
= symbolicName
.substring(0, symbolicName
.length() - ".bnd".length());
243 mergeProps
.put(BUNDLE_SYMBOLICNAME
.toString(), symbolicName
);
246 String repoStr
= mergeProps
.containsKey(SLC_ORIGIN_M2_REPO
.toString())
247 ? mergeProps
.getProperty(SLC_ORIGIN_M2_REPO
.toString())
251 URL url
= MavenConventionsUtils
.mavenRepoUrl(repoStr
, artifact
);
252 Path downloaded
= download(url
, originBase
, artifact
);
254 Path targetBundleDir
= processBndJar(downloaded
, targetCategoryBase
, mergeProps
, artifact
);
255 // logger.log(Level.DEBUG, () -> "Processed " + downloaded);
258 downloadAndProcessM2Sources(repoStr
, artifact
, targetBundleDir
);
260 createJar(targetBundleDir
);
262 } catch (IOException e
) {
263 throw new RuntimeException("Cannot process " + duDir
, e
);
268 /** Merge multiple Maven artifacts. */
269 protected void mergeM2Artifacts(Path mergeBnd
) throws IOException
{
270 Path duDir
= mergeBnd
.getParent();
271 String category
= duDir
.getParent().getFileName().toString();
272 Path targetCategoryBase
= a2Base
.resolve(category
);
274 Properties mergeProps
= new Properties();
275 try (InputStream in
= Files
.newInputStream(mergeBnd
)) {
280 String m2Version
= mergeProps
.getProperty(SLC_ORIGIN_M2
.toString());
281 if (m2Version
== null) {
282 logger
.log(Level
.WARNING
, "Ignoring " + duDir
+ " as it is not an M2-based distribution unit");
283 return;// ignore, this is probably an Eclipse archive
285 if (!m2Version
.startsWith(":")) {
286 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version
);
288 m2Version
= m2Version
.substring(1);
289 mergeProps
.put(ManifestConstants
.BUNDLE_VERSION
.toString(), m2Version
);
291 String artifactsStr
= mergeProps
.getProperty(ManifestConstants
.SLC_ORIGIN_M2_MERGE
.toString());
292 String repoStr
= mergeProps
.containsKey(SLC_ORIGIN_M2_REPO
.toString())
293 ? mergeProps
.getProperty(SLC_ORIGIN_M2_REPO
.toString())
296 String bundleSymbolicName
= mergeProps
.getProperty(ManifestConstants
.BUNDLE_SYMBOLICNAME
.toString());
297 if (bundleSymbolicName
== null)
298 throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd
);
299 DefaultCategoryNameVersion nameVersion
= new DefaultArtifact(
300 category
+ ":" + bundleSymbolicName
+ ":" + m2Version
);
301 Path targetBundleDir
= targetCategoryBase
.resolve(bundleSymbolicName
+ "." + nameVersion
.getBranch());
303 String
[] artifacts
= artifactsStr
.split(",");
304 artifacts
: for (String str
: artifacts
) {
305 String m2Coordinates
= str
.trim();
306 if ("".equals(m2Coordinates
))
308 DefaultArtifact artifact
= new DefaultArtifact(m2Coordinates
.trim());
309 if (artifact
.getVersion() == null)
310 artifact
.setVersion(m2Version
);
311 URL url
= MavenConventionsUtils
.mavenRepoUrl(repoStr
, artifact
);
312 Path downloaded
= download(url
, originBase
, artifact
);
314 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(downloaded
), false)) {
315 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
316 if (entry
.isDirectory())
318 else if (entry
.getName().endsWith(".RSA") || entry
.getName().endsWith(".SF"))
320 else if (entry
.getName().startsWith("META-INF/versions/"))
322 else if (entry
.getName().startsWith("META-INF/maven/"))
324 else if (entry
.getName().equals("module-info.class"))
326 else if (entry
.getName().equals("META-INF/NOTICE"))
328 else if (entry
.getName().equals("META-INF/NOTICE.txt"))
330 else if (entry
.getName().equals("META-INF/LICENSE"))
332 else if (entry
.getName().equals("META-INF/LICENSE.md"))
334 else if (entry
.getName().equals("META-INF/LICENSE-notice.md"))
336 else if (entry
.getName().equals("META-INF/DEPENDENCIES"))
338 if (entry
.getName().startsWith(".cache/")) // Apache SSHD
340 Path target
= targetBundleDir
.resolve(entry
.getName());
341 Files
.createDirectories(target
.getParent());
342 if (!Files
.exists(target
)) {
343 Files
.copy(jarIn
, target
);
345 if (entry
.getName().startsWith("META-INF/services/")) {
346 try (OutputStream out
= Files
.newOutputStream(target
, StandardOpenOption
.APPEND
)) {
347 out
.write("\n".getBytes());
348 jarIn
.transferTo(out
);
349 if (logger
.isLoggable(DEBUG
))
350 logger
.log(DEBUG
, artifact
.getArtifactId() + " - Appended " + entry
.getName());
352 } else if (entry
.getName().startsWith("org/apache/batik/")) {
353 logger
.log(Level
.WARNING
, "Skip " + entry
.getName());
356 throw new IllegalStateException("File " + target
+ " from " + artifact
+ " already exists");
359 logger
.log(Level
.TRACE
, () -> "Copied " + target
);
363 downloadAndProcessM2Sources(repoStr
, artifact
, targetBundleDir
);
366 // additional service files
367 Path servicesDir
= duDir
.resolve("services");
368 if (Files
.exists(servicesDir
)) {
369 for (Path p
: Files
.newDirectoryStream(servicesDir
)) {
370 Path target
= targetBundleDir
.resolve("META-INF/services/").resolve(p
.getFileName());
371 try (InputStream in
= Files
.newInputStream(p
);
372 OutputStream out
= Files
.newOutputStream(target
, StandardOpenOption
.APPEND
);) {
373 out
.write("\n".getBytes());
375 if (logger
.isLoggable(DEBUG
))
376 logger
.log(DEBUG
, "Appended " + p
);
381 Map
<String
, String
> entries
= new TreeMap
<>();
382 try (Analyzer bndAnalyzer
= new Analyzer()) {
383 bndAnalyzer
.setProperties(mergeProps
);
384 Jar jar
= new Jar(targetBundleDir
.toFile());
385 bndAnalyzer
.setJar(jar
);
386 Manifest manifest
= bndAnalyzer
.calcManifest();
388 keys
: for (Object key
: manifest
.getMainAttributes().keySet()) {
389 Object value
= manifest
.getMainAttributes().get(key
);
391 switch (key
.toString()) {
393 case "Bnd-LastModified":
397 if ("Require-Capability".equals(key
.toString())
398 && value
.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
399 continue keys
;// hack for very old classes
400 entries
.put(key
.toString(), value
.toString());
401 // logger.log(DEBUG, () -> key + "=" + value);
404 } catch (Exception e
) {
405 throw new RuntimeException("Cannot process " + mergeBnd
, e
);
408 Manifest manifest
= new Manifest();
409 Path manifestPath
= targetBundleDir
.resolve("META-INF/MANIFEST.MF");
410 Files
.createDirectories(manifestPath
.getParent());
411 for (String key
: entries
.keySet()) {
412 String value
= entries
.get(key
);
413 manifest
.getMainAttributes().putValue(key
, value
);
416 // // Use Maven version as Bundle-Version
417 // String bundleVersion = manifest.getMainAttributes().getValue(ManifestConstants.BUNDLE_VERSION.toString());
418 // if (bundleVersion == null || bundleVersion.trim().equals("0")) {
419 // // TODO check why it is sometimes set to "0"
420 // manifest.getMainAttributes().putValue(ManifestConstants.BUNDLE_VERSION.toString(), m2Version);
422 try (OutputStream out
= Files
.newOutputStream(manifestPath
)) {
426 createJar(targetBundleDir
);
430 /** Generate MANIFEST using BND. */
431 protected Path
processBndJar(Path downloaded
, Path targetCategoryBase
, Properties fileProps
,
432 DefaultArtifact artifact
) {
435 Map
<String
, String
> additionalEntries
= new TreeMap
<>();
436 boolean doNotModify
= Boolean
.parseBoolean(fileProps
437 .getOrDefault(ManifestConstants
.SLC_ORIGIN_MANIFEST_NOT_MODIFIED
.toString(), "false").toString());
439 // we always force the symbolic name
442 fileEntries
: for (Object key
: fileProps
.keySet()) {
443 if (ManifestConstants
.SLC_ORIGIN_M2
.toString().equals(key
))
444 continue fileEntries
;
445 String value
= fileProps
.getProperty(key
.toString());
446 additionalEntries
.put(key
.toString(), value
);
449 if (artifact
!= null) {
450 if (!fileProps
.containsKey(BUNDLE_SYMBOLICNAME
.toString())) {
451 fileProps
.put(BUNDLE_SYMBOLICNAME
.toString(), artifact
.getName());
453 if (!fileProps
.containsKey(BUNDLE_VERSION
.toString())) {
454 fileProps
.put(BUNDLE_VERSION
.toString(), artifact
.getVersion());
458 if (!fileProps
.containsKey(EXPORT_PACKAGE
.toString())) {
459 fileProps
.put(EXPORT_PACKAGE
.toString(),
460 "*;version=\"" + fileProps
.getProperty(BUNDLE_VERSION
.toString()) + "\"");
462 // if (!fileProps.contains(IMPORT_PACKAGE.toString())) {
463 // fileProps.put(IMPORT_PACKAGE.toString(), "*");
466 try (Analyzer bndAnalyzer
= new Analyzer()) {
467 bndAnalyzer
.setProperties(fileProps
);
468 Jar jar
= new Jar(downloaded
.toFile());
469 bndAnalyzer
.setJar(jar
);
470 Manifest manifest
= bndAnalyzer
.calcManifest();
472 keys
: for (Object key
: manifest
.getMainAttributes().keySet()) {
473 Object value
= manifest
.getMainAttributes().get(key
);
475 switch (key
.toString()) {
477 case "Bnd-LastModified":
481 if ("Require-Capability".equals(key
.toString())
482 && value
.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
483 continue keys
;// hack for very old classes
484 additionalEntries
.put(key
.toString(), value
.toString());
485 // logger.log(DEBUG, () -> key + "=" + value);
490 // try (Builder bndBuilder = new Builder()) {
491 // Jar jar = new Jar(downloaded.toFile());
492 // bndBuilder.addClasspath(jar);
493 // Path targetBundleDir = targetCategoryBase.resolve(artifact.getName() + "." + artifact.getBranch());
495 // Jar target = new Jar(targetBundleDir.toFile());
496 // bndBuilder.setJar(target);
497 // return targetBundleDir;
500 Path targetBundleDir
= processBundleJar(downloaded
, targetCategoryBase
, additionalEntries
);
501 logger
.log(Level
.DEBUG
, () -> "Processed " + downloaded
);
502 return targetBundleDir
;
503 } catch (Exception e
) {
504 throw new RuntimeException("Cannot BND process " + downloaded
, e
);
509 /** Download and integrates sources for a single Maven artifact. */
510 protected void downloadAndProcessM2Sources(String repoStr
, DefaultArtifact artifact
, Path targetBundleDir
)
514 DefaultArtifact sourcesArtifact
= new DefaultArtifact(artifact
.toM2Coordinates(), "sources");
515 URL sourcesUrl
= MavenConventionsUtils
.mavenRepoUrl(repoStr
, sourcesArtifact
);
516 Path sourcesDownloaded
= download(sourcesUrl
, originBase
, artifact
, true);
517 processM2SourceJar(sourcesDownloaded
, targetBundleDir
);
518 logger
.log(Level
.TRACE
, () -> "Processed source " + sourcesDownloaded
);
522 /** Integrate sources from a downloaded jar file. */
523 protected void processM2SourceJar(Path file
, Path targetBundleDir
) throws IOException
{
524 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
525 Path targetSourceDir
= targetBundleDir
.resolve("OSGI-OPT/src");
527 // TODO make it less dangerous?
528 if (Files
.exists(targetSourceDir
)) {
529 // deleteDirectory(targetSourceDir);
531 Files
.createDirectories(targetSourceDir
);
536 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
537 if (entry
.isDirectory())
539 if (entry
.getName().startsWith("META-INF"))// skip META-INF entries
541 if (entry
.getName().startsWith("module-info.java"))// skip META-INF entries
543 if (entry
.getName().startsWith("/")) // absolute paths
545 Path target
= targetSourceDir
.resolve(entry
.getName());
546 Files
.createDirectories(target
.getParent());
547 if (!Files
.exists(target
)) {
548 Files
.copy(jarIn
, target
);
549 logger
.log(Level
.TRACE
, () -> "Copied source " + target
);
551 logger
.log(Level
.WARNING
, () -> target
+ " already exists, skipping...");
558 /** Download a Maven artifact. */
559 protected Path
download(URL url
, Path dir
, Artifact artifact
) throws IOException
{
560 return download(url
, dir
, artifact
, false);
563 /** Download a Maven artifact. */
564 protected Path
download(URL url
, Path dir
, Artifact artifact
, boolean sources
) throws IOException
{
565 return download(url
, dir
, artifact
.getGroupId() + '/' + artifact
.getArtifactId() + "-" + artifact
.getVersion()
566 + (sources ?
"-sources" : "") + ".jar");
573 /** Process an archive in Eclipse format. */
574 public void processEclipseArchive(Path relativePath
) {
576 Path categoryRelativePath
= relativePath
.getParent();
577 // String category = categoryRelativePath.getFileName().toString();
578 Path targetCategoryBase
= a2Base
.resolve(categoryRelativePath
);
579 Files
.createDirectories(targetCategoryBase
);
580 // first delete all directories from previous builds
581 for (Path dir
: Files
.newDirectoryStream(targetCategoryBase
, (p
) -> Files
.isDirectory(p
))) {
582 deleteDirectory(dir
);
585 Files
.createDirectories(originBase
);
587 Path duDir
= descriptorsBase
.resolve(relativePath
);
588 Path commonBnd
= duDir
.resolve(COMMON_BND
);
589 Properties commonProps
= new Properties();
590 try (InputStream in
= Files
.newInputStream(commonBnd
)) {
591 commonProps
.load(in
);
593 String url
= commonProps
.getProperty(ManifestConstants
.SLC_ORIGIN_URI
.toString());
595 url
= uris
.getProperty(relativePath
.getFileName().toString());
597 throw new IllegalStateException("No url available for " + duDir
);
598 commonProps
.put(ManifestConstants
.SLC_ORIGIN_URI
.toString(), url
);
600 Path downloaded
= tryDownload(url
, originBase
);
602 FileSystem zipFs
= FileSystems
.newFileSystem(downloaded
, (ClassLoader
) null);
605 List
<PathMatcher
> includeMatchers
= new ArrayList
<>();
606 Properties includes
= new Properties();
607 try (InputStream in
= Files
.newInputStream(duDir
.resolve("includes.properties"))) {
610 for (Object pattern
: includes
.keySet()) {
611 PathMatcher pathMatcher
= zipFs
.getPathMatcher("glob:/" + pattern
);
612 includeMatchers
.add(pathMatcher
);
615 List
<PathMatcher
> excludeMatchers
= new ArrayList
<>();
616 Path excludeFile
= duDir
.resolve("excludes.properties");
617 if (Files
.exists(excludeFile
)) {
618 Properties excludes
= new Properties();
619 try (InputStream in
= Files
.newInputStream(excludeFile
)) {
622 for (Object pattern
: excludes
.keySet()) {
623 PathMatcher pathMatcher
= zipFs
.getPathMatcher("glob:/" + pattern
);
624 excludeMatchers
.add(pathMatcher
);
628 Files
.walkFileTree(zipFs
.getRootDirectories().iterator().next(), new SimpleFileVisitor
<Path
>() {
631 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
632 includeMatchers
: for (PathMatcher includeMatcher
: includeMatchers
) {
633 if (includeMatcher
.matches(file
)) {
634 for (PathMatcher excludeMatcher
: excludeMatchers
) {
635 if (excludeMatcher
.matches(file
)) {
636 logger
.log(Level
.WARNING
, "Skipping excluded " + file
);
637 return FileVisitResult
.CONTINUE
;
640 if (includeSources
&& file
.getFileName().toString().contains(".source_")) {
641 processEclipseSourceJar(file
, targetCategoryBase
);
642 logger
.log(Level
.DEBUG
, () -> "Processed source " + file
);
645 processBundleJar(file
, targetCategoryBase
, new HashMap(commonProps
));
646 logger
.log(Level
.DEBUG
, () -> "Processed " + file
);
648 break includeMatchers
;
651 return FileVisitResult
.CONTINUE
;
655 DirectoryStream
<Path
> dirs
= Files
.newDirectoryStream(targetCategoryBase
,
656 (p
) -> Files
.isDirectory(p
) && p
.getFileName().toString().indexOf('.') >= 0);
657 for (Path dir
: dirs
) {
660 } catch (IOException e
) {
661 throw new RuntimeException("Cannot process " + relativePath
, e
);
666 /** Process sources in Eclipse format. */
667 protected void processEclipseSourceJar(Path file
, Path targetBase
) throws IOException
{
669 Path targetBundleDir
;
670 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
671 Manifest manifest
= jarIn
.getManifest();
673 String
[] relatedBundle
= manifest
.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
674 String version
= relatedBundle
[1].substring("version=\"".length());
675 version
= version
.substring(0, version
.length() - 1);
676 NameVersion nameVersion
= new DefaultNameVersion(relatedBundle
[0], version
);
677 targetBundleDir
= targetBase
.resolve(nameVersion
.getName() + "." + nameVersion
.getBranch());
679 Path targetSourceDir
= targetBundleDir
.resolve("OSGI-OPT/src");
681 // TODO make it less dangerous?
682 if (Files
.exists(targetSourceDir
)) {
683 // deleteDirectory(targetSourceDir);
685 Files
.createDirectories(targetSourceDir
);
690 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
691 if (entry
.isDirectory())
693 if (entry
.getName().startsWith("META-INF"))// skip META-INF entries
695 Path target
= targetSourceDir
.resolve(entry
.getName());
696 Files
.createDirectories(target
.getParent());
697 Files
.copy(jarIn
, target
);
698 logger
.log(Level
.TRACE
, () -> "Copied source " + target
);
702 // Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
703 // Files.createDirectories(manifestPath.getParent());
704 // try (OutputStream out = Files.newOutputStream(manifestPath)) {
705 // manifest.write(out);
708 } catch (IOException e
) {
709 throw new IllegalStateException("Cannot process " + file
, e
);
717 /** Normalise a bundle. */
718 protected Path
processBundleJar(Path file
, Path targetBase
, Map
<String
, String
> entries
) throws IOException
{
719 DefaultNameVersion nameVersion
;
720 Path targetBundleDir
;
721 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
722 Manifest sourceManifest
= jarIn
.getManifest();
723 Manifest manifest
= sourceManifest
!= null ?
new Manifest(sourceManifest
) : new Manifest();
725 // remove problematic entries in MANIFEST
726 manifest
.getEntries().clear();
727 // Set<String> entriesToDelete = new HashSet<>();
728 // for (String key : manifest.getEntries().keySet()) {
729 //// logger.log(DEBUG, "## " + key);
730 // Attributes attrs = manifest.getAttributes(key);
731 // for (Object attrName : attrs.keySet()) {
732 //// logger.log(DEBUG, attrName + "=" + attrs.get(attrName));
733 // if ("Specification-Version".equals(attrName.toString())
734 // || "Implementation-Version".equals(attrName.toString())) {
735 // entriesToDelete.add(key);
740 // for (String key : entriesToDelete) {
741 // manifest.getEntries().remove(key);
744 String ourSymbolicName
= entries
.get(BUNDLE_SYMBOLICNAME
.toString());
745 String ourVersion
= entries
.get(BUNDLE_VERSION
.toString());
747 if (ourSymbolicName
!= null && ourVersion
!= null) {
748 nameVersion
= new DefaultNameVersion(ourSymbolicName
, ourVersion
);
750 nameVersion
= nameVersionFromManifest(manifest
);
751 if (ourVersion
!= null && !nameVersion
.getVersion().equals(ourVersion
)) {
752 logger
.log(Level
.WARNING
,
753 "Original version is " + nameVersion
.getVersion() + " while new version is " + ourVersion
);
754 entries
.put(BUNDLE_VERSION
.toString(), ourVersion
);
756 if (ourSymbolicName
!= null) {
757 // we always force our symbolic name
758 nameVersion
.setName(ourSymbolicName
);
761 targetBundleDir
= targetBase
.resolve(nameVersion
.getName() + "." + nameVersion
.getBranch());
763 // force Java 9 module name
764 entries
.put(ManifestConstants
.AUTOMATIC_MODULE_NAME
.toString(), nameVersion
.getName());
766 // TODO make it less dangerous?
767 // if (Files.exists(targetBundleDir)) {
768 // deleteDirectory(targetBundleDir);
771 boolean isNative
= false;
774 if (targetBundleDir
.startsWith(a2LibBase
)) {
776 Path libRelativePath
= a2LibBase
.relativize(targetBundleDir
);
777 os
= libRelativePath
.getName(0).toString();
778 arch
= libRelativePath
.getName(1).toString();
783 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
784 if (entry
.isDirectory())
786 if (entry
.getName().endsWith(".RSA") || entry
.getName().endsWith(".SF"))
788 if (entry
.getName().endsWith("module-info.class")) // skip Java 9 module info
790 if (entry
.getName().startsWith("META-INF/versions/")) // skip multi-version
792 // skip file system providers as they cause issues with native image
793 if (entry
.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider"))
795 if (entry
.getName().startsWith("OSGI-OPT/src/")) // skip embedded sources
797 Path target
= targetBundleDir
.resolve(entry
.getName());
798 Files
.createDirectories(target
.getParent());
799 Files
.copy(jarIn
, target
);
802 if (isNative
&& (entry
.getName().endsWith(".so") || entry
.getName().endsWith(".dll")
803 || entry
.getName().endsWith(".jnilib"))) {
804 Path categoryDir
= targetBundleDir
.getParent();
805 // String[] segments = categoryDir.getFileName().toString().split("\\.");
806 // String arch = segments[segments.length - 1];
807 // String os = segments[segments.length - 2];
808 boolean copyDll
= false;
809 Path targetDll
= categoryDir
.resolve(targetBundleDir
.relativize(target
));
810 if (nameVersion
.getName().equals("com.sun.jna")) {
811 if (arch
.equals("x86_64"))
813 if (os
.equals("macosx"))
815 if (target
.getParent().getFileName().toString().equals(os
+ "-" + arch
)) {
818 targetDll
= categoryDir
.resolve(target
.getFileName());
823 Files
.createDirectories(targetDll
.getParent());
824 if (Files
.exists(targetDll
))
825 Files
.delete(targetDll
);
826 Files
.copy(target
, targetDll
);
828 Files
.delete(target
);
830 logger
.log(Level
.TRACE
, () -> "Copied " + target
);
834 Path manifestPath
= targetBundleDir
.resolve("META-INF/MANIFEST.MF");
835 Files
.createDirectories(manifestPath
.getParent());
836 for (String key
: entries
.keySet()) {
837 String value
= entries
.get(key
);
838 Object previousValue
= manifest
.getMainAttributes().putValue(key
, value
);
839 if (previousValue
!= null && !previousValue
.equals(value
)) {
840 if (ManifestConstants
.IMPORT_PACKAGE
.toString().equals(key
)
841 || ManifestConstants
.EXPORT_PACKAGE
.toString().equals(key
))
842 logger
.log(Level
.TRACE
, file
.getFileName() + ": " + key
+ " was modified");
844 logger
.log(Level
.WARNING
, file
.getFileName() + ": " + key
+ " was " + previousValue
845 + ", overridden with " + value
);
848 // hack to remove unresolvable
849 if (key
.equals("Provide-Capability") || key
.equals("Require-Capability"))
850 if (nameVersion
.getName().equals("osgi.core") || nameVersion
.getName().equals("osgi.cmpn")) {
851 manifest
.getMainAttributes().remove(key
);
854 try (OutputStream out
= Files
.newOutputStream(manifestPath
)) {
858 return targetBundleDir
;
865 /** Recursively deletes a directory. */
866 private static void deleteDirectory(Path path
) throws IOException
{
867 if (!Files
.exists(path
))
869 Files
.walkFileTree(path
, new SimpleFileVisitor
<Path
>() {
871 public FileVisitResult
postVisitDirectory(Path directory
, IOException e
) throws IOException
{
874 Files
.delete(directory
);
875 return FileVisitResult
.CONTINUE
;
879 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
881 return FileVisitResult
.CONTINUE
;
886 /** Extract name/version from a MANIFEST. */
887 protected DefaultNameVersion
nameVersionFromManifest(Manifest manifest
) {
888 Attributes attrs
= manifest
.getMainAttributes();
890 String symbolicName
= attrs
.getValue(ManifestConstants
.BUNDLE_SYMBOLICNAME
.toString());
891 if (symbolicName
== null)
893 // make sure there is no directive
894 symbolicName
= symbolicName
.split(";")[0];
896 String version
= attrs
.getValue(ManifestConstants
.BUNDLE_VERSION
.toString());
897 return new DefaultNameVersion(symbolicName
, version
);
900 /** Try to download from an URI. */
901 protected Path
tryDownload(String uri
, Path dir
) throws IOException
{
903 List
<String
> urlBases
= null;
904 String uriPrefix
= null;
905 uriPrefixes
: for (String uriPref
: mirrors
.keySet()) {
906 if (uri
.startsWith(uriPref
)) {
907 if (mirrors
.get(uriPref
).size() > 0) {
908 urlBases
= mirrors
.get(uriPref
);
914 if (urlBases
== null)
916 return download(new URL(uri
), dir
);
917 } catch (FileNotFoundException e
) {
918 throw new FileNotFoundException("Cannot find " + uri
);
922 for (String urlBase
: urlBases
) {
923 String relativePath
= uri
.substring(uriPrefix
.length());
924 URL url
= new URL(urlBase
+ relativePath
);
926 return download(url
, dir
);
927 } catch (FileNotFoundException e
) {
928 logger
.log(Level
.WARNING
, "Cannot download " + url
+ ", trying another mirror");
931 throw new FileNotFoundException("Cannot find " + uri
);
934 // protected String simplifyName(URL u) {
935 // String name = u.getPath().substring(u.getPath().lastIndexOf('/') + 1);
939 /** Effectively download. */
940 protected Path
download(URL url
, Path dir
) throws IOException
{
941 return download(url
, dir
, (String
) null);
944 /** Effectively download. */
945 protected Path
download(URL url
, Path dir
, String name
) throws IOException
{
949 name
= url
.getPath().substring(url
.getPath().lastIndexOf('/') + 1);
952 dest
= dir
.resolve(name
);
953 if (Files
.exists(dest
)) {
954 logger
.log(Level
.TRACE
, () -> "File " + dest
+ " already exists for " + url
+ ", not downloading again");
957 Files
.createDirectories(dest
.getParent());
960 try (InputStream in
= url
.openStream()) {
961 Files
.copy(in
, dest
);
962 logger
.log(Level
.DEBUG
, () -> "Downloaded " + dest
+ " from " + url
);
967 /** Create a JAR file from a directory. */
968 protected Path
createJar(Path bundleDir
) throws IOException
{
969 // Remove from source the resources already in the jar file
970 // Path srcDir = bundleDir.resolve("OSGI-OPT/src");
971 // Files.walkFileTree(srcDir, new SimpleFileVisitor<Path>() {
974 // public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
975 // // always keep .java file
976 // if (file.getFileName().toString().endsWith(".java"))
977 // return super.visitFile(file, attrs);
979 // Path relPath = srcDir.relativize(file);
980 // Path bundlePath = bundleDir.resolve(relPath);
981 // if (Files.exists(bundlePath)) {
982 // Files.delete(file);
983 // logger.log(DEBUG, () -> "Removed " + file + " from sources.");
985 // return super.visitFile(file, attrs);
991 Path jarPath
= bundleDir
.getParent().resolve(bundleDir
.getFileName() + ".jar");
992 Path manifestPath
= bundleDir
.resolve("META-INF/MANIFEST.MF");
994 try (InputStream in
= Files
.newInputStream(manifestPath
)) {
995 manifest
= new Manifest(in
);
997 try (JarOutputStream jarOut
= new JarOutputStream(Files
.newOutputStream(jarPath
), manifest
)) {
998 jarOut
.setLevel(Deflater
.DEFAULT_COMPRESSION
);
999 Files
.walkFileTree(bundleDir
, new SimpleFileVisitor
<Path
>() {
1002 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
1003 if (file
.getFileName().toString().equals("MANIFEST.MF"))
1004 return super.visitFile(file
, attrs
);
1005 JarEntry entry
= new JarEntry(bundleDir
.relativize(file
).toString());
1006 jarOut
.putNextEntry(entry
);
1007 Files
.copy(file
, jarOut
);
1008 return super.visitFile(file
, attrs
);
1013 deleteDirectory(bundleDir
);
1017 /** For development purproses. */
1018 public static void main(String
[] args
) {
1019 Path factoryBase
= Paths
.get("../../output/a2").toAbsolutePath().normalize();
1020 Path descriptorsBase
= Paths
.get("../tp").toAbsolutePath().normalize();
1021 A2Factory factory
= new A2Factory(factoryBase
, descriptorsBase
, true);
1023 // factory.processSingleM2ArtifactDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache").resolve("org.apache.xml.resolver.bnd"));
1024 factory
.processM2BasedDistributionUnit(descriptorsBase
.resolve("org.argeo.tp.apache/apache-sshd"));
1025 // factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty"));
1026 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.osgi"));
1027 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp"));
1028 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp"));
1029 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache"));
1030 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk"));
1031 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats"));
1032 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis"));
1036 factory
.processCategory(Paths
.get("org.argeo.tp.sdk"));
1039 factory
.processCategory(Paths
.get("osgi/api/org.argeo.tp.osgi"));
1040 factory
.processEclipseArchive(Paths
.get("osgi/equinox/org.argeo.tp.osgi", "eclipse-equinox"));
1041 factory
.processEclipseArchive(Paths
.get("osgi/equinox/org.argeo.tp.eclipse", "eclipse-equinox"));
1043 factory
.processEclipseArchive(Paths
.get("swt/rap/org.argeo.tp.swt", "eclipse-rap"));
1044 factory
.processEclipseArchive(Paths
.get("swt/rap/org.argeo.tp.swt.workbench", "eclipse-rap"));
1046 factory
.processEclipseArchive(Paths
.get("swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1048 factory
.processEclipseArchive(Paths
.get("lib/linux/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1049 factory
.processEclipseArchive(Paths
.get("lib/linux/aarch64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1050 factory
.processEclipseArchive(Paths
.get("lib/win32/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1051 factory
.processEclipseArchive(Paths
.get("lib/macosx/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1053 factory
.processEclipseArchive(Paths
.get("swt/rcp/org.argeo.tp.swt", "eclipse-nebula"));
1054 factory
.processEclipseArchive(Paths
.get("swt/rcp/org.argeo.tp.swt.workbench", "eclipse-rcp"));
1055 factory
.processCategory(Paths
.get("swt/rcp/org.argeo.tp.swt.workbench"));
1058 factory
.processCategory(Paths
.get("org.argeo.tp"));
1059 factory
.processCategory(Paths
.get("org.argeo.tp.apache"));
1060 factory
.processCategory(Paths
.get("org.argeo.tp.jetty"));
1061 factory
.processCategory(Paths
.get("org.argeo.tp.jcr"));
1062 factory
.processCategory(Paths
.get("org.argeo.tp.formats"));
1063 factory
.processCategory(Paths
.get("org.argeo.tp.poi"));
1064 factory
.processCategory(Paths
.get("org.argeo.tp.gis"));