1 package org
.argeo
.slc
.factory
;
3 import static java
.lang
.System
.Logger
.Level
.DEBUG
;
4 import static org
.argeo
.slc
.ManifestConstants
.BUNDLE_SYMBOLICNAME
;
5 import static org
.argeo
.slc
.ManifestConstants
.BUNDLE_VERSION
;
6 import static org
.argeo
.slc
.ManifestConstants
.EXPORT_PACKAGE
;
7 import static org
.argeo
.slc
.ManifestConstants
.SLC_ORIGIN_M2
;
8 import static org
.argeo
.slc
.ManifestConstants
.SLC_ORIGIN_M2_REPO
;
10 import java
.io
.FileNotFoundException
;
11 import java
.io
.IOException
;
12 import java
.io
.InputStream
;
13 import java
.io
.OutputStream
;
14 import java
.lang
.System
.Logger
;
15 import java
.lang
.System
.Logger
.Level
;
17 import java
.nio
.file
.DirectoryStream
;
18 import java
.nio
.file
.FileSystem
;
19 import java
.nio
.file
.FileSystems
;
20 import java
.nio
.file
.FileVisitResult
;
21 import java
.nio
.file
.Files
;
22 import java
.nio
.file
.Path
;
23 import java
.nio
.file
.PathMatcher
;
24 import java
.nio
.file
.Paths
;
25 import java
.nio
.file
.SimpleFileVisitor
;
26 import java
.nio
.file
.StandardOpenOption
;
27 import java
.nio
.file
.attribute
.BasicFileAttributes
;
28 import java
.util
.ArrayList
;
29 import java
.util
.HashMap
;
30 import java
.util
.List
;
32 import java
.util
.Properties
;
33 import java
.util
.TreeMap
;
34 import java
.util
.jar
.Attributes
;
35 import java
.util
.jar
.JarEntry
;
36 import java
.util
.jar
.JarInputStream
;
37 import java
.util
.jar
.JarOutputStream
;
38 import java
.util
.jar
.Manifest
;
39 import java
.util
.zip
.Deflater
;
41 import org
.argeo
.slc
.DefaultCategoryNameVersion
;
42 import org
.argeo
.slc
.DefaultNameVersion
;
43 import org
.argeo
.slc
.ManifestConstants
;
44 import org
.argeo
.slc
.NameVersion
;
45 import org
.argeo
.slc
.factory
.m2
.Artifact
;
46 import org
.argeo
.slc
.factory
.m2
.DefaultArtifact
;
47 import org
.argeo
.slc
.factory
.m2
.MavenConventionsUtils
;
49 import aQute
.bnd
.osgi
.Analyzer
;
50 import aQute
.bnd
.osgi
.Jar
;
52 /** The central class for A2 packaging. */
53 public class A2Factory
{
54 private final static Logger logger
= System
.getLogger(A2Factory
.class.getName());
56 private final static String COMMON_BND
= "common.bnd";
57 private final static String MERGE_BND
= "merge.bnd";
59 private Path originBase
;
62 /** key is URI prefix, value list of base URLs */
63 private Map
<String
, List
<String
>> mirrors
= new HashMap
<String
, List
<String
>>();
65 public A2Factory(Path a2Base
) {
66 this.originBase
= Paths
.get(System
.getProperty("user.home"), ".cache", "argeo/slc/origin");
69 // TODO make it configurable
70 List
<String
> eclipseMirrors
= new ArrayList
<>();
71 eclipseMirrors
.add("https://archive.eclipse.org/");
72 eclipseMirrors
.add("http://ftp-stud.hs-esslingen.de/Mirrors/eclipse/");
73 eclipseMirrors
.add("http://ftp.fau.de/eclipse/");
75 mirrors
.put("http://www.eclipse.org/downloads", eclipseMirrors
);
82 /** Process a whole category/group id. */
83 public void processCategory(Path targetCategoryBase
) {
85 DirectoryStream
<Path
> bnds
= Files
.newDirectoryStream(targetCategoryBase
,
86 (p
) -> p
.getFileName().toString().endsWith(".bnd") && !p
.getFileName().toString().equals(COMMON_BND
)
87 && !p
.getFileName().toString().equals(MERGE_BND
));
89 processSingleM2ArtifactDistributionUnit(p
);
92 DirectoryStream
<Path
> dus
= Files
.newDirectoryStream(targetCategoryBase
, (p
) -> Files
.isDirectory(p
));
93 for (Path duDir
: dus
) {
94 processM2BasedDistributionUnit(duDir
);
96 } catch (IOException e
) {
97 throw new RuntimeException("Cannot process category " + targetCategoryBase
, e
);
101 /** Process a standalone Maven artifact. */
102 public void processSingleM2ArtifactDistributionUnit(Path bndFile
) {
104 String category
= bndFile
.getParent().getFileName().toString();
105 Path targetCategoryBase
= a2Base
.resolve(category
);
106 Properties fileProps
= new Properties();
107 try (InputStream in
= Files
.newInputStream(bndFile
)) {
110 String repoStr
= fileProps
.containsKey(SLC_ORIGIN_M2_REPO
.toString())
111 ? fileProps
.getProperty(SLC_ORIGIN_M2_REPO
.toString())
114 if (!fileProps
.containsKey(BUNDLE_SYMBOLICNAME
.toString())
115 && !fileProps
.containsKey(ManifestConstants
.SLC_ORIGIN_MANIFEST_NOT_MODIFIED
.toString())) {
116 // use file name as symbolic name
117 String symbolicName
= bndFile
.getFileName().toString();
118 symbolicName
= symbolicName
.substring(0, symbolicName
.length() - ".bnd".length());
119 fileProps
.put(BUNDLE_SYMBOLICNAME
.toString(), symbolicName
);
122 String m2Coordinates
= fileProps
.getProperty(SLC_ORIGIN_M2
.toString());
123 if (m2Coordinates
== null)
124 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile
);
125 DefaultArtifact artifact
= new DefaultArtifact(m2Coordinates
);
126 URL url
= MavenConventionsUtils
.mavenRepoUrl(repoStr
, artifact
);
127 Path downloaded
= download(url
, originBase
, artifact
);
129 Path targetBundleDir
= processBndJar(downloaded
, targetCategoryBase
, fileProps
, artifact
);
131 downloadAndProcessM2Sources(repoStr
, artifact
, targetBundleDir
);
133 createJar(targetBundleDir
);
134 } catch (Exception e
) {
135 throw new RuntimeException("Cannot process " + bndFile
, e
);
139 /** Process multiple Maven artifacts. */
140 public void processM2BasedDistributionUnit(Path duDir
) {
142 String category
= duDir
.getParent().getFileName().toString();
143 Path targetCategoryBase
= a2Base
.resolve(category
);
146 Path mergeBnd
= duDir
.resolve(MERGE_BND
);
147 if (Files
.exists(mergeBnd
)) {
148 mergeM2Artifacts(mergeBnd
);
152 Path commonBnd
= duDir
.resolve(COMMON_BND
);
153 if (!Files
.exists(commonBnd
)) {
156 Properties commonProps
= new Properties();
157 try (InputStream in
= Files
.newInputStream(commonBnd
)) {
158 commonProps
.load(in
);
161 String m2Version
= commonProps
.getProperty(SLC_ORIGIN_M2
.toString());
162 if (m2Version
== null) {
163 logger
.log(Level
.WARNING
, "Ignoring " + duDir
+ " as it is not an M2-based distribution unit");
164 return;// ignore, this is probably an Eclipse archive
166 if (!m2Version
.startsWith(":")) {
167 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version
);
169 m2Version
= m2Version
.substring(1);
171 DirectoryStream
<Path
> ds
= Files
.newDirectoryStream(duDir
,
172 (p
) -> p
.getFileName().toString().endsWith(".bnd") && !p
.getFileName().toString().equals(COMMON_BND
)
173 && !p
.getFileName().toString().equals(MERGE_BND
));
175 Properties fileProps
= new Properties();
176 try (InputStream in
= Files
.newInputStream(p
)) {
179 String m2Coordinates
= fileProps
.getProperty(SLC_ORIGIN_M2
.toString());
180 DefaultArtifact artifact
= new DefaultArtifact(m2Coordinates
);
182 // temporary rewrite, for migration
183 // String localLicense = fileProps.getProperty(BUNDLE_LICENSE.toString());
184 // if (localLicense != null || artifact.getVersion() != null) {
185 // fileProps.remove(BUNDLE_LICENSE.toString());
186 // fileProps.put(SLC_ORIGIN_M2.toString(), artifact.getGroupId() + ":" + artifact.getArtifactId());
187 // try (Writer writer = Files.newBufferedWriter(p)) {
188 // for (Object key : fileProps.keySet()) {
189 // String value = fileProps.getProperty(key.toString());
190 // writer.write(key + ": " + value + '\n');
192 // logger.log(DEBUG, () -> "Migrated " + p);
196 artifact
.setVersion(m2Version
);
198 // prepare manifest entries
199 Properties mergeProps
= new Properties();
200 mergeProps
.putAll(commonProps
);
202 fileEntries
: for (Object key
: fileProps
.keySet()) {
203 if (ManifestConstants
.SLC_ORIGIN_M2
.toString().equals(key
))
204 continue fileEntries
;
205 String value
= fileProps
.getProperty(key
.toString());
206 Object previousValue
= mergeProps
.put(key
.toString(), value
);
207 if (previousValue
!= null) {
208 logger
.log(Level
.WARNING
,
209 commonBnd
+ ": " + key
+ " was " + previousValue
+ ", overridden with " + value
);
212 mergeProps
.put(ManifestConstants
.SLC_ORIGIN_M2
.toString(), artifact
.toM2Coordinates());
213 if (!mergeProps
.containsKey(BUNDLE_SYMBOLICNAME
.toString())
214 && !mergeProps
.containsKey(ManifestConstants
.SLC_ORIGIN_MANIFEST_NOT_MODIFIED
.toString())) {
215 // use file name as symbolic name
216 String symbolicName
= p
.getFileName().toString();
217 symbolicName
= symbolicName
.substring(0, symbolicName
.length() - ".bnd".length());
218 mergeProps
.put(BUNDLE_SYMBOLICNAME
.toString(), symbolicName
);
221 String repoStr
= mergeProps
.containsKey(SLC_ORIGIN_M2_REPO
.toString())
222 ? mergeProps
.getProperty(SLC_ORIGIN_M2_REPO
.toString())
226 URL url
= MavenConventionsUtils
.mavenRepoUrl(repoStr
, artifact
);
227 Path downloaded
= download(url
, originBase
, artifact
);
229 Path targetBundleDir
= processBndJar(downloaded
, targetCategoryBase
, mergeProps
, artifact
);
230 // logger.log(Level.DEBUG, () -> "Processed " + downloaded);
233 downloadAndProcessM2Sources(repoStr
, artifact
, targetBundleDir
);
235 createJar(targetBundleDir
);
237 } catch (IOException e
) {
238 throw new RuntimeException("Cannot process " + duDir
, e
);
243 /** Merge multiple Maven artifacts. */
244 protected void mergeM2Artifacts(Path mergeBnd
) throws IOException
{
245 Path duDir
= mergeBnd
.getParent();
246 String category
= duDir
.getParent().getFileName().toString();
247 Path targetCategoryBase
= a2Base
.resolve(category
);
249 Properties mergeProps
= new Properties();
250 try (InputStream in
= Files
.newInputStream(mergeBnd
)) {
255 String m2Version
= mergeProps
.getProperty(SLC_ORIGIN_M2
.toString());
256 if (m2Version
== null) {
257 logger
.log(Level
.WARNING
, "Ignoring " + duDir
+ " as it is not an M2-based distribution unit");
258 return;// ignore, this is probably an Eclipse archive
260 if (!m2Version
.startsWith(":")) {
261 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version
);
263 m2Version
= m2Version
.substring(1);
264 mergeProps
.put(ManifestConstants
.BUNDLE_VERSION
.toString(), m2Version
);
266 String artifactsStr
= mergeProps
.getProperty(ManifestConstants
.SLC_ORIGIN_M2_MERGE
.toString());
267 String repoStr
= mergeProps
.containsKey(SLC_ORIGIN_M2_REPO
.toString())
268 ? mergeProps
.getProperty(SLC_ORIGIN_M2_REPO
.toString())
271 String bundleSymbolicName
= mergeProps
.getProperty(ManifestConstants
.BUNDLE_SYMBOLICNAME
.toString());
272 if (bundleSymbolicName
== null)
273 throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd
);
274 DefaultCategoryNameVersion nameVersion
= new DefaultArtifact(
275 category
+ ":" + bundleSymbolicName
+ ":" + m2Version
);
276 Path targetBundleDir
= targetCategoryBase
.resolve(bundleSymbolicName
+ "." + nameVersion
.getBranch());
278 String
[] artifacts
= artifactsStr
.split(",");
279 artifacts
: for (String str
: artifacts
) {
280 String m2Coordinates
= str
.trim();
281 if ("".equals(m2Coordinates
))
283 DefaultArtifact artifact
= new DefaultArtifact(m2Coordinates
.trim());
284 if (artifact
.getVersion() == null)
285 artifact
.setVersion(m2Version
);
286 URL url
= MavenConventionsUtils
.mavenRepoUrl(repoStr
, artifact
);
287 Path downloaded
= download(url
, originBase
, artifact
);
289 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(downloaded
), false)) {
290 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
291 if (entry
.isDirectory())
293 else if (entry
.getName().endsWith(".RSA") || entry
.getName().endsWith(".SF"))
295 else if (entry
.getName().startsWith("META-INF/versions/"))
297 else if (entry
.getName().startsWith("META-INF/maven/"))
299 else if (entry
.getName().equals("module-info.class"))
301 else if (entry
.getName().equals("META-INF/NOTICE"))
303 else if (entry
.getName().equals("META-INF/NOTICE.txt"))
305 else if (entry
.getName().equals("META-INF/LICENSE"))
307 else if (entry
.getName().equals("META-INF/LICENSE.md"))
309 else if (entry
.getName().equals("META-INF/LICENSE-notice.md"))
311 Path target
= targetBundleDir
.resolve(entry
.getName());
312 Files
.createDirectories(target
.getParent());
313 if (!Files
.exists(target
)) {
314 Files
.copy(jarIn
, target
);
316 if (entry
.getName().startsWith("META-INF/services/")) {
317 try (OutputStream out
= Files
.newOutputStream(target
, StandardOpenOption
.APPEND
)) {
318 out
.write("\n".getBytes());
319 jarIn
.transferTo(out
);
320 if (logger
.isLoggable(DEBUG
))
321 logger
.log(DEBUG
, artifact
.getArtifactId() + " - Appended " + entry
.getName());
323 } else if (entry
.getName().startsWith("org/apache/batik/")) {
324 logger
.log(Level
.WARNING
, "Skip " + entry
.getName());
327 throw new IllegalStateException("File " + target
+ " from " + artifact
+ " already exists");
330 logger
.log(Level
.TRACE
, () -> "Copied " + target
);
334 downloadAndProcessM2Sources(repoStr
, artifact
, targetBundleDir
);
337 // additional service files
338 Path servicesDir
= duDir
.resolve("services");
339 if (Files
.exists(servicesDir
)) {
340 for (Path p
: Files
.newDirectoryStream(servicesDir
)) {
341 Path target
= targetBundleDir
.resolve("META-INF/services/").resolve(p
.getFileName());
342 try (InputStream in
= Files
.newInputStream(p
);
343 OutputStream out
= Files
.newOutputStream(target
, StandardOpenOption
.APPEND
);) {
344 out
.write("\n".getBytes());
346 if (logger
.isLoggable(DEBUG
))
347 logger
.log(DEBUG
, "Appended " + p
);
352 Map
<String
, String
> entries
= new TreeMap
<>();
353 try (Analyzer bndAnalyzer
= new Analyzer()) {
354 bndAnalyzer
.setProperties(mergeProps
);
355 Jar jar
= new Jar(targetBundleDir
.toFile());
356 bndAnalyzer
.setJar(jar
);
357 Manifest manifest
= bndAnalyzer
.calcManifest();
359 keys
: for (Object key
: manifest
.getMainAttributes().keySet()) {
360 Object value
= manifest
.getMainAttributes().get(key
);
362 switch (key
.toString()) {
364 case "Bnd-LastModified":
368 if ("Require-Capability".equals(key
.toString())
369 && value
.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
370 continue keys
;// hack for very old classes
371 entries
.put(key
.toString(), value
.toString());
372 // logger.log(DEBUG, () -> key + "=" + value);
375 } catch (Exception e
) {
376 throw new RuntimeException("Cannot process " + mergeBnd
, e
);
379 Manifest manifest
= new Manifest();
380 Path manifestPath
= targetBundleDir
.resolve("META-INF/MANIFEST.MF");
381 Files
.createDirectories(manifestPath
.getParent());
382 for (String key
: entries
.keySet()) {
383 String value
= entries
.get(key
);
384 manifest
.getMainAttributes().putValue(key
, value
);
387 // // Use Maven version as Bundle-Version
388 // String bundleVersion = manifest.getMainAttributes().getValue(ManifestConstants.BUNDLE_VERSION.toString());
389 // if (bundleVersion == null || bundleVersion.trim().equals("0")) {
390 // // TODO check why it is sometimes set to "0"
391 // manifest.getMainAttributes().putValue(ManifestConstants.BUNDLE_VERSION.toString(), m2Version);
393 try (OutputStream out
= Files
.newOutputStream(manifestPath
)) {
397 createJar(targetBundleDir
);
401 /** Generate MANIFEST using BND. */
402 protected Path
processBndJar(Path downloaded
, Path targetCategoryBase
, Properties fileProps
,
403 DefaultArtifact artifact
) {
406 Map
<String
, String
> additionalEntries
= new TreeMap
<>();
407 boolean doNotModify
= Boolean
.parseBoolean(fileProps
408 .getOrDefault(ManifestConstants
.SLC_ORIGIN_MANIFEST_NOT_MODIFIED
.toString(), "false").toString());
410 // we always force the symbolic name
413 fileEntries
: for (Object key
: fileProps
.keySet()) {
414 if (ManifestConstants
.SLC_ORIGIN_M2
.toString().equals(key
))
415 continue fileEntries
;
416 String value
= fileProps
.getProperty(key
.toString());
417 additionalEntries
.put(key
.toString(), value
);
420 if (artifact
!= null) {
421 if (!fileProps
.containsKey(BUNDLE_SYMBOLICNAME
.toString())) {
422 fileProps
.put(BUNDLE_SYMBOLICNAME
.toString(), artifact
.getName());
424 if (!fileProps
.containsKey(BUNDLE_VERSION
.toString())) {
425 fileProps
.put(BUNDLE_VERSION
.toString(), artifact
.getVersion());
429 if (!fileProps
.containsKey(EXPORT_PACKAGE
.toString())) {
430 fileProps
.put(EXPORT_PACKAGE
.toString(),
431 "*;version=\"" + fileProps
.getProperty(BUNDLE_VERSION
.toString()) + "\"");
433 // if (!fileProps.contains(IMPORT_PACKAGE.toString())) {
434 // fileProps.put(IMPORT_PACKAGE.toString(), "*");
437 try (Analyzer bndAnalyzer
= new Analyzer()) {
438 bndAnalyzer
.setProperties(fileProps
);
439 Jar jar
= new Jar(downloaded
.toFile());
440 bndAnalyzer
.setJar(jar
);
441 Manifest manifest
= bndAnalyzer
.calcManifest();
443 keys
: for (Object key
: manifest
.getMainAttributes().keySet()) {
444 Object value
= manifest
.getMainAttributes().get(key
);
446 switch (key
.toString()) {
448 case "Bnd-LastModified":
452 if ("Require-Capability".equals(key
.toString())
453 && value
.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
454 continue keys
;// hack for very old classes
455 additionalEntries
.put(key
.toString(), value
.toString());
456 // logger.log(DEBUG, () -> key + "=" + value);
461 // try (Builder bndBuilder = new Builder()) {
462 // Jar jar = new Jar(downloaded.toFile());
463 // bndBuilder.addClasspath(jar);
464 // Path targetBundleDir = targetCategoryBase.resolve(artifact.getName() + "." + artifact.getBranch());
466 // Jar target = new Jar(targetBundleDir.toFile());
467 // bndBuilder.setJar(target);
468 // return targetBundleDir;
471 Path targetBundleDir
= processBundleJar(downloaded
, targetCategoryBase
, additionalEntries
);
472 logger
.log(Level
.DEBUG
, () -> "Processed " + downloaded
);
473 return targetBundleDir
;
474 } catch (Exception e
) {
475 throw new RuntimeException("Cannot BND process " + downloaded
, e
);
480 /** Download and integrates sources for a single Maven artifact. */
481 protected void downloadAndProcessM2Sources(String repoStr
, DefaultArtifact artifact
, Path targetBundleDir
)
483 DefaultArtifact sourcesArtifact
= new DefaultArtifact(artifact
.toM2Coordinates(), "sources");
484 URL sourcesUrl
= MavenConventionsUtils
.mavenRepoUrl(repoStr
, sourcesArtifact
);
485 Path sourcesDownloaded
= download(sourcesUrl
, originBase
, artifact
, true);
486 processM2SourceJar(sourcesDownloaded
, targetBundleDir
);
487 logger
.log(Level
.TRACE
, () -> "Processed source " + sourcesDownloaded
);
491 /** Integrate sources from a downloaded jar file. */
492 protected void processM2SourceJar(Path file
, Path targetBundleDir
) throws IOException
{
493 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
494 Path targetSourceDir
= targetBundleDir
.resolve("OSGI-OPT/src");
496 // TODO make it less dangerous?
497 if (Files
.exists(targetSourceDir
)) {
498 // deleteDirectory(targetSourceDir);
500 Files
.createDirectories(targetSourceDir
);
505 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
506 if (entry
.isDirectory())
508 if (entry
.getName().startsWith("META-INF"))// skip META-INF entries
510 if (entry
.getName().startsWith("module-info.java"))// skip META-INF entries
512 if (entry
.getName().startsWith("/")) // absolute paths
514 Path target
= targetSourceDir
.resolve(entry
.getName());
515 Files
.createDirectories(target
.getParent());
516 if (!Files
.exists(target
)) {
517 Files
.copy(jarIn
, target
);
518 logger
.log(Level
.TRACE
, () -> "Copied source " + target
);
520 logger
.log(Level
.WARNING
, () -> target
+ " already exists, skipping...");
527 /** Download a Maven artifact. */
528 protected Path
download(URL url
, Path dir
, Artifact artifact
) throws IOException
{
529 return download(url
, dir
, artifact
, false);
532 /** Download a Maven artifact. */
533 protected Path
download(URL url
, Path dir
, Artifact artifact
, boolean sources
) throws IOException
{
534 return download(url
, dir
, artifact
.getGroupId() + '/' + artifact
.getArtifactId() + "-" + artifact
.getVersion()
535 + (sources ?
"-sources" : "") + ".jar");
542 /** Process an archive in Eclipse format. */
543 public void processEclipseArchive(Path duDir
) {
545 String category
= duDir
.getParent().getFileName().toString();
546 Path targetCategoryBase
= a2Base
.resolve(category
);
547 Files
.createDirectories(targetCategoryBase
);
548 // first delete all directories from previous builds
549 for (Path dir
: Files
.newDirectoryStream(targetCategoryBase
, (p
) -> Files
.isDirectory(p
))) {
550 deleteDirectory(dir
);
553 Files
.createDirectories(originBase
);
555 Path commonBnd
= duDir
.resolve(COMMON_BND
);
556 Properties commonProps
= new Properties();
557 try (InputStream in
= Files
.newInputStream(commonBnd
)) {
558 commonProps
.load(in
);
560 String url
= commonProps
.getProperty(ManifestConstants
.SLC_ORIGIN_URI
.toString());
561 Path downloaded
= tryDownload(url
, originBase
);
563 FileSystem zipFs
= FileSystems
.newFileSystem(downloaded
, (ClassLoader
) null);
566 List
<PathMatcher
> includeMatchers
= new ArrayList
<>();
567 Properties includes
= new Properties();
568 try (InputStream in
= Files
.newInputStream(duDir
.resolve("includes.properties"))) {
571 for (Object pattern
: includes
.keySet()) {
572 PathMatcher pathMatcher
= zipFs
.getPathMatcher("glob:/" + pattern
);
573 includeMatchers
.add(pathMatcher
);
576 List
<PathMatcher
> excludeMatchers
= new ArrayList
<>();
577 Path excludeFile
= duDir
.resolve("excludes.properties");
578 if (Files
.exists(excludeFile
)) {
579 Properties excludes
= new Properties();
580 try (InputStream in
= Files
.newInputStream(excludeFile
)) {
583 for (Object pattern
: excludes
.keySet()) {
584 PathMatcher pathMatcher
= zipFs
.getPathMatcher("glob:/" + pattern
);
585 excludeMatchers
.add(pathMatcher
);
589 Files
.walkFileTree(zipFs
.getRootDirectories().iterator().next(), new SimpleFileVisitor
<Path
>() {
592 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
593 includeMatchers
: for (PathMatcher includeMatcher
: includeMatchers
) {
594 if (includeMatcher
.matches(file
)) {
595 for (PathMatcher excludeMatcher
: excludeMatchers
) {
596 if (excludeMatcher
.matches(file
)) {
597 logger
.log(Level
.WARNING
, "Skipping excluded " + file
);
598 return FileVisitResult
.CONTINUE
;
601 if (file
.getFileName().toString().contains(".source_")) {
602 processEclipseSourceJar(file
, targetCategoryBase
);
603 logger
.log(Level
.DEBUG
, () -> "Processed source " + file
);
606 processBundleJar(file
, targetCategoryBase
, new HashMap
<>());
607 logger
.log(Level
.DEBUG
, () -> "Processed " + file
);
609 break includeMatchers
;
612 return FileVisitResult
.CONTINUE
;
616 DirectoryStream
<Path
> dirs
= Files
.newDirectoryStream(targetCategoryBase
, (p
) -> Files
.isDirectory(p
));
617 for (Path dir
: dirs
) {
620 } catch (IOException e
) {
621 throw new RuntimeException("Cannot process " + duDir
, e
);
626 /** Process sources in Eclipse format. */
627 protected void processEclipseSourceJar(Path file
, Path targetBase
) throws IOException
{
629 Path targetBundleDir
;
630 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
631 Manifest manifest
= jarIn
.getManifest();
633 String
[] relatedBundle
= manifest
.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
634 String version
= relatedBundle
[1].substring("version=\"".length());
635 version
= version
.substring(0, version
.length() - 1);
636 NameVersion nameVersion
= new DefaultNameVersion(relatedBundle
[0], version
);
637 targetBundleDir
= targetBase
.resolve(nameVersion
.getName() + "." + nameVersion
.getBranch());
639 Path targetSourceDir
= targetBundleDir
.resolve("OSGI-OPT/src");
641 // TODO make it less dangerous?
642 if (Files
.exists(targetSourceDir
)) {
643 // deleteDirectory(targetSourceDir);
645 Files
.createDirectories(targetSourceDir
);
650 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
651 if (entry
.isDirectory())
653 if (entry
.getName().startsWith("META-INF"))// skip META-INF entries
655 Path target
= targetSourceDir
.resolve(entry
.getName());
656 Files
.createDirectories(target
.getParent());
657 Files
.copy(jarIn
, target
);
658 logger
.log(Level
.TRACE
, () -> "Copied source " + target
);
662 // Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
663 // Files.createDirectories(manifestPath.getParent());
664 // try (OutputStream out = Files.newOutputStream(manifestPath)) {
665 // manifest.write(out);
668 } catch (IOException e
) {
669 throw new IllegalStateException("Cannot process " + file
, e
);
677 /** Normalise a bundle. */
678 protected Path
processBundleJar(Path file
, Path targetBase
, Map
<String
, String
> entries
) throws IOException
{
679 DefaultNameVersion nameVersion
;
680 Path targetBundleDir
;
681 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
682 Manifest sourceManifest
= jarIn
.getManifest();
683 Manifest manifest
= sourceManifest
!= null ?
new Manifest(sourceManifest
) : new Manifest();
685 // remove problematic entries in MANIFEST
686 manifest
.getEntries().clear();
687 // Set<String> entriesToDelete = new HashSet<>();
688 // for (String key : manifest.getEntries().keySet()) {
689 //// logger.log(DEBUG, "## " + key);
690 // Attributes attrs = manifest.getAttributes(key);
691 // for (Object attrName : attrs.keySet()) {
692 //// logger.log(DEBUG, attrName + "=" + attrs.get(attrName));
693 // if ("Specification-Version".equals(attrName.toString())
694 // || "Implementation-Version".equals(attrName.toString())) {
695 // entriesToDelete.add(key);
700 // for (String key : entriesToDelete) {
701 // manifest.getEntries().remove(key);
704 String symbolicNameFromEntries
= entries
.get(BUNDLE_SYMBOLICNAME
.toString());
705 String versionFromEntries
= entries
.get(BUNDLE_VERSION
.toString());
707 if (symbolicNameFromEntries
!= null && versionFromEntries
!= null) {
708 nameVersion
= new DefaultNameVersion(symbolicNameFromEntries
, versionFromEntries
);
710 nameVersion
= nameVersionFromManifest(manifest
);
711 if (versionFromEntries
!= null && !nameVersion
.getVersion().equals(versionFromEntries
)) {
712 logger
.log(Level
.WARNING
, "Original version is " + nameVersion
.getVersion()
713 + " while new version is " + versionFromEntries
);
715 if (symbolicNameFromEntries
!= null) {
716 // we always force our symbolic name
717 nameVersion
.setName(symbolicNameFromEntries
);
720 targetBundleDir
= targetBase
.resolve(nameVersion
.getName() + "." + nameVersion
.getBranch());
722 // TODO make it less dangerous?
723 // if (Files.exists(targetBundleDir)) {
724 // deleteDirectory(targetBundleDir);
729 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
730 if (entry
.isDirectory())
732 if (entry
.getName().endsWith(".RSA") || entry
.getName().endsWith(".SF"))
734 Path target
= targetBundleDir
.resolve(entry
.getName());
735 Files
.createDirectories(target
.getParent());
736 Files
.copy(jarIn
, target
);
737 logger
.log(Level
.TRACE
, () -> "Copied " + target
);
741 Path manifestPath
= targetBundleDir
.resolve("META-INF/MANIFEST.MF");
742 Files
.createDirectories(manifestPath
.getParent());
743 for (String key
: entries
.keySet()) {
744 String value
= entries
.get(key
);
745 Object previousValue
= manifest
.getMainAttributes().putValue(key
, value
);
746 if (previousValue
!= null && !previousValue
.equals(value
)) {
747 if (ManifestConstants
.IMPORT_PACKAGE
.toString().equals(key
)
748 || ManifestConstants
.EXPORT_PACKAGE
.toString().equals(key
))
749 logger
.log(Level
.WARNING
, file
.getFileName() + ": " + key
+ " was modified");
752 logger
.log(Level
.WARNING
, file
.getFileName() + ": " + key
+ " was " + previousValue
753 + ", overridden with " + value
);
756 try (OutputStream out
= Files
.newOutputStream(manifestPath
)) {
760 return targetBundleDir
;
767 /** Recursively deletes a directory. */
768 private static void deleteDirectory(Path path
) throws IOException
{
769 if (!Files
.exists(path
))
771 Files
.walkFileTree(path
, new SimpleFileVisitor
<Path
>() {
773 public FileVisitResult
postVisitDirectory(Path directory
, IOException e
) throws IOException
{
776 Files
.delete(directory
);
777 return FileVisitResult
.CONTINUE
;
781 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
783 return FileVisitResult
.CONTINUE
;
788 /** Extract name/version from a MANIFEST. */
789 protected DefaultNameVersion
nameVersionFromManifest(Manifest manifest
) {
790 Attributes attrs
= manifest
.getMainAttributes();
792 String symbolicName
= attrs
.getValue(ManifestConstants
.BUNDLE_SYMBOLICNAME
.toString());
793 if (symbolicName
== null)
795 // make sure there is no directive
796 symbolicName
= symbolicName
.split(";")[0];
798 String version
= attrs
.getValue(ManifestConstants
.BUNDLE_VERSION
.toString());
799 return new DefaultNameVersion(symbolicName
, version
);
802 /** Try to download from an URI. */
803 protected Path
tryDownload(String uri
, Path dir
) throws IOException
{
805 List
<String
> urlBases
= null;
806 String uriPrefix
= null;
807 uriPrefixes
: for (String uriPref
: mirrors
.keySet()) {
808 if (uri
.startsWith(uriPref
)) {
809 if (mirrors
.get(uriPref
).size() > 0) {
810 urlBases
= mirrors
.get(uriPref
);
816 if (urlBases
== null)
818 return download(new URL(uri
), dir
);
819 } catch (FileNotFoundException e
) {
820 throw new FileNotFoundException("Cannot find " + uri
);
824 for (String urlBase
: urlBases
) {
825 String relativePath
= uri
.substring(uriPrefix
.length());
826 URL url
= new URL(urlBase
+ relativePath
);
828 return download(url
, dir
);
829 } catch (FileNotFoundException e
) {
830 logger
.log(Level
.WARNING
, "Cannot download " + url
+ ", trying another mirror");
833 throw new FileNotFoundException("Cannot find " + uri
);
836 // protected String simplifyName(URL u) {
837 // String name = u.getPath().substring(u.getPath().lastIndexOf('/') + 1);
841 /** Effectively download. */
842 protected Path
download(URL url
, Path dir
) throws IOException
{
843 return download(url
, dir
, (String
) null);
846 /** Effectively download. */
847 protected Path
download(URL url
, Path dir
, String name
) throws IOException
{
851 name
= url
.getPath().substring(url
.getPath().lastIndexOf('/') + 1);
854 dest
= dir
.resolve(name
);
855 if (Files
.exists(dest
)) {
856 logger
.log(Level
.TRACE
, () -> "File " + dest
+ " already exists for " + url
+ ", not downloading again");
859 Files
.createDirectories(dest
.getParent());
862 try (InputStream in
= url
.openStream()) {
863 Files
.copy(in
, dest
);
864 logger
.log(Level
.DEBUG
, () -> "Downloaded " + dest
+ " from " + url
);
869 /** Create a JAR file from a directory. */
870 protected Path
createJar(Path bundleDir
) throws IOException
{
871 // Remove from source the resources already in the jar file
872 // Path srcDir = bundleDir.resolve("OSGI-OPT/src");
873 // Files.walkFileTree(srcDir, new SimpleFileVisitor<Path>() {
876 // public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
877 // // always keep .java file
878 // if (file.getFileName().toString().endsWith(".java"))
879 // return super.visitFile(file, attrs);
881 // Path relPath = srcDir.relativize(file);
882 // Path bundlePath = bundleDir.resolve(relPath);
883 // if (Files.exists(bundlePath)) {
884 // Files.delete(file);
885 // logger.log(DEBUG, () -> "Removed " + file + " from sources.");
887 // return super.visitFile(file, attrs);
893 Path jarPath
= bundleDir
.getParent().resolve(bundleDir
.getFileName() + ".jar");
894 Path manifestPath
= bundleDir
.resolve("META-INF/MANIFEST.MF");
896 try (InputStream in
= Files
.newInputStream(manifestPath
)) {
897 manifest
= new Manifest(in
);
899 try (JarOutputStream jarOut
= new JarOutputStream(Files
.newOutputStream(jarPath
), manifest
)) {
900 jarOut
.setLevel(Deflater
.DEFAULT_COMPRESSION
);
901 Files
.walkFileTree(bundleDir
, new SimpleFileVisitor
<Path
>() {
904 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
905 if (file
.getFileName().toString().equals("MANIFEST.MF"))
906 return super.visitFile(file
, attrs
);
907 JarEntry entry
= new JarEntry(bundleDir
.relativize(file
).toString());
908 jarOut
.putNextEntry(entry
);
909 Files
.copy(file
, jarOut
);
910 return super.visitFile(file
, attrs
);
915 deleteDirectory(bundleDir
);
919 /** For development purproses. */
920 public static void main(String
[] args
) {
921 Path factoryBase
= Paths
.get("../../output/a2").toAbsolutePath().normalize();
922 A2Factory factory
= new A2Factory(factoryBase
);
924 Path descriptorsBase
= Paths
.get("../tp").toAbsolutePath().normalize();
926 // factory.processSingleM2ArtifactDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache").resolve("org.apache.xml.resolver.bnd"));
927 // factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache/apache-sshd"));
928 // factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty"));
929 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.osgi"));
930 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp"));
931 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp"));
932 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.apache"));
933 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk"));
934 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats"));
935 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis"));
939 factory
.processEclipseArchive(
940 descriptorsBase
.resolve("org.argeo.tp.eclipse.equinox").resolve("eclipse-equinox"));
941 factory
.processEclipseArchive(descriptorsBase
.resolve("org.argeo.tp.eclipse.rwt").resolve("eclipse-rwt"));
942 factory
.processEclipseArchive(descriptorsBase
.resolve("org.argeo.tp.eclipse.rap").resolve("eclipse-rap"));
943 factory
.processEclipseArchive(descriptorsBase
.resolve("org.argeo.tp.eclipse.swt").resolve("eclipse-swt"));
944 factory
.processEclipseArchive(descriptorsBase
.resolve("org.argeo.tp.eclipse.swt").resolve("eclipse-nebula"));
945 factory
.processEclipseArchive(descriptorsBase
.resolve("org.argeo.tp.eclipse.swt").resolve("eclipse-equinox"));
946 factory
.processEclipseArchive(descriptorsBase
.resolve("org.argeo.tp.eclipse.rcp").resolve("eclipse-rcp"));
947 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.eclipse.rcp"));
950 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.sdk"));
951 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp"));
952 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.apache"));
953 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.jetty"));
954 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.jcr"));
955 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.formats"));
956 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.poi"));
957 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.gis"));