1 package org
.argeo
.slc
.build
;
3 import static java
.lang
.System
.Logger
.Level
.DEBUG
;
4 import static org
.argeo
.slc
.ManifestConstants
.BUNDLE_LICENSE
;
5 import static org
.argeo
.slc
.ManifestConstants
.BUNDLE_SYMBOLICNAME
;
6 import static org
.argeo
.slc
.ManifestConstants
.BUNDLE_VERSION
;
7 import static org
.argeo
.slc
.ManifestConstants
.SLC_ORIGIN_M2
;
9 import java
.io
.FileNotFoundException
;
10 import java
.io
.IOException
;
11 import java
.io
.InputStream
;
12 import java
.io
.OutputStream
;
13 import java
.io
.Writer
;
14 import java
.lang
.System
.Logger
;
15 import java
.lang
.System
.Logger
.Level
;
17 import java
.nio
.file
.DirectoryStream
;
18 import java
.nio
.file
.FileSystem
;
19 import java
.nio
.file
.FileSystems
;
20 import java
.nio
.file
.FileVisitResult
;
21 import java
.nio
.file
.Files
;
22 import java
.nio
.file
.Path
;
23 import java
.nio
.file
.PathMatcher
;
24 import java
.nio
.file
.Paths
;
25 import java
.nio
.file
.SimpleFileVisitor
;
26 import java
.nio
.file
.attribute
.BasicFileAttributes
;
27 import java
.util
.ArrayList
;
28 import java
.util
.HashMap
;
29 import java
.util
.List
;
31 import java
.util
.Properties
;
32 import java
.util
.TreeMap
;
33 import java
.util
.jar
.Attributes
;
34 import java
.util
.jar
.JarEntry
;
35 import java
.util
.jar
.JarInputStream
;
36 import java
.util
.jar
.JarOutputStream
;
37 import java
.util
.jar
.Manifest
;
39 import org
.argeo
.slc
.DefaultNameVersion
;
40 import org
.argeo
.slc
.ManifestConstants
;
41 import org
.argeo
.slc
.NameVersion
;
42 import org
.argeo
.slc
.build
.m2
.DefaultArtifact
;
43 import org
.argeo
.slc
.build
.m2
.MavenConventionsUtils
;
45 import aQute
.bnd
.osgi
.Analyzer
;
46 import aQute
.bnd
.osgi
.Jar
;
48 public class A2Factory
{
49 private final static Logger logger
= System
.getLogger(A2Factory
.class.getName());
51 private final static String COMMON_BND
= "common.bnd";
53 private Path originBase
;
54 private Path factoryBase
;
56 /** key is URI prefix, value list of base URLs */
57 private Map
<String
, List
<String
>> mirrors
= new HashMap
<String
, List
<String
>>();
59 public A2Factory(Path originBase
, Path factoryBase
) {
61 this.originBase
= originBase
;
62 this.factoryBase
= factoryBase
;
64 // TODO make it configurable
65 List
<String
> eclipseMirrors
= new ArrayList
<>();
66 eclipseMirrors
.add("https://archive.eclipse.org/");
68 mirrors
.put("http://www.eclipse.org/downloads", eclipseMirrors
);
71 public void processCategory(Path targetCategoryBase
) {
73 DirectoryStream
<Path
> bnds
= Files
.newDirectoryStream(targetCategoryBase
,
74 (p
) -> p
.getFileName().toString().endsWith(".bnd")
75 && !p
.getFileName().toString().equals(COMMON_BND
));
77 processSingleM2ArtifactDistributionUnit(p
);
80 DirectoryStream
<Path
> dus
= Files
.newDirectoryStream(targetCategoryBase
, (p
) -> Files
.isDirectory(p
));
81 for (Path duDir
: dus
) {
82 processM2BasedDistributionUnit(duDir
);
84 } catch (IOException e
) {
85 throw new RuntimeException("Cannot process category " + targetCategoryBase
, e
);
89 public void processSingleM2ArtifactDistributionUnit(Path bndFile
) {
91 String category
= bndFile
.getParent().getFileName().toString();
92 Path targetCategoryBase
= factoryBase
.resolve(category
);
93 Properties fileProps
= new Properties();
94 try (InputStream in
= Files
.newInputStream(bndFile
)) {
98 String m2Coordinates
= fileProps
.getProperty(SLC_ORIGIN_M2
.toString());
99 if (m2Coordinates
== null)
100 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile
);
101 DefaultArtifact artifact
= new DefaultArtifact(m2Coordinates
);
102 URL url
= MavenConventionsUtils
.mavenCentralUrl(artifact
);
103 Path downloaded
= download(url
, originBase
, artifact
.toM2Coordinates() + ".jar");
105 Path targetBundleDir
= processBndJar(downloaded
, targetCategoryBase
, fileProps
);
107 downloadAndProcessM2Sources(artifact
, targetBundleDir
);
109 createJar(targetBundleDir
);
110 } catch (Exception e
) {
111 throw new RuntimeException("Cannot process " + bndFile
, e
);
115 public void processM2BasedDistributionUnit(Path duDir
) {
117 String category
= duDir
.getParent().getFileName().toString();
118 Path targetCategoryBase
= factoryBase
.resolve(category
);
119 Path commonBnd
= duDir
.resolve(COMMON_BND
);
120 Properties commonProps
= new Properties();
121 try (InputStream in
= Files
.newInputStream(commonBnd
)) {
122 commonProps
.load(in
);
125 String m2Version
= commonProps
.getProperty(SLC_ORIGIN_M2
.toString());
126 if (!m2Version
.startsWith(":")) {
127 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version
);
129 m2Version
= m2Version
.substring(1);
131 // String license = commonProps.getProperty(BUNDLE_LICENSE.toString());
133 DirectoryStream
<Path
> ds
= Files
.newDirectoryStream(duDir
,
134 (p
) -> p
.getFileName().toString().endsWith(".bnd")
135 && !p
.getFileName().toString().equals(COMMON_BND
));
137 Properties fileProps
= new Properties();
138 try (InputStream in
= Files
.newInputStream(p
)) {
141 String m2Coordinates
= fileProps
.getProperty(SLC_ORIGIN_M2
.toString());
142 DefaultArtifact artifact
= new DefaultArtifact(m2Coordinates
);
144 // temporary rewrite, for migration
145 String localLicense
= fileProps
.getProperty(BUNDLE_LICENSE
.toString());
146 if (localLicense
!= null || artifact
.getVersion() != null) {
147 fileProps
.remove(BUNDLE_LICENSE
.toString());
148 fileProps
.put(SLC_ORIGIN_M2
.toString(), artifact
.getGroupId() + ":" + artifact
.getArtifactId());
149 try (Writer writer
= Files
.newBufferedWriter(p
)) {
150 for (Object key
: fileProps
.keySet()) {
151 String value
= fileProps
.getProperty(key
.toString());
152 writer
.write(key
+ ": " + value
+ '\n');
154 logger
.log(DEBUG
, () -> "Migrated " + p
);
158 artifact
.setVersion(m2Version
);
159 URL url
= MavenConventionsUtils
.mavenCentralUrl(artifact
);
160 Path downloaded
= download(url
, originBase
, artifact
.toM2Coordinates() + ".jar");
162 // prepare manifest entries
163 Properties mergeProps
= new Properties();
164 mergeProps
.putAll(commonProps
);
166 // Map<String, String> entries = new HashMap<>();
167 // for (Object key : commonProps.keySet()) {
168 // entries.put(key.toString(), commonProps.getProperty(key.toString()));
170 fileEntries
: for (Object key
: fileProps
.keySet()) {
171 if (ManifestConstants
.SLC_ORIGIN_M2
.toString().equals(key
))
172 continue fileEntries
;
173 String value
= fileProps
.getProperty(key
.toString());
174 Object previousValue
= mergeProps
.put(key
.toString(), value
);
175 if (previousValue
!= null) {
176 logger
.log(Level
.WARNING
,
177 downloaded
+ ": " + key
+ " was " + previousValue
+ ", overridden with " + value
);
180 mergeProps
.put(ManifestConstants
.SLC_ORIGIN_M2
.toString(), artifact
.toM2Coordinates());
181 Path targetBundleDir
= processBndJar(downloaded
, targetCategoryBase
, mergeProps
);
182 // logger.log(Level.DEBUG, () -> "Processed " + downloaded);
185 downloadAndProcessM2Sources(artifact
, targetBundleDir
);
187 createJar(targetBundleDir
);
189 } catch (IOException e
) {
190 throw new RuntimeException("Cannot process " + duDir
, e
);
195 protected void downloadAndProcessM2Sources(DefaultArtifact artifact
, Path targetBundleDir
) throws IOException
{
196 DefaultArtifact sourcesArtifact
= new DefaultArtifact(artifact
.toM2Coordinates(), "sources");
197 URL sourcesUrl
= MavenConventionsUtils
.mavenCentralUrl(sourcesArtifact
);
198 Path sourcesDownloaded
= download(sourcesUrl
, originBase
, artifact
.toM2Coordinates() + ".sources.jar");
199 processM2SourceJar(sourcesDownloaded
, targetBundleDir
);
200 logger
.log(Level
.DEBUG
, () -> "Processed source " + sourcesDownloaded
);
204 protected Path
processBndJar(Path downloaded
, Path targetCategoryBase
, Properties fileProps
) {
207 Map
<String
, String
> additionalEntries
= new TreeMap
<>();
208 boolean doNotModify
= Boolean
.parseBoolean(fileProps
209 .getOrDefault(ManifestConstants
.SLC_ORIGIN_MANIFEST_NOT_MODIFIED
.toString(), "false").toString());
212 fileEntries
: for (Object key
: fileProps
.keySet()) {
213 if (ManifestConstants
.SLC_ORIGIN_M2
.toString().equals(key
))
214 continue fileEntries
;
215 String value
= fileProps
.getProperty(key
.toString());
216 additionalEntries
.put(key
.toString(), value
);
220 try (Analyzer bndAnalyzer
= new Analyzer()) {
221 bndAnalyzer
.setProperties(fileProps
);
222 Jar jar
= new Jar(downloaded
.toFile());
223 bndAnalyzer
.setJar(jar
);
224 Manifest manifest
= bndAnalyzer
.calcManifest();
226 keys
: for (Object key
: manifest
.getMainAttributes().keySet()) {
227 Object value
= manifest
.getMainAttributes().get(key
);
229 switch (key
.toString()) {
231 case "Bnd-LastModified":
235 additionalEntries
.put(key
.toString(), value
.toString());
236 // logger.log(DEBUG, () -> key + "=" + value);
241 Path targetBundleDir
= processBundleJar(downloaded
, targetCategoryBase
, additionalEntries
);
242 logger
.log(Level
.DEBUG
, () -> "Processed " + downloaded
);
243 return targetBundleDir
;
244 } catch (Exception e
) {
245 throw new RuntimeException("Cannot BND process " + downloaded
, e
);
250 protected void processM2SourceJar(Path file
, Path targetBundleDir
) throws IOException
{
251 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
252 Path targetSourceDir
= targetBundleDir
.resolve("OSGI-OPT/src");
254 // TODO make it less dangerous?
255 if (Files
.exists(targetSourceDir
)) {
256 deleteDirectory(targetSourceDir
);
258 Files
.createDirectories(targetSourceDir
);
263 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
264 if (entry
.isDirectory())
266 if (entry
.getName().startsWith("META-INF"))// skip META-INF entries
268 Path target
= targetSourceDir
.resolve(entry
.getName());
269 Files
.createDirectories(target
.getParent());
270 Files
.copy(jarIn
, target
);
271 logger
.log(Level
.TRACE
, () -> "Copied source " + target
);
277 public void processEclipseArchive(Path duDir
) {
279 String category
= duDir
.getParent().getFileName().toString();
280 Path targetCategoryBase
= factoryBase
.resolve(category
);
281 Files
.createDirectories(targetCategoryBase
);
282 Files
.createDirectories(originBase
);
284 Path commonBnd
= duDir
.resolve(COMMON_BND
);
285 Properties commonProps
= new Properties();
286 try (InputStream in
= Files
.newInputStream(commonBnd
)) {
287 commonProps
.load(in
);
289 Properties includes
= new Properties();
290 try (InputStream in
= Files
.newInputStream(duDir
.resolve("includes.properties"))) {
293 String url
= commonProps
.getProperty(ManifestConstants
.SLC_ORIGIN_URI
.toString());
294 Path downloaded
= tryDownload(url
, originBase
);
296 FileSystem zipFs
= FileSystems
.newFileSystem(downloaded
, null);
298 List
<PathMatcher
> pathMatchers
= new ArrayList
<>();
299 for (Object pattern
: includes
.keySet()) {
300 PathMatcher pathMatcher
= zipFs
.getPathMatcher("glob:/" + pattern
);
301 pathMatchers
.add(pathMatcher
);
304 Files
.walkFileTree(zipFs
.getRootDirectories().iterator().next(), new SimpleFileVisitor
<Path
>() {
307 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
308 pathMatchers
: for (PathMatcher pathMatcher
: pathMatchers
) {
309 if (pathMatcher
.matches(file
)) {
310 // Path target = targetBase.resolve(file.getFileName().toString());
311 // if (!Files.exists(target)) {
312 // Files.copy(file, target);
313 // logger.log(Level.DEBUG, () -> "Copied " + target + " from " + downloaded);
315 // logger.log(Level.DEBUG, () -> target + " already exists.");
318 if (file
.getFileName().toString().contains(".source_")) {
319 processEclipseSourceJar(file
, targetCategoryBase
);
320 logger
.log(Level
.DEBUG
, () -> "Processed source " + file
);
323 processBundleJar(file
, targetCategoryBase
, new HashMap
<>());
324 logger
.log(Level
.DEBUG
, () -> "Processed " + file
);
326 continue pathMatchers
;
329 return super.visitFile(file
, attrs
);
333 DirectoryStream
<Path
> dirs
= Files
.newDirectoryStream(targetCategoryBase
, (p
) -> Files
.isDirectory(p
));
334 for (Path dir
: dirs
) {
337 } catch (IOException e
) {
338 throw new RuntimeException("Cannot process " + duDir
, e
);
343 protected Path
processBundleJar(Path file
, Path targetBase
, Map
<String
, String
> entries
) throws IOException
{
344 NameVersion nameVersion
;
345 Path targetBundleDir
;
346 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
347 Manifest manifest
= new Manifest(jarIn
.getManifest());
349 // remove problematic entries in MANIFEST
350 // Set<String> entriesToDelete = new HashSet<>();
351 // for (String key : manifest.getEntries().keySet()) {
352 //// logger.log(DEBUG, "## " + key);
353 // Attributes attrs = manifest.getAttributes(key);
354 // for (Object attrName : attrs.keySet()) {
355 //// logger.log(DEBUG, attrName + "=" + attrs.get(attrName));
356 // if ("Specification-Version".equals(attrName.toString())
357 // || "Implementation-Version".equals(attrName.toString())) {
358 // entriesToDelete.add(key);
363 // for (String key : entriesToDelete) {
364 // manifest.getEntries().remove(key);
367 String symbolicNameFromEntries
= entries
.get(BUNDLE_SYMBOLICNAME
.toString());
368 String versionFromEntries
= entries
.get(BUNDLE_VERSION
.toString());
370 if (symbolicNameFromEntries
!= null && versionFromEntries
!= null) {
371 nameVersion
= new DefaultNameVersion(symbolicNameFromEntries
, versionFromEntries
);
373 nameVersion
= nameVersionFromManifest(manifest
);
374 if (versionFromEntries
!= null && !nameVersion
.getVersion().equals(versionFromEntries
)) {
375 logger
.log(Level
.WARNING
, "Original version is " + nameVersion
.getVersion()
376 + " while new version is " + versionFromEntries
);
379 targetBundleDir
= targetBase
.resolve(nameVersion
.getName() + "." + nameVersion
.getBranch());
381 // TODO make it less dangerous?
382 if (Files
.exists(targetBundleDir
)) {
383 deleteDirectory(targetBundleDir
);
388 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
389 if (entry
.isDirectory())
391 Path target
= targetBundleDir
.resolve(entry
.getName());
392 Files
.createDirectories(target
.getParent());
393 Files
.copy(jarIn
, target
);
394 logger
.log(Level
.TRACE
, () -> "Copied " + target
);
398 Path manifestPath
= targetBundleDir
.resolve("META-INF/MANIFEST.MF");
399 Files
.createDirectories(manifestPath
.getParent());
400 for (String key
: entries
.keySet()) {
401 String value
= entries
.get(key
);
402 Object previousValue
= manifest
.getMainAttributes().putValue(key
, value
);
403 if (previousValue
!= null && !previousValue
.equals(value
)) {
404 logger
.log(Level
.WARNING
,
405 file
.getFileName() + ": " + key
+ " was " + previousValue
+ ", overridden with " + value
);
408 try (OutputStream out
= Files
.newOutputStream(manifestPath
)) {
412 return targetBundleDir
;
415 protected void processEclipseSourceJar(Path file
, Path targetBase
) throws IOException
{
416 // NameVersion nameVersion;
417 Path targetBundleDir
;
418 try (JarInputStream jarIn
= new JarInputStream(Files
.newInputStream(file
), false)) {
419 Manifest manifest
= jarIn
.getManifest();
420 // nameVersion = nameVersionFromManifest(manifest);
422 String
[] relatedBundle
= manifest
.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
423 String version
= relatedBundle
[1].substring("version=\"".length());
424 version
= version
.substring(0, version
.length() - 1);
425 NameVersion nameVersion
= new DefaultNameVersion(relatedBundle
[0], version
);
426 targetBundleDir
= targetBase
.resolve(nameVersion
.getName() + "." + nameVersion
.getBranch());
428 Path targetSourceDir
= targetBundleDir
.resolve("OSGI-OPT/src");
430 // TODO make it less dangerous?
431 if (Files
.exists(targetSourceDir
)) {
432 deleteDirectory(targetSourceDir
);
434 Files
.createDirectories(targetSourceDir
);
439 entries
: while ((entry
= jarIn
.getNextJarEntry()) != null) {
440 if (entry
.isDirectory())
442 if (entry
.getName().startsWith("META-INF"))// skip META-INF entries
444 Path target
= targetSourceDir
.resolve(entry
.getName());
445 Files
.createDirectories(target
.getParent());
446 Files
.copy(jarIn
, target
);
447 logger
.log(Level
.TRACE
, () -> "Copied source " + target
);
451 // Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
452 // Files.createDirectories(manifestPath.getParent());
453 // try (OutputStream out = Files.newOutputStream(manifestPath)) {
454 // manifest.write(out);
460 static void deleteDirectory(Path path
) throws IOException
{
461 if (!Files
.exists(path
))
463 Files
.walkFileTree(path
, new SimpleFileVisitor
<Path
>() {
465 public FileVisitResult
postVisitDirectory(Path directory
, IOException e
) throws IOException
{
468 Files
.delete(directory
);
469 return FileVisitResult
.CONTINUE
;
473 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
475 return FileVisitResult
.CONTINUE
;
480 protected NameVersion
nameVersionFromManifest(Manifest manifest
) {
481 Attributes attrs
= manifest
.getMainAttributes();
483 String symbolicName
= attrs
.getValue(ManifestConstants
.BUNDLE_SYMBOLICNAME
.toString());
484 if (symbolicName
== null)
486 // make sure there is no directive
487 symbolicName
= symbolicName
.split(";")[0];
489 String version
= attrs
.getValue(ManifestConstants
.BUNDLE_VERSION
.toString());
490 return new DefaultNameVersion(symbolicName
, version
);
493 protected Path
tryDownload(String uri
, Path dir
) throws IOException
{
495 List
<String
> urlBases
= null;
496 String uriPrefix
= null;
497 uriPrefixes
: for (String uriPref
: mirrors
.keySet()) {
498 if (uri
.startsWith(uriPref
)) {
499 if (mirrors
.get(uriPref
).size() > 0) {
500 urlBases
= mirrors
.get(uriPref
);
506 if (urlBases
== null)
508 return download(new URL(uri
), dir
, null);
509 } catch (FileNotFoundException e
) {
510 throw new FileNotFoundException("Cannot find " + uri
);
514 for (String urlBase
: urlBases
) {
515 String relativePath
= uri
.substring(uriPrefix
.length());
516 URL url
= new URL(urlBase
+ relativePath
);
518 return download(url
, dir
, null);
519 } catch (FileNotFoundException e
) {
520 logger
.log(Level
.WARNING
, "Cannot download " + url
+ ", trying another mirror");
523 throw new FileNotFoundException("Cannot find " + uri
);
526 // protected String simplifyName(URL u) {
527 // String name = u.getPath().substring(u.getPath().lastIndexOf('/') + 1);
531 protected Path
download(URL url
, Path dir
, String name
) throws IOException
{
535 name
= url
.getPath().substring(url
.getPath().lastIndexOf('/') + 1);
538 dest
= dir
.resolve(name
);
539 if (Files
.exists(dest
)) {
540 logger
.log(Level
.TRACE
, () -> "File " + dest
+ " already exists for " + url
+ ", not downloading again");
544 try (InputStream in
= url
.openStream()) {
545 Files
.copy(in
, dest
);
546 logger
.log(Level
.DEBUG
, () -> "Downloaded " + dest
+ " from " + url
);
551 protected Path
createJar(Path bundleDir
) throws IOException
{
552 Path jarPath
= bundleDir
.getParent().resolve(bundleDir
.getFileName() + ".jar");
553 Path manifestPath
= bundleDir
.resolve("META-INF/MANIFEST.MF");
555 try (InputStream in
= Files
.newInputStream(manifestPath
)) {
556 manifest
= new Manifest(in
);
558 try (JarOutputStream jarOut
= new JarOutputStream(Files
.newOutputStream(jarPath
), manifest
)) {
559 Files
.walkFileTree(bundleDir
, new SimpleFileVisitor
<Path
>() {
562 public FileVisitResult
visitFile(Path file
, BasicFileAttributes attrs
) throws IOException
{
563 if (file
.getFileName().toString().equals("MANIFEST.MF"))
564 return super.visitFile(file
, attrs
);
565 JarEntry entry
= new JarEntry(bundleDir
.relativize(file
).toString());
566 jarOut
.putNextEntry(entry
);
567 Files
.copy(file
, jarOut
);
568 return super.visitFile(file
, attrs
);
573 deleteDirectory(bundleDir
);
577 public static void main(String
[] args
) {
578 Path originBase
= Paths
.get("../output/origin").toAbsolutePath().normalize();
579 Path factoryBase
= Paths
.get("../output/a2").toAbsolutePath().normalize();
580 A2Factory factory
= new A2Factory(originBase
, factoryBase
);
582 Path descriptorsBase
= Paths
.get("../tp").toAbsolutePath().normalize();
585 factory
.processEclipseArchive(
586 descriptorsBase
.resolve("org.argeo.tp.eclipse.equinox").resolve("eclipse-equinox"));
587 factory
.processEclipseArchive(descriptorsBase
.resolve("org.argeo.tp.eclipse.rap").resolve("eclipse-rap"));
588 factory
.processEclipseArchive(descriptorsBase
.resolve("org.argeo.tp.eclipse.rcp").resolve("eclipse-rcp"));
591 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp"));
592 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.apache"));
593 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.jetty"));
594 factory
.processCategory(descriptorsBase
.resolve("org.argeo.tp.sdk"));