]> git.argeo.org Git - gpl/argeo-slc.git/blob - factory/A2Factory.java
Prepare next development cycle
[gpl/argeo-slc.git] / factory / A2Factory.java
1 package org.argeo.slc.factory;
2
3 import static java.lang.System.Logger.Level.DEBUG;
4 import static org.argeo.slc.ManifestConstants.BUNDLE_SYMBOLICNAME;
5 import static org.argeo.slc.ManifestConstants.BUNDLE_VERSION;
6 import static org.argeo.slc.ManifestConstants.EXPORT_PACKAGE;
7 import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2;
8 import static org.argeo.slc.ManifestConstants.SLC_ORIGIN_M2_REPO;
9
10 import java.io.FileNotFoundException;
11 import java.io.IOException;
12 import java.io.InputStream;
13 import java.io.OutputStream;
14 import java.lang.System.Logger;
15 import java.lang.System.Logger.Level;
16 import java.net.URL;
17 import java.nio.file.DirectoryStream;
18 import java.nio.file.FileSystem;
19 import java.nio.file.FileSystems;
20 import java.nio.file.FileVisitResult;
21 import java.nio.file.Files;
22 import java.nio.file.Path;
23 import java.nio.file.PathMatcher;
24 import java.nio.file.Paths;
25 import java.nio.file.SimpleFileVisitor;
26 import java.nio.file.StandardOpenOption;
27 import java.nio.file.attribute.BasicFileAttributes;
28 import java.util.ArrayList;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Objects;
33 import java.util.Properties;
34 import java.util.TreeMap;
35 import java.util.jar.Attributes;
36 import java.util.jar.JarEntry;
37 import java.util.jar.JarInputStream;
38 import java.util.jar.JarOutputStream;
39 import java.util.jar.Manifest;
40 import java.util.zip.Deflater;
41
42 import org.argeo.slc.DefaultCategoryNameVersion;
43 import org.argeo.slc.DefaultNameVersion;
44 import org.argeo.slc.ManifestConstants;
45 import org.argeo.slc.NameVersion;
46 import org.argeo.slc.factory.m2.Artifact;
47 import org.argeo.slc.factory.m2.DefaultArtifact;
48 import org.argeo.slc.factory.m2.MavenConventionsUtils;
49
50 import aQute.bnd.osgi.Analyzer;
51 import aQute.bnd.osgi.Jar;
52
53 /** The central class for A2 packaging. */
54 public class A2Factory {
55 private final static Logger logger = System.getLogger(A2Factory.class.getName());
56
57 private final static String COMMON_BND = "common.bnd";
58 private final static String MERGE_BND = "merge.bnd";
59
60 private Path originBase;
61 private Path a2Base;
62 private Path a2LibBase;
63 private Path descriptorsBase;
64
65 private Properties uris = new Properties();
66
67 private boolean includeSources = true;
68
69 /** key is URI prefix, value list of base URLs */
70 private Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
71
72 public A2Factory(Path a2Base, Path descriptorsBase, boolean includeSources) {
73 Objects.requireNonNull(a2Base);
74 Objects.requireNonNull(descriptorsBase);
75 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/slc/origin");
76 this.a2Base = a2Base;
77 this.a2LibBase = a2Base.resolve("lib");
78 this.descriptorsBase = descriptorsBase;
79 if (!Files.exists(this.descriptorsBase))
80 throw new IllegalArgumentException(this.descriptorsBase + " does not exist");
81 this.includeSources = includeSources;
82
83 Path urisPath = this.descriptorsBase.resolve("uris.properties");
84 if (Files.exists(urisPath)) {
85 try (InputStream in = Files.newInputStream(urisPath)) {
86 uris.load(in);
87 } catch (IOException e) {
88 throw new IllegalStateException("Cannot load " + urisPath, e);
89 }
90 }
91
92 // TODO make it configurable
93 List<String> eclipseMirrors = new ArrayList<>();
94 eclipseMirrors.add("https://archive.eclipse.org/");
95 eclipseMirrors.add("http://ftp-stud.hs-esslingen.de/Mirrors/eclipse/");
96 eclipseMirrors.add("http://ftp.fau.de/eclipse/");
97
98 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
99 }
100
101 /*
102 * MAVEN ORIGIN
103 */
104
105 /** Process a whole category/group id. */
106 public void processCategory(Path categoryRelativePath) {
107 try {
108 Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath);
109 DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
110 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
111 && !p.getFileName().toString().equals(MERGE_BND));
112 for (Path p : bnds) {
113 processSingleM2ArtifactDistributionUnit(p);
114 }
115
116 DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
117 for (Path duDir : dus) {
118 processM2BasedDistributionUnit(duDir);
119 }
120 } catch (IOException e) {
121 throw new RuntimeException("Cannot process category " + categoryRelativePath, e);
122 }
123 }
124
125 /** Process a standalone Maven artifact. */
126 public void processSingleM2ArtifactDistributionUnit(Path bndFile) {
127 try {
128 // String category = bndFile.getParent().getFileName().toString();
129 Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent());
130 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
131
132 Properties fileProps = new Properties();
133 try (InputStream in = Files.newInputStream(bndFile)) {
134 fileProps.load(in);
135 }
136 String repoStr = fileProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
137 ? fileProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
138 : null;
139
140 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
141 // use file name as symbolic name
142 String symbolicName = bndFile.getFileName().toString();
143 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
144 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
145 }
146
147 String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
148 if (m2Coordinates == null)
149 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
150 DefaultArtifact artifact = new DefaultArtifact(m2Coordinates);
151 URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
152 Path downloaded = download(url, originBase, artifact);
153
154 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact);
155
156 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
157
158 createJar(targetBundleDir);
159 } catch (Exception e) {
160 throw new RuntimeException("Cannot process " + bndFile, e);
161 }
162 }
163
164 /** Process multiple Maven artifacts. */
165 public void processM2BasedDistributionUnit(Path duDir) {
166 try {
167 // String category = duDir.getParent().getFileName().toString();
168 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
169 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
170
171 // merge
172 Path mergeBnd = duDir.resolve(MERGE_BND);
173 if (Files.exists(mergeBnd)) {
174 mergeM2Artifacts(mergeBnd);
175 // return;
176 }
177
178 Path commonBnd = duDir.resolve(COMMON_BND);
179 if (!Files.exists(commonBnd)) {
180 return;
181 }
182 Properties commonProps = new Properties();
183 try (InputStream in = Files.newInputStream(commonBnd)) {
184 commonProps.load(in);
185 }
186
187 String m2Version = commonProps.getProperty(SLC_ORIGIN_M2.toString());
188 if (m2Version == null) {
189 logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
190 return;// ignore, this is probably an Eclipse archive
191 }
192 if (!m2Version.startsWith(":")) {
193 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
194 }
195 m2Version = m2Version.substring(1);
196
197 DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
198 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
199 && !p.getFileName().toString().equals(MERGE_BND));
200 for (Path p : ds) {
201 Properties fileProps = new Properties();
202 try (InputStream in = Files.newInputStream(p)) {
203 fileProps.load(in);
204 }
205 String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
206 DefaultArtifact artifact = new DefaultArtifact(m2Coordinates);
207
208 // temporary rewrite, for migration
209 // String localLicense = fileProps.getProperty(BUNDLE_LICENSE.toString());
210 // if (localLicense != null || artifact.getVersion() != null) {
211 // fileProps.remove(BUNDLE_LICENSE.toString());
212 // fileProps.put(SLC_ORIGIN_M2.toString(), artifact.getGroupId() + ":" + artifact.getArtifactId());
213 // try (Writer writer = Files.newBufferedWriter(p)) {
214 // for (Object key : fileProps.keySet()) {
215 // String value = fileProps.getProperty(key.toString());
216 // writer.write(key + ": " + value + '\n');
217 // }
218 // logger.log(DEBUG, () -> "Migrated " + p);
219 // }
220 // }
221
222 artifact.setVersion(m2Version);
223
224 // prepare manifest entries
225 Properties mergeProps = new Properties();
226 mergeProps.putAll(commonProps);
227
228 fileEntries: for (Object key : fileProps.keySet()) {
229 if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
230 continue fileEntries;
231 String value = fileProps.getProperty(key.toString());
232 Object previousValue = mergeProps.put(key.toString(), value);
233 if (previousValue != null) {
234 logger.log(Level.WARNING,
235 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
236 }
237 }
238 mergeProps.put(ManifestConstants.SLC_ORIGIN_M2.toString(), artifact.toM2Coordinates());
239 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
240 // use file name as symbolic name
241 String symbolicName = p.getFileName().toString();
242 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
243 mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
244 }
245
246 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
247 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
248 : null;
249
250 // download
251 URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
252 Path downloaded = download(url, originBase, artifact);
253
254 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact);
255 // logger.log(Level.DEBUG, () -> "Processed " + downloaded);
256
257 // sources
258 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
259
260 createJar(targetBundleDir);
261 }
262 } catch (IOException e) {
263 throw new RuntimeException("Cannot process " + duDir, e);
264 }
265
266 }
267
268 /** Merge multiple Maven artifacts. */
269 protected void mergeM2Artifacts(Path mergeBnd) throws IOException {
270 Path duDir = mergeBnd.getParent();
271 String category = duDir.getParent().getFileName().toString();
272 Path targetCategoryBase = a2Base.resolve(category);
273
274 Properties mergeProps = new Properties();
275 try (InputStream in = Files.newInputStream(mergeBnd)) {
276 mergeProps.load(in);
277 }
278
279 // Version
280 String m2Version = mergeProps.getProperty(SLC_ORIGIN_M2.toString());
281 if (m2Version == null) {
282 logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
283 return;// ignore, this is probably an Eclipse archive
284 }
285 if (!m2Version.startsWith(":")) {
286 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
287 }
288 m2Version = m2Version.substring(1);
289 mergeProps.put(ManifestConstants.BUNDLE_VERSION.toString(), m2Version);
290
291 String artifactsStr = mergeProps.getProperty(ManifestConstants.SLC_ORIGIN_M2_MERGE.toString());
292 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
293 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
294 : null;
295
296 String bundleSymbolicName = mergeProps.getProperty(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
297 if (bundleSymbolicName == null)
298 throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
299 DefaultCategoryNameVersion nameVersion = new DefaultArtifact(
300 category + ":" + bundleSymbolicName + ":" + m2Version);
301 Path targetBundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
302
303 String[] artifacts = artifactsStr.split(",");
304 artifacts: for (String str : artifacts) {
305 String m2Coordinates = str.trim();
306 if ("".equals(m2Coordinates))
307 continue artifacts;
308 DefaultArtifact artifact = new DefaultArtifact(m2Coordinates.trim());
309 if (artifact.getVersion() == null)
310 artifact.setVersion(m2Version);
311 URL url = MavenConventionsUtils.mavenRepoUrl(repoStr, artifact);
312 Path downloaded = download(url, originBase, artifact);
313 JarEntry entry;
314 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
315 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
316 if (entry.isDirectory())
317 continue entries;
318 else if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
319 continue entries;
320 else if (entry.getName().startsWith("META-INF/versions/"))
321 continue entries;
322 else if (entry.getName().startsWith("META-INF/maven/"))
323 continue entries;
324 else if (entry.getName().equals("module-info.class"))
325 continue entries;
326 else if (entry.getName().equals("META-INF/NOTICE"))
327 continue entries;
328 else if (entry.getName().equals("META-INF/NOTICE.txt"))
329 continue entries;
330 else if (entry.getName().equals("META-INF/LICENSE"))
331 continue entries;
332 else if (entry.getName().equals("META-INF/LICENSE.md"))
333 continue entries;
334 else if (entry.getName().equals("META-INF/LICENSE-notice.md"))
335 continue entries;
336 else if (entry.getName().equals("META-INF/DEPENDENCIES"))
337 continue entries;
338 if (entry.getName().startsWith(".cache/")) // Apache SSHD
339 continue entries;
340 Path target = targetBundleDir.resolve(entry.getName());
341 Files.createDirectories(target.getParent());
342 if (!Files.exists(target)) {
343 Files.copy(jarIn, target);
344 } else {
345 if (entry.getName().startsWith("META-INF/services/")) {
346 try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
347 out.write("\n".getBytes());
348 jarIn.transferTo(out);
349 if (logger.isLoggable(DEBUG))
350 logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName());
351 }
352 } else if (entry.getName().startsWith("org/apache/batik/")) {
353 logger.log(Level.WARNING, "Skip " + entry.getName());
354 continue entries;
355 } else {
356 throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
357 }
358 }
359 logger.log(Level.TRACE, () -> "Copied " + target);
360 }
361
362 }
363 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
364 }
365
366 // additional service files
367 Path servicesDir = duDir.resolve("services");
368 if (Files.exists(servicesDir)) {
369 for (Path p : Files.newDirectoryStream(servicesDir)) {
370 Path target = targetBundleDir.resolve("META-INF/services/").resolve(p.getFileName());
371 try (InputStream in = Files.newInputStream(p);
372 OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) {
373 out.write("\n".getBytes());
374 in.transferTo(out);
375 if (logger.isLoggable(DEBUG))
376 logger.log(DEBUG, "Appended " + p);
377 }
378 }
379 }
380
381 Map<String, String> entries = new TreeMap<>();
382 try (Analyzer bndAnalyzer = new Analyzer()) {
383 bndAnalyzer.setProperties(mergeProps);
384 Jar jar = new Jar(targetBundleDir.toFile());
385 bndAnalyzer.setJar(jar);
386 Manifest manifest = bndAnalyzer.calcManifest();
387
388 keys: for (Object key : manifest.getMainAttributes().keySet()) {
389 Object value = manifest.getMainAttributes().get(key);
390
391 switch (key.toString()) {
392 case "Tool":
393 case "Bnd-LastModified":
394 case "Created-By":
395 continue keys;
396 }
397 if ("Require-Capability".equals(key.toString())
398 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
399 continue keys;// hack for very old classes
400 entries.put(key.toString(), value.toString());
401 // logger.log(DEBUG, () -> key + "=" + value);
402
403 }
404 } catch (Exception e) {
405 throw new RuntimeException("Cannot process " + mergeBnd, e);
406 }
407
408 Manifest manifest = new Manifest();
409 Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
410 Files.createDirectories(manifestPath.getParent());
411 for (String key : entries.keySet()) {
412 String value = entries.get(key);
413 manifest.getMainAttributes().putValue(key, value);
414 }
415
416 // // Use Maven version as Bundle-Version
417 // String bundleVersion = manifest.getMainAttributes().getValue(ManifestConstants.BUNDLE_VERSION.toString());
418 // if (bundleVersion == null || bundleVersion.trim().equals("0")) {
419 // // TODO check why it is sometimes set to "0"
420 // manifest.getMainAttributes().putValue(ManifestConstants.BUNDLE_VERSION.toString(), m2Version);
421 // }
422 try (OutputStream out = Files.newOutputStream(manifestPath)) {
423 manifest.write(out);
424 }
425
426 createJar(targetBundleDir);
427
428 }
429
430 /** Generate MANIFEST using BND. */
431 protected Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps,
432 DefaultArtifact artifact) {
433
434 try {
435 Map<String, String> additionalEntries = new TreeMap<>();
436 boolean doNotModify = Boolean.parseBoolean(fileProps
437 .getOrDefault(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString());
438
439 // we always force the symbolic name
440
441 if (doNotModify) {
442 fileEntries: for (Object key : fileProps.keySet()) {
443 if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
444 continue fileEntries;
445 String value = fileProps.getProperty(key.toString());
446 additionalEntries.put(key.toString(), value);
447 }
448 } else {
449 if (artifact != null) {
450 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
451 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
452 }
453 if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
454 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
455 }
456 }
457
458 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
459 fileProps.put(EXPORT_PACKAGE.toString(),
460 "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
461 }
462 // if (!fileProps.contains(IMPORT_PACKAGE.toString())) {
463 // fileProps.put(IMPORT_PACKAGE.toString(), "*");
464 // }
465
466 try (Analyzer bndAnalyzer = new Analyzer()) {
467 bndAnalyzer.setProperties(fileProps);
468 Jar jar = new Jar(downloaded.toFile());
469 bndAnalyzer.setJar(jar);
470 Manifest manifest = bndAnalyzer.calcManifest();
471
472 keys: for (Object key : manifest.getMainAttributes().keySet()) {
473 Object value = manifest.getMainAttributes().get(key);
474
475 switch (key.toString()) {
476 case "Tool":
477 case "Bnd-LastModified":
478 case "Created-By":
479 continue keys;
480 }
481 if ("Require-Capability".equals(key.toString())
482 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
483 continue keys;// hack for very old classes
484 additionalEntries.put(key.toString(), value.toString());
485 // logger.log(DEBUG, () -> key + "=" + value);
486
487 }
488 }
489
490 // try (Builder bndBuilder = new Builder()) {
491 // Jar jar = new Jar(downloaded.toFile());
492 // bndBuilder.addClasspath(jar);
493 // Path targetBundleDir = targetCategoryBase.resolve(artifact.getName() + "." + artifact.getBranch());
494 //
495 // Jar target = new Jar(targetBundleDir.toFile());
496 // bndBuilder.setJar(target);
497 // return targetBundleDir;
498 // }
499 }
500 Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries);
501 logger.log(Level.DEBUG, () -> "Processed " + downloaded);
502 return targetBundleDir;
503 } catch (Exception e) {
504 throw new RuntimeException("Cannot BND process " + downloaded, e);
505 }
506
507 }
508
509 /** Download and integrates sources for a single Maven artifact. */
510 protected void downloadAndProcessM2Sources(String repoStr, DefaultArtifact artifact, Path targetBundleDir)
511 throws IOException {
512 if (!includeSources)
513 return;
514 DefaultArtifact sourcesArtifact = new DefaultArtifact(artifact.toM2Coordinates(), "sources");
515 URL sourcesUrl = MavenConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
516 Path sourcesDownloaded = download(sourcesUrl, originBase, artifact, true);
517 processM2SourceJar(sourcesDownloaded, targetBundleDir);
518 logger.log(Level.TRACE, () -> "Processed source " + sourcesDownloaded);
519
520 }
521
522 /** Integrate sources from a downloaded jar file. */
523 protected void processM2SourceJar(Path file, Path targetBundleDir) throws IOException {
524 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
525 Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
526
527 // TODO make it less dangerous?
528 if (Files.exists(targetSourceDir)) {
529 // deleteDirectory(targetSourceDir);
530 } else {
531 Files.createDirectories(targetSourceDir);
532 }
533
534 // copy entries
535 JarEntry entry;
536 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
537 if (entry.isDirectory())
538 continue entries;
539 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
540 continue entries;
541 if (entry.getName().startsWith("module-info.java"))// skip META-INF entries
542 continue entries;
543 if (entry.getName().startsWith("/")) // absolute paths
544 continue entries;
545 Path target = targetSourceDir.resolve(entry.getName());
546 Files.createDirectories(target.getParent());
547 if (!Files.exists(target)) {
548 Files.copy(jarIn, target);
549 logger.log(Level.TRACE, () -> "Copied source " + target);
550 } else {
551 logger.log(Level.WARNING, () -> target + " already exists, skipping...");
552 }
553 }
554 }
555
556 }
557
558 /** Download a Maven artifact. */
559 protected Path download(URL url, Path dir, Artifact artifact) throws IOException {
560 return download(url, dir, artifact, false);
561 }
562
563 /** Download a Maven artifact. */
564 protected Path download(URL url, Path dir, Artifact artifact, boolean sources) throws IOException {
565 return download(url, dir, artifact.getGroupId() + '/' + artifact.getArtifactId() + "-" + artifact.getVersion()
566 + (sources ? "-sources" : "") + ".jar");
567 }
568
569 /*
570 * ECLIPSE ORIGIN
571 */
572
573 /** Process an archive in Eclipse format. */
574 public void processEclipseArchive(Path relativePath) {
575 try {
576 Path categoryRelativePath = relativePath.getParent();
577 // String category = categoryRelativePath.getFileName().toString();
578 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
579 Files.createDirectories(targetCategoryBase);
580 // first delete all directories from previous builds
581 for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) {
582 deleteDirectory(dir);
583 }
584
585 Files.createDirectories(originBase);
586
587 Path duDir = descriptorsBase.resolve(relativePath);
588 Path commonBnd = duDir.resolve(COMMON_BND);
589 Properties commonProps = new Properties();
590 try (InputStream in = Files.newInputStream(commonBnd)) {
591 commonProps.load(in);
592 }
593 String url = commonProps.getProperty(ManifestConstants.SLC_ORIGIN_URI.toString());
594 if (url == null) {
595 url = uris.getProperty(relativePath.getFileName().toString());
596 if (url == null)
597 throw new IllegalStateException("No url available for " + duDir);
598 commonProps.put(ManifestConstants.SLC_ORIGIN_URI.toString(), url);
599 }
600 Path downloaded = tryDownload(url, originBase);
601
602 FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
603
604 // filters
605 List<PathMatcher> includeMatchers = new ArrayList<>();
606 Properties includes = new Properties();
607 try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
608 includes.load(in);
609 }
610 for (Object pattern : includes.keySet()) {
611 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
612 includeMatchers.add(pathMatcher);
613 }
614
615 List<PathMatcher> excludeMatchers = new ArrayList<>();
616 Path excludeFile = duDir.resolve("excludes.properties");
617 if (Files.exists(excludeFile)) {
618 Properties excludes = new Properties();
619 try (InputStream in = Files.newInputStream(excludeFile)) {
620 excludes.load(in);
621 }
622 for (Object pattern : excludes.keySet()) {
623 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
624 excludeMatchers.add(pathMatcher);
625 }
626 }
627
628 Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
629
630 @Override
631 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
632 includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
633 if (includeMatcher.matches(file)) {
634 for (PathMatcher excludeMatcher : excludeMatchers) {
635 if (excludeMatcher.matches(file)) {
636 logger.log(Level.WARNING, "Skipping excluded " + file);
637 return FileVisitResult.CONTINUE;
638 }
639 }
640 if (includeSources && file.getFileName().toString().contains(".source_")) {
641 processEclipseSourceJar(file, targetCategoryBase);
642 logger.log(Level.DEBUG, () -> "Processed source " + file);
643
644 } else {
645 processBundleJar(file, targetCategoryBase, new HashMap(commonProps));
646 logger.log(Level.DEBUG, () -> "Processed " + file);
647 }
648 break includeMatchers;
649 }
650 }
651 return FileVisitResult.CONTINUE;
652 }
653 });
654
655 DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase,
656 (p) -> Files.isDirectory(p) && p.getFileName().toString().indexOf('.') >= 0);
657 for (Path dir : dirs) {
658 createJar(dir);
659 }
660 } catch (IOException e) {
661 throw new RuntimeException("Cannot process " + relativePath, e);
662 }
663
664 }
665
666 /** Process sources in Eclipse format. */
667 protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
668 try {
669 Path targetBundleDir;
670 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
671 Manifest manifest = jarIn.getManifest();
672
673 String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
674 String version = relatedBundle[1].substring("version=\"".length());
675 version = version.substring(0, version.length() - 1);
676 NameVersion nameVersion = new DefaultNameVersion(relatedBundle[0], version);
677 targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
678
679 Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
680
681 // TODO make it less dangerous?
682 if (Files.exists(targetSourceDir)) {
683 // deleteDirectory(targetSourceDir);
684 } else {
685 Files.createDirectories(targetSourceDir);
686 }
687
688 // copy entries
689 JarEntry entry;
690 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
691 if (entry.isDirectory())
692 continue entries;
693 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
694 continue entries;
695 Path target = targetSourceDir.resolve(entry.getName());
696 Files.createDirectories(target.getParent());
697 Files.copy(jarIn, target);
698 logger.log(Level.TRACE, () -> "Copied source " + target);
699 }
700
701 // copy MANIFEST
702 // Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
703 // Files.createDirectories(manifestPath.getParent());
704 // try (OutputStream out = Files.newOutputStream(manifestPath)) {
705 // manifest.write(out);
706 // }
707 }
708 } catch (IOException e) {
709 throw new IllegalStateException("Cannot process " + file, e);
710 }
711
712 }
713
714 /*
715 * COMMON PROCESSING
716 */
717 /** Normalise a bundle. */
718 protected Path processBundleJar(Path file, Path targetBase, Map<String, String> entries) throws IOException {
719 DefaultNameVersion nameVersion;
720 Path targetBundleDir;
721 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
722 Manifest sourceManifest = jarIn.getManifest();
723 Manifest manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
724
725 // remove problematic entries in MANIFEST
726 manifest.getEntries().clear();
727 // Set<String> entriesToDelete = new HashSet<>();
728 // for (String key : manifest.getEntries().keySet()) {
729 //// logger.log(DEBUG, "## " + key);
730 // Attributes attrs = manifest.getAttributes(key);
731 // for (Object attrName : attrs.keySet()) {
732 //// logger.log(DEBUG, attrName + "=" + attrs.get(attrName));
733 // if ("Specification-Version".equals(attrName.toString())
734 // || "Implementation-Version".equals(attrName.toString())) {
735 // entriesToDelete.add(key);
736 //
737 // }
738 // }
739 // }
740 // for (String key : entriesToDelete) {
741 // manifest.getEntries().remove(key);
742 // }
743
744 String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString());
745 String ourVersion = entries.get(BUNDLE_VERSION.toString());
746
747 if (ourSymbolicName != null && ourVersion != null) {
748 nameVersion = new DefaultNameVersion(ourSymbolicName, ourVersion);
749 } else {
750 nameVersion = nameVersionFromManifest(manifest);
751 if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) {
752 logger.log(Level.WARNING,
753 "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion);
754 entries.put(BUNDLE_VERSION.toString(), ourVersion);
755 }
756 if (ourSymbolicName != null) {
757 // we always force our symbolic name
758 nameVersion.setName(ourSymbolicName);
759 }
760 }
761 targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
762
763 // force Java 9 module name
764 entries.put(ManifestConstants.AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName());
765
766 // TODO make it less dangerous?
767 // if (Files.exists(targetBundleDir)) {
768 // deleteDirectory(targetBundleDir);
769 // }
770
771 boolean isNative = false;
772 String os = null;
773 String arch = null;
774 if (targetBundleDir.startsWith(a2LibBase)) {
775 isNative = true;
776 Path libRelativePath = a2LibBase.relativize(targetBundleDir);
777 os = libRelativePath.getName(0).toString();
778 arch = libRelativePath.getName(1).toString();
779 }
780
781 // copy entries
782 JarEntry entry;
783 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
784 if (entry.isDirectory())
785 continue entries;
786 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
787 continue entries;
788 if (entry.getName().endsWith("module-info.class")) // skip Java 9 module info
789 continue entries;
790 if (entry.getName().startsWith("META-INF/versions/")) // skip multi-version
791 continue entries;
792 // skip file system providers as they cause issues with native image
793 if (entry.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider"))
794 continue entries;
795 if (entry.getName().startsWith("OSGI-OPT/src/")) // skip embedded sources
796 continue entries;
797 Path target = targetBundleDir.resolve(entry.getName());
798 Files.createDirectories(target.getParent());
799 Files.copy(jarIn, target);
800
801 // native libraries
802 if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll")
803 || entry.getName().endsWith(".jnilib"))) {
804 Path categoryDir = targetBundleDir.getParent();
805 // String[] segments = categoryDir.getFileName().toString().split("\\.");
806 // String arch = segments[segments.length - 1];
807 // String os = segments[segments.length - 2];
808 boolean copyDll = false;
809 Path targetDll = categoryDir.resolve(targetBundleDir.relativize(target));
810 if (nameVersion.getName().equals("com.sun.jna")) {
811 if (arch.equals("x86_64"))
812 arch = "x86-64";
813 if (os.equals("macosx"))
814 os = "darwin";
815 if (target.getParent().getFileName().toString().equals(os + "-" + arch)) {
816 copyDll = true;
817 }
818 targetDll = categoryDir.resolve(target.getFileName());
819 } else {
820 copyDll = true;
821 }
822 if (copyDll) {
823 Files.createDirectories(targetDll.getParent());
824 if (Files.exists(targetDll))
825 Files.delete(targetDll);
826 Files.copy(target, targetDll);
827 }
828 Files.delete(target);
829 }
830 logger.log(Level.TRACE, () -> "Copied " + target);
831 }
832
833 // copy MANIFEST
834 Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
835 Files.createDirectories(manifestPath.getParent());
836 for (String key : entries.keySet()) {
837 String value = entries.get(key);
838 Object previousValue = manifest.getMainAttributes().putValue(key, value);
839 if (previousValue != null && !previousValue.equals(value)) {
840 if (ManifestConstants.IMPORT_PACKAGE.toString().equals(key)
841 || ManifestConstants.EXPORT_PACKAGE.toString().equals(key))
842 logger.log(Level.TRACE, file.getFileName() + ": " + key + " was modified");
843 else
844 logger.log(Level.WARNING, file.getFileName() + ": " + key + " was " + previousValue
845 + ", overridden with " + value);
846 }
847
848 // hack to remove unresolvable
849 if (key.equals("Provide-Capability") || key.equals("Require-Capability"))
850 if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) {
851 manifest.getMainAttributes().remove(key);
852 }
853 }
854 try (OutputStream out = Files.newOutputStream(manifestPath)) {
855 manifest.write(out);
856 }
857 }
858 return targetBundleDir;
859 }
860
861 /*
862 * UTILITIES
863 */
864
865 /** Recursively deletes a directory. */
866 private static void deleteDirectory(Path path) throws IOException {
867 if (!Files.exists(path))
868 return;
869 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
870 @Override
871 public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
872 if (e != null)
873 throw e;
874 Files.delete(directory);
875 return FileVisitResult.CONTINUE;
876 }
877
878 @Override
879 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
880 Files.delete(file);
881 return FileVisitResult.CONTINUE;
882 }
883 });
884 }
885
886 /** Extract name/version from a MANIFEST. */
887 protected DefaultNameVersion nameVersionFromManifest(Manifest manifest) {
888 Attributes attrs = manifest.getMainAttributes();
889 // symbolic name
890 String symbolicName = attrs.getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
891 if (symbolicName == null)
892 return null;
893 // make sure there is no directive
894 symbolicName = symbolicName.split(";")[0];
895
896 String version = attrs.getValue(ManifestConstants.BUNDLE_VERSION.toString());
897 return new DefaultNameVersion(symbolicName, version);
898 }
899
900 /** Try to download from an URI. */
901 protected Path tryDownload(String uri, Path dir) throws IOException {
902 // find mirror
903 List<String> urlBases = null;
904 String uriPrefix = null;
905 uriPrefixes: for (String uriPref : mirrors.keySet()) {
906 if (uri.startsWith(uriPref)) {
907 if (mirrors.get(uriPref).size() > 0) {
908 urlBases = mirrors.get(uriPref);
909 uriPrefix = uriPref;
910 break uriPrefixes;
911 }
912 }
913 }
914 if (urlBases == null)
915 try {
916 return download(new URL(uri), dir);
917 } catch (FileNotFoundException e) {
918 throw new FileNotFoundException("Cannot find " + uri);
919 }
920
921 // try to download
922 for (String urlBase : urlBases) {
923 String relativePath = uri.substring(uriPrefix.length());
924 URL url = new URL(urlBase + relativePath);
925 try {
926 return download(url, dir);
927 } catch (FileNotFoundException e) {
928 logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror");
929 }
930 }
931 throw new FileNotFoundException("Cannot find " + uri);
932 }
933
934 // protected String simplifyName(URL u) {
935 // String name = u.getPath().substring(u.getPath().lastIndexOf('/') + 1);
936 //
937 // }
938
939 /** Effectively download. */
940 protected Path download(URL url, Path dir) throws IOException {
941 return download(url, dir, (String) null);
942 }
943
944 /** Effectively download. */
945 protected Path download(URL url, Path dir, String name) throws IOException {
946
947 Path dest;
948 if (name == null) {
949 name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1);
950 }
951
952 dest = dir.resolve(name);
953 if (Files.exists(dest)) {
954 logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
955 return dest;
956 } else {
957 Files.createDirectories(dest.getParent());
958 }
959
960 try (InputStream in = url.openStream()) {
961 Files.copy(in, dest);
962 logger.log(Level.DEBUG, () -> "Downloaded " + dest + " from " + url);
963 }
964 return dest;
965 }
966
967 /** Create a JAR file from a directory. */
968 protected Path createJar(Path bundleDir) throws IOException {
969 // Remove from source the resources already in the jar file
970 // Path srcDir = bundleDir.resolve("OSGI-OPT/src");
971 // Files.walkFileTree(srcDir, new SimpleFileVisitor<Path>() {
972 //
973 // @Override
974 // public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
975 // // always keep .java file
976 // if (file.getFileName().toString().endsWith(".java"))
977 // return super.visitFile(file, attrs);
978 //
979 // Path relPath = srcDir.relativize(file);
980 // Path bundlePath = bundleDir.resolve(relPath);
981 // if (Files.exists(bundlePath)) {
982 // Files.delete(file);
983 // logger.log(DEBUG, () -> "Removed " + file + " from sources.");
984 // }
985 // return super.visitFile(file, attrs);
986 // }
987 //
988 // });
989
990 // Create the jar
991 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
992 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
993 Manifest manifest;
994 try (InputStream in = Files.newInputStream(manifestPath)) {
995 manifest = new Manifest(in);
996 }
997 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
998 jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
999 Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
1000
1001 @Override
1002 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1003 if (file.getFileName().toString().equals("MANIFEST.MF"))
1004 return super.visitFile(file, attrs);
1005 JarEntry entry = new JarEntry(bundleDir.relativize(file).toString());
1006 jarOut.putNextEntry(entry);
1007 Files.copy(file, jarOut);
1008 return super.visitFile(file, attrs);
1009 }
1010
1011 });
1012 }
1013 deleteDirectory(bundleDir);
1014 return jarPath;
1015 }
1016
1017 /** For development purproses. */
1018 public static void main(String[] args) {
1019 Path factoryBase = Paths.get("../../output/a2").toAbsolutePath().normalize();
1020 Path descriptorsBase = Paths.get("../tp").toAbsolutePath().normalize();
1021 A2Factory factory = new A2Factory(factoryBase, descriptorsBase, true);
1022
1023 // factory.processSingleM2ArtifactDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache").resolve("org.apache.xml.resolver.bnd"));
1024 factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.apache/apache-sshd"));
1025 // factory.processM2BasedDistributionUnit(descriptorsBase.resolve("org.argeo.tp.jetty/jetty"));
1026 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.osgi"));
1027 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.eclipse.rcp"));
1028 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp"));
1029 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.apache"));
1030 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.sdk"));
1031 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.formats"));
1032 // factory.processCategory(descriptorsBase.resolve("org.argeo.tp.gis"));
1033 System.exit(1);
1034
1035 // SDK
1036 factory.processCategory(Paths.get("org.argeo.tp.sdk"));
1037
1038 // Eclipse
1039 factory.processCategory(Paths.get("osgi/api/org.argeo.tp.osgi"));
1040 factory.processEclipseArchive(Paths.get("osgi/equinox/org.argeo.tp.osgi", "eclipse-equinox"));
1041
1042 factory.processEclipseArchive(Paths.get("org.argeo.tp.eclipse", "eclipse-equinox"));
1043
1044 factory.processEclipseArchive(Paths.get("swt/rap/org.argeo.tp.swt", "eclipse-rap"));
1045 factory.processEclipseArchive(Paths.get("swt/rap/org.argeo.tp.swt.workbench", "eclipse-rap"));
1046
1047 factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1048
1049 factory.processEclipseArchive(Paths.get("lib/linux/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1050 factory.processEclipseArchive(Paths.get("lib/linux/aarch64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1051 factory.processEclipseArchive(Paths.get("lib/win32/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1052 factory.processEclipseArchive(Paths.get("lib/macosx/x86_64/swt/rcp/org.argeo.tp.swt", "eclipse-rcp"));
1053
1054 System.exit(1);
1055
1056 factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt", "eclipse-nebula"));
1057 // factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt",
1058 // "eclipse-equinox"));
1059 factory.processEclipseArchive(Paths.get("swt/rcp/org.argeo.tp.swt.workbench", "eclipse-rcp"));
1060 factory.processCategory(Paths.get("swt/rcp/org.argeo.tp.swt.workbench"));
1061
1062 // Maven
1063 factory.processCategory(Paths.get("org.argeo.tp"));
1064 factory.processCategory(Paths.get("org.argeo.tp.apache"));
1065 factory.processCategory(Paths.get("org.argeo.tp.jetty"));
1066 factory.processCategory(Paths.get("org.argeo.tp.jcr"));
1067 factory.processCategory(Paths.get("org.argeo.tp.formats"));
1068 factory.processCategory(Paths.get("org.argeo.tp.poi"));
1069 factory.processCategory(Paths.get("org.argeo.tp.gis"));
1070 }
1071 }