]> git.argeo.org Git - cc0/argeo-build.git/blob - org/argeo/build/Repackage.java
projects / cc0 / argeo-build.git / blob
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Releasing
[cc0/argeo-build.git] / org / argeo / build / Repackage.java
1 package org.argeo.build;
2
3 import static java.lang.System.Logger.Level.DEBUG;
4 import static java.lang.System.Logger.Level.ERROR;
5 import static java.lang.System.Logger.Level.INFO;
6 import static java.lang.System.Logger.Level.TRACE;
7 import static java.lang.System.Logger.Level.WARNING;
8 import static java.nio.file.FileVisitResult.CONTINUE;
9 import static java.nio.file.StandardOpenOption.APPEND;
10 import static java.nio.file.StandardOpenOption.CREATE;
11 import static java.util.jar.Attributes.Name.MANIFEST_VERSION;
12 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2;
13 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_MERGE;
14 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_REPO;
15 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_NO_METADATA_GENERATION;
16 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_URI;
17 import static org.argeo.build.Repackage.ManifestHeader.AUTOMATIC_MODULE_NAME;
18 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_LICENSE;
19 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_SYMBOLICNAME;
20 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_VERSION;
21 import static org.argeo.build.Repackage.ManifestHeader.ECLIPSE_SOURCE_BUNDLE;
22 import static org.argeo.build.Repackage.ManifestHeader.EXPORT_PACKAGE;
23 import static org.argeo.build.Repackage.ManifestHeader.IMPORT_PACKAGE;
24 import static org.argeo.build.Repackage.ManifestHeader.SPDX_LICENSE_IDENTIFIER;
25
26 import java.io.BufferedWriter;
27 import java.io.File;
28 import java.io.FileNotFoundException;
29 import java.io.IOException;
30 import java.io.InputStream;
31 import java.io.OutputStream;
32 import java.lang.System.Logger;
33 import java.net.MalformedURLException;
34 import java.net.URL;
35 import java.nio.charset.StandardCharsets;
36 import java.nio.file.DirectoryStream;
37 import java.nio.file.FileSystem;
38 import java.nio.file.FileSystems;
39 import java.nio.file.FileVisitResult;
40 import java.nio.file.Files;
41 import java.nio.file.Path;
42 import java.nio.file.PathMatcher;
43 import java.nio.file.Paths;
44 import java.nio.file.SimpleFileVisitor;
45 import java.nio.file.StandardCopyOption;
46 import java.nio.file.StandardOpenOption;
47 import java.nio.file.attribute.BasicFileAttributes;
48 import java.util.ArrayList;
49 import java.util.HashMap;
50 import java.util.Iterator;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.Objects;
54 import java.util.Properties;
55 import java.util.Set;
56 import java.util.StringJoiner;
57 import java.util.TreeMap;
58 import java.util.TreeSet;
59 import java.util.concurrent.CompletableFuture;
60 import java.util.jar.Attributes;
61 import java.util.jar.JarEntry;
62 import java.util.jar.JarInputStream;
63 import java.util.jar.JarOutputStream;
64 import java.util.jar.Manifest;
65 import java.util.zip.Deflater;
66
67 import aQute.bnd.osgi.Analyzer;
68 import aQute.bnd.osgi.Jar;
69
70 /** Repackages existing jar files into OSGi bundles in an A2 repository. */
71 public class Repackage {
72 final static Logger logger = System.getLogger(Repackage.class.getName());
73
74 /**
75 * Environment variable on whether sources should be packaged separately or
76 * integrated in the bundles.
77 */
78 final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES";
79 /** Environment variable on whether operations should be parallelised. */
80 final static String ENV_ARGEO_BUILD_SEQUENTIAL = "ARGEO_BUILD_SEQUENTIAL";
81
82 /** Whether repackaging should run in parallel (default) or sequentially. */
83 final static boolean sequential = Boolean.parseBoolean(System.getenv(ENV_ARGEO_BUILD_SEQUENTIAL));
84
85 /** Main entry point. */
86 public static void main(String[] args) {
87 if (sequential)
88 logger.log(INFO, "Build will be sequential");
89 if (args.length < 2) {
90 System.err.println("Usage: <path to a2 output dir> <category1> <category2> ...");
91 System.exit(1);
92 }
93 Path a2Base = Paths.get(args[0]).toAbsolutePath().normalize();
94 Path descriptorsBase = Paths.get(".").toAbsolutePath().normalize();
95 Repackage factory = new Repackage(a2Base, descriptorsBase);
96
97 List<CompletableFuture<Void>> toDos = new ArrayList<>();
98 for (int i = 1; i < args.length; i++) {
99 Path p = Paths.get(args[i]);
100 if (sequential)
101 factory.processCategory(p);
102 else
103 toDos.add(CompletableFuture.runAsync(() -> factory.processCategory(p)));
104 }
105 if (!sequential)
106 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
107
108 // Summary
109 StringBuilder sb = new StringBuilder();
110 for (String licenseId : licensesUsed.keySet())
111 for (String name : licensesUsed.get(licenseId))
112 sb.append((licenseId.equals("") ? "Proprietary" : licenseId) + "\t\t" + name + "\n");
113 logger.log(INFO, "# License summary:\n" + sb);
114 }
115
116 /** MANIFEST headers. */
117 enum ManifestHeader {
118 // OSGi
119 /** OSGi bundle symbolic name. */
120 BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), //
121 /** OSGi bundle version. */
122 BUNDLE_VERSION("Bundle-Version"), //
123 /** OSGi bundle license. */
124 BUNDLE_LICENSE("Bundle-License"), //
125 /** OSGi exported packages list. */
126 EXPORT_PACKAGE("Export-Package"), //
127 /** OSGi imported packages list. */
128 IMPORT_PACKAGE("Import-Package"), //
129 /** OSGi path to embedded jar. */
130 BUNDLE_CLASSPATH("Bundle-Classpath"), //
131 // Java
132 /** Java module name. */
133 AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), //
134 // Eclipse
135 /** Eclipse source bundle. */
136 ECLIPSE_SOURCE_BUNDLE("Eclipse-SourceBundle"), //
137 // SPDX
138 /**
139 * SPDX license identifier.
140 *
141 * @see https://spdx.org/licenses/
142 */
143 SPDX_LICENSE_IDENTIFIER("SPDX-License-Identifier"), //
144 // Argeo Origin
145 /**
146 * Maven coordinates of the origin, possibly partial when using common.bnd or
147 * merge.bnd.
148 */
149 ARGEO_ORIGIN_M2("Argeo-Origin-M2"), //
150 /** List of Maven coordinates to merge. */
151 ARGEO_ORIGIN_M2_MERGE("Argeo-Origin-M2-Merge"), //
152 /** Maven repository, if not the default one. */
153 ARGEO_ORIGIN_M2_REPO("Argeo-Origin-M2-Repo"), //
154 /**
155 * Do not perform BND analysis of the origin component. Typically Import_package
156 * and Export-Package will be kept untouched.
157 */
158 ARGEO_ORIGIN_NO_METADATA_GENERATION("Argeo-Origin-NoMetadataGeneration"), //
159 // /**
160 // * Embed the original jar without modifying it (may be required by some
161 // * proprietary licenses, such as JCR Day License).
162 // */
163 // ARGEO_ORIGIN_EMBED("Argeo-Origin-Embed"), //
164 /**
165 * Do not modify original jar (may be required by some proprietary licenses,
166 * such as JCR Day License).
167 */
168 ARGEO_DO_NOT_MODIFY("Argeo-Origin-Do-Not-Modify"), //
169 /**
170 * Origin (non-Maven) URI of the component. It may be anything (jar, archive,
171 * etc.).
172 */
173 ARGEO_ORIGIN_URI("Argeo-Origin-URI"), //
174 ;
175
176 final String value;
177
178 private ManifestHeader(String value) {
179 this.value = value;
180 }
181
182 @Override
183 public String toString() {
184 return value;
185 }
186 }
187
188 /** Name of the file centralising information for multiple M2 artifacts. */
189 final static String COMMON_BND = "common.bnd";
190 /** Name of the file centralising information for mergin M2 artifacts. */
191 final static String MERGE_BND = "merge.bnd";
192 /**
193 * Subdirectory of the jar file where origin informations (changes, legal
194 * notices etc. are stored)
195 */
196 final static String ARGEO_ORIGIN = "ARGEO-ORIGIN";
197 /** File detailing modifications to the original component. */
198 final static String CHANGES = ARGEO_ORIGIN + "/changes";
199 /**
200 * Name of the file at the root of the repackaged jar, which prominently
201 * notifies that the component has be repackaged.
202 */
203 final static String README_REPACKAGED = "README.repackaged";
204
205 // cache
206 /** Summary of all license seen during the repackaging. */
207 final static Map<String, Set<String>> licensesUsed = new TreeMap<>();
208
209 /** Directory where to download archives */
210 final Path originBase;
211 /** Directory where to download Maven artifacts */
212 final Path mavenBase;
213
214 /** A2 repository base for binary bundles */
215 final Path a2Base;
216 /** A2 repository base for source bundles */
217 final Path a2SrcBase;
218 /** A2 base for native components */
219 final Path a2LibBase;
220 /** Location of the descriptors driving the packaging */
221 final Path descriptorsBase;
222 /** URIs of archives to download */
223 final Properties uris = new Properties();
224 /** Mirrors for archive download. Key is URI prefix, value list of base URLs */
225 final Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
226
227 /** Whether sources should be packaged separately */
228 final boolean sourceBundles;
229
230 /** Constructor initialises the various variables */
231 public Repackage(Path a2Base, Path descriptorsBase) {
232 sourceBundles = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES));
233 if (sourceBundles)
234 logger.log(INFO, "Sources will be packaged separately");
235
236 Objects.requireNonNull(a2Base);
237 Objects.requireNonNull(descriptorsBase);
238 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/build/origin");
239 this.mavenBase = Paths.get(System.getProperty("user.home"), ".m2", "repository");
240
241 // TODO define and use a build base
242 this.a2Base = a2Base;
243 this.a2SrcBase = a2Base.getParent().resolve(a2Base.getFileName() + ".src");
244 this.a2LibBase = a2Base.resolve("lib");
245 this.descriptorsBase = descriptorsBase;
246 if (!Files.exists(this.descriptorsBase))
247 throw new IllegalArgumentException(this.descriptorsBase + " does not exist");
248
249 // URIs mapping
250 Path urisPath = this.descriptorsBase.resolve("uris.properties");
251 if (Files.exists(urisPath)) {
252 try (InputStream in = Files.newInputStream(urisPath)) {
253 uris.load(in);
254 } catch (IOException e) {
255 throw new IllegalStateException("Cannot load " + urisPath, e);
256 }
257 }
258
259 // Eclipse mirrors
260 Path eclipseMirrorsPath = this.descriptorsBase.resolve("eclipse.mirrors.txt");
261 List<String> eclipseMirrors = new ArrayList<>();
262 if (Files.exists(eclipseMirrorsPath)) {
263 try {
264 eclipseMirrors = Files.readAllLines(eclipseMirrorsPath, StandardCharsets.UTF_8);
265 } catch (IOException e) {
266 throw new IllegalStateException("Cannot load " + eclipseMirrorsPath, e);
267 }
268 for (Iterator<String> it = eclipseMirrors.iterator(); it.hasNext();) {
269 String value = it.next();
270 if (value.strip().equals(""))
271 it.remove();
272 }
273 }
274 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
275 }
276
277 /*
278 * MAVEN ORIGIN
279 */
280 /** Process a whole category/group id. */
281 void processCategory(Path categoryRelativePath) {
282 try {
283 Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath);
284 DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
285 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
286 && !p.getFileName().toString().equals(MERGE_BND));
287 for (Path p : bnds) {
288 processSingleM2ArtifactDistributionUnit(p);
289 }
290
291 DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
292 for (Path duDir : dus) {
293 if (duDir.getFileName().toString().startsWith("eclipse-")) {
294 processEclipseArchive(duDir);
295 } else {
296 processM2BasedDistributionUnit(duDir);
297 }
298 }
299 } catch (IOException e) {
300 throw new RuntimeException("Cannot process category " + categoryRelativePath, e);
301 }
302 }
303
304 /** Process a standalone Maven artifact. */
305 void processSingleM2ArtifactDistributionUnit(Path bndFile) {
306 try {
307 Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent());
308 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
309
310 Properties fileProps = new Properties();
311 try (InputStream in = Files.newInputStream(bndFile)) {
312 fileProps.load(in);
313 }
314 String repoStr = fileProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
315 ? fileProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
316 : null;
317
318 // use file name as symbolic name
319 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
320 String symbolicName = bndFile.getFileName().toString();
321 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
322 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
323 }
324
325 String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
326 if (m2Coordinates == null)
327 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
328 M2Artifact artifact = new M2Artifact(m2Coordinates);
329 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
330 Path downloaded = downloadMaven(url, artifact);
331
332 // some proprietary artifacts do not allow any modification
333 // when releasing (with separate sources) we just copy it
334 boolean doNotModify = Boolean.parseBoolean(
335 fileProps.getOrDefault(ManifestHeader.ARGEO_DO_NOT_MODIFY.toString(), "false").toString());
336 if (doNotModify && sourceBundles) {
337 Path unmodifiedTarget = targetCategoryBase.resolve(
338 fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch() + ".jar");
339 Files.copy(downloaded, unmodifiedTarget, StandardCopyOption.REPLACE_EXISTING);
340 Path bundleDir = targetCategoryBase
341 .resolve(fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch());
342 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
343 Manifest manifest;
344 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(unmodifiedTarget))) {
345 manifest = jarIn.getManifest();
346 }
347 createSourceJar(bundleDir, manifest);
348 return;
349 }
350
351 // regular processing
352 A2Origin origin = new A2Origin();
353 Path bundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact, origin);
354 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
355 createJar(bundleDir, origin);
356 } catch (Exception e) {
357 throw new RuntimeException("Cannot process " + bndFile, e);
358 }
359 }
360
361 /**
362 * Process multiple Maven artifacts coming from a same project and therefore
363 * with information in common (typically the version), generating single bundles
364 * or merging them if necessary.
365 *
366 * @see #COMMON_BND
367 * @see #MERGE_BND
368 */
369 void processM2BasedDistributionUnit(Path duDir) {
370 try {
371 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
372 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
373
374 Path mergeBnd = duDir.resolve(MERGE_BND);
375 if (Files.exists(mergeBnd)) // merge
376 mergeM2Artifacts(mergeBnd);
377
378 Path commonBnd = duDir.resolve(COMMON_BND);
379 if (!Files.exists(commonBnd))
380 return;
381
382 Properties commonProps = new Properties();
383 try (InputStream in = Files.newInputStream(commonBnd)) {
384 commonProps.load(in);
385 }
386
387 String m2Version = commonProps.getProperty(ARGEO_ORIGIN_M2.toString());
388 if (m2Version == null) {
389 logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
390 return;// ignore, this is probably an Eclipse archive
391 }
392 if (!m2Version.startsWith(":")) {
393 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
394 }
395 m2Version = m2Version.substring(1);
396
397 DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
398 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
399 && !p.getFileName().toString().equals(MERGE_BND));
400 for (Path p : ds) {
401 Properties fileProps = new Properties();
402 try (InputStream in = Files.newInputStream(p)) {
403 fileProps.load(in);
404 }
405 String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
406 M2Artifact artifact = new M2Artifact(m2Coordinates);
407 if (artifact.getVersion() == null) {
408 artifact.setVersion(m2Version);
409 } else {
410 logger.log(WARNING, p.getFileName() + " : Using version " + artifact.getVersion()
411 + " specified in descriptor rather than " + m2Version + " specified in " + COMMON_BND);
412 }
413
414 // prepare manifest entries
415 Properties mergeProps = new Properties();
416 mergeProps.putAll(commonProps);
417
418 fileEntries: for (Object key : fileProps.keySet()) {
419 if (ARGEO_ORIGIN_M2.toString().equals(key))
420 continue fileEntries;
421 String value = fileProps.getProperty(key.toString());
422 Object previousValue = mergeProps.put(key.toString(), value);
423 if (previousValue != null) {
424 logger.log(WARNING,
425 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
426 }
427 }
428 mergeProps.put(ARGEO_ORIGIN_M2.toString(), artifact.toM2Coordinates());
429 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
430 // use file name as symbolic name
431 String symbolicName = p.getFileName().toString();
432 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
433 mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
434 }
435
436 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
437 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
438 : null;
439
440 // download
441 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
442 Path downloaded = downloadMaven(url, artifact);
443
444 A2Origin origin = new A2Origin();
445 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact, origin);
446 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir, false);
447 createJar(targetBundleDir, origin);
448 }
449 } catch (IOException e) {
450 throw new RuntimeException("Cannot process " + duDir, e);
451 }
452 }
453
454 /** Merge multiple Maven artifacts. */
455 void mergeM2Artifacts(Path mergeBnd) throws IOException {
456 Path duDir = mergeBnd.getParent();
457 String category = duDir.getParent().getFileName().toString();
458 Path targetCategoryBase = a2Base.resolve(category);
459
460 Properties mergeProps = new Properties();
461 // first, load common properties
462 Path commonBnd = duDir.resolve(COMMON_BND);
463 if (Files.exists(commonBnd))
464 try (InputStream in = Files.newInputStream(commonBnd)) {
465 mergeProps.load(in);
466 }
467 // then, the merge properties themselves
468 try (InputStream in = Files.newInputStream(mergeBnd)) {
469 mergeProps.load(in);
470 }
471
472 String m2Version = mergeProps.getProperty(ARGEO_ORIGIN_M2.toString());
473 if (m2Version == null) {
474 logger.log(WARNING, "Ignoring merging in " + duDir + " as it is not an M2-based distribution unit");
475 return;// ignore, this is probably an Eclipse archive
476 }
477 if (!m2Version.startsWith(":")) {
478 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
479 }
480 m2Version = m2Version.substring(1);
481 mergeProps.put(BUNDLE_VERSION.toString(), m2Version);
482
483 String artifactsStr = mergeProps.getProperty(ARGEO_ORIGIN_M2_MERGE.toString());
484 if (artifactsStr == null)
485 throw new IllegalArgumentException(mergeBnd + ": " + ARGEO_ORIGIN_M2_MERGE + " must be set");
486
487 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
488 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
489 : null;
490
491 String bundleSymbolicName = mergeProps.getProperty(BUNDLE_SYMBOLICNAME.toString());
492 if (bundleSymbolicName == null)
493 throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
494 CategoryNameVersion nameVersion = new M2Artifact(category + ":" + bundleSymbolicName + ":" + m2Version);
495
496 A2Origin origin = new A2Origin();
497 Path bundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
498
499 StringJoiner originDesc = new StringJoiner(",");
500 String[] artifacts = artifactsStr.split(",");
501 artifacts: for (String str : artifacts) {
502 String m2Coordinates = str.trim();
503 if ("".equals(m2Coordinates))
504 continue artifacts;
505 M2Artifact artifact = new M2Artifact(m2Coordinates.trim());
506 if (artifact.getVersion() == null)
507 artifact.setVersion(m2Version);
508 originDesc.add(artifact.toString());
509 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
510 Path downloaded = downloadMaven(url, artifact);
511 JarEntry entry;
512 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
513 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
514 if (entry.isDirectory())
515 continue entries;
516 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
517 || entry.getName().endsWith(".SF")) {
518 origin.deleted.add("cryptographic signatures from " + artifact);
519 continue entries;
520 }
521 if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
522 origin.deleted.add("Java module information (module-info.class) from " + artifact);
523 continue entries;
524 }
525 if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
526 origin.deleted.add("additional Java versions (META-INF/versions) from " + artifact);
527 continue entries;
528 }
529 if (entry.getName().startsWith("META-INF/maven/")) {
530 origin.deleted.add("Maven information (META-INF/maven) from " + artifact);
531 continue entries;
532 }
533 if (entry.getName().startsWith(".cache/")) { // Apache SSHD
534 origin.deleted.add("cache directory (.cache) from " + artifact);
535 continue entries;
536 }
537 if (entry.getName().equals("META-INF/DEPENDENCIES")) {
538 origin.deleted.add("Dependencies (META-INF/DEPENDENCIES) from " + artifact);
539 continue entries;
540 }
541 if (entry.getName().equals("META-INF/MANIFEST.MF")) {
542 Path originalManifest = bundleDir.resolve(ARGEO_ORIGIN).resolve(artifact.getGroupId())
543 .resolve(artifact.getArtifactId()).resolve("MANIFEST.MF");
544 Files.createDirectories(originalManifest.getParent());
545 try (OutputStream out = Files.newOutputStream(originalManifest)) {
546 Files.copy(jarIn, originalManifest);
547 }
548 origin.added.add(
549 "original MANIFEST (" + bundleDir.relativize(originalManifest) + ") from " + artifact);
550 continue entries;
551 }
552
553 if (entry.getName().endsWith("NOTICE") || entry.getName().endsWith("NOTICE.txt")
554 || entry.getName().endsWith("LICENSE") || entry.getName().endsWith("LICENSE.md")
555 || entry.getName().endsWith("LICENSE-notice.md") || entry.getName().endsWith("COPYING")
556 || entry.getName().endsWith("COPYING.LESSER")) {
557 Path artifactOriginDir = bundleDir.resolve(ARGEO_ORIGIN).resolve(artifact.getGroupId())
558 .resolve(artifact.getArtifactId());
559 Path target = artifactOriginDir.resolve(entry.getName());
560 Files.createDirectories(target.getParent());
561 Files.copy(jarIn, target);
562 origin.moved.add(entry.getName() + " in " + artifact + " to " + bundleDir.relativize(target));
563 continue entries;
564 }
565 Path target = bundleDir.resolve(entry.getName());
566 Files.createDirectories(target.getParent());
567 if (!Files.exists(target)) {
568 Files.copy(jarIn, target);
569 } else {
570 if (entry.getName().startsWith("META-INF/services/")) {
571 try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
572 out.write("\n".getBytes());
573 jarIn.transferTo(out);
574 logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName());
575 }
576 origin.modified.add(entry.getName() + ", merging from " + artifact);
577 } else if (entry.getName().startsWith("org/apache/batik/")) {
578 logger.log(TRACE, "Skip " + entry.getName());
579 continue entries;
580 } else {
581 throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
582 }
583 }
584 logger.log(TRACE, () -> "Copied " + target);
585 }
586 }
587 origin.added.add("binary content of " + artifact);
588
589 // process sources
590 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, true);
591 }
592
593 // additional service files
594 Path servicesDir = duDir.resolve("services");
595 if (Files.exists(servicesDir)) {
596 for (Path p : Files.newDirectoryStream(servicesDir)) {
597 Path target = bundleDir.resolve("META-INF/services/").resolve(p.getFileName());
598 try (InputStream in = Files.newInputStream(p);
599 OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) {
600 out.write("\n".getBytes());
601 in.transferTo(out);
602 logger.log(DEBUG, "Appended " + p);
603 }
604 origin.added.add(bundleDir.relativize(target).toString());
605 }
606 }
607
608 // BND analysis
609 Map<String, String> entries = new TreeMap<>();
610 try (Analyzer bndAnalyzer = new Analyzer()) {
611 bndAnalyzer.setProperties(mergeProps);
612 Jar jar = new Jar(bundleDir.toFile());
613 bndAnalyzer.setJar(jar);
614 Manifest manifest = bndAnalyzer.calcManifest();
615
616 keys: for (Object key : manifest.getMainAttributes().keySet()) {
617 Object value = manifest.getMainAttributes().get(key);
618
619 switch (key.toString()) {
620 case "Tool":
621 case "Bnd-LastModified":
622 case "Created-By":
623 continue keys;
624 }
625 if ("Require-Capability".equals(key.toString())
626 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) {
627 origin.deleted.add("MANIFEST header " + key);
628 continue keys;// hack for very old classes
629 }
630 entries.put(key.toString(), value.toString());
631 }
632 } catch (Exception e) {
633 throw new RuntimeException("Cannot process " + mergeBnd, e);
634 }
635
636 Manifest manifest = new Manifest();
637 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
638 Files.createDirectories(manifestPath.getParent());
639 for (String key : entries.keySet()) {
640 String value = entries.get(key);
641 manifest.getMainAttributes().putValue(key, value);
642 }
643 manifest.getMainAttributes().putValue(ARGEO_ORIGIN_M2.toString(), originDesc.toString());
644
645 processLicense(bundleDir, manifest);
646
647 // write MANIFEST
648 try (OutputStream out = Files.newOutputStream(manifestPath)) {
649 manifest.write(out);
650 }
651 createJar(bundleDir, origin);
652 }
653
654 /** Generates MANIFEST using BND. */
655 Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, M2Artifact artifact,
656 A2Origin origin) {
657 try {
658 Map<String, String> additionalEntries = new TreeMap<>();
659 boolean doNotModifyManifest = Boolean.parseBoolean(
660 fileProps.getOrDefault(ARGEO_ORIGIN_NO_METADATA_GENERATION.toString(), "false").toString());
661
662 // Note: we always force the symbolic name
663 if (doNotModifyManifest) {
664 for (Object key : fileProps.keySet()) {
665 String value = fileProps.getProperty(key.toString());
666 additionalEntries.put(key.toString(), value);
667 }
668 } else {
669 if (artifact != null) {
670 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
671 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
672 }
673 if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
674 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
675 }
676 }
677
678 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
679 fileProps.put(EXPORT_PACKAGE.toString(),
680 "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
681 }
682
683 // BND analysis
684 try (Analyzer bndAnalyzer = new Analyzer()) {
685 bndAnalyzer.setProperties(fileProps);
686 Jar jar = new Jar(downloaded.toFile());
687 bndAnalyzer.setJar(jar);
688 Manifest manifest = bndAnalyzer.calcManifest();
689
690 keys: for (Object key : manifest.getMainAttributes().keySet()) {
691 Object value = manifest.getMainAttributes().get(key);
692
693 switch (key.toString()) {
694 case "Tool":
695 case "Bnd-LastModified":
696 case "Created-By":
697 continue keys;
698 }
699 if ("Require-Capability".equals(key.toString())
700 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) {
701 origin.deleted.add("MANIFEST header " + key);
702 continue keys;// !! hack for very old classes
703 }
704 additionalEntries.put(key.toString(), value.toString());
705 }
706 }
707 }
708 Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries, origin);
709 logger.log(DEBUG, () -> "Processed " + downloaded);
710 return targetBundleDir;
711 } catch (Exception e) {
712 throw new RuntimeException("Cannot BND process " + downloaded, e);
713 }
714
715 }
716
717 /** Download and integrates sources for a single Maven artifact. */
718 void downloadAndProcessM2Sources(String repoStr, M2Artifact artifact, Path targetBundleDir, boolean merging)
719 throws IOException {
720 try {
721 M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources");
722 URL sourcesUrl = M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
723 Path sourcesDownloaded = downloadMaven(sourcesUrl, artifact, true);
724 processM2SourceJar(sourcesDownloaded, targetBundleDir, merging ? artifact : null);
725 logger.log(TRACE, () -> "Processed source " + sourcesDownloaded);
726 } catch (Exception e) {
727 logger.log(ERROR, () -> "Cannot download source for " + artifact);
728 }
729
730 }
731
732 /** Integrate sources from a downloaded jar file. */
733 void processM2SourceJar(Path file, Path bundleDir, M2Artifact mergingFrom) throws IOException {
734 A2Origin origin = new A2Origin();
735 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
736 : bundleDir.resolve("OSGI-OPT/src");
737 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
738
739 String mergingMsg = "";
740 if (mergingFrom != null)
741 mergingMsg = " of " + mergingFrom;
742
743 Files.createDirectories(sourceDir);
744 JarEntry entry;
745 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
746 String relPath = entry.getName();
747 if (entry.isDirectory())
748 continue entries;
749 if (entry.getName().startsWith("META-INF")) {// skip META-INF entries
750 origin.deleted.add("META-INF directory from the sources" + mergingMsg);
751 continue entries;
752 }
753 if (entry.getName().startsWith("module-info.java")) {// skip Java module information
754 origin.deleted.add("Java module information from the sources (module-info.java)" + mergingMsg);
755 continue entries;
756 }
757 if (entry.getName().startsWith("/")) { // absolute paths
758 int metaInfIndex = entry.getName().indexOf("META-INF");
759 if (metaInfIndex >= 0) {
760 relPath = entry.getName().substring(metaInfIndex);
761 origin.moved.add(" to " + relPath + " entry with absolute path " + entry.getName());
762 } else {
763 logger.log(WARNING, entry.getName() + " has an absolute path");
764 origin.deleted.add(entry.getName() + " from the sources" + mergingMsg);
765 }
766 continue entries;
767 }
768 Path target = sourceDir.resolve(relPath);
769 Files.createDirectories(target.getParent());
770 if (!Files.exists(target)) {
771 Files.copy(jarIn, target);
772 logger.log(TRACE, () -> "Copied source " + target);
773 } else {
774 logger.log(TRACE, () -> target + " already exists, skipping...");
775 }
776 }
777 }
778 // write the changes
779 if (sourceBundles) {
780 origin.appendChanges(sourceDir);
781 } else {
782 origin.added.add("source code under OSGI-OPT/src");
783 origin.appendChanges(bundleDir);
784 }
785 }
786
787 /** Download a Maven artifact. */
788 Path downloadMaven(URL url, M2Artifact artifact) throws IOException {
789 return downloadMaven(url, artifact, false);
790 }
791
792 /** Download a Maven artifact. */
793 Path downloadMaven(URL url, M2Artifact artifact, boolean sources) throws IOException {
794 return download(url, mavenBase, artifact.getGroupId().replace(".", "/") //
795 + '/' + artifact.getArtifactId() + '/' + artifact.getVersion() //
796 + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() + (sources ? "-sources" : "") + ".jar");
797 }
798
799 /*
800 * ECLIPSE ORIGIN
801 */
802 /** Process an archive in Eclipse format. */
803 void processEclipseArchive(Path duDir) {
804 try {
805 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
806 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
807 Files.createDirectories(targetCategoryBase);
808 // first delete all directories from previous builds
809 for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)))
810 deleteDirectory(dir);
811
812 Files.createDirectories(originBase);
813
814 Path commonBnd = duDir.resolve(COMMON_BND);
815 Properties commonProps = new Properties();
816 try (InputStream in = Files.newInputStream(commonBnd)) {
817 commonProps.load(in);
818 }
819 String url = commonProps.getProperty(ARGEO_ORIGIN_URI.toString());
820 if (url == null) {
821 url = uris.getProperty(duDir.getFileName().toString());
822 if (url == null)
823 throw new IllegalStateException("No url available for " + duDir);
824 commonProps.put(ARGEO_ORIGIN_URI.toString(), url);
825 }
826 Path downloaded = tryDownloadArchive(url, originBase);
827
828 FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
829
830 // filters
831 List<PathMatcher> includeMatchers = new ArrayList<>();
832 Properties includes = new Properties();
833 try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
834 includes.load(in);
835 }
836 for (Object pattern : includes.keySet()) {
837 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
838 includeMatchers.add(pathMatcher);
839 }
840
841 List<PathMatcher> excludeMatchers = new ArrayList<>();
842 Path excludeFile = duDir.resolve("excludes.properties");
843 if (Files.exists(excludeFile)) {
844 Properties excludes = new Properties();
845 try (InputStream in = Files.newInputStream(excludeFile)) {
846 excludes.load(in);
847 }
848 for (Object pattern : excludes.keySet()) {
849 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
850 excludeMatchers.add(pathMatcher);
851 }
852 }
853
854 // keys are the bundle directories
855 Map<Path, A2Origin> origins = new HashMap<>();
856 Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
857
858 @Override
859 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
860 includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
861 if (includeMatcher.matches(file)) {
862 for (PathMatcher excludeMatcher : excludeMatchers) {
863 if (excludeMatcher.matches(file)) {
864 logger.log(TRACE, "Skipping excluded " + file);
865 return FileVisitResult.CONTINUE;
866 }
867 }
868 if (file.getFileName().toString().contains(".source_")) {
869 processEclipseSourceJar(file, targetCategoryBase);
870 logger.log(DEBUG, () -> "Processed source " + file);
871 } else {
872 Map<String, String> map = new HashMap<>();
873 for (Object key : commonProps.keySet())
874 map.put(key.toString(), commonProps.getProperty(key.toString()));
875 A2Origin origin = new A2Origin();
876 Path bundleDir = processBundleJar(file, targetCategoryBase, map, origin);
877 origins.put(bundleDir, origin);
878 logger.log(DEBUG, () -> "Processed " + file);
879 }
880 break includeMatchers;
881 }
882 }
883 return FileVisitResult.CONTINUE;
884 }
885 });
886
887 DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)
888 && p.getFileName().toString().indexOf('.') >= 0 && !p.getFileName().toString().endsWith(".src"));
889 for (Path bundleDir : dirs) {
890 A2Origin origin = origins.get(bundleDir);
891 Objects.requireNonNull(origin, "No A2 origin found for " + bundleDir);
892 createJar(bundleDir, origin);
893 }
894 } catch (IOException e) {
895 throw new RuntimeException("Cannot process " + duDir, e);
896 }
897
898 }
899
900 /** Process sources in Eclipse format. */
901 void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
902 try {
903 A2Origin origin = new A2Origin();
904 Path bundleDir;
905 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
906 Manifest manifest = jarIn.getManifest();
907
908 String[] relatedBundle = manifest.getMainAttributes().getValue(ECLIPSE_SOURCE_BUNDLE.toString())
909 .split(";");
910 String version = relatedBundle[1].substring("version=\"".length());
911 version = version.substring(0, version.length() - 1);
912 NameVersion nameVersion = new NameVersion(relatedBundle[0], version);
913 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
914
915 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
916 : bundleDir.resolve("OSGI-OPT/src");
917
918 Files.createDirectories(sourceDir);
919 JarEntry entry;
920 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
921 if (entry.isDirectory())
922 continue entries;
923 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
924 continue entries;
925 Path target = sourceDir.resolve(entry.getName());
926 Files.createDirectories(target.getParent());
927 Files.copy(jarIn, target);
928 logger.log(TRACE, () -> "Copied source " + target);
929 }
930
931 // write the changes
932 if (sourceBundles) {
933 origin.appendChanges(sourceDir);
934 } else {
935 origin.added.add("source code under OSGI-OPT/src");
936 origin.appendChanges(bundleDir);
937 }
938 }
939 } catch (IOException e) {
940 throw new IllegalStateException("Cannot process " + file, e);
941 }
942 }
943
944 /*
945 * COMMON PROCESSING
946 */
947 /** Normalise a single (that is, non-merged) bundle. */
948 Path processBundleJar(Path file, Path targetBase, Map<String, String> entries, A2Origin origin) throws IOException {
949 // boolean embed = Boolean.parseBoolean(entries.getOrDefault(ARGEO_ORIGIN_EMBED.toString(), "false").toString());
950 boolean doNotModify = Boolean
951 .parseBoolean(entries.getOrDefault(ManifestHeader.ARGEO_DO_NOT_MODIFY.toString(), "false").toString());
952 NameVersion nameVersion;
953 Path bundleDir;
954 // singleton
955 boolean isSingleton = false;
956 Manifest manifest;
957 Manifest sourceManifest;
958 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
959 sourceManifest = jarIn.getManifest();
960 manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
961
962 String rawSourceSymbolicName = manifest.getMainAttributes().getValue(BUNDLE_SYMBOLICNAME.toString());
963 if (rawSourceSymbolicName != null) {
964 // make sure there is no directive
965 String[] arr = rawSourceSymbolicName.split(";");
966 for (int i = 1; i < arr.length; i++) {
967 if (arr[i].trim().equals("singleton:=true"))
968 isSingleton = true;
969 logger.log(DEBUG, file.getFileName() + " is a singleton");
970 }
971 }
972 // remove problematic entries in MANIFEST
973 manifest.getEntries().clear();
974
975 String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString());
976 String ourVersion = entries.get(BUNDLE_VERSION.toString());
977
978 if (ourSymbolicName != null && ourVersion != null) {
979 nameVersion = new NameVersion(ourSymbolicName, ourVersion);
980 } else {
981 nameVersion = nameVersionFromManifest(manifest);
982 if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) {
983 logger.log(WARNING,
984 "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion);
985 entries.put(BUNDLE_VERSION.toString(), ourVersion);
986 }
987 if (ourSymbolicName != null) {
988 // we always force our symbolic name
989 nameVersion.setName(ourSymbolicName);
990 }
991 }
992 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
993
994 // copy original MANIFEST
995 if (sourceManifest != null) {
996 Path originalManifest = bundleDir.resolve(ARGEO_ORIGIN).resolve("MANIFEST.MF");
997 Files.createDirectories(originalManifest.getParent());
998 try (OutputStream out = Files.newOutputStream(originalManifest)) {
999 sourceManifest.write(out);
1000 }
1001 origin.moved.add("original MANIFEST to " + bundleDir.relativize(originalManifest));
1002 }
1003
1004 // force Java 9 module name
1005 entries.put(ManifestHeader.AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName());
1006
1007 boolean isNative = false;
1008 String os = null;
1009 String arch = null;
1010 if (bundleDir.startsWith(a2LibBase)) {
1011 isNative = true;
1012 Path libRelativePath = a2LibBase.relativize(bundleDir);
1013 os = libRelativePath.getName(0).toString();
1014 arch = libRelativePath.getName(1).toString();
1015 }
1016
1017 // if (!embed) {
1018 // copy entries
1019 JarEntry entry;
1020 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
1021 if (entry.isDirectory())
1022 continue entries;
1023 if (!doNotModify) {
1024 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
1025 || entry.getName().endsWith(".SF")) {
1026 origin.deleted.add("cryptographic signatures");
1027 continue entries;
1028 }
1029 if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
1030 origin.deleted.add("Java module information (module-info.class)");
1031 continue entries;
1032 }
1033 if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
1034 origin.deleted.add("additional Java versions (META-INF/versions)");
1035 continue entries;
1036 }
1037 if (entry.getName().startsWith("META-INF/maven/")) {
1038 origin.deleted.add("Maven information (META-INF/maven)");
1039 continue entries;
1040 }
1041 // skip file system providers as they cause issues with native image
1042 if (entry.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider")) {
1043 origin.deleted
1044 .add("file system providers (META-INF/services/java.nio.file.spi.FileSystemProvider)");
1045 continue entries;
1046 }
1047 }
1048 if (entry.getName().startsWith("OSGI-OPT/src/")) { // skip embedded sources
1049 origin.deleted.add("embedded sources");
1050 continue entries;
1051 }
1052 Path target = bundleDir.resolve(entry.getName());
1053 Files.createDirectories(target.getParent());
1054 Files.copy(jarIn, target);
1055
1056 // native libraries
1057 if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll")
1058 || entry.getName().endsWith(".jnilib"))) {
1059 Path categoryDir = bundleDir.getParent();
1060 boolean copyDll = false;
1061 Path targetDll = categoryDir.resolve(bundleDir.relativize(target));
1062 if (nameVersion.getName().equals("com.sun.jna")) {
1063 if (arch.equals("x86_64"))
1064 arch = "x86-64";
1065 if (os.equals("macosx"))
1066 os = "darwin";
1067 if (target.getParent().getFileName().toString().equals(os + "-" + arch)) {
1068 copyDll = true;
1069 }
1070 targetDll = categoryDir.resolve(target.getFileName());
1071 } else {
1072 copyDll = true;
1073 }
1074 if (copyDll) {
1075 Files.createDirectories(targetDll.getParent());
1076 if (Files.exists(targetDll))
1077 Files.delete(targetDll);
1078 Files.copy(target, targetDll);
1079 }
1080 Files.delete(target);
1081 origin.deleted.add(bundleDir.relativize(target).toString());
1082 }
1083 logger.log(TRACE, () -> "Copied " + target);
1084 // }
1085 }
1086 }
1087
1088 // copy MANIFEST
1089 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1090 Files.createDirectories(manifestPath.getParent());
1091
1092 if (isSingleton && entries.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
1093 entries.put(BUNDLE_SYMBOLICNAME.toString(),
1094 entries.get(BUNDLE_SYMBOLICNAME.toString()) + ";singleton:=true");
1095 }
1096
1097 // if (embed) {// copy embedded jar
1098 // Files.copy(file, bundleDir.resolve(file.getFileName()));
1099 // entries.put(ManifestHeader.BUNDLE_CLASSPATH.toString(), file.getFileName().toString());
1100 // }
1101
1102 // Final MANIFEST decisions
1103 // We also check the original OSGi metadata and compare with our changes
1104 for (String key : entries.keySet()) {
1105 String value = entries.get(key);
1106 String previousValue = manifest.getMainAttributes().getValue(key);
1107 boolean wasDifferent = previousValue != null && !previousValue.equals(value);
1108 boolean keepPrevious = false;
1109 if (wasDifferent) {
1110 if (SPDX_LICENSE_IDENTIFIER.toString().equals(key) && previousValue != null)
1111 keepPrevious = true;
1112 else if (BUNDLE_VERSION.toString().equals(key) && wasDifferent)
1113 if (previousValue.equals(value + ".0")) // typically a Maven first release
1114 keepPrevious = true;
1115
1116 if (keepPrevious) {
1117 if (logger.isLoggable(DEBUG))
1118 logger.log(DEBUG, file.getFileName() + ": " + key + " was NOT modified, value kept is "
1119 + previousValue + ", not overriden with " + value);
1120 value = previousValue;
1121 }
1122 }
1123
1124 manifest.getMainAttributes().putValue(key, value);
1125 if (wasDifferent && !keepPrevious) {
1126 if (IMPORT_PACKAGE.toString().equals(key) || EXPORT_PACKAGE.toString().equals(key))
1127 logger.log(TRACE, () -> file.getFileName() + ": " + key + " was modified");
1128 else if (BUNDLE_SYMBOLICNAME.toString().equals(key) || AUTOMATIC_MODULE_NAME.toString().equals(key))
1129 logger.log(DEBUG,
1130 file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value);
1131 else
1132 logger.log(WARNING,
1133 file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value);
1134 origin.modified.add("MANIFEST header " + key);
1135 }
1136
1137 // !! hack to remove unresolvable
1138 if (key.equals("Provide-Capability") || key.equals("Require-Capability"))
1139 if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) {
1140 manifest.getMainAttributes().remove(key);
1141 origin.deleted.add("MANIFEST header " + key);
1142 }
1143 }
1144
1145 // de-pollute MANIFEST
1146 for (Iterator<Map.Entry<Object, Object>> manifestEntries = manifest.getMainAttributes().entrySet()
1147 .iterator(); manifestEntries.hasNext();) {
1148 Map.Entry<Object, Object> manifestEntry = manifestEntries.next();
1149 switch (manifestEntry.getKey().toString()) {
1150 case "Archiver-Version":
1151 case "Build-By":
1152 case "Created-By":
1153 case "Originally-Created-By":
1154 case "Tool":
1155 case "Bnd-LastModified":
1156 manifestEntries.remove();
1157 origin.deleted.add("MANIFEST header " + manifestEntry.getKey());
1158 break;
1159 default:
1160 if (sourceManifest != null && !sourceManifest.getMainAttributes().containsKey(manifestEntry.getKey()))
1161 origin.added.add("MANIFEST header " + manifestEntry.getKey());
1162 }
1163 }
1164
1165 processLicense(bundleDir, manifest);
1166
1167 origin.modified.add("MANIFEST (META-INF/MANIFEST.MF)");
1168 // write the MANIFEST
1169 try (OutputStream out = Files.newOutputStream(manifestPath)) {
1170 manifest.write(out);
1171 }
1172 return bundleDir;
1173 }
1174
1175 /** Process SPDX license identifier. */
1176 void processLicense(Path bundleDir, Manifest manifest) {
1177 String spdxLicenceId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString());
1178 String bundleLicense = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString());
1179 if (spdxLicenceId == null) {
1180 logger.log(ERROR, bundleDir.getFileName() + ": " + SPDX_LICENSE_IDENTIFIER + " not available, "
1181 + BUNDLE_LICENSE + " is " + bundleLicense);
1182 } else {
1183 // only use the first licensing option
1184 int orIndex = spdxLicenceId.indexOf(" OR ");
1185 if (orIndex >= 0)
1186 spdxLicenceId = spdxLicenceId.substring(0, orIndex).trim();
1187
1188 String bundleDirName = bundleDir.getFileName().toString();
1189 // force licenses of some well-known components
1190 // even if we say otherwise (typically because from an Eclipse archive)
1191 if (bundleDirName.startsWith("org.apache."))
1192 spdxLicenceId = "Apache-2.0";
1193 if (bundleDirName.startsWith("com.sun.jna."))
1194 spdxLicenceId = "Apache-2.0";
1195 if (bundleDirName.startsWith("com.ibm.icu."))
1196 spdxLicenceId = "ICU";
1197 if (bundleDirName.startsWith("javax.annotation."))
1198 spdxLicenceId = "GPL-2.0-only WITH Classpath-exception-2.0";
1199 if (bundleDirName.startsWith("javax.inject."))
1200 spdxLicenceId = "Apache-2.0";
1201 if (bundleDirName.startsWith("org.osgi."))
1202 spdxLicenceId = "Apache-2.0";
1203
1204 manifest.getMainAttributes().putValue(SPDX_LICENSE_IDENTIFIER.toString(), spdxLicenceId);
1205 if (!licensesUsed.containsKey(spdxLicenceId))
1206 licensesUsed.put(spdxLicenceId, new TreeSet<>());
1207 licensesUsed.get(spdxLicenceId).add(bundleDir.getParent().getFileName() + "/" + bundleDir.getFileName());
1208 }
1209 }
1210
1211 /*
1212 * UTILITIES
1213 */
1214 /** Recursively deletes a directory. */
1215 static void deleteDirectory(Path path) throws IOException {
1216 if (!Files.exists(path))
1217 return;
1218 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
1219 @Override
1220 public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
1221 if (e != null)
1222 throw e;
1223 Files.delete(directory);
1224 return CONTINUE;
1225 }
1226
1227 @Override
1228 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1229 Files.delete(file);
1230 return CONTINUE;
1231 }
1232 });
1233 }
1234
1235 /** Extract name/version from a MANIFEST. */
1236 NameVersion nameVersionFromManifest(Manifest manifest) {
1237 Attributes attrs = manifest.getMainAttributes();
1238 // symbolic name
1239 String symbolicName = attrs.getValue(ManifestHeader.BUNDLE_SYMBOLICNAME.toString());
1240 if (symbolicName == null)
1241 return null;
1242 // make sure there is no directive
1243 symbolicName = symbolicName.split(";")[0];
1244
1245 String version = attrs.getValue(ManifestHeader.BUNDLE_VERSION.toString());
1246 return new NameVersion(symbolicName, version);
1247 }
1248
1249 /** Try to download from an URI. */
1250 Path tryDownloadArchive(String uri, Path dir) throws IOException {
1251 // find mirror
1252 List<String> urlBases = null;
1253 String uriPrefix = null;
1254 uriPrefixes: for (String uriPref : mirrors.keySet()) {
1255 if (uri.startsWith(uriPref)) {
1256 if (mirrors.get(uriPref).size() > 0) {
1257 urlBases = mirrors.get(uriPref);
1258 uriPrefix = uriPref;
1259 break uriPrefixes;
1260 }
1261 }
1262 }
1263 if (urlBases == null)
1264 try {
1265 return downloadArchive(new URL(uri), dir);
1266 } catch (FileNotFoundException e) {
1267 throw new FileNotFoundException("Cannot find " + uri);
1268 }
1269
1270 // try to download
1271 for (String urlBase : urlBases) {
1272 String relativePath = uri.substring(uriPrefix.length());
1273 URL url = new URL(urlBase + relativePath);
1274 try {
1275 return downloadArchive(url, dir);
1276 } catch (FileNotFoundException e) {
1277 logger.log(WARNING, "Cannot download " + url + ", trying another mirror");
1278 }
1279 }
1280 throw new FileNotFoundException("Cannot find " + uri);
1281 }
1282
1283 /**
1284 * Effectively download. Synchronised in order to avoid downloading twice in
1285 * parallel.
1286 */
1287 synchronized Path downloadArchive(URL url, Path dir) throws IOException {
1288 return download(url, dir, (String) null);
1289 }
1290
1291 /** Effectively download. */
1292 Path download(URL url, Path dir, String name) throws IOException {
1293
1294 Path dest;
1295 if (name == null) {
1296 // We use also use parent directory in case the archive itself has a fixed name
1297 String[] segments = url.getPath().split("/");
1298 name = segments.length > 1 ? segments[segments.length - 2] + '-' + segments[segments.length - 1]
1299 : segments[segments.length - 1];
1300 }
1301
1302 dest = dir.resolve(name);
1303 if (Files.exists(dest)) {
1304 logger.log(TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
1305 return dest;
1306 } else {
1307 Files.createDirectories(dest.getParent());
1308 }
1309
1310 try (InputStream in = url.openStream()) {
1311 Files.copy(in, dest);
1312 logger.log(DEBUG, () -> "Downloaded " + dest + " from " + url);
1313 }
1314 return dest;
1315 }
1316
1317 /** Create a JAR file from a directory. */
1318 Path createJar(Path bundleDir, A2Origin origin) throws IOException {
1319 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1320 Manifest manifest;
1321 try (InputStream in = Files.newInputStream(manifestPath)) {
1322 manifest = new Manifest(in);
1323 }
1324 // legal requirements
1325 origin.appendChanges(bundleDir);
1326 createReadMe(bundleDir, manifest);
1327
1328 // create the jar
1329 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
1330 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
1331 jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
1332 Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
1333
1334 @Override
1335 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1336 if (file.getFileName().toString().equals("MANIFEST.MF"))
1337 return super.visitFile(file, attrs);
1338 JarEntry entry = new JarEntry(
1339 bundleDir.relativize(file).toString().replace(File.separatorChar, '/'));
1340 jarOut.putNextEntry(entry);
1341 Files.copy(file, jarOut);
1342 return super.visitFile(file, attrs);
1343 }
1344
1345 });
1346 }
1347 deleteDirectory(bundleDir);
1348
1349 if (sourceBundles)
1350 createSourceJar(bundleDir, manifest);
1351
1352 return jarPath;
1353 }
1354
1355 /** Package sources separately, in the Eclipse-SourceBundle format. */
1356 void createSourceJar(Path bundleDir, Manifest manifest) throws IOException {
1357 Path bundleCategoryDir = bundleDir.getParent();
1358 Path sourceDir = bundleCategoryDir.resolve(bundleDir.toString() + ".src");
1359 if (!Files.exists(sourceDir)) {
1360 logger.log(WARNING, sourceDir + " does not exist, skipping...");
1361 return;
1362 }
1363 createReadMe(sourceDir, manifest);
1364
1365 Path relPath = a2Base.relativize(bundleCategoryDir);
1366 Path srcCategoryDir = a2SrcBase.resolve(relPath);
1367 Path srcJarP = srcCategoryDir.resolve(sourceDir.getFileName() + ".jar");
1368 Files.createDirectories(srcJarP.getParent());
1369
1370 String bundleSymbolicName = manifest.getMainAttributes().getValue("Bundle-SymbolicName").toString();
1371 // in case there are additional directives
1372 bundleSymbolicName = bundleSymbolicName.split(";")[0];
1373 Manifest srcManifest = new Manifest();
1374 srcManifest.getMainAttributes().put(MANIFEST_VERSION, "1.0");
1375 srcManifest.getMainAttributes().putValue(BUNDLE_SYMBOLICNAME.toString(), bundleSymbolicName + ".src");
1376 srcManifest.getMainAttributes().putValue(BUNDLE_VERSION.toString(),
1377 manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()).toString());
1378 srcManifest.getMainAttributes().putValue(ECLIPSE_SOURCE_BUNDLE.toString(), bundleSymbolicName + ";version=\""
1379 + manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()) + "\"");
1380
1381 try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
1382 srcJarOut.setLevel(Deflater.BEST_COMPRESSION);
1383 Files.walkFileTree(sourceDir, new SimpleFileVisitor<Path>() {
1384
1385 @Override
1386 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1387 if (file.getFileName().toString().equals("MANIFEST.MF"))
1388 return super.visitFile(file, attrs);
1389 JarEntry entry = new JarEntry(
1390 sourceDir.relativize(file).toString().replace(File.separatorChar, '/'));
1391 srcJarOut.putNextEntry(entry);
1392 Files.copy(file, srcJarOut);
1393 return super.visitFile(file, attrs);
1394 }
1395
1396 });
1397 }
1398 deleteDirectory(sourceDir);
1399 }
1400
1401 /**
1402 * Generate a readme clarifying and prominently notifying of the repackaging and
1403 * modifications.
1404 */
1405 void createReadMe(Path jarDir, Manifest manifest) throws IOException {
1406 // write repackaged README
1407 try (BufferedWriter writer = Files.newBufferedWriter(jarDir.resolve(README_REPACKAGED))) {
1408 boolean merged = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2_MERGE.toString()) != null;
1409 if (merged)
1410 writer.append("This component is a merging of third party components"
1411 + " in order to comply with A2 packaging standards.\n");
1412 else
1413 writer.append("This component is a repackaging of a third party component"
1414 + " in order to comply with A2 packaging standards.\n");
1415
1416 // license
1417 String spdxLicenseId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString());
1418 if (spdxLicenseId == null)
1419 throw new IllegalStateException("An SPDX license id must have beend defined at this stage.");
1420 writer.append("\nIt is redistributed under the following license:\n\n");
1421 writer.append("SPDX-Identifier: " + spdxLicenseId + "\n\n");
1422
1423 if (!spdxLicenseId.startsWith("LicenseRef")) {// standard
1424 int withIndex = spdxLicenseId.indexOf(" WITH ");
1425 if (withIndex >= 0) {
1426 String simpleId = spdxLicenseId.substring(0, withIndex).trim();
1427 String exception = spdxLicenseId.substring(withIndex + " WITH ".length());
1428 writer.append("which are available here: https://spdx.org/licenses/" + simpleId
1429 + "\nand here: https://spdx.org/licenses/" + exception + "\n");
1430 } else {
1431 writer.append("which is available here: https://spdx.org/licenses/" + spdxLicenseId + "\n");
1432 }
1433 } else {
1434 String url = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString());
1435 if (url != null) {
1436 writer.write("which is available here: " + url + "\n");
1437 } else {
1438 logger.log(ERROR, "No licne URL for " + jarDir);
1439 }
1440 }
1441 writer.write("\n");
1442
1443 // origin
1444 String m2Repo = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2_REPO.toString());
1445 String originDesc = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2.toString());
1446 if (originDesc != null)
1447 writer.append("The original component has M2 coordinates:\n" + originDesc.replace(',', '\n') + "\n"
1448 + (m2Repo != null ? "\nin M2 repository " + m2Repo + "\n" : ""));
1449 else {
1450 originDesc = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_URI.toString());
1451 if (originDesc != null)
1452 writer.append("The original component comes from " + originDesc + ".\n");
1453 else
1454 logger.log(ERROR, "Cannot find origin information in " + jarDir);
1455 }
1456
1457 writer.append("\nA detailed list of changes is available under " + CHANGES + ".\n");
1458 if (!jarDir.getFileName().endsWith(".src")) {// binary archive
1459 if (sourceBundles)
1460 writer.append("Corresponding sources are available in the related archive named "
1461 + jarDir.toString() + ".src.jar.\n");
1462 else
1463 writer.append("Corresponding sources are available under OSGI-OPT/src.\n");
1464 }
1465 }
1466
1467 }
1468
1469 /**
1470 * Gathers modifications performed on the original binaries and sources,
1471 * especially in order to comply with their license requirements.
1472 */
1473 class A2Origin {
1474 A2Origin() {
1475
1476 }
1477
1478 Set<String> modified = new TreeSet<>();
1479 Set<String> deleted = new TreeSet<>();
1480 Set<String> added = new TreeSet<>();
1481 Set<String> moved = new TreeSet<>();
1482
1483 /** Append changes to the A2-ORIGIN/changes file. */
1484 void appendChanges(Path baseDirectory) throws IOException {
1485 Path changesFile = baseDirectory.resolve(CHANGES);
1486 Files.createDirectories(changesFile.getParent());
1487 try (BufferedWriter writer = Files.newBufferedWriter(changesFile, APPEND, CREATE)) {
1488 for (String msg : added)
1489 writer.write("- Added " + msg + ".\n");
1490 for (String msg : modified)
1491 writer.write("- Modified " + msg + ".\n");
1492 for (String msg : moved)
1493 writer.write("- Moved " + msg + ".\n");
1494 for (String msg : deleted)
1495 writer.write("- Deleted " + msg + ".\n");
1496 }
1497 }
1498 }
1499 }
1500
1501 /** Simple representation of an M2 artifact. */
1502 class M2Artifact extends CategoryNameVersion {
1503 private String classifier;
1504
1505 M2Artifact(String m2coordinates) {
1506 this(m2coordinates, null);
1507 }
1508
1509 M2Artifact(String m2coordinates, String classifier) {
1510 String[] parts = m2coordinates.split(":");
1511 setCategory(parts[0]);
1512 setName(parts[1]);
1513 if (parts.length > 2) {
1514 setVersion(parts[2]);
1515 }
1516 this.classifier = classifier;
1517 }
1518
1519 String getGroupId() {
1520 return super.getCategory();
1521 }
1522
1523 String getArtifactId() {
1524 return super.getName();
1525 }
1526
1527 String toM2Coordinates() {
1528 return getCategory() + ":" + getName() + (getVersion() != null ? ":" + getVersion() : "");
1529 }
1530
1531 String getClassifier() {
1532 return classifier != null ? classifier : "";
1533 }
1534
1535 String getExtension() {
1536 return "jar";
1537 }
1538 }
1539
1540 /** Utilities around Maven (conventions based). */
1541 class M2ConventionsUtils {
1542 final static String MAVEN_CENTRAL_BASE_URL = "https://repo1.maven.org/maven2/";
1543
1544 /** The file name of this artifact when stored */
1545 static String artifactFileName(M2Artifact artifact) {
1546 return artifact.getArtifactId() + '-' + artifact.getVersion()
1547 + (artifact.getClassifier().equals("") ? "" : '-' + artifact.getClassifier()) + '.'
1548 + artifact.getExtension();
1549 }
1550
1551 /** Absolute path to the file */
1552 static String artifactPath(String artifactBasePath, M2Artifact artifact) {
1553 return artifactParentPath(artifactBasePath, artifact) + '/' + artifactFileName(artifact);
1554 }
1555
1556 /** Absolute path to the file */
1557 static String artifactUrl(String repoUrl, M2Artifact artifact) {
1558 if (repoUrl.endsWith("/"))
1559 return repoUrl + artifactPath("/", artifact).substring(1);
1560 else
1561 return repoUrl + artifactPath("/", artifact);
1562 }
1563
1564 /** Absolute path to the file */
1565 static URL mavenRepoUrl(String repoBase, M2Artifact artifact) {
1566 String url = artifactUrl(repoBase == null ? MAVEN_CENTRAL_BASE_URL : repoBase, artifact);
1567 try {
1568 return new URL(url);
1569 } catch (MalformedURLException e) {
1570 // it should not happen
1571 throw new IllegalStateException(e);
1572 }
1573 }
1574
1575 /** Absolute path to the directories where the files will be stored */
1576 static String artifactParentPath(String artifactBasePath, M2Artifact artifact) {
1577 return artifactBasePath + (artifactBasePath.endsWith("/") ? "" : "/") + artifactParentPath(artifact);
1578 }
1579
1580 /** Relative path to the directories where the files will be stored */
1581 static String artifactParentPath(M2Artifact artifact) {
1582 return artifact.getGroupId().replace('.', '/') + '/' + artifact.getArtifactId() + '/' + artifact.getVersion();
1583 }
1584
1585 /** Singleton */
1586 private M2ConventionsUtils() {
1587 }
1588 }
1589
1590 /** Combination of a category, a name and a version. */
1591 class CategoryNameVersion extends NameVersion {
1592 private String category;
1593
1594 CategoryNameVersion() {
1595 }
1596
1597 CategoryNameVersion(String category, String name, String version) {
1598 super(name, version);
1599 this.category = category;
1600 }
1601
1602 CategoryNameVersion(String category, NameVersion nameVersion) {
1603 super(nameVersion);
1604 this.category = category;
1605 }
1606
1607 String getCategory() {
1608 return category;
1609 }
1610
1611 void setCategory(String category) {
1612 this.category = category;
1613 }
1614
1615 @Override
1616 public String toString() {
1617 return category + ":" + super.toString();
1618 }
1619
1620 }
1621
1622 /** Combination of a name and a version. */
1623 class NameVersion implements Comparable<NameVersion> {
1624 private String name;
1625 private String version;
1626
1627 NameVersion() {
1628 }
1629
1630 /** Interprets string in OSGi-like format my.module.name;version=0.0.0 */
1631 NameVersion(String nameVersion) {
1632 int index = nameVersion.indexOf(";version=");
1633 if (index < 0) {
1634 setName(nameVersion);
1635 setVersion(null);
1636 } else {
1637 setName(nameVersion.substring(0, index));
1638 setVersion(nameVersion.substring(index + ";version=".length()));
1639 }
1640 }
1641
1642 NameVersion(String name, String version) {
1643 this.name = name;
1644 this.version = version;
1645 }
1646
1647 NameVersion(NameVersion nameVersion) {
1648 this.name = nameVersion.getName();
1649 this.version = nameVersion.getVersion();
1650 }
1651
1652 String getName() {
1653 return name;
1654 }
1655
1656 void setName(String name) {
1657 this.name = name;
1658 }
1659
1660 String getVersion() {
1661 return version;
1662 }
1663
1664 void setVersion(String version) {
1665 this.version = version;
1666 }
1667
1668 String getBranch() {
1669 String[] parts = getVersion().split("\\.");
1670 if (parts.length < 2)
1671 throw new IllegalStateException("Version " + getVersion() + " cannot be interpreted as branch.");
1672 return parts[0] + "." + parts[1];
1673 }
1674
1675 @Override
1676 public boolean equals(Object obj) {
1677 if (obj instanceof NameVersion) {
1678 NameVersion nameVersion = (NameVersion) obj;
1679 return name.equals(nameVersion.getName()) && version.equals(nameVersion.getVersion());
1680 } else
1681 return false;
1682 }
1683
1684 @Override
1685 public int hashCode() {
1686 return name.hashCode();
1687 }
1688
1689 @Override
1690 public String toString() {
1691 return name + ":" + version;
1692 }
1693
1694 public int compareTo(NameVersion o) {
1695 if (o.getName().equals(name))
1696 return version.compareTo(o.getVersion());
1697 else
1698 return name.compareTo(o.getName());
1699 }
1700 }
Argeo Build - Build scripts and configuration for Argeo projects