]> git.argeo.org Git - cc0/argeo-build.git/blob - src/org/argeo/build/Repackage.java
projects / cc0 / argeo-build.git / blob


9e2c14fd409f36e62ec13e03845d16ee64b52a02
[cc0/argeo-build.git] / src / org / argeo / build / Repackage.java
1 package org.argeo.build;
2
3 import static java.lang.System.Logger.Level.DEBUG;
4 import static java.lang.System.Logger.Level.ERROR;
5 import static java.lang.System.Logger.Level.INFO;
6 import static java.lang.System.Logger.Level.TRACE;
7 import static java.lang.System.Logger.Level.WARNING;
8 import static java.nio.file.FileVisitResult.CONTINUE;
9 import static java.nio.file.StandardOpenOption.APPEND;
10 import static java.nio.file.StandardOpenOption.CREATE;
11 import static java.util.jar.Attributes.Name.MANIFEST_VERSION;
12 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_EMBED;
13 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2;
14 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_MERGE;
15 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_REPO;
16 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_NO_METADATA_GENERATION;
17 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_URI;
18 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_LICENSE;
19 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_SYMBOLICNAME;
20 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_VERSION;
21 import static org.argeo.build.Repackage.ManifestHeader.ECLIPSE_SOURCE_BUNDLE;
22 import static org.argeo.build.Repackage.ManifestHeader.EXPORT_PACKAGE;
23 import static org.argeo.build.Repackage.ManifestHeader.IMPORT_PACKAGE;
24 import static org.argeo.build.Repackage.ManifestHeader.SPDX_LICENSE_IDENTIFIER;
25
26 import java.io.BufferedWriter;
27 import java.io.File;
28 import java.io.FileNotFoundException;
29 import java.io.IOException;
30 import java.io.InputStream;
31 import java.io.OutputStream;
32 import java.lang.System.Logger;
33 import java.net.MalformedURLException;
34 import java.net.URL;
35 import java.nio.charset.StandardCharsets;
36 import java.nio.file.DirectoryStream;
37 import java.nio.file.FileSystem;
38 import java.nio.file.FileSystems;
39 import java.nio.file.FileVisitResult;
40 import java.nio.file.Files;
41 import java.nio.file.Path;
42 import java.nio.file.PathMatcher;
43 import java.nio.file.Paths;
44 import java.nio.file.SimpleFileVisitor;
45 import java.nio.file.StandardCopyOption;
46 import java.nio.file.StandardOpenOption;
47 import java.nio.file.attribute.BasicFileAttributes;
48 import java.util.ArrayList;
49 import java.util.HashMap;
50 import java.util.Iterator;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.Objects;
54 import java.util.Properties;
55 import java.util.Set;
56 import java.util.StringJoiner;
57 import java.util.TreeMap;
58 import java.util.TreeSet;
59 import java.util.concurrent.CompletableFuture;
60 import java.util.jar.Attributes;
61 import java.util.jar.JarEntry;
62 import java.util.jar.JarInputStream;
63 import java.util.jar.JarOutputStream;
64 import java.util.jar.Manifest;
65 import java.util.zip.Deflater;
66
67 import aQute.bnd.osgi.Analyzer;
68 import aQute.bnd.osgi.Jar;
69
70 /** Repackages existing jar files into OSGi bundles in an A2 repository. */
71 public class Repackage {
72 final static Logger logger = System.getLogger(Repackage.class.getName());
73
74 /**
75 * Environment variable on whether sources should be packaged separately or
76 * integrated in the bundles.
77 */
78 final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES";
79 /** Environment variable on whether operations should be parallelised. */
80 final static String ENV_ARGEO_BUILD_SEQUENTIAL = "ARGEO_BUILD_SEQUENTIAL";
81
82 /** Whether repackaging should run in parallel (default) or sequentially. */
83 final static boolean sequential = Boolean.parseBoolean(System.getenv(ENV_ARGEO_BUILD_SEQUENTIAL));
84
85 /** Main entry point. */
86 public static void main(String[] args) {
87 if (sequential)
88 logger.log(INFO, "Build will be sequential");
89 if (args.length < 2) {
90 System.err.println("Usage: <path to a2 output dir> <category1> <category2> ...");
91 System.exit(1);
92 }
93 Path a2Base = Paths.get(args[0]).toAbsolutePath().normalize();
94 Path descriptorsBase = Paths.get(".").toAbsolutePath().normalize();
95 Repackage factory = new Repackage(a2Base, descriptorsBase);
96
97 List<CompletableFuture<Void>> toDos = new ArrayList<>();
98 for (int i = 1; i < args.length; i++) {
99 Path p = Paths.get(args[i]);
100 if (sequential)
101 factory.processCategory(p);
102 else
103 toDos.add(CompletableFuture.runAsync(() -> factory.processCategory(p)));
104 }
105 if (!sequential)
106 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
107
108 // Summary
109 StringBuilder sb = new StringBuilder();
110 for (String licenseId : licensesUsed.keySet())
111 for (String name : licensesUsed.get(licenseId))
112 sb.append((licenseId.equals("") ? "Proprietary" : licenseId) + "\t\t" + name + "\n");
113 logger.log(INFO, "# License summary:\n" + sb);
114 }
115
116 /** MANIFEST headers. */
117 enum ManifestHeader {
118 // OSGi
119 /** OSGi bundle symbolic name. */
120 BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), //
121 /** OSGi bundle version. */
122 BUNDLE_VERSION("Bundle-Version"), //
123 /** OSGi bundle license. */
124 BUNDLE_LICENSE("Bundle-License"), //
125 /** OSGi exported packages list. */
126 EXPORT_PACKAGE("Export-Package"), //
127 /** OSGi imported packages list. */
128 IMPORT_PACKAGE("Import-Package"), //
129 /** OSGi path to embedded jar. */
130 BUNDLE_CLASSPATH("Bundle-Classpath"), //
131 // Java
132 /** Java module name. */
133 AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), //
134 // Eclipse
135 /** Eclipse source bundle. */
136 ECLIPSE_SOURCE_BUNDLE("Eclipse-SourceBundle"), //
137 // SPDX
138 /**
139 * SPDX license identifier.
140 *
141 * @see https://spdx.org/licenses/
142 */
143 SPDX_LICENSE_IDENTIFIER("SPDX-License-Identifier"), //
144 // Argeo Origin
145 /**
146 * Maven coordinates of the origin, possibly partial when using common.bnd or
147 * merge.bnd.
148 */
149 ARGEO_ORIGIN_M2("Argeo-Origin-M2"), //
150 /** List of Maven coordinates to merge. */
151 ARGEO_ORIGIN_M2_MERGE("Argeo-Origin-M2-Merge"), //
152 /** Maven repository, if not the default one. */
153 ARGEO_ORIGIN_M2_REPO("Argeo-Origin-M2-Repo"), //
154 /**
155 * Do not perform BND analysis of the origin component. Typically Import_package
156 * and Export-Package will be kept untouched.
157 */
158 ARGEO_ORIGIN_NO_METADATA_GENERATION("Argeo-Origin-NoMetadataGeneration"), //
159 /**
160 * Embed the original jar without modifying it (may be required by some
161 * proprietary licenses, such as JCR Day License).
162 */
163 ARGEO_ORIGIN_EMBED("Argeo-Origin-Embed"), //
164 /**
165 * Do not modify original jar (may be required by some proprietary licenses,
166 * such as JCR Day License).
167 */
168 ARGEO_DO_NOT_MODIFY("Argeo-Origin-Do-Not-Modify"), //
169 /**
170 * Origin (non-Maven) URI of the component. It may be anything (jar, archive,
171 * etc.).
172 */
173 ARGEO_ORIGIN_URI("Argeo-Origin-URI"), //
174 ;
175
176 final String value;
177
178 private ManifestHeader(String value) {
179 this.value = value;
180 }
181
182 @Override
183 public String toString() {
184 return value;
185 }
186 }
187
188 /** Name of the file centralising information for multiple M2 artifacts. */
189 final static String COMMON_BND = "common.bnd";
190 /** Name of the file centralising information for mergin M2 artifacts. */
191 final static String MERGE_BND = "merge.bnd";
192 /**
193 * Subdirectory of the jar file where origin informations (changes, legal
194 * notices etc. are stored)
195 */
196 final static String A2_ORIGIN = "A2-ORIGIN";
197 /** File detailing modifications to the original component. */
198 final static String CHANGES = A2_ORIGIN + "/changes";
199 /**
200 * Name of the file at the root of the repackaged jar, which prominently
201 * notifies that the component has be repackaged.
202 */
203 final static String README_REPACKAGED = "README.repackaged";
204
205 // cache
206 /** Summary of all license seen during the repackaging. */
207 final static Map<String, Set<String>> licensesUsed = new TreeMap<>();
208
209 /** Directory where to download archives */
210 final Path originBase;
211 /** Directory where to download Maven artifacts */
212 final Path mavenBase;
213
214 /** A2 repository base for binary bundles */
215 final Path a2Base;
216 /** A2 repository base for source bundles */
217 final Path a2SrcBase;
218 /** A2 base for native components */
219 final Path a2LibBase;
220 /** Location of the descriptors driving the packaging */
221 final Path descriptorsBase;
222 /** URIs of archives to download */
223 final Properties uris = new Properties();
224 /** Mirrors for archive download. Key is URI prefix, value list of base URLs */
225 final Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
226
227 /** Whether sources should be packaged separately */
228 final boolean sourceBundles;
229
230 /** Constructor initialises the various variables */
231 public Repackage(Path a2Base, Path descriptorsBase) {
232 sourceBundles = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES));
233 if (sourceBundles)
234 logger.log(INFO, "Sources will be packaged separately");
235
236 Objects.requireNonNull(a2Base);
237 Objects.requireNonNull(descriptorsBase);
238 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/build/origin");
239 this.mavenBase = Paths.get(System.getProperty("user.home"), ".m2", "repository");
240
241 // TODO define and use a build base
242 this.a2Base = a2Base;
243 this.a2SrcBase = a2Base.getParent().resolve(a2Base.getFileName() + ".src");
244 this.a2LibBase = a2Base.resolve("lib");
245 this.descriptorsBase = descriptorsBase;
246 if (!Files.exists(this.descriptorsBase))
247 throw new IllegalArgumentException(this.descriptorsBase + " does not exist");
248
249 // URIs mapping
250 Path urisPath = this.descriptorsBase.resolve("uris.properties");
251 if (Files.exists(urisPath)) {
252 try (InputStream in = Files.newInputStream(urisPath)) {
253 uris.load(in);
254 } catch (IOException e) {
255 throw new IllegalStateException("Cannot load " + urisPath, e);
256 }
257 }
258
259 // Eclipse mirrors
260 Path eclipseMirrorsPath = this.descriptorsBase.resolve("eclipse.mirrors.txt");
261 List<String> eclipseMirrors = new ArrayList<>();
262 if (Files.exists(eclipseMirrorsPath)) {
263 try {
264 eclipseMirrors = Files.readAllLines(eclipseMirrorsPath, StandardCharsets.UTF_8);
265 } catch (IOException e) {
266 throw new IllegalStateException("Cannot load " + eclipseMirrorsPath, e);
267 }
268 for (Iterator<String> it = eclipseMirrors.iterator(); it.hasNext();) {
269 String value = it.next();
270 if (value.strip().equals(""))
271 it.remove();
272 }
273 }
274 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
275 }
276
277 /*
278 * MAVEN ORIGIN
279 */
280 /** Process a whole category/group id. */
281 void processCategory(Path categoryRelativePath) {
282 try {
283 Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath);
284 DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
285 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
286 && !p.getFileName().toString().equals(MERGE_BND));
287 for (Path p : bnds) {
288 processSingleM2ArtifactDistributionUnit(p);
289 }
290
291 DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
292 for (Path duDir : dus) {
293 if (duDir.getFileName().toString().startsWith("eclipse-")) {
294 processEclipseArchive(duDir);
295 } else {
296 processM2BasedDistributionUnit(duDir);
297 }
298 }
299 } catch (IOException e) {
300 throw new RuntimeException("Cannot process category " + categoryRelativePath, e);
301 }
302 }
303
304 /** Process a standalone Maven artifact. */
305 void processSingleM2ArtifactDistributionUnit(Path bndFile) {
306 try {
307 Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent());
308 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
309
310 Properties fileProps = new Properties();
311 try (InputStream in = Files.newInputStream(bndFile)) {
312 fileProps.load(in);
313 }
314 String repoStr = fileProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
315 ? fileProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
316 : null;
317
318 // use file name as symbolic name
319 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
320 String symbolicName = bndFile.getFileName().toString();
321 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
322 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
323 }
324
325 String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
326 if (m2Coordinates == null)
327 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
328 M2Artifact artifact = new M2Artifact(m2Coordinates);
329 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
330 Path downloaded = downloadMaven(url, artifact);
331
332 // some proprietary artifacts do not allow any modification
333 // when releasing (with separate sources) we just copy it
334 boolean doNotModify = Boolean.parseBoolean(
335 fileProps.getOrDefault(ManifestHeader.ARGEO_DO_NOT_MODIFY.toString(), "false").toString());
336 if (doNotModify && sourceBundles) {
337 Path unmodifiedTarget = targetCategoryBase.resolve(
338 fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch() + ".jar");
339 Files.copy(downloaded, unmodifiedTarget, StandardCopyOption.REPLACE_EXISTING);
340 Path bundleDir = targetCategoryBase
341 .resolve(fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch());
342 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
343 Manifest manifest;
344 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(unmodifiedTarget))) {
345 manifest = jarIn.getManifest();
346 }
347 createSourceJar(bundleDir, manifest);
348 return;
349 }
350
351 // regular processing
352 A2Origin origin = new A2Origin();
353 Path bundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact, origin);
354 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
355 createJar(bundleDir, origin);
356 } catch (Exception e) {
357 throw new RuntimeException("Cannot process " + bndFile, e);
358 }
359 }
360
361 /**
362 * Process multiple Maven artifacts coming from a same project and therefore
363 * with information in common (typically the version), generating single bundles
364 * or merging them if necessary.
365 *
366 * @see #COMMON_BND
367 * @see #MERGE_BND
368 */
369 void processM2BasedDistributionUnit(Path duDir) {
370 try {
371 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
372 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
373
374 Path mergeBnd = duDir.resolve(MERGE_BND);
375 if (Files.exists(mergeBnd)) // merge
376 mergeM2Artifacts(mergeBnd);
377
378 Path commonBnd = duDir.resolve(COMMON_BND);
379 if (!Files.exists(commonBnd))
380 return;
381
382 Properties commonProps = new Properties();
383 try (InputStream in = Files.newInputStream(commonBnd)) {
384 commonProps.load(in);
385 }
386
387 String m2Version = commonProps.getProperty(ARGEO_ORIGIN_M2.toString());
388 if (m2Version == null) {
389 logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
390 return;// ignore, this is probably an Eclipse archive
391 }
392 if (!m2Version.startsWith(":")) {
393 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
394 }
395 m2Version = m2Version.substring(1);
396
397 DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
398 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
399 && !p.getFileName().toString().equals(MERGE_BND));
400 for (Path p : ds) {
401 Properties fileProps = new Properties();
402 try (InputStream in = Files.newInputStream(p)) {
403 fileProps.load(in);
404 }
405 String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
406 M2Artifact artifact = new M2Artifact(m2Coordinates);
407 artifact.setVersion(m2Version);
408
409 // prepare manifest entries
410 Properties mergeProps = new Properties();
411 mergeProps.putAll(commonProps);
412
413 fileEntries: for (Object key : fileProps.keySet()) {
414 if (ARGEO_ORIGIN_M2.toString().equals(key))
415 continue fileEntries;
416 String value = fileProps.getProperty(key.toString());
417 Object previousValue = mergeProps.put(key.toString(), value);
418 if (previousValue != null) {
419 logger.log(WARNING,
420 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
421 }
422 }
423 mergeProps.put(ARGEO_ORIGIN_M2.toString(), artifact.toM2Coordinates());
424 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
425 // use file name as symbolic name
426 String symbolicName = p.getFileName().toString();
427 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
428 mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
429 }
430
431 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
432 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
433 : null;
434
435 // download
436 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
437 Path downloaded = downloadMaven(url, artifact);
438
439 A2Origin origin = new A2Origin();
440 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact, origin);
441 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir, false);
442 createJar(targetBundleDir, origin);
443 }
444 } catch (IOException e) {
445 throw new RuntimeException("Cannot process " + duDir, e);
446 }
447 }
448
449 /** Merge multiple Maven artifacts. */
450 void mergeM2Artifacts(Path mergeBnd) throws IOException {
451 Path duDir = mergeBnd.getParent();
452 String category = duDir.getParent().getFileName().toString();
453 Path targetCategoryBase = a2Base.resolve(category);
454
455 Properties mergeProps = new Properties();
456 try (InputStream in = Files.newInputStream(mergeBnd)) {
457 mergeProps.load(in);
458 }
459
460 String m2Version = mergeProps.getProperty(ARGEO_ORIGIN_M2.toString());
461 if (m2Version == null) {
462 logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
463 return;// ignore, this is probably an Eclipse archive
464 }
465 if (!m2Version.startsWith(":")) {
466 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
467 }
468 m2Version = m2Version.substring(1);
469 mergeProps.put(BUNDLE_VERSION.toString(), m2Version);
470
471 String artifactsStr = mergeProps.getProperty(ARGEO_ORIGIN_M2_MERGE.toString());
472 if (artifactsStr == null)
473 throw new IllegalArgumentException(mergeBnd + ": " + ARGEO_ORIGIN_M2_MERGE + " must be set");
474
475 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
476 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
477 : null;
478
479 String bundleSymbolicName = mergeProps.getProperty(BUNDLE_SYMBOLICNAME.toString());
480 if (bundleSymbolicName == null)
481 throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
482 CategoryNameVersion nameVersion = new M2Artifact(category + ":" + bundleSymbolicName + ":" + m2Version);
483
484 A2Origin origin = new A2Origin();
485 Path bundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
486
487 StringJoiner originDesc = new StringJoiner(",");
488 String[] artifacts = artifactsStr.split(",");
489 artifacts: for (String str : artifacts) {
490 String m2Coordinates = str.trim();
491 if ("".equals(m2Coordinates))
492 continue artifacts;
493 M2Artifact artifact = new M2Artifact(m2Coordinates.trim());
494 if (artifact.getVersion() == null)
495 artifact.setVersion(m2Version);
496 originDesc.add(artifact.toString());
497 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
498 Path downloaded = downloadMaven(url, artifact);
499 JarEntry entry;
500 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
501 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
502 if (entry.isDirectory())
503 continue entries;
504 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
505 || entry.getName().endsWith(".SF")) {
506 origin.deleted.add("cryptographic signatures from " + artifact);
507 continue entries;
508 }
509 if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
510 origin.deleted.add("Java module information (module-info.class) from " + artifact);
511 continue entries;
512 }
513 if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
514 origin.deleted.add("additional Java versions (META-INF/versions) from " + artifact);
515 continue entries;
516 }
517 if (entry.getName().startsWith("META-INF/maven/")) {
518 origin.deleted.add("Maven information (META-INF/maven) from " + artifact);
519 continue entries;
520 }
521 if (entry.getName().startsWith(".cache/")) { // Apache SSHD
522 origin.deleted.add("cache directory (.cache) from " + artifact);
523 continue entries;
524 }
525 if (entry.getName().equals("META-INF/DEPENDENCIES")) {
526 origin.deleted.add("Dependencies (META-INF/DEPENDENCIES) from " + artifact);
527 continue entries;
528 }
529 if (entry.getName().equals("META-INF/MANIFEST.MF")) {
530 Path originalManifest = bundleDir.resolve(A2_ORIGIN).resolve(artifact.getGroupId())
531 .resolve(artifact.getArtifactId()).resolve("MANIFEST.MF");
532 Files.createDirectories(originalManifest.getParent());
533 try (OutputStream out = Files.newOutputStream(originalManifest)) {
534 Files.copy(jarIn, originalManifest);
535 }
536 origin.added.add(
537 "original MANIFEST (" + bundleDir.relativize(originalManifest) + ") from " + artifact);
538 continue entries;
539 }
540
541 if (entry.getName().endsWith("NOTICE") || entry.getName().endsWith("NOTICE.txt")
542 || entry.getName().endsWith("LICENSE") || entry.getName().endsWith("LICENSE.md")
543 || entry.getName().endsWith("LICENSE-notice.md") || entry.getName().endsWith("COPYING")
544 || entry.getName().endsWith("COPYING.LESSER")) {
545 Path artifactOriginDir = bundleDir.resolve(A2_ORIGIN).resolve(artifact.getGroupId())
546 .resolve(artifact.getArtifactId());
547 Path target = artifactOriginDir.resolve(entry.getName());
548 Files.createDirectories(target.getParent());
549 Files.copy(jarIn, target);
550 origin.moved.add(entry.getName() + " in " + artifact + " to " + bundleDir.relativize(target));
551 continue entries;
552 }
553 Path target = bundleDir.resolve(entry.getName());
554 Files.createDirectories(target.getParent());
555 if (!Files.exists(target)) {
556 Files.copy(jarIn, target);
557 } else {
558 if (entry.getName().startsWith("META-INF/services/")) {
559 try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
560 out.write("\n".getBytes());
561 jarIn.transferTo(out);
562 logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName());
563 }
564 origin.modified.add(entry.getName() + ", merging from " + artifact);
565 } else if (entry.getName().startsWith("org/apache/batik/")) {
566 logger.log(TRACE, "Skip " + entry.getName());
567 continue entries;
568 } else {
569 throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
570 }
571 }
572 logger.log(TRACE, () -> "Copied " + target);
573 }
574 }
575 origin.added.add("binary content of " + artifact);
576
577 // process sources
578 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, true);
579 }
580
581 // additional service files
582 Path servicesDir = duDir.resolve("services");
583 if (Files.exists(servicesDir)) {
584 for (Path p : Files.newDirectoryStream(servicesDir)) {
585 Path target = bundleDir.resolve("META-INF/services/").resolve(p.getFileName());
586 try (InputStream in = Files.newInputStream(p);
587 OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) {
588 out.write("\n".getBytes());
589 in.transferTo(out);
590 logger.log(DEBUG, "Appended " + p);
591 }
592 origin.added.add(bundleDir.relativize(target).toString());
593 }
594 }
595
596 // BND analysis
597 Map<String, String> entries = new TreeMap<>();
598 try (Analyzer bndAnalyzer = new Analyzer()) {
599 bndAnalyzer.setProperties(mergeProps);
600 Jar jar = new Jar(bundleDir.toFile());
601 bndAnalyzer.setJar(jar);
602 Manifest manifest = bndAnalyzer.calcManifest();
603
604 keys: for (Object key : manifest.getMainAttributes().keySet()) {
605 Object value = manifest.getMainAttributes().get(key);
606
607 switch (key.toString()) {
608 case "Tool":
609 case "Bnd-LastModified":
610 case "Created-By":
611 continue keys;
612 }
613 if ("Require-Capability".equals(key.toString())
614 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
615 continue keys;// hack for very old classes
616 entries.put(key.toString(), value.toString());
617 }
618 } catch (Exception e) {
619 throw new RuntimeException("Cannot process " + mergeBnd, e);
620 }
621
622 Manifest manifest = new Manifest();
623 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
624 Files.createDirectories(manifestPath.getParent());
625 for (String key : entries.keySet()) {
626 String value = entries.get(key);
627 manifest.getMainAttributes().putValue(key, value);
628 }
629 manifest.getMainAttributes().putValue(ARGEO_ORIGIN_M2.toString(), originDesc.toString());
630
631 processLicense(bundleDir, manifest);
632
633 // write MANIFEST
634 try (OutputStream out = Files.newOutputStream(manifestPath)) {
635 manifest.write(out);
636 }
637 createJar(bundleDir, origin);
638 }
639
640 /** Generates MANIFEST using BND. */
641 Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, M2Artifact artifact,
642 A2Origin origin) {
643 try {
644 Map<String, String> additionalEntries = new TreeMap<>();
645 boolean doNotModifyManifest = Boolean.parseBoolean(
646 fileProps.getOrDefault(ARGEO_ORIGIN_NO_METADATA_GENERATION.toString(), "false").toString());
647
648 // Note: we always force the symbolic name
649 if (doNotModifyManifest) {
650 for (Object key : fileProps.keySet()) {
651 String value = fileProps.getProperty(key.toString());
652 additionalEntries.put(key.toString(), value);
653 }
654 } else {
655 if (artifact != null) {
656 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
657 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
658 }
659 if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
660 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
661 }
662 }
663
664 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
665 fileProps.put(EXPORT_PACKAGE.toString(),
666 "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
667 }
668
669 // BND analysis
670 try (Analyzer bndAnalyzer = new Analyzer()) {
671 bndAnalyzer.setProperties(fileProps);
672 Jar jar = new Jar(downloaded.toFile());
673 bndAnalyzer.setJar(jar);
674 Manifest manifest = bndAnalyzer.calcManifest();
675
676 keys: for (Object key : manifest.getMainAttributes().keySet()) {
677 Object value = manifest.getMainAttributes().get(key);
678
679 switch (key.toString()) {
680 case "Tool":
681 case "Bnd-LastModified":
682 case "Created-By":
683 continue keys;
684 }
685 if ("Require-Capability".equals(key.toString())
686 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) {
687 origin.deleted.add("MANIFEST header " + key);
688 continue keys;// !! hack for very old classes
689 }
690 additionalEntries.put(key.toString(), value.toString());
691 }
692 }
693 }
694 Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries, origin);
695 logger.log(DEBUG, () -> "Processed " + downloaded);
696 return targetBundleDir;
697 } catch (Exception e) {
698 throw new RuntimeException("Cannot BND process " + downloaded, e);
699 }
700
701 }
702
703 /** Download and integrates sources for a single Maven artifact. */
704 void downloadAndProcessM2Sources(String repoStr, M2Artifact artifact, Path targetBundleDir, boolean merging)
705 throws IOException {
706 try {
707 M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources");
708 URL sourcesUrl = M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
709 Path sourcesDownloaded = downloadMaven(sourcesUrl, artifact, true);
710 processM2SourceJar(sourcesDownloaded, targetBundleDir, merging ? artifact : null);
711 logger.log(TRACE, () -> "Processed source " + sourcesDownloaded);
712 } catch (Exception e) {
713 logger.log(ERROR, () -> "Cannot download source for " + artifact);
714 }
715
716 }
717
718 /** Integrate sources from a downloaded jar file. */
719 void processM2SourceJar(Path file, Path bundleDir, M2Artifact mergingFrom) throws IOException {
720 A2Origin origin = new A2Origin();
721 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
722 : bundleDir.resolve("OSGI-OPT/src");
723 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
724
725 String mergingMsg = "";
726 if (mergingFrom != null)
727 mergingMsg = " of " + mergingFrom;
728
729 Files.createDirectories(sourceDir);
730 JarEntry entry;
731 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
732 String relPath = entry.getName();
733 if (entry.isDirectory())
734 continue entries;
735 if (entry.getName().startsWith("META-INF")) {// skip META-INF entries
736 origin.deleted.add("META-INF directory from the sources" + mergingMsg);
737 continue entries;
738 }
739 if (entry.getName().startsWith("module-info.java")) {// skip Java module information
740 origin.deleted.add("Java module information from the sources (module-info.java)" + mergingMsg);
741 continue entries;
742 }
743 if (entry.getName().startsWith("/")) { // absolute paths
744 int metaInfIndex = entry.getName().indexOf("META-INF");
745 if (metaInfIndex >= 0) {
746 relPath = entry.getName().substring(metaInfIndex);
747 origin.moved.add(" to " + relPath + " entry with absolute path " + entry.getName());
748 } else {
749 logger.log(WARNING, entry.getName() + " has an absolute path");
750 origin.deleted.add(entry.getName() + " from the sources" + mergingMsg);
751 }
752 continue entries;
753 }
754 Path target = sourceDir.resolve(relPath);
755 Files.createDirectories(target.getParent());
756 if (!Files.exists(target)) {
757 Files.copy(jarIn, target);
758 logger.log(TRACE, () -> "Copied source " + target);
759 } else {
760 logger.log(TRACE, () -> target + " already exists, skipping...");
761 }
762 }
763 }
764 // write the changes
765 if (sourceBundles) {
766 origin.appendChanges(sourceDir);
767 } else {
768 origin.added.add("source code under OSGI-OPT/src");
769 origin.appendChanges(bundleDir);
770 }
771 }
772
773 /** Download a Maven artifact. */
774 Path downloadMaven(URL url, M2Artifact artifact) throws IOException {
775 return downloadMaven(url, artifact, false);
776 }
777
778 /** Download a Maven artifact. */
779 Path downloadMaven(URL url, M2Artifact artifact, boolean sources) throws IOException {
780 return download(url, mavenBase, artifact.getGroupId().replace(".", "/") //
781 + '/' + artifact.getArtifactId() + '/' + artifact.getVersion() //
782 + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() + (sources ? "-sources" : "") + ".jar");
783 }
784
785 /*
786 * ECLIPSE ORIGIN
787 */
788 /** Process an archive in Eclipse format. */
789 void processEclipseArchive(Path duDir) {
790 try {
791 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
792 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
793 Files.createDirectories(targetCategoryBase);
794 // first delete all directories from previous builds
795 for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)))
796 deleteDirectory(dir);
797
798 Files.createDirectories(originBase);
799
800 Path commonBnd = duDir.resolve(COMMON_BND);
801 Properties commonProps = new Properties();
802 try (InputStream in = Files.newInputStream(commonBnd)) {
803 commonProps.load(in);
804 }
805 String url = commonProps.getProperty(ARGEO_ORIGIN_URI.toString());
806 if (url == null) {
807 url = uris.getProperty(duDir.getFileName().toString());
808 if (url == null)
809 throw new IllegalStateException("No url available for " + duDir);
810 commonProps.put(ARGEO_ORIGIN_URI.toString(), url);
811 }
812 Path downloaded = tryDownloadArchive(url, originBase);
813
814 FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
815
816 // filters
817 List<PathMatcher> includeMatchers = new ArrayList<>();
818 Properties includes = new Properties();
819 try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
820 includes.load(in);
821 }
822 for (Object pattern : includes.keySet()) {
823 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
824 includeMatchers.add(pathMatcher);
825 }
826
827 List<PathMatcher> excludeMatchers = new ArrayList<>();
828 Path excludeFile = duDir.resolve("excludes.properties");
829 if (Files.exists(excludeFile)) {
830 Properties excludes = new Properties();
831 try (InputStream in = Files.newInputStream(excludeFile)) {
832 excludes.load(in);
833 }
834 for (Object pattern : excludes.keySet()) {
835 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
836 excludeMatchers.add(pathMatcher);
837 }
838 }
839
840 // keys are the bundle directories
841 Map<Path, A2Origin> origins = new HashMap<>();
842 Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
843
844 @Override
845 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
846 includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
847 if (includeMatcher.matches(file)) {
848 for (PathMatcher excludeMatcher : excludeMatchers) {
849 if (excludeMatcher.matches(file)) {
850 logger.log(TRACE, "Skipping excluded " + file);
851 return FileVisitResult.CONTINUE;
852 }
853 }
854 if (file.getFileName().toString().contains(".source_")) {
855 processEclipseSourceJar(file, targetCategoryBase);
856 logger.log(DEBUG, () -> "Processed source " + file);
857 } else {
858 Map<String, String> map = new HashMap<>();
859 for (Object key : commonProps.keySet())
860 map.put(key.toString(), commonProps.getProperty(key.toString()));
861 A2Origin origin = new A2Origin();
862 Path bundleDir = processBundleJar(file, targetCategoryBase, map, origin);
863 origins.put(bundleDir, origin);
864 logger.log(DEBUG, () -> "Processed " + file);
865 }
866 break includeMatchers;
867 }
868 }
869 return FileVisitResult.CONTINUE;
870 }
871 });
872
873 DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)
874 && p.getFileName().toString().indexOf('.') >= 0 && !p.getFileName().toString().endsWith(".src"));
875 for (Path bundleDir : dirs) {
876 A2Origin origin = origins.get(bundleDir);
877 Objects.requireNonNull(origin, "No A2 origin found for " + bundleDir);
878 createJar(bundleDir, origin);
879 }
880 } catch (IOException e) {
881 throw new RuntimeException("Cannot process " + duDir, e);
882 }
883
884 }
885
886 /** Process sources in Eclipse format. */
887 void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
888 try {
889 A2Origin origin = new A2Origin();
890 Path bundleDir;
891 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
892 Manifest manifest = jarIn.getManifest();
893
894 String[] relatedBundle = manifest.getMainAttributes().getValue(ECLIPSE_SOURCE_BUNDLE.toString())
895 .split(";");
896 String version = relatedBundle[1].substring("version=\"".length());
897 version = version.substring(0, version.length() - 1);
898 NameVersion nameVersion = new NameVersion(relatedBundle[0], version);
899 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
900
901 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
902 : bundleDir.resolve("OSGI-OPT/src");
903
904 Files.createDirectories(sourceDir);
905 JarEntry entry;
906 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
907 if (entry.isDirectory())
908 continue entries;
909 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
910 continue entries;
911 Path target = sourceDir.resolve(entry.getName());
912 Files.createDirectories(target.getParent());
913 Files.copy(jarIn, target);
914 logger.log(TRACE, () -> "Copied source " + target);
915 }
916
917 // write the changes
918 if (sourceBundles) {
919 origin.appendChanges(sourceDir);
920 } else {
921 origin.added.add("source code under OSGI-OPT/src");
922 origin.appendChanges(bundleDir);
923 }
924 }
925 } catch (IOException e) {
926 throw new IllegalStateException("Cannot process " + file, e);
927 }
928 }
929
930 /*
931 * COMMON PROCESSING
932 */
933 /** Normalise a single (that is, non-merged) bundle. */
934 Path processBundleJar(Path file, Path targetBase, Map<String, String> entries, A2Origin origin) throws IOException {
935 boolean embed = Boolean.parseBoolean(entries.getOrDefault(ARGEO_ORIGIN_EMBED.toString(), "false").toString());
936 NameVersion nameVersion;
937 Path bundleDir;
938 // singleton
939 boolean isSingleton = false;
940 Manifest manifest;
941 Manifest sourceManifest;
942 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
943 sourceManifest = jarIn.getManifest();
944 manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
945
946 String rawSourceSymbolicName = manifest.getMainAttributes().getValue(BUNDLE_SYMBOLICNAME.toString());
947 if (rawSourceSymbolicName != null) {
948 // make sure there is no directive
949 String[] arr = rawSourceSymbolicName.split(";");
950 for (int i = 1; i < arr.length; i++) {
951 if (arr[i].trim().equals("singleton:=true"))
952 isSingleton = true;
953 logger.log(DEBUG, file.getFileName() + " is a singleton");
954 }
955 }
956 // remove problematic entries in MANIFEST
957 manifest.getEntries().clear();
958
959 String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString());
960 String ourVersion = entries.get(BUNDLE_VERSION.toString());
961
962 if (ourSymbolicName != null && ourVersion != null) {
963 nameVersion = new NameVersion(ourSymbolicName, ourVersion);
964 } else {
965 nameVersion = nameVersionFromManifest(manifest);
966 if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) {
967 logger.log(WARNING,
968 "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion);
969 entries.put(BUNDLE_VERSION.toString(), ourVersion);
970 }
971 if (ourSymbolicName != null) {
972 // we always force our symbolic name
973 nameVersion.setName(ourSymbolicName);
974 }
975 }
976 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
977
978 // copy original MANIFEST
979 if (sourceManifest != null) {
980 Path originalManifest = bundleDir.resolve(A2_ORIGIN).resolve("MANIFEST.MF");
981 Files.createDirectories(originalManifest.getParent());
982 try (OutputStream out = Files.newOutputStream(originalManifest)) {
983 sourceManifest.write(out);
984 }
985 origin.moved.add("original MANIFEST to " + bundleDir.relativize(originalManifest));
986 }
987
988 // force Java 9 module name
989 entries.put(ManifestHeader.AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName());
990
991 boolean isNative = false;
992 String os = null;
993 String arch = null;
994 if (bundleDir.startsWith(a2LibBase)) {
995 isNative = true;
996 Path libRelativePath = a2LibBase.relativize(bundleDir);
997 os = libRelativePath.getName(0).toString();
998 arch = libRelativePath.getName(1).toString();
999 }
1000
1001 if (!embed) {
1002 // copy entries
1003 JarEntry entry;
1004 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
1005 if (entry.isDirectory())
1006 continue entries;
1007 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
1008 || entry.getName().endsWith(".SF")) {
1009 origin.deleted.add("cryptographic signatures");
1010 continue entries;
1011 }
1012 if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
1013 origin.deleted.add("Java module information (module-info.class)");
1014 continue entries;
1015 }
1016 if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
1017 origin.deleted.add("additional Java versions (META-INF/versions)");
1018 continue entries;
1019 }
1020 if (entry.getName().startsWith("META-INF/maven/")) {
1021 origin.deleted.add("Maven information (META-INF/maven)");
1022 continue entries;
1023 }
1024 // skip file system providers as they cause issues with native image
1025 if (entry.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider")) {
1026 origin.deleted
1027 .add("file system providers (META-INF/services/java.nio.file.spi.FileSystemProvider)");
1028 continue entries;
1029 }
1030 if (entry.getName().startsWith("OSGI-OPT/src/")) { // skip embedded sources
1031 origin.deleted.add("embedded sources");
1032 continue entries;
1033 }
1034 Path target = bundleDir.resolve(entry.getName());
1035 Files.createDirectories(target.getParent());
1036 Files.copy(jarIn, target);
1037
1038 // native libraries
1039 if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll")
1040 || entry.getName().endsWith(".jnilib"))) {
1041 Path categoryDir = bundleDir.getParent();
1042 boolean copyDll = false;
1043 Path targetDll = categoryDir.resolve(bundleDir.relativize(target));
1044 if (nameVersion.getName().equals("com.sun.jna")) {
1045 if (arch.equals("x86_64"))
1046 arch = "x86-64";
1047 if (os.equals("macosx"))
1048 os = "darwin";
1049 if (target.getParent().getFileName().toString().equals(os + "-" + arch)) {
1050 copyDll = true;
1051 }
1052 targetDll = categoryDir.resolve(target.getFileName());
1053 } else {
1054 copyDll = true;
1055 }
1056 if (copyDll) {
1057 Files.createDirectories(targetDll.getParent());
1058 if (Files.exists(targetDll))
1059 Files.delete(targetDll);
1060 Files.copy(target, targetDll);
1061 }
1062 Files.delete(target);
1063 origin.deleted.add(bundleDir.relativize(target).toString());
1064 }
1065 logger.log(TRACE, () -> "Copied " + target);
1066 }
1067 }
1068 }
1069
1070 // copy MANIFEST
1071 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1072 Files.createDirectories(manifestPath.getParent());
1073
1074 if (isSingleton && entries.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
1075 entries.put(BUNDLE_SYMBOLICNAME.toString(),
1076 entries.get(BUNDLE_SYMBOLICNAME.toString()) + ";singleton:=true");
1077 }
1078
1079 if (embed) {// copy embedded jar
1080 Files.copy(file, bundleDir.resolve(file.getFileName()));
1081 entries.put(ManifestHeader.BUNDLE_CLASSPATH.toString(), file.getFileName().toString());
1082 }
1083
1084 // Final MANIFEST decisions
1085 // We also check the original OSGi metadata and compare with our changes
1086 for (String key : entries.keySet()) {
1087 String value = entries.get(key);
1088 String previousValue = manifest.getMainAttributes().getValue(key);
1089 boolean wasDifferent = previousValue != null && !previousValue.equals(value);
1090 boolean keepPrevious = false;
1091 if (wasDifferent) {
1092 if (SPDX_LICENSE_IDENTIFIER.toString().equals(key) && previousValue != null)
1093 keepPrevious = true;
1094 else if (BUNDLE_VERSION.toString().equals(key) && wasDifferent)
1095 if (previousValue.equals(value + ".0")) // typically a Maven first release
1096 keepPrevious = true;
1097
1098 if (keepPrevious) {
1099 if (logger.isLoggable(DEBUG))
1100 logger.log(DEBUG, file.getFileName() + ": " + key + " was NOT modified, value kept is "
1101 + previousValue + ", not overriden with " + value);
1102 value = previousValue;
1103 }
1104 }
1105
1106 manifest.getMainAttributes().putValue(key, value);
1107 if (wasDifferent && !keepPrevious) {
1108 if (IMPORT_PACKAGE.toString().equals(key) || EXPORT_PACKAGE.toString().equals(key))
1109 logger.log(TRACE, () -> file.getFileName() + ": " + key + " was modified");
1110 else
1111 logger.log(WARNING,
1112 file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value);
1113 origin.modified.add("MANIFEST header " + key);
1114 }
1115
1116 // !! hack to remove unresolvable
1117 if (key.equals("Provide-Capability") || key.equals("Require-Capability"))
1118 if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) {
1119 manifest.getMainAttributes().remove(key);
1120 origin.deleted.add("MANIFEST header " + key);
1121 }
1122 }
1123
1124 // de-pollute MANIFEST
1125 for (Iterator<Map.Entry<Object, Object>> manifestEntries = manifest.getMainAttributes().entrySet()
1126 .iterator(); manifestEntries.hasNext();) {
1127 Map.Entry<Object, Object> manifestEntry = manifestEntries.next();
1128 switch (manifestEntry.getKey().toString()) {
1129 case "Archiver-Version":
1130 case "Build-By":
1131 case "Created-By":
1132 case "Originally-Created-By":
1133 case "Tool":
1134 case "Bnd-LastModified":
1135 manifestEntries.remove();
1136 origin.deleted.add("MANIFEST header " + manifestEntry.getKey());
1137 break;
1138 default:
1139 if (sourceManifest != null && !sourceManifest.getMainAttributes().containsKey(manifestEntry.getKey()))
1140 origin.added.add("MANIFEST header " + manifestEntry.getKey());
1141 }
1142 }
1143
1144 processLicense(bundleDir, manifest);
1145
1146 origin.modified.add("MANIFEST (META-INF/MANIFEST.MF)");
1147 // write the MANIFEST
1148 try (OutputStream out = Files.newOutputStream(manifestPath)) {
1149 manifest.write(out);
1150 }
1151 return bundleDir;
1152 }
1153
1154 /** Process SPDX license identifier. */
1155 void processLicense(Path bundleDir, Manifest manifest) {
1156 String spdxLicenceId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString());
1157 String bundleLicense = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString());
1158 if (spdxLicenceId == null) {
1159 logger.log(ERROR, bundleDir.getFileName() + ": " + SPDX_LICENSE_IDENTIFIER + " not available, "
1160 + BUNDLE_LICENSE + " is " + bundleLicense);
1161 } else {
1162 // only use the first licensing option
1163 int orIndex = spdxLicenceId.indexOf(" OR ");
1164 if (orIndex >= 0)
1165 spdxLicenceId = spdxLicenceId.substring(0, orIndex).trim();
1166
1167 // set licenses of some well-known components
1168 // even if we say otherwise (typically because coming from an Eclipse archive)
1169 if (bundleDir.getFileName().startsWith("org.apache."))
1170 spdxLicenceId = "Apache-2.0";
1171 if (bundleDir.getFileName().startsWith("com.sun.jna."))
1172 spdxLicenceId = "Apache-2.0";
1173 if (bundleDir.getFileName().startsWith("com.ibm.icu."))
1174 spdxLicenceId = "ICU";
1175 if (bundleDir.getFileName().startsWith("javax.annotation."))
1176 spdxLicenceId = "GPL-2.0-only WITH Classpath-exception-2.0";
1177 if (bundleDir.getFileName().startsWith("javax.inject."))
1178 spdxLicenceId = "Apache-2.0";
1179
1180 manifest.getMainAttributes().putValue(SPDX_LICENSE_IDENTIFIER.toString(), spdxLicenceId);
1181 if (!licensesUsed.containsKey(spdxLicenceId))
1182 licensesUsed.put(spdxLicenceId, new TreeSet<>());
1183 licensesUsed.get(spdxLicenceId).add(bundleDir.getParent().getFileName() + "/" + bundleDir.getFileName());
1184 }
1185 }
1186
1187 /*
1188 * UTILITIES
1189 */
1190 /** Recursively deletes a directory. */
1191 static void deleteDirectory(Path path) throws IOException {
1192 if (!Files.exists(path))
1193 return;
1194 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
1195 @Override
1196 public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
1197 if (e != null)
1198 throw e;
1199 Files.delete(directory);
1200 return CONTINUE;
1201 }
1202
1203 @Override
1204 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1205 Files.delete(file);
1206 return CONTINUE;
1207 }
1208 });
1209 }
1210
1211 /** Extract name/version from a MANIFEST. */
1212 NameVersion nameVersionFromManifest(Manifest manifest) {
1213 Attributes attrs = manifest.getMainAttributes();
1214 // symbolic name
1215 String symbolicName = attrs.getValue(ManifestHeader.BUNDLE_SYMBOLICNAME.toString());
1216 if (symbolicName == null)
1217 return null;
1218 // make sure there is no directive
1219 symbolicName = symbolicName.split(";")[0];
1220
1221 String version = attrs.getValue(ManifestHeader.BUNDLE_VERSION.toString());
1222 return new NameVersion(symbolicName, version);
1223 }
1224
1225 /** Try to download from an URI. */
1226 Path tryDownloadArchive(String uri, Path dir) throws IOException {
1227 // find mirror
1228 List<String> urlBases = null;
1229 String uriPrefix = null;
1230 uriPrefixes: for (String uriPref : mirrors.keySet()) {
1231 if (uri.startsWith(uriPref)) {
1232 if (mirrors.get(uriPref).size() > 0) {
1233 urlBases = mirrors.get(uriPref);
1234 uriPrefix = uriPref;
1235 break uriPrefixes;
1236 }
1237 }
1238 }
1239 if (urlBases == null)
1240 try {
1241 return downloadArchive(new URL(uri), dir);
1242 } catch (FileNotFoundException e) {
1243 throw new FileNotFoundException("Cannot find " + uri);
1244 }
1245
1246 // try to download
1247 for (String urlBase : urlBases) {
1248 String relativePath = uri.substring(uriPrefix.length());
1249 URL url = new URL(urlBase + relativePath);
1250 try {
1251 return downloadArchive(url, dir);
1252 } catch (FileNotFoundException e) {
1253 logger.log(WARNING, "Cannot download " + url + ", trying another mirror");
1254 }
1255 }
1256 throw new FileNotFoundException("Cannot find " + uri);
1257 }
1258
1259 /**
1260 * Effectively download. Synchronised in order to avoid downloading twice in
1261 * parallel.
1262 */
1263 synchronized Path downloadArchive(URL url, Path dir) throws IOException {
1264 return download(url, dir, (String) null);
1265 }
1266
1267 /** Effectively download. */
1268 Path download(URL url, Path dir, String name) throws IOException {
1269
1270 Path dest;
1271 if (name == null) {
1272 // We use also use parent directory in case the archive itself has a fixed name
1273 String[] segments = url.getPath().split("/");
1274 name = segments.length > 1 ? segments[segments.length - 2] + '-' + segments[segments.length - 1]
1275 : segments[segments.length - 1];
1276 }
1277
1278 dest = dir.resolve(name);
1279 if (Files.exists(dest)) {
1280 logger.log(TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
1281 return dest;
1282 } else {
1283 Files.createDirectories(dest.getParent());
1284 }
1285
1286 try (InputStream in = url.openStream()) {
1287 Files.copy(in, dest);
1288 logger.log(DEBUG, () -> "Downloaded " + dest + " from " + url);
1289 }
1290 return dest;
1291 }
1292
1293 /** Create a JAR file from a directory. */
1294 Path createJar(Path bundleDir, A2Origin origin) throws IOException {
1295 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1296 Manifest manifest;
1297 try (InputStream in = Files.newInputStream(manifestPath)) {
1298 manifest = new Manifest(in);
1299 }
1300 // legal requirements
1301 origin.appendChanges(bundleDir);
1302 createReadMe(bundleDir, manifest);
1303
1304 // create the jar
1305 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
1306 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
1307 jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
1308 Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
1309
1310 @Override
1311 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1312 if (file.getFileName().toString().equals("MANIFEST.MF"))
1313 return super.visitFile(file, attrs);
1314 JarEntry entry = new JarEntry(
1315 bundleDir.relativize(file).toString().replace(File.separatorChar, '/'));
1316 jarOut.putNextEntry(entry);
1317 Files.copy(file, jarOut);
1318 return super.visitFile(file, attrs);
1319 }
1320
1321 });
1322 }
1323 deleteDirectory(bundleDir);
1324
1325 if (sourceBundles)
1326 createSourceJar(bundleDir, manifest);
1327
1328 return jarPath;
1329 }
1330
1331 /** Package sources separately, in the Eclipse-SourceBundle format. */
1332 void createSourceJar(Path bundleDir, Manifest manifest) throws IOException {
1333 Path bundleCategoryDir = bundleDir.getParent();
1334 Path sourceDir = bundleCategoryDir.resolve(bundleDir.toString() + ".src");
1335 if (!Files.exists(sourceDir)) {
1336 logger.log(WARNING, sourceDir + " does not exist, skipping...");
1337 return;
1338 }
1339 createReadMe(sourceDir, manifest);
1340
1341 Path relPath = a2Base.relativize(bundleCategoryDir);
1342 Path srcCategoryDir = a2SrcBase.resolve(relPath);
1343 Path srcJarP = srcCategoryDir.resolve(sourceDir.getFileName() + ".jar");
1344 Files.createDirectories(srcJarP.getParent());
1345
1346 String bundleSymbolicName = manifest.getMainAttributes().getValue("Bundle-SymbolicName").toString();
1347 // in case there are additional directives
1348 bundleSymbolicName = bundleSymbolicName.split(";")[0];
1349 Manifest srcManifest = new Manifest();
1350 srcManifest.getMainAttributes().put(MANIFEST_VERSION, "1.0");
1351 srcManifest.getMainAttributes().putValue(BUNDLE_SYMBOLICNAME.toString(), bundleSymbolicName + ".src");
1352 srcManifest.getMainAttributes().putValue(BUNDLE_VERSION.toString(),
1353 manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()).toString());
1354 srcManifest.getMainAttributes().putValue(ECLIPSE_SOURCE_BUNDLE.toString(), bundleSymbolicName + ";version=\""
1355 + manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()) + "\"");
1356
1357 try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
1358 srcJarOut.setLevel(Deflater.BEST_COMPRESSION);
1359 Files.walkFileTree(sourceDir, new SimpleFileVisitor<Path>() {
1360
1361 @Override
1362 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1363 if (file.getFileName().toString().equals("MANIFEST.MF"))
1364 return super.visitFile(file, attrs);
1365 JarEntry entry = new JarEntry(
1366 sourceDir.relativize(file).toString().replace(File.separatorChar, '/'));
1367 srcJarOut.putNextEntry(entry);
1368 Files.copy(file, srcJarOut);
1369 return super.visitFile(file, attrs);
1370 }
1371
1372 });
1373 }
1374 deleteDirectory(sourceDir);
1375 }
1376
1377 void createReadMe(Path jarDir, Manifest manifest) throws IOException {
1378 // write repackaged README
1379 try (BufferedWriter writer = Files.newBufferedWriter(jarDir.resolve(README_REPACKAGED))) {
1380 boolean merged = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2_MERGE.toString()) != null;
1381 if (merged)
1382 writer.append("This component is a merging of third party components"
1383 + " in order to comply with A2 packaging standards.\n");
1384 else
1385 writer.append("This component is a repackaging of a third party component"
1386 + " in order to comply with A2 packaging standards.\n");
1387
1388 // license
1389 String spdxLicenseId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString());
1390 if (spdxLicenseId == null)
1391 throw new IllegalStateException("An SPDX license id must have beend defined at this stage.");
1392 writer.append("\nIt is redistributed under the following license:\n\n");
1393 writer.append("SPDX-Identifier: " + spdxLicenseId + "\n\n");
1394
1395 if (!spdxLicenseId.startsWith("LicenseRef")) {// standard
1396 int withIndex = spdxLicenseId.indexOf(" WITH ");
1397 if (withIndex >= 0) {
1398 String simpleId = spdxLicenseId.substring(0, withIndex).trim();
1399 String exception = spdxLicenseId.substring(withIndex + " WITH ".length());
1400 writer.append("which are available here: https://spdx.org/licenses/" + simpleId
1401 + "\nand here: https://spdx.org/licenses/" + exception + "\n");
1402 } else {
1403 writer.append("which is available here: https://spdx.org/licenses/" + spdxLicenseId + "\n");
1404 }
1405 } else {
1406 String url = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString());
1407 if (url != null) {
1408 writer.write("which is avaliabel here: " + url + "\n");
1409 } else {
1410 logger.log(ERROR, "No licne URL for " + jarDir);
1411 }
1412 }
1413 writer.write("\n");
1414
1415 String m2Repo = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2_REPO.toString());
1416 String originDesc = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2.toString());
1417 if (originDesc != null)
1418 writer.append("The original component has Maven coordinates " + originDesc
1419 + (m2Repo != null ? " in M2 repository" + m2Repo : "") + ".\n");
1420 else
1421 originDesc = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_URI.toString());
1422 if (originDesc != null)
1423 writer.append("The original component comes from " + originDesc + ".\n");
1424 else
1425 logger.log(ERROR, "Cannot find origin information in " + jarDir);
1426
1427 writer.append("A detailed list of changes is available under " + CHANGES + ".\n");
1428 if (!jarDir.getFileName().endsWith(".src")) {// binary archive
1429 if (sourceBundles)
1430 writer.append("Corresponding sources are available in the related archive named "
1431 + jarDir.toString() + ".src.jar.\n");
1432 else
1433 writer.append("Corresponding sources are available under OSGI-OPT/src.\n");
1434 }
1435 }
1436
1437 }
1438
1439 /**
1440 * Gathers modifications performed on the original binaries and sources,
1441 * especially in order to comply with their license requirements.
1442 */
1443 class A2Origin {
1444 A2Origin() {
1445
1446 }
1447
1448 Set<String> modified = new TreeSet<>();
1449 Set<String> deleted = new TreeSet<>();
1450 Set<String> added = new TreeSet<>();
1451 Set<String> moved = new TreeSet<>();
1452
1453 /** Append changes to the A2-ORIGIN/changes file. */
1454 void appendChanges(Path baseDirectory) throws IOException {
1455 Path changesFile = baseDirectory.resolve(CHANGES);
1456 Files.createDirectories(changesFile.getParent());
1457 try (BufferedWriter writer = Files.newBufferedWriter(changesFile, APPEND, CREATE)) {
1458 for (String msg : added)
1459 writer.write("- Added " + msg + ".\n");
1460 for (String msg : modified)
1461 writer.write("- Modified " + msg + ".\n");
1462 for (String msg : moved)
1463 writer.write("- Moved " + msg + ".\n");
1464 for (String msg : deleted)
1465 writer.write("- Deleted " + msg + ".\n");
1466 }
1467 }
1468 }
1469 }
1470
1471 /** Simple representation of an M2 artifact. */
1472 class M2Artifact extends CategoryNameVersion {
1473 private String classifier;
1474
1475 M2Artifact(String m2coordinates) {
1476 this(m2coordinates, null);
1477 }
1478
1479 M2Artifact(String m2coordinates, String classifier) {
1480 String[] parts = m2coordinates.split(":");
1481 setCategory(parts[0]);
1482 setName(parts[1]);
1483 if (parts.length > 2) {
1484 setVersion(parts[2]);
1485 }
1486 this.classifier = classifier;
1487 }
1488
1489 String getGroupId() {
1490 return super.getCategory();
1491 }
1492
1493 String getArtifactId() {
1494 return super.getName();
1495 }
1496
1497 String toM2Coordinates() {
1498 return getCategory() + ":" + getName() + (getVersion() != null ? ":" + getVersion() : "");
1499 }
1500
1501 String getClassifier() {
1502 return classifier != null ? classifier : "";
1503 }
1504
1505 String getExtension() {
1506 return "jar";
1507 }
1508 }
1509
1510 /** Utilities around Maven (conventions based). */
1511 class M2ConventionsUtils {
1512 final static String MAVEN_CENTRAL_BASE_URL = "https://repo1.maven.org/maven2/";
1513
1514 /** The file name of this artifact when stored */
1515 static String artifactFileName(M2Artifact artifact) {
1516 return artifact.getArtifactId() + '-' + artifact.getVersion()
1517 + (artifact.getClassifier().equals("") ? "" : '-' + artifact.getClassifier()) + '.'
1518 + artifact.getExtension();
1519 }
1520
1521 /** Absolute path to the file */
1522 static String artifactPath(String artifactBasePath, M2Artifact artifact) {
1523 return artifactParentPath(artifactBasePath, artifact) + '/' + artifactFileName(artifact);
1524 }
1525
1526 /** Absolute path to the file */
1527 static String artifactUrl(String repoUrl, M2Artifact artifact) {
1528 if (repoUrl.endsWith("/"))
1529 return repoUrl + artifactPath("/", artifact).substring(1);
1530 else
1531 return repoUrl + artifactPath("/", artifact);
1532 }
1533
1534 /** Absolute path to the file */
1535 static URL mavenRepoUrl(String repoBase, M2Artifact artifact) {
1536 String url = artifactUrl(repoBase == null ? MAVEN_CENTRAL_BASE_URL : repoBase, artifact);
1537 try {
1538 return new URL(url);
1539 } catch (MalformedURLException e) {
1540 // it should not happen
1541 throw new IllegalStateException(e);
1542 }
1543 }
1544
1545 /** Absolute path to the directories where the files will be stored */
1546 static String artifactParentPath(String artifactBasePath, M2Artifact artifact) {
1547 return artifactBasePath + (artifactBasePath.endsWith("/") ? "" : "/") + artifactParentPath(artifact);
1548 }
1549
1550 /** Relative path to the directories where the files will be stored */
1551 static String artifactParentPath(M2Artifact artifact) {
1552 return artifact.getGroupId().replace('.', '/') + '/' + artifact.getArtifactId() + '/' + artifact.getVersion();
1553 }
1554
1555 /** Singleton */
1556 private M2ConventionsUtils() {
1557 }
1558 }
1559
1560 /** Combination of a category, a name and a version. */
1561 class CategoryNameVersion extends NameVersion {
1562 private String category;
1563
1564 CategoryNameVersion() {
1565 }
1566
1567 CategoryNameVersion(String category, String name, String version) {
1568 super(name, version);
1569 this.category = category;
1570 }
1571
1572 CategoryNameVersion(String category, NameVersion nameVersion) {
1573 super(nameVersion);
1574 this.category = category;
1575 }
1576
1577 String getCategory() {
1578 return category;
1579 }
1580
1581 void setCategory(String category) {
1582 this.category = category;
1583 }
1584
1585 @Override
1586 public String toString() {
1587 return category + ":" + super.toString();
1588 }
1589
1590 }
1591
1592 /** Combination of a name and a version. */
1593 class NameVersion implements Comparable<NameVersion> {
1594 private String name;
1595 private String version;
1596
1597 NameVersion() {
1598 }
1599
1600 /** Interprets string in OSGi-like format my.module.name;version=0.0.0 */
1601 NameVersion(String nameVersion) {
1602 int index = nameVersion.indexOf(";version=");
1603 if (index < 0) {
1604 setName(nameVersion);
1605 setVersion(null);
1606 } else {
1607 setName(nameVersion.substring(0, index));
1608 setVersion(nameVersion.substring(index + ";version=".length()));
1609 }
1610 }
1611
1612 NameVersion(String name, String version) {
1613 this.name = name;
1614 this.version = version;
1615 }
1616
1617 NameVersion(NameVersion nameVersion) {
1618 this.name = nameVersion.getName();
1619 this.version = nameVersion.getVersion();
1620 }
1621
1622 String getName() {
1623 return name;
1624 }
1625
1626 void setName(String name) {
1627 this.name = name;
1628 }
1629
1630 String getVersion() {
1631 return version;
1632 }
1633
1634 void setVersion(String version) {
1635 this.version = version;
1636 }
1637
1638 String getBranch() {
1639 String[] parts = getVersion().split("\\.");
1640 if (parts.length < 2)
1641 throw new IllegalStateException("Version " + getVersion() + " cannot be interpreted as branch.");
1642 return parts[0] + "." + parts[1];
1643 }
1644
1645 @Override
1646 public boolean equals(Object obj) {
1647 if (obj instanceof NameVersion) {
1648 NameVersion nameVersion = (NameVersion) obj;
1649 return name.equals(nameVersion.getName()) && version.equals(nameVersion.getVersion());
1650 } else
1651 return false;
1652 }
1653
1654 @Override
1655 public int hashCode() {
1656 return name.hashCode();
1657 }
1658
1659 @Override
1660 public String toString() {
1661 return name + ":" + version;
1662 }
1663
1664 public int compareTo(NameVersion o) {
1665 if (o.getName().equals(name))
1666 return version.compareTo(o.getVersion());
1667 else
1668 return name.compareTo(o.getName());
1669 }
1670 }
Argeo Build - Build scripts and configuration for Argeo projects