]> git.argeo.org Git - cc0/argeo-build.git/blob - src/org/argeo/build/Repackage.java
projects / cc0 / argeo-build.git / blob
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Clarify copyright and waiver
[cc0/argeo-build.git] / src / org / argeo / build / Repackage.java
1 package org.argeo.build;
2
3 import static java.lang.System.Logger.Level.DEBUG;
4 import static java.lang.System.Logger.Level.ERROR;
5 import static java.lang.System.Logger.Level.INFO;
6 import static java.lang.System.Logger.Level.TRACE;
7 import static java.lang.System.Logger.Level.WARNING;
8 import static java.nio.file.FileVisitResult.CONTINUE;
9 import static java.nio.file.StandardOpenOption.APPEND;
10 import static java.nio.file.StandardOpenOption.CREATE;
11 import static java.util.jar.Attributes.Name.MANIFEST_VERSION;
12 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_EMBED;
13 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2;
14 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_MERGE;
15 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_REPO;
16 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_NO_METADATA_GENERATION;
17 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_URI;
18 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_LICENSE;
19 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_SYMBOLICNAME;
20 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_VERSION;
21 import static org.argeo.build.Repackage.ManifestHeader.ECLIPSE_SOURCE_BUNDLE;
22 import static org.argeo.build.Repackage.ManifestHeader.EXPORT_PACKAGE;
23 import static org.argeo.build.Repackage.ManifestHeader.IMPORT_PACKAGE;
24 import static org.argeo.build.Repackage.ManifestHeader.SPDX_LICENSE_IDENTIFIER;
25
26 import java.io.BufferedWriter;
27 import java.io.File;
28 import java.io.FileNotFoundException;
29 import java.io.IOException;
30 import java.io.InputStream;
31 import java.io.OutputStream;
32 import java.lang.System.Logger;
33 import java.net.MalformedURLException;
34 import java.net.URL;
35 import java.nio.charset.StandardCharsets;
36 import java.nio.file.DirectoryStream;
37 import java.nio.file.FileSystem;
38 import java.nio.file.FileSystems;
39 import java.nio.file.FileVisitResult;
40 import java.nio.file.Files;
41 import java.nio.file.Path;
42 import java.nio.file.PathMatcher;
43 import java.nio.file.Paths;
44 import java.nio.file.SimpleFileVisitor;
45 import java.nio.file.StandardCopyOption;
46 import java.nio.file.StandardOpenOption;
47 import java.nio.file.attribute.BasicFileAttributes;
48 import java.util.ArrayList;
49 import java.util.HashMap;
50 import java.util.Iterator;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.Objects;
54 import java.util.Properties;
55 import java.util.Set;
56 import java.util.StringJoiner;
57 import java.util.TreeMap;
58 import java.util.TreeSet;
59 import java.util.concurrent.CompletableFuture;
60 import java.util.jar.Attributes;
61 import java.util.jar.JarEntry;
62 import java.util.jar.JarInputStream;
63 import java.util.jar.JarOutputStream;
64 import java.util.jar.Manifest;
65 import java.util.zip.Deflater;
66
67 import aQute.bnd.osgi.Analyzer;
68 import aQute.bnd.osgi.Jar;
69
70 /** Repackages existing jar files into OSGi bundles in an A2 repository. */
71 public class Repackage {
72 final static Logger logger = System.getLogger(Repackage.class.getName());
73
74 /**
75 * Environment variable on whether sources should be packaged separately or
76 * integrated in the bundles.
77 */
78 final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES";
79 /** Environment variable on whether operations should be parallelised. */
80 final static String ENV_ARGEO_BUILD_SEQUENTIAL = "ARGEO_BUILD_SEQUENTIAL";
81
82 /** Whether repackaging should run in parallel (default) or sequentially. */
83 final static boolean sequential = Boolean.parseBoolean(System.getenv(ENV_ARGEO_BUILD_SEQUENTIAL));
84
85 /** Main entry point. */
86 public static void main(String[] args) {
87 if (sequential)
88 logger.log(INFO, "Build will be sequential");
89 if (args.length < 2) {
90 System.err.println("Usage: <path to a2 output dir> <category1> <category2> ...");
91 System.exit(1);
92 }
93 Path a2Base = Paths.get(args[0]).toAbsolutePath().normalize();
94 Path descriptorsBase = Paths.get(".").toAbsolutePath().normalize();
95 Repackage factory = new Repackage(a2Base, descriptorsBase);
96
97 List<CompletableFuture<Void>> toDos = new ArrayList<>();
98 for (int i = 1; i < args.length; i++) {
99 Path p = Paths.get(args[i]);
100 if (sequential)
101 factory.processCategory(p);
102 else
103 toDos.add(CompletableFuture.runAsync(() -> factory.processCategory(p)));
104 }
105 if (!sequential)
106 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
107
108 // Summary
109 StringBuilder sb = new StringBuilder();
110 for (String licenseId : licensesUsed.keySet())
111 for (String name : licensesUsed.get(licenseId))
112 sb.append((licenseId.equals("") ? "Proprietary" : licenseId) + "\t\t" + name + "\n");
113 logger.log(INFO, "# License summary:\n" + sb);
114 }
115
116 /** MANIFEST headers. */
117 enum ManifestHeader {
118 // OSGi
119 /** OSGi bundle symbolic name. */
120 BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), //
121 /** OSGi bundle version. */
122 BUNDLE_VERSION("Bundle-Version"), //
123 /** OSGi bundle license. */
124 BUNDLE_LICENSE("Bundle-License"), //
125 /** OSGi exported packages list. */
126 EXPORT_PACKAGE("Export-Package"), //
127 /** OSGi imported packages list. */
128 IMPORT_PACKAGE("Import-Package"), //
129 /** OSGi path to embedded jar. */
130 BUNDLE_CLASSPATH("Bundle-Classpath"), //
131 // Java
132 /** Java module name. */
133 AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), //
134 // Eclipse
135 /** Eclipse source bundle. */
136 ECLIPSE_SOURCE_BUNDLE("Eclipse-SourceBundle"), //
137 // SPDX
138 /**
139 * SPDX license identifier.
140 *
141 * @see https://spdx.org/licenses/
142 */
143 SPDX_LICENSE_IDENTIFIER("SPDX-License-Identifier"), //
144 // Argeo Origin
145 /**
146 * Maven coordinates of the origin, possibly partial when using common.bnd or
147 * merge.bnd.
148 */
149 ARGEO_ORIGIN_M2("Argeo-Origin-M2"), //
150 /** List of Maven coordinates to merge. */
151 ARGEO_ORIGIN_M2_MERGE("Argeo-Origin-M2-Merge"), //
152 /** Maven repository, if not the default one. */
153 ARGEO_ORIGIN_M2_REPO("Argeo-Origin-M2-Repo"), //
154 /**
155 * Do not perform BND analysis of the origin component. Typically Import_package
156 * and Export-Package will be kept untouched.
157 */
158 ARGEO_ORIGIN_NO_METADATA_GENERATION("Argeo-Origin-NoMetadataGeneration"), //
159 /**
160 * Embed the original jar without modifying it (may be required by some
161 * proprietary licenses, such as JCR Day License).
162 */
163 ARGEO_ORIGIN_EMBED("Argeo-Origin-Embed"), //
164 /**
165 * Do not modify original jar (may be required by some proprietary licenses,
166 * such as JCR Day License).
167 */
168 ARGEO_DO_NOT_MODIFY("Argeo-Origin-Do-Not-Modify"), //
169 /**
170 * Origin (non-Maven) URI of the component. It may be anything (jar, archive,
171 * etc.).
172 */
173 ARGEO_ORIGIN_URI("Argeo-Origin-URI"), //
174 ;
175
176 final String value;
177
178 private ManifestHeader(String value) {
179 this.value = value;
180 }
181
182 @Override
183 public String toString() {
184 return value;
185 }
186 }
187
188 /** Name of the file centralising information for multiple M2 artifacts. */
189 final static String COMMON_BND = "common.bnd";
190 /** Name of the file centralising information for mergin M2 artifacts. */
191 final static String MERGE_BND = "merge.bnd";
192 /**
193 * Subdirectory of the jar file where origin informations (changes, legal
194 * notices etc. are stored)
195 */
196 final static String A2_ORIGIN = "A2-ORIGIN";
197 /** File detailing modifications to the original component. */
198 final static String CHANGES = A2_ORIGIN + "/changes";
199 /**
200 * Name of the file at the root of the repackaged jar, which prominently
201 * notifies that the component has be repackaged.
202 */
203 final static String README_REPACKAGED = "README.repackaged";
204
205 // cache
206 /** Summary of all license seen during the repackaging. */
207 final static Map<String, Set<String>> licensesUsed = new TreeMap<>();
208
209 /** Directory where to download archives */
210 final Path originBase;
211 /** Directory where to download Maven artifacts */
212 final Path mavenBase;
213
214 /** A2 repository base for binary bundles */
215 final Path a2Base;
216 /** A2 repository base for source bundles */
217 final Path a2SrcBase;
218 /** A2 base for native components */
219 final Path a2LibBase;
220 /** Location of the descriptors driving the packaging */
221 final Path descriptorsBase;
222 /** URIs of archives to download */
223 final Properties uris = new Properties();
224 /** Mirrors for archive download. Key is URI prefix, value list of base URLs */
225 final Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
226
227 /** Whether sources should be packaged separately */
228 final boolean sourceBundles;
229
230 /** Constructor initialises the various variables */
231 public Repackage(Path a2Base, Path descriptorsBase) {
232 sourceBundles = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES));
233 if (sourceBundles)
234 logger.log(INFO, "Sources will be packaged separately");
235
236 Objects.requireNonNull(a2Base);
237 Objects.requireNonNull(descriptorsBase);
238 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/build/origin");
239 this.mavenBase = Paths.get(System.getProperty("user.home"), ".m2", "repository");
240
241 // TODO define and use a build base
242 this.a2Base = a2Base;
243 this.a2SrcBase = a2Base.getParent().resolve(a2Base.getFileName() + ".src");
244 this.a2LibBase = a2Base.resolve("lib");
245 this.descriptorsBase = descriptorsBase;
246 if (!Files.exists(this.descriptorsBase))
247 throw new IllegalArgumentException(this.descriptorsBase + " does not exist");
248
249 // URIs mapping
250 Path urisPath = this.descriptorsBase.resolve("uris.properties");
251 if (Files.exists(urisPath)) {
252 try (InputStream in = Files.newInputStream(urisPath)) {
253 uris.load(in);
254 } catch (IOException e) {
255 throw new IllegalStateException("Cannot load " + urisPath, e);
256 }
257 }
258
259 // Eclipse mirrors
260 Path eclipseMirrorsPath = this.descriptorsBase.resolve("eclipse.mirrors.txt");
261 List<String> eclipseMirrors = new ArrayList<>();
262 if (Files.exists(eclipseMirrorsPath)) {
263 try {
264 eclipseMirrors = Files.readAllLines(eclipseMirrorsPath, StandardCharsets.UTF_8);
265 } catch (IOException e) {
266 throw new IllegalStateException("Cannot load " + eclipseMirrorsPath, e);
267 }
268 for (Iterator<String> it = eclipseMirrors.iterator(); it.hasNext();) {
269 String value = it.next();
270 if (value.strip().equals(""))
271 it.remove();
272 }
273 }
274 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
275 }
276
277 /*
278 * MAVEN ORIGIN
279 */
280 /** Process a whole category/group id. */
281 void processCategory(Path categoryRelativePath) {
282 try {
283 Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath);
284 DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
285 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
286 && !p.getFileName().toString().equals(MERGE_BND));
287 for (Path p : bnds) {
288 processSingleM2ArtifactDistributionUnit(p);
289 }
290
291 DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
292 for (Path duDir : dus) {
293 if (duDir.getFileName().toString().startsWith("eclipse-")) {
294 processEclipseArchive(duDir);
295 } else {
296 processM2BasedDistributionUnit(duDir);
297 }
298 }
299 } catch (IOException e) {
300 throw new RuntimeException("Cannot process category " + categoryRelativePath, e);
301 }
302 }
303
304 /** Process a standalone Maven artifact. */
305 void processSingleM2ArtifactDistributionUnit(Path bndFile) {
306 try {
307 Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent());
308 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
309
310 Properties fileProps = new Properties();
311 try (InputStream in = Files.newInputStream(bndFile)) {
312 fileProps.load(in);
313 }
314 String repoStr = fileProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
315 ? fileProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
316 : null;
317
318 // use file name as symbolic name
319 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
320 String symbolicName = bndFile.getFileName().toString();
321 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
322 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
323 }
324
325 String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
326 if (m2Coordinates == null)
327 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
328 M2Artifact artifact = new M2Artifact(m2Coordinates);
329 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
330 Path downloaded = downloadMaven(url, artifact);
331
332 // some proprietary artifacts do not allow any modification
333 // when releasing (with separate sources) we just copy it
334 boolean doNotModify = Boolean.parseBoolean(
335 fileProps.getOrDefault(ManifestHeader.ARGEO_DO_NOT_MODIFY.toString(), "false").toString());
336 if (doNotModify && sourceBundles) {
337 Path unmodifiedTarget = targetCategoryBase.resolve(
338 fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch() + ".jar");
339 Files.copy(downloaded, unmodifiedTarget, StandardCopyOption.REPLACE_EXISTING);
340 Path bundleDir = targetCategoryBase
341 .resolve(fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch());
342 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
343 Manifest manifest;
344 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(unmodifiedTarget))) {
345 manifest = jarIn.getManifest();
346 }
347 createSourceJar(bundleDir, manifest);
348 return;
349 }
350
351 // regular processing
352 A2Origin origin = new A2Origin();
353 Path bundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact, origin);
354 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
355 createJar(bundleDir, origin);
356 } catch (Exception e) {
357 throw new RuntimeException("Cannot process " + bndFile, e);
358 }
359 }
360
361 /**
362 * Process multiple Maven artifacts coming from a same project and therefore
363 * with information in common (typically the version), generating single bundles
364 * or merging them if necessary.
365 *
366 * @see #COMMON_BND
367 * @see #MERGE_BND
368 */
369 void processM2BasedDistributionUnit(Path duDir) {
370 try {
371 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
372 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
373
374 Path mergeBnd = duDir.resolve(MERGE_BND);
375 if (Files.exists(mergeBnd)) // merge
376 mergeM2Artifacts(mergeBnd);
377
378 Path commonBnd = duDir.resolve(COMMON_BND);
379 if (!Files.exists(commonBnd))
380 return;
381
382 Properties commonProps = new Properties();
383 try (InputStream in = Files.newInputStream(commonBnd)) {
384 commonProps.load(in);
385 }
386
387 String m2Version = commonProps.getProperty(ARGEO_ORIGIN_M2.toString());
388 if (m2Version == null) {
389 logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
390 return;// ignore, this is probably an Eclipse archive
391 }
392 if (!m2Version.startsWith(":")) {
393 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
394 }
395 m2Version = m2Version.substring(1);
396
397 DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
398 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
399 && !p.getFileName().toString().equals(MERGE_BND));
400 for (Path p : ds) {
401 Properties fileProps = new Properties();
402 try (InputStream in = Files.newInputStream(p)) {
403 fileProps.load(in);
404 }
405 String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
406 M2Artifact artifact = new M2Artifact(m2Coordinates);
407 artifact.setVersion(m2Version);
408
409 // prepare manifest entries
410 Properties mergeProps = new Properties();
411 mergeProps.putAll(commonProps);
412
413 fileEntries: for (Object key : fileProps.keySet()) {
414 if (ARGEO_ORIGIN_M2.toString().equals(key))
415 continue fileEntries;
416 String value = fileProps.getProperty(key.toString());
417 Object previousValue = mergeProps.put(key.toString(), value);
418 if (previousValue != null) {
419 logger.log(WARNING,
420 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
421 }
422 }
423 mergeProps.put(ARGEO_ORIGIN_M2.toString(), artifact.toM2Coordinates());
424 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
425 // use file name as symbolic name
426 String symbolicName = p.getFileName().toString();
427 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
428 mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
429 }
430
431 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
432 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
433 : null;
434
435 // download
436 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
437 Path downloaded = downloadMaven(url, artifact);
438
439 A2Origin origin = new A2Origin();
440 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact, origin);
441 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir, false);
442 createJar(targetBundleDir, origin);
443 }
444 } catch (IOException e) {
445 throw new RuntimeException("Cannot process " + duDir, e);
446 }
447 }
448
449 /** Merge multiple Maven artifacts. */
450 void mergeM2Artifacts(Path mergeBnd) throws IOException {
451 Path duDir = mergeBnd.getParent();
452 String category = duDir.getParent().getFileName().toString();
453 Path targetCategoryBase = a2Base.resolve(category);
454
455 Properties mergeProps = new Properties();
456 try (InputStream in = Files.newInputStream(mergeBnd)) {
457 mergeProps.load(in);
458 }
459
460 String m2Version = mergeProps.getProperty(ARGEO_ORIGIN_M2.toString());
461 if (m2Version == null) {
462 logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
463 return;// ignore, this is probably an Eclipse archive
464 }
465 if (!m2Version.startsWith(":")) {
466 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
467 }
468 m2Version = m2Version.substring(1);
469 mergeProps.put(BUNDLE_VERSION.toString(), m2Version);
470
471 String artifactsStr = mergeProps.getProperty(ARGEO_ORIGIN_M2_MERGE.toString());
472 if (artifactsStr == null)
473 throw new IllegalArgumentException(mergeBnd + ": " + ARGEO_ORIGIN_M2_MERGE + " must be set");
474
475 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
476 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
477 : null;
478
479 String bundleSymbolicName = mergeProps.getProperty(BUNDLE_SYMBOLICNAME.toString());
480 if (bundleSymbolicName == null)
481 throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
482 CategoryNameVersion nameVersion = new M2Artifact(category + ":" + bundleSymbolicName + ":" + m2Version);
483
484 A2Origin origin = new A2Origin();
485 Path bundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
486
487 StringJoiner originDesc = new StringJoiner(",");
488 String[] artifacts = artifactsStr.split(",");
489 artifacts: for (String str : artifacts) {
490 String m2Coordinates = str.trim();
491 if ("".equals(m2Coordinates))
492 continue artifacts;
493 M2Artifact artifact = new M2Artifact(m2Coordinates.trim());
494 if (artifact.getVersion() == null)
495 artifact.setVersion(m2Version);
496 originDesc.add(artifact.toString());
497 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
498 Path downloaded = downloadMaven(url, artifact);
499 JarEntry entry;
500 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
501 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
502 if (entry.isDirectory())
503 continue entries;
504 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
505 || entry.getName().endsWith(".SF")) {
506 origin.deleted.add("cryptographic signatures from " + artifact);
507 continue entries;
508 }
509 if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
510 origin.deleted.add("Java module information (module-info.class) from " + artifact);
511 continue entries;
512 }
513 if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
514 origin.deleted.add("additional Java versions (META-INF/versions) from " + artifact);
515 continue entries;
516 }
517 if (entry.getName().startsWith("META-INF/maven/")) {
518 origin.deleted.add("Maven information (META-INF/maven) from " + artifact);
519 continue entries;
520 }
521 if (entry.getName().startsWith(".cache/")) { // Apache SSHD
522 origin.deleted.add("cache directory (.cache) from " + artifact);
523 continue entries;
524 }
525 if (entry.getName().equals("META-INF/DEPENDENCIES")) {
526 origin.deleted.add("Dependencies (META-INF/DEPENDENCIES) from " + artifact);
527 continue entries;
528 }
529 if (entry.getName().equals("META-INF/MANIFEST.MF")) {
530 Path originalManifest = bundleDir.resolve(A2_ORIGIN).resolve(artifact.getGroupId())
531 .resolve(artifact.getArtifactId()).resolve("MANIFEST.MF");
532 Files.createDirectories(originalManifest.getParent());
533 try (OutputStream out = Files.newOutputStream(originalManifest)) {
534 Files.copy(jarIn, originalManifest);
535 }
536 origin.added.add(
537 "original MANIFEST (" + bundleDir.relativize(originalManifest) + ") from " + artifact);
538 continue entries;
539 }
540
541 if (entry.getName().endsWith("NOTICE") || entry.getName().endsWith("NOTICE.txt")
542 || entry.getName().endsWith("LICENSE") || entry.getName().endsWith("LICENSE.md")
543 || entry.getName().endsWith("LICENSE-notice.md") || entry.getName().endsWith("COPYING")
544 || entry.getName().endsWith("COPYING.LESSER")) {
545 Path artifactOriginDir = bundleDir.resolve(A2_ORIGIN).resolve(artifact.getGroupId())
546 .resolve(artifact.getArtifactId());
547 Path target = artifactOriginDir.resolve(entry.getName());
548 Files.createDirectories(target.getParent());
549 Files.copy(jarIn, target);
550 origin.moved.add(entry.getName() + " in " + artifact + " to " + bundleDir.relativize(target));
551 continue entries;
552 }
553 Path target = bundleDir.resolve(entry.getName());
554 Files.createDirectories(target.getParent());
555 if (!Files.exists(target)) {
556 Files.copy(jarIn, target);
557 } else {
558 if (entry.getName().startsWith("META-INF/services/")) {
559 try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
560 out.write("\n".getBytes());
561 jarIn.transferTo(out);
562 logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName());
563 }
564 origin.modified.add(entry.getName() + ", merging from " + artifact);
565 } else if (entry.getName().startsWith("org/apache/batik/")) {
566 logger.log(TRACE, "Skip " + entry.getName());
567 continue entries;
568 } else {
569 throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
570 }
571 }
572 logger.log(TRACE, () -> "Copied " + target);
573 }
574 }
575 origin.added.add("binary content of " + artifact);
576
577 // process sources
578 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, true);
579 }
580
581 // additional service files
582 Path servicesDir = duDir.resolve("services");
583 if (Files.exists(servicesDir)) {
584 for (Path p : Files.newDirectoryStream(servicesDir)) {
585 Path target = bundleDir.resolve("META-INF/services/").resolve(p.getFileName());
586 try (InputStream in = Files.newInputStream(p);
587 OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) {
588 out.write("\n".getBytes());
589 in.transferTo(out);
590 logger.log(DEBUG, "Appended " + p);
591 }
592 origin.added.add(bundleDir.relativize(target).toString());
593 }
594 }
595
596 // BND analysis
597 Map<String, String> entries = new TreeMap<>();
598 try (Analyzer bndAnalyzer = new Analyzer()) {
599 bndAnalyzer.setProperties(mergeProps);
600 Jar jar = new Jar(bundleDir.toFile());
601 bndAnalyzer.setJar(jar);
602 Manifest manifest = bndAnalyzer.calcManifest();
603
604 keys: for (Object key : manifest.getMainAttributes().keySet()) {
605 Object value = manifest.getMainAttributes().get(key);
606
607 switch (key.toString()) {
608 case "Tool":
609 case "Bnd-LastModified":
610 case "Created-By":
611 continue keys;
612 }
613 if ("Require-Capability".equals(key.toString())
614 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
615 continue keys;// hack for very old classes
616 entries.put(key.toString(), value.toString());
617 }
618 } catch (Exception e) {
619 throw new RuntimeException("Cannot process " + mergeBnd, e);
620 }
621
622 Manifest manifest = new Manifest();
623 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
624 Files.createDirectories(manifestPath.getParent());
625 for (String key : entries.keySet()) {
626 String value = entries.get(key);
627 manifest.getMainAttributes().putValue(key, value);
628 }
629 manifest.getMainAttributes().putValue(ARGEO_ORIGIN_M2.toString(), originDesc.toString());
630
631 processLicense(bundleDir, manifest);
632
633 // write MANIFEST
634 try (OutputStream out = Files.newOutputStream(manifestPath)) {
635 manifest.write(out);
636 }
637 createJar(bundleDir, origin);
638 }
639
640 /** Generates MANIFEST using BND. */
641 Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, M2Artifact artifact,
642 A2Origin origin) {
643 try {
644 Map<String, String> additionalEntries = new TreeMap<>();
645 boolean doNotModifyManifest = Boolean.parseBoolean(
646 fileProps.getOrDefault(ARGEO_ORIGIN_NO_METADATA_GENERATION.toString(), "false").toString());
647
648 // Note: we always force the symbolic name
649 if (doNotModifyManifest) {
650 for (Object key : fileProps.keySet()) {
651 String value = fileProps.getProperty(key.toString());
652 additionalEntries.put(key.toString(), value);
653 }
654 } else {
655 if (artifact != null) {
656 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
657 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
658 }
659 if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
660 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
661 }
662 }
663
664 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
665 fileProps.put(EXPORT_PACKAGE.toString(),
666 "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
667 }
668
669 // BND analysis
670 try (Analyzer bndAnalyzer = new Analyzer()) {
671 bndAnalyzer.setProperties(fileProps);
672 Jar jar = new Jar(downloaded.toFile());
673 bndAnalyzer.setJar(jar);
674 Manifest manifest = bndAnalyzer.calcManifest();
675
676 keys: for (Object key : manifest.getMainAttributes().keySet()) {
677 Object value = manifest.getMainAttributes().get(key);
678
679 switch (key.toString()) {
680 case "Tool":
681 case "Bnd-LastModified":
682 case "Created-By":
683 continue keys;
684 }
685 if ("Require-Capability".equals(key.toString())
686 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) {
687 origin.deleted.add("MANIFEST header " + key);
688 continue keys;// !! hack for very old classes
689 }
690 additionalEntries.put(key.toString(), value.toString());
691 }
692 }
693 }
694 Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries, origin);
695 logger.log(DEBUG, () -> "Processed " + downloaded);
696 return targetBundleDir;
697 } catch (Exception e) {
698 throw new RuntimeException("Cannot BND process " + downloaded, e);
699 }
700
701 }
702
703 /** Download and integrates sources for a single Maven artifact. */
704 void downloadAndProcessM2Sources(String repoStr, M2Artifact artifact, Path targetBundleDir, boolean merging)
705 throws IOException {
706 try {
707 M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources");
708 URL sourcesUrl = M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
709 Path sourcesDownloaded = downloadMaven(sourcesUrl, artifact, true);
710 processM2SourceJar(sourcesDownloaded, targetBundleDir, merging ? artifact : null);
711 logger.log(TRACE, () -> "Processed source " + sourcesDownloaded);
712 } catch (Exception e) {
713 logger.log(ERROR, () -> "Cannot download source for " + artifact);
714 }
715
716 }
717
718 /** Integrate sources from a downloaded jar file. */
719 void processM2SourceJar(Path file, Path bundleDir, M2Artifact mergingFrom) throws IOException {
720 A2Origin origin = new A2Origin();
721 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
722 : bundleDir.resolve("OSGI-OPT/src");
723 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
724
725 String mergingMsg = "";
726 if (mergingFrom != null)
727 mergingMsg = " of " + mergingFrom;
728
729 Files.createDirectories(sourceDir);
730 JarEntry entry;
731 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
732 String relPath = entry.getName();
733 if (entry.isDirectory())
734 continue entries;
735 if (entry.getName().startsWith("META-INF")) {// skip META-INF entries
736 origin.deleted.add("META-INF directory from the sources" + mergingMsg);
737 continue entries;
738 }
739 if (entry.getName().startsWith("module-info.java")) {// skip Java module information
740 origin.deleted.add("Java module information from the sources (module-info.java)" + mergingMsg);
741 continue entries;
742 }
743 if (entry.getName().startsWith("/")) { // absolute paths
744 int metaInfIndex = entry.getName().indexOf("META-INF");
745 if (metaInfIndex >= 0) {
746 relPath = entry.getName().substring(metaInfIndex);
747 origin.moved.add(" to " + relPath + " entry with absolute path " + entry.getName());
748 } else {
749 logger.log(WARNING, entry.getName() + " has an absolute path");
750 origin.deleted.add(entry.getName() + " from the sources" + mergingMsg);
751 }
752 continue entries;
753 }
754 Path target = sourceDir.resolve(relPath);
755 Files.createDirectories(target.getParent());
756 if (!Files.exists(target)) {
757 Files.copy(jarIn, target);
758 logger.log(TRACE, () -> "Copied source " + target);
759 } else {
760 logger.log(TRACE, () -> target + " already exists, skipping...");
761 }
762 }
763 }
764 // write the changes
765 if (sourceBundles) {
766 origin.appendChanges(sourceDir);
767 } else {
768 origin.added.add("source code under OSGI-OPT/src");
769 origin.appendChanges(bundleDir);
770 }
771 }
772
773 /** Download a Maven artifact. */
774 Path downloadMaven(URL url, M2Artifact artifact) throws IOException {
775 return downloadMaven(url, artifact, false);
776 }
777
778 /** Download a Maven artifact. */
779 Path downloadMaven(URL url, M2Artifact artifact, boolean sources) throws IOException {
780 return download(url, mavenBase, artifact.getGroupId().replace(".", "/") //
781 + '/' + artifact.getArtifactId() + '/' + artifact.getVersion() //
782 + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() + (sources ? "-sources" : "") + ".jar");
783 }
784
785 /*
786 * ECLIPSE ORIGIN
787 */
788 /** Process an archive in Eclipse format. */
789 void processEclipseArchive(Path duDir) {
790 try {
791 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
792 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
793 Files.createDirectories(targetCategoryBase);
794 // first delete all directories from previous builds
795 for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)))
796 deleteDirectory(dir);
797
798 Files.createDirectories(originBase);
799
800 Path commonBnd = duDir.resolve(COMMON_BND);
801 Properties commonProps = new Properties();
802 try (InputStream in = Files.newInputStream(commonBnd)) {
803 commonProps.load(in);
804 }
805 String url = commonProps.getProperty(ARGEO_ORIGIN_URI.toString());
806 if (url == null) {
807 url = uris.getProperty(duDir.getFileName().toString());
808 if (url == null)
809 throw new IllegalStateException("No url available for " + duDir);
810 commonProps.put(ARGEO_ORIGIN_URI.toString(), url);
811 }
812 Path downloaded = tryDownloadArchive(url, originBase);
813
814 FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
815
816 // filters
817 List<PathMatcher> includeMatchers = new ArrayList<>();
818 Properties includes = new Properties();
819 try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
820 includes.load(in);
821 }
822 for (Object pattern : includes.keySet()) {
823 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
824 includeMatchers.add(pathMatcher);
825 }
826
827 List<PathMatcher> excludeMatchers = new ArrayList<>();
828 Path excludeFile = duDir.resolve("excludes.properties");
829 if (Files.exists(excludeFile)) {
830 Properties excludes = new Properties();
831 try (InputStream in = Files.newInputStream(excludeFile)) {
832 excludes.load(in);
833 }
834 for (Object pattern : excludes.keySet()) {
835 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
836 excludeMatchers.add(pathMatcher);
837 }
838 }
839
840 // keys are the bundle directories
841 Map<Path, A2Origin> origins = new HashMap<>();
842 Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
843
844 @Override
845 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
846 includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
847 if (includeMatcher.matches(file)) {
848 for (PathMatcher excludeMatcher : excludeMatchers) {
849 if (excludeMatcher.matches(file)) {
850 logger.log(TRACE, "Skipping excluded " + file);
851 return FileVisitResult.CONTINUE;
852 }
853 }
854 if (file.getFileName().toString().contains(".source_")) {
855 processEclipseSourceJar(file, targetCategoryBase);
856 logger.log(DEBUG, () -> "Processed source " + file);
857 } else {
858 Map<String, String> map = new HashMap<>();
859 for (Object key : commonProps.keySet())
860 map.put(key.toString(), commonProps.getProperty(key.toString()));
861 A2Origin origin = new A2Origin();
862 Path bundleDir = processBundleJar(file, targetCategoryBase, map, origin);
863 origins.put(bundleDir, origin);
864 logger.log(DEBUG, () -> "Processed " + file);
865 }
866 break includeMatchers;
867 }
868 }
869 return FileVisitResult.CONTINUE;
870 }
871 });
872
873 DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)
874 && p.getFileName().toString().indexOf('.') >= 0 && !p.getFileName().toString().endsWith(".src"));
875 for (Path bundleDir : dirs) {
876 A2Origin origin = origins.get(bundleDir);
877 Objects.requireNonNull(origin, "No A2 origin found for " + bundleDir);
878 createJar(bundleDir, origin);
879 }
880 } catch (IOException e) {
881 throw new RuntimeException("Cannot process " + duDir, e);
882 }
883
884 }
885
886 /** Process sources in Eclipse format. */
887 void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
888 try {
889 A2Origin origin = new A2Origin();
890 Path bundleDir;
891 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
892 Manifest manifest = jarIn.getManifest();
893
894 String[] relatedBundle = manifest.getMainAttributes().getValue(ECLIPSE_SOURCE_BUNDLE.toString())
895 .split(";");
896 String version = relatedBundle[1].substring("version=\"".length());
897 version = version.substring(0, version.length() - 1);
898 NameVersion nameVersion = new NameVersion(relatedBundle[0], version);
899 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
900
901 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
902 : bundleDir.resolve("OSGI-OPT/src");
903
904 Files.createDirectories(sourceDir);
905 JarEntry entry;
906 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
907 if (entry.isDirectory())
908 continue entries;
909 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
910 continue entries;
911 Path target = sourceDir.resolve(entry.getName());
912 Files.createDirectories(target.getParent());
913 Files.copy(jarIn, target);
914 logger.log(TRACE, () -> "Copied source " + target);
915 }
916
917 // write the changes
918 if (sourceBundles) {
919 origin.appendChanges(sourceDir);
920 } else {
921 origin.added.add("source code under OSGI-OPT/src");
922 origin.appendChanges(bundleDir);
923 }
924 }
925 } catch (IOException e) {
926 throw new IllegalStateException("Cannot process " + file, e);
927 }
928 }
929
930 /*
931 * COMMON PROCESSING
932 */
933 /** Normalise a single (that is, non-merged) bundle. */
934 Path processBundleJar(Path file, Path targetBase, Map<String, String> entries, A2Origin origin) throws IOException {
935 boolean embed = Boolean.parseBoolean(entries.getOrDefault(ARGEO_ORIGIN_EMBED.toString(), "false").toString());
936 NameVersion nameVersion;
937 Path bundleDir;
938 // singleton
939 boolean isSingleton = false;
940 Manifest manifest;
941 Manifest sourceManifest;
942 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
943 sourceManifest = jarIn.getManifest();
944 manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
945
946 String rawSourceSymbolicName = manifest.getMainAttributes().getValue(BUNDLE_SYMBOLICNAME.toString());
947 if (rawSourceSymbolicName != null) {
948 // make sure there is no directive
949 String[] arr = rawSourceSymbolicName.split(";");
950 for (int i = 1; i < arr.length; i++) {
951 if (arr[i].trim().equals("singleton:=true"))
952 isSingleton = true;
953 logger.log(DEBUG, file.getFileName() + " is a singleton");
954 }
955 }
956 // remove problematic entries in MANIFEST
957 manifest.getEntries().clear();
958
959 String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString());
960 String ourVersion = entries.get(BUNDLE_VERSION.toString());
961
962 if (ourSymbolicName != null && ourVersion != null) {
963 nameVersion = new NameVersion(ourSymbolicName, ourVersion);
964 } else {
965 nameVersion = nameVersionFromManifest(manifest);
966 if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) {
967 logger.log(WARNING,
968 "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion);
969 entries.put(BUNDLE_VERSION.toString(), ourVersion);
970 }
971 if (ourSymbolicName != null) {
972 // we always force our symbolic name
973 nameVersion.setName(ourSymbolicName);
974 }
975 }
976 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
977
978 // copy original MANIFEST
979 if (sourceManifest != null) {
980 Path originalManifest = bundleDir.resolve(A2_ORIGIN).resolve("MANIFEST.MF");
981 Files.createDirectories(originalManifest.getParent());
982 try (OutputStream out = Files.newOutputStream(originalManifest)) {
983 sourceManifest.write(out);
984 }
985 origin.moved.add("original MANIFEST to " + bundleDir.relativize(originalManifest));
986 }
987
988 // force Java 9 module name
989 entries.put(ManifestHeader.AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName());
990
991 boolean isNative = false;
992 String os = null;
993 String arch = null;
994 if (bundleDir.startsWith(a2LibBase)) {
995 isNative = true;
996 Path libRelativePath = a2LibBase.relativize(bundleDir);
997 os = libRelativePath.getName(0).toString();
998 arch = libRelativePath.getName(1).toString();
999 }
1000
1001 if (!embed) {
1002 // copy entries
1003 JarEntry entry;
1004 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
1005 if (entry.isDirectory())
1006 continue entries;
1007 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
1008 || entry.getName().endsWith(".SF")) {
1009 origin.deleted.add("cryptographic signatures");
1010 continue entries;
1011 }
1012 if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
1013 origin.deleted.add("Java module information (module-info.class)");
1014 continue entries;
1015 }
1016 if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
1017 origin.deleted.add("additional Java versions (META-INF/versions)");
1018 continue entries;
1019 }
1020 if (entry.getName().startsWith("META-INF/maven/")) {
1021 origin.deleted.add("Maven information (META-INF/maven)");
1022 continue entries;
1023 }
1024 // skip file system providers as they cause issues with native image
1025 if (entry.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider")) {
1026 origin.deleted
1027 .add("file system providers (META-INF/services/java.nio.file.spi.FileSystemProvider)");
1028 continue entries;
1029 }
1030 if (entry.getName().startsWith("OSGI-OPT/src/")) { // skip embedded sources
1031 origin.deleted.add("embedded sources");
1032 continue entries;
1033 }
1034 Path target = bundleDir.resolve(entry.getName());
1035 Files.createDirectories(target.getParent());
1036 Files.copy(jarIn, target);
1037
1038 // native libraries
1039 if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll")
1040 || entry.getName().endsWith(".jnilib"))) {
1041 Path categoryDir = bundleDir.getParent();
1042 boolean copyDll = false;
1043 Path targetDll = categoryDir.resolve(bundleDir.relativize(target));
1044 if (nameVersion.getName().equals("com.sun.jna")) {
1045 if (arch.equals("x86_64"))
1046 arch = "x86-64";
1047 if (os.equals("macosx"))
1048 os = "darwin";
1049 if (target.getParent().getFileName().toString().equals(os + "-" + arch)) {
1050 copyDll = true;
1051 }
1052 targetDll = categoryDir.resolve(target.getFileName());
1053 } else {
1054 copyDll = true;
1055 }
1056 if (copyDll) {
1057 Files.createDirectories(targetDll.getParent());
1058 if (Files.exists(targetDll))
1059 Files.delete(targetDll);
1060 Files.copy(target, targetDll);
1061 }
1062 Files.delete(target);
1063 origin.deleted.add(bundleDir.relativize(target).toString());
1064 }
1065 logger.log(TRACE, () -> "Copied " + target);
1066 }
1067 }
1068 }
1069
1070 // copy MANIFEST
1071 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1072 Files.createDirectories(manifestPath.getParent());
1073
1074 if (isSingleton && entries.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
1075 entries.put(BUNDLE_SYMBOLICNAME.toString(),
1076 entries.get(BUNDLE_SYMBOLICNAME.toString()) + ";singleton:=true");
1077 }
1078
1079 if (embed) {// copy embedded jar
1080 Files.copy(file, bundleDir.resolve(file.getFileName()));
1081 entries.put(ManifestHeader.BUNDLE_CLASSPATH.toString(), file.getFileName().toString());
1082 }
1083
1084 // Final MANIFEST decisions
1085 // We also check the original OSGi metadata and compare with our changes
1086 for (String key : entries.keySet()) {
1087 String value = entries.get(key);
1088 String previousValue = manifest.getMainAttributes().getValue(key);
1089 boolean wasDifferent = previousValue != null && !previousValue.equals(value);
1090 boolean keepPrevious = false;
1091 if (wasDifferent) {
1092 if (SPDX_LICENSE_IDENTIFIER.toString().equals(key) && previousValue != null)
1093 keepPrevious = true;
1094 else if (BUNDLE_VERSION.toString().equals(key) && wasDifferent)
1095 if (previousValue.equals(value + ".0")) // typically a Maven first release
1096 keepPrevious = true;
1097
1098 if (keepPrevious) {
1099 if (logger.isLoggable(DEBUG))
1100 logger.log(DEBUG, file.getFileName() + ": " + key + " was NOT modified, value kept is "
1101 + previousValue + ", not overriden with " + value);
1102 value = previousValue;
1103 }
1104 }
1105
1106 manifest.getMainAttributes().putValue(key, value);
1107 if (wasDifferent && !keepPrevious) {
1108 if (IMPORT_PACKAGE.toString().equals(key) || EXPORT_PACKAGE.toString().equals(key))
1109 logger.log(TRACE, () -> file.getFileName() + ": " + key + " was modified");
1110 else
1111 logger.log(WARNING,
1112 file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value);
1113 origin.modified.add("MANIFEST header " + key);
1114 }
1115
1116 // !! hack to remove unresolvable
1117 if (key.equals("Provide-Capability") || key.equals("Require-Capability"))
1118 if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) {
1119 manifest.getMainAttributes().remove(key);
1120 origin.deleted.add("MANIFEST header " + key);
1121 }
1122 }
1123
1124 // de-pollute MANIFEST
1125 for (Iterator<Map.Entry<Object, Object>> manifestEntries = manifest.getMainAttributes().entrySet()
1126 .iterator(); manifestEntries.hasNext();) {
1127 Map.Entry<Object, Object> manifestEntry = manifestEntries.next();
1128 switch (manifestEntry.getKey().toString()) {
1129 case "Archiver-Version":
1130 case "Build-By":
1131 case "Created-By":
1132 case "Originally-Created-By":
1133 case "Tool":
1134 case "Bnd-LastModified":
1135 manifestEntries.remove();
1136 origin.deleted.add("MANIFEST header " + manifestEntry.getKey());
1137 break;
1138 default:
1139 if (sourceManifest != null && !sourceManifest.getMainAttributes().containsKey(manifestEntry.getKey()))
1140 origin.added.add("MANIFEST header " + manifestEntry.getKey());
1141 }
1142 }
1143
1144 processLicense(bundleDir, manifest);
1145
1146 origin.modified.add("MANIFEST (META-INF/MANIFEST.MF)");
1147 // write the MANIFEST
1148 try (OutputStream out = Files.newOutputStream(manifestPath)) {
1149 manifest.write(out);
1150 }
1151 return bundleDir;
1152 }
1153
1154 /** Process SPDX license identifier. */
1155 void processLicense(Path bundleDir, Manifest manifest) {
1156 String spdxLicenceId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString());
1157 String bundleLicense = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString());
1158 if (spdxLicenceId == null) {
1159 logger.log(ERROR, bundleDir.getFileName() + ": " + SPDX_LICENSE_IDENTIFIER + " not available, "
1160 + BUNDLE_LICENSE + " is " + bundleLicense);
1161 } else {
1162 // only use the first licensing option
1163 int orIndex = spdxLicenceId.indexOf(" OR ");
1164 if (orIndex >= 0)
1165 spdxLicenceId = spdxLicenceId.substring(0, orIndex).trim();
1166
1167 String bundleDirName = bundleDir.getFileName().toString();
1168 // force licenses of some well-known components
1169 // even if we say otherwise (typically because from an Eclipse archive)
1170 if (bundleDirName.startsWith("org.apache."))
1171 spdxLicenceId = "Apache-2.0";
1172 if (bundleDirName.startsWith("com.sun.jna."))
1173 spdxLicenceId = "Apache-2.0";
1174 if (bundleDirName.startsWith("com.ibm.icu."))
1175 spdxLicenceId = "ICU";
1176 if (bundleDirName.startsWith("javax.annotation."))
1177 spdxLicenceId = "GPL-2.0-only WITH Classpath-exception-2.0";
1178 if (bundleDirName.startsWith("javax.inject."))
1179 spdxLicenceId = "Apache-2.0";
1180 if (bundleDirName.startsWith("org.osgi."))
1181 spdxLicenceId = "Apache-2.0";
1182
1183 manifest.getMainAttributes().putValue(SPDX_LICENSE_IDENTIFIER.toString(), spdxLicenceId);
1184 if (!licensesUsed.containsKey(spdxLicenceId))
1185 licensesUsed.put(spdxLicenceId, new TreeSet<>());
1186 licensesUsed.get(spdxLicenceId).add(bundleDir.getParent().getFileName() + "/" + bundleDir.getFileName());
1187 }
1188 }
1189
1190 /*
1191 * UTILITIES
1192 */
1193 /** Recursively deletes a directory. */
1194 static void deleteDirectory(Path path) throws IOException {
1195 if (!Files.exists(path))
1196 return;
1197 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
1198 @Override
1199 public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
1200 if (e != null)
1201 throw e;
1202 Files.delete(directory);
1203 return CONTINUE;
1204 }
1205
1206 @Override
1207 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1208 Files.delete(file);
1209 return CONTINUE;
1210 }
1211 });
1212 }
1213
1214 /** Extract name/version from a MANIFEST. */
1215 NameVersion nameVersionFromManifest(Manifest manifest) {
1216 Attributes attrs = manifest.getMainAttributes();
1217 // symbolic name
1218 String symbolicName = attrs.getValue(ManifestHeader.BUNDLE_SYMBOLICNAME.toString());
1219 if (symbolicName == null)
1220 return null;
1221 // make sure there is no directive
1222 symbolicName = symbolicName.split(";")[0];
1223
1224 String version = attrs.getValue(ManifestHeader.BUNDLE_VERSION.toString());
1225 return new NameVersion(symbolicName, version);
1226 }
1227
1228 /** Try to download from an URI. */
1229 Path tryDownloadArchive(String uri, Path dir) throws IOException {
1230 // find mirror
1231 List<String> urlBases = null;
1232 String uriPrefix = null;
1233 uriPrefixes: for (String uriPref : mirrors.keySet()) {
1234 if (uri.startsWith(uriPref)) {
1235 if (mirrors.get(uriPref).size() > 0) {
1236 urlBases = mirrors.get(uriPref);
1237 uriPrefix = uriPref;
1238 break uriPrefixes;
1239 }
1240 }
1241 }
1242 if (urlBases == null)
1243 try {
1244 return downloadArchive(new URL(uri), dir);
1245 } catch (FileNotFoundException e) {
1246 throw new FileNotFoundException("Cannot find " + uri);
1247 }
1248
1249 // try to download
1250 for (String urlBase : urlBases) {
1251 String relativePath = uri.substring(uriPrefix.length());
1252 URL url = new URL(urlBase + relativePath);
1253 try {
1254 return downloadArchive(url, dir);
1255 } catch (FileNotFoundException e) {
1256 logger.log(WARNING, "Cannot download " + url + ", trying another mirror");
1257 }
1258 }
1259 throw new FileNotFoundException("Cannot find " + uri);
1260 }
1261
1262 /**
1263 * Effectively download. Synchronised in order to avoid downloading twice in
1264 * parallel.
1265 */
1266 synchronized Path downloadArchive(URL url, Path dir) throws IOException {
1267 return download(url, dir, (String) null);
1268 }
1269
1270 /** Effectively download. */
1271 Path download(URL url, Path dir, String name) throws IOException {
1272
1273 Path dest;
1274 if (name == null) {
1275 // We use also use parent directory in case the archive itself has a fixed name
1276 String[] segments = url.getPath().split("/");
1277 name = segments.length > 1 ? segments[segments.length - 2] + '-' + segments[segments.length - 1]
1278 : segments[segments.length - 1];
1279 }
1280
1281 dest = dir.resolve(name);
1282 if (Files.exists(dest)) {
1283 logger.log(TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
1284 return dest;
1285 } else {
1286 Files.createDirectories(dest.getParent());
1287 }
1288
1289 try (InputStream in = url.openStream()) {
1290 Files.copy(in, dest);
1291 logger.log(DEBUG, () -> "Downloaded " + dest + " from " + url);
1292 }
1293 return dest;
1294 }
1295
1296 /** Create a JAR file from a directory. */
1297 Path createJar(Path bundleDir, A2Origin origin) throws IOException {
1298 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1299 Manifest manifest;
1300 try (InputStream in = Files.newInputStream(manifestPath)) {
1301 manifest = new Manifest(in);
1302 }
1303 // legal requirements
1304 origin.appendChanges(bundleDir);
1305 createReadMe(bundleDir, manifest);
1306
1307 // create the jar
1308 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
1309 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
1310 jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
1311 Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
1312
1313 @Override
1314 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1315 if (file.getFileName().toString().equals("MANIFEST.MF"))
1316 return super.visitFile(file, attrs);
1317 JarEntry entry = new JarEntry(
1318 bundleDir.relativize(file).toString().replace(File.separatorChar, '/'));
1319 jarOut.putNextEntry(entry);
1320 Files.copy(file, jarOut);
1321 return super.visitFile(file, attrs);
1322 }
1323
1324 });
1325 }
1326 deleteDirectory(bundleDir);
1327
1328 if (sourceBundles)
1329 createSourceJar(bundleDir, manifest);
1330
1331 return jarPath;
1332 }
1333
1334 /** Package sources separately, in the Eclipse-SourceBundle format. */
1335 void createSourceJar(Path bundleDir, Manifest manifest) throws IOException {
1336 Path bundleCategoryDir = bundleDir.getParent();
1337 Path sourceDir = bundleCategoryDir.resolve(bundleDir.toString() + ".src");
1338 if (!Files.exists(sourceDir)) {
1339 logger.log(WARNING, sourceDir + " does not exist, skipping...");
1340 return;
1341 }
1342 createReadMe(sourceDir, manifest);
1343
1344 Path relPath = a2Base.relativize(bundleCategoryDir);
1345 Path srcCategoryDir = a2SrcBase.resolve(relPath);
1346 Path srcJarP = srcCategoryDir.resolve(sourceDir.getFileName() + ".jar");
1347 Files.createDirectories(srcJarP.getParent());
1348
1349 String bundleSymbolicName = manifest.getMainAttributes().getValue("Bundle-SymbolicName").toString();
1350 // in case there are additional directives
1351 bundleSymbolicName = bundleSymbolicName.split(";")[0];
1352 Manifest srcManifest = new Manifest();
1353 srcManifest.getMainAttributes().put(MANIFEST_VERSION, "1.0");
1354 srcManifest.getMainAttributes().putValue(BUNDLE_SYMBOLICNAME.toString(), bundleSymbolicName + ".src");
1355 srcManifest.getMainAttributes().putValue(BUNDLE_VERSION.toString(),
1356 manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()).toString());
1357 srcManifest.getMainAttributes().putValue(ECLIPSE_SOURCE_BUNDLE.toString(), bundleSymbolicName + ";version=\""
1358 + manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()) + "\"");
1359
1360 try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
1361 srcJarOut.setLevel(Deflater.BEST_COMPRESSION);
1362 Files.walkFileTree(sourceDir, new SimpleFileVisitor<Path>() {
1363
1364 @Override
1365 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1366 if (file.getFileName().toString().equals("MANIFEST.MF"))
1367 return super.visitFile(file, attrs);
1368 JarEntry entry = new JarEntry(
1369 sourceDir.relativize(file).toString().replace(File.separatorChar, '/'));
1370 srcJarOut.putNextEntry(entry);
1371 Files.copy(file, srcJarOut);
1372 return super.visitFile(file, attrs);
1373 }
1374
1375 });
1376 }
1377 deleteDirectory(sourceDir);
1378 }
1379
1380 /**
1381 * Generate a readme clarifying and prominently notifying of the repackaging and
1382 * modifications.
1383 */
1384 void createReadMe(Path jarDir, Manifest manifest) throws IOException {
1385 // write repackaged README
1386 try (BufferedWriter writer = Files.newBufferedWriter(jarDir.resolve(README_REPACKAGED))) {
1387 boolean merged = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2_MERGE.toString()) != null;
1388 if (merged)
1389 writer.append("This component is a merging of third party components"
1390 + " in order to comply with A2 packaging standards.\n");
1391 else
1392 writer.append("This component is a repackaging of a third party component"
1393 + " in order to comply with A2 packaging standards.\n");
1394
1395 // license
1396 String spdxLicenseId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString());
1397 if (spdxLicenseId == null)
1398 throw new IllegalStateException("An SPDX license id must have beend defined at this stage.");
1399 writer.append("\nIt is redistributed under the following license:\n\n");
1400 writer.append("SPDX-Identifier: " + spdxLicenseId + "\n\n");
1401
1402 if (!spdxLicenseId.startsWith("LicenseRef")) {// standard
1403 int withIndex = spdxLicenseId.indexOf(" WITH ");
1404 if (withIndex >= 0) {
1405 String simpleId = spdxLicenseId.substring(0, withIndex).trim();
1406 String exception = spdxLicenseId.substring(withIndex + " WITH ".length());
1407 writer.append("which are available here: https://spdx.org/licenses/" + simpleId
1408 + "\nand here: https://spdx.org/licenses/" + exception + "\n");
1409 } else {
1410 writer.append("which is available here: https://spdx.org/licenses/" + spdxLicenseId + "\n");
1411 }
1412 } else {
1413 String url = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString());
1414 if (url != null) {
1415 writer.write("which is available here: " + url + "\n");
1416 } else {
1417 logger.log(ERROR, "No licne URL for " + jarDir);
1418 }
1419 }
1420 writer.write("\n");
1421
1422 // origin
1423 String m2Repo = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2_REPO.toString());
1424 String originDesc = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2.toString());
1425 if (originDesc != null)
1426 writer.append("The original component has M2 coordinates:\n" + originDesc.replace(',', '\n') + "\n"
1427 + (m2Repo != null ? "\nin M2 repository " + m2Repo + "\n" : ""));
1428 else {
1429 originDesc = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_URI.toString());
1430 if (originDesc != null)
1431 writer.append("The original component comes from " + originDesc + ".\n");
1432 else
1433 logger.log(ERROR, "Cannot find origin information in " + jarDir);
1434 }
1435
1436 writer.append("\nA detailed list of changes is available under " + CHANGES + ".\n");
1437 if (!jarDir.getFileName().endsWith(".src")) {// binary archive
1438 if (sourceBundles)
1439 writer.append("Corresponding sources are available in the related archive named "
1440 + jarDir.toString() + ".src.jar.\n");
1441 else
1442 writer.append("Corresponding sources are available under OSGI-OPT/src.\n");
1443 }
1444 }
1445
1446 }
1447
1448 /**
1449 * Gathers modifications performed on the original binaries and sources,
1450 * especially in order to comply with their license requirements.
1451 */
1452 class A2Origin {
1453 A2Origin() {
1454
1455 }
1456
1457 Set<String> modified = new TreeSet<>();
1458 Set<String> deleted = new TreeSet<>();
1459 Set<String> added = new TreeSet<>();
1460 Set<String> moved = new TreeSet<>();
1461
1462 /** Append changes to the A2-ORIGIN/changes file. */
1463 void appendChanges(Path baseDirectory) throws IOException {
1464 Path changesFile = baseDirectory.resolve(CHANGES);
1465 Files.createDirectories(changesFile.getParent());
1466 try (BufferedWriter writer = Files.newBufferedWriter(changesFile, APPEND, CREATE)) {
1467 for (String msg : added)
1468 writer.write("- Added " + msg + ".\n");
1469 for (String msg : modified)
1470 writer.write("- Modified " + msg + ".\n");
1471 for (String msg : moved)
1472 writer.write("- Moved " + msg + ".\n");
1473 for (String msg : deleted)
1474 writer.write("- Deleted " + msg + ".\n");
1475 }
1476 }
1477 }
1478 }
1479
1480 /** Simple representation of an M2 artifact. */
1481 class M2Artifact extends CategoryNameVersion {
1482 private String classifier;
1483
1484 M2Artifact(String m2coordinates) {
1485 this(m2coordinates, null);
1486 }
1487
1488 M2Artifact(String m2coordinates, String classifier) {
1489 String[] parts = m2coordinates.split(":");
1490 setCategory(parts[0]);
1491 setName(parts[1]);
1492 if (parts.length > 2) {
1493 setVersion(parts[2]);
1494 }
1495 this.classifier = classifier;
1496 }
1497
1498 String getGroupId() {
1499 return super.getCategory();
1500 }
1501
1502 String getArtifactId() {
1503 return super.getName();
1504 }
1505
1506 String toM2Coordinates() {
1507 return getCategory() + ":" + getName() + (getVersion() != null ? ":" + getVersion() : "");
1508 }
1509
1510 String getClassifier() {
1511 return classifier != null ? classifier : "";
1512 }
1513
1514 String getExtension() {
1515 return "jar";
1516 }
1517 }
1518
1519 /** Utilities around Maven (conventions based). */
1520 class M2ConventionsUtils {
1521 final static String MAVEN_CENTRAL_BASE_URL = "https://repo1.maven.org/maven2/";
1522
1523 /** The file name of this artifact when stored */
1524 static String artifactFileName(M2Artifact artifact) {
1525 return artifact.getArtifactId() + '-' + artifact.getVersion()
1526 + (artifact.getClassifier().equals("") ? "" : '-' + artifact.getClassifier()) + '.'
1527 + artifact.getExtension();
1528 }
1529
1530 /** Absolute path to the file */
1531 static String artifactPath(String artifactBasePath, M2Artifact artifact) {
1532 return artifactParentPath(artifactBasePath, artifact) + '/' + artifactFileName(artifact);
1533 }
1534
1535 /** Absolute path to the file */
1536 static String artifactUrl(String repoUrl, M2Artifact artifact) {
1537 if (repoUrl.endsWith("/"))
1538 return repoUrl + artifactPath("/", artifact).substring(1);
1539 else
1540 return repoUrl + artifactPath("/", artifact);
1541 }
1542
1543 /** Absolute path to the file */
1544 static URL mavenRepoUrl(String repoBase, M2Artifact artifact) {
1545 String url = artifactUrl(repoBase == null ? MAVEN_CENTRAL_BASE_URL : repoBase, artifact);
1546 try {
1547 return new URL(url);
1548 } catch (MalformedURLException e) {
1549 // it should not happen
1550 throw new IllegalStateException(e);
1551 }
1552 }
1553
1554 /** Absolute path to the directories where the files will be stored */
1555 static String artifactParentPath(String artifactBasePath, M2Artifact artifact) {
1556 return artifactBasePath + (artifactBasePath.endsWith("/") ? "" : "/") + artifactParentPath(artifact);
1557 }
1558
1559 /** Relative path to the directories where the files will be stored */
1560 static String artifactParentPath(M2Artifact artifact) {
1561 return artifact.getGroupId().replace('.', '/') + '/' + artifact.getArtifactId() + '/' + artifact.getVersion();
1562 }
1563
1564 /** Singleton */
1565 private M2ConventionsUtils() {
1566 }
1567 }
1568
1569 /** Combination of a category, a name and a version. */
1570 class CategoryNameVersion extends NameVersion {
1571 private String category;
1572
1573 CategoryNameVersion() {
1574 }
1575
1576 CategoryNameVersion(String category, String name, String version) {
1577 super(name, version);
1578 this.category = category;
1579 }
1580
1581 CategoryNameVersion(String category, NameVersion nameVersion) {
1582 super(nameVersion);
1583 this.category = category;
1584 }
1585
1586 String getCategory() {
1587 return category;
1588 }
1589
1590 void setCategory(String category) {
1591 this.category = category;
1592 }
1593
1594 @Override
1595 public String toString() {
1596 return category + ":" + super.toString();
1597 }
1598
1599 }
1600
1601 /** Combination of a name and a version. */
1602 class NameVersion implements Comparable<NameVersion> {
1603 private String name;
1604 private String version;
1605
1606 NameVersion() {
1607 }
1608
1609 /** Interprets string in OSGi-like format my.module.name;version=0.0.0 */
1610 NameVersion(String nameVersion) {
1611 int index = nameVersion.indexOf(";version=");
1612 if (index < 0) {
1613 setName(nameVersion);
1614 setVersion(null);
1615 } else {
1616 setName(nameVersion.substring(0, index));
1617 setVersion(nameVersion.substring(index + ";version=".length()));
1618 }
1619 }
1620
1621 NameVersion(String name, String version) {
1622 this.name = name;
1623 this.version = version;
1624 }
1625
1626 NameVersion(NameVersion nameVersion) {
1627 this.name = nameVersion.getName();
1628 this.version = nameVersion.getVersion();
1629 }
1630
1631 String getName() {
1632 return name;
1633 }
1634
1635 void setName(String name) {
1636 this.name = name;
1637 }
1638
1639 String getVersion() {
1640 return version;
1641 }
1642
1643 void setVersion(String version) {
1644 this.version = version;
1645 }
1646
1647 String getBranch() {
1648 String[] parts = getVersion().split("\\.");
1649 if (parts.length < 2)
1650 throw new IllegalStateException("Version " + getVersion() + " cannot be interpreted as branch.");
1651 return parts[0] + "." + parts[1];
1652 }
1653
1654 @Override
1655 public boolean equals(Object obj) {
1656 if (obj instanceof NameVersion) {
1657 NameVersion nameVersion = (NameVersion) obj;
1658 return name.equals(nameVersion.getName()) && version.equals(nameVersion.getVersion());
1659 } else
1660 return false;
1661 }
1662
1663 @Override
1664 public int hashCode() {
1665 return name.hashCode();
1666 }
1667
1668 @Override
1669 public String toString() {
1670 return name + ":" + version;
1671 }
1672
1673 public int compareTo(NameVersion o) {
1674 if (o.getName().equals(name))
1675 return version.compareTo(o.getVersion());
1676 else
1677 return name.compareTo(o.getName());
1678 }
1679 }
Argeo Build - Build scripts and configuration for Argeo projects