]> git.argeo.org Git - cc0/argeo-build.git/blob - Repackage.java
projects / cc0 / argeo-build.git / blob


e9f8a3370f26890521d01c5ad4ec84d0833390ae
[cc0/argeo-build.git] / Repackage.java
1 package org.argeo.build;
2
3 import static java.lang.System.Logger.Level.DEBUG;
4 import static java.lang.System.Logger.Level.ERROR;
5 import static java.lang.System.Logger.Level.INFO;
6 import static java.lang.System.Logger.Level.TRACE;
7 import static java.lang.System.Logger.Level.WARNING;
8 import static java.nio.file.FileVisitResult.CONTINUE;
9 import static java.nio.file.StandardOpenOption.APPEND;
10 import static java.nio.file.StandardOpenOption.CREATE;
11 import static java.util.jar.Attributes.Name.MANIFEST_VERSION;
12 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_EMBED;
13 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2;
14 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_MERGE;
15 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_M2_REPO;
16 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_MANIFEST_NOT_MODIFIED;
17 import static org.argeo.build.Repackage.ManifestHeader.ARGEO_ORIGIN_URI;
18 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_LICENSE;
19 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_SYMBOLICNAME;
20 import static org.argeo.build.Repackage.ManifestHeader.BUNDLE_VERSION;
21 import static org.argeo.build.Repackage.ManifestHeader.ECLIPSE_SOURCE_BUNDLE;
22 import static org.argeo.build.Repackage.ManifestHeader.EXPORT_PACKAGE;
23 import static org.argeo.build.Repackage.ManifestHeader.IMPORT_PACKAGE;
24 import static org.argeo.build.Repackage.ManifestHeader.SPDX_LICENSE_IDENTIFIER;
25
26 import java.io.BufferedWriter;
27 import java.io.File;
28 import java.io.FileNotFoundException;
29 import java.io.IOException;
30 import java.io.InputStream;
31 import java.io.OutputStream;
32 import java.lang.System.Logger;
33 import java.net.MalformedURLException;
34 import java.net.URL;
35 import java.nio.charset.StandardCharsets;
36 import java.nio.file.DirectoryStream;
37 import java.nio.file.FileSystem;
38 import java.nio.file.FileSystems;
39 import java.nio.file.FileVisitResult;
40 import java.nio.file.Files;
41 import java.nio.file.Path;
42 import java.nio.file.PathMatcher;
43 import java.nio.file.Paths;
44 import java.nio.file.SimpleFileVisitor;
45 import java.nio.file.StandardCopyOption;
46 import java.nio.file.StandardOpenOption;
47 import java.nio.file.attribute.BasicFileAttributes;
48 import java.util.ArrayList;
49 import java.util.HashMap;
50 import java.util.Iterator;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.Objects;
54 import java.util.Properties;
55 import java.util.Set;
56 import java.util.StringJoiner;
57 import java.util.TreeMap;
58 import java.util.TreeSet;
59 import java.util.concurrent.CompletableFuture;
60 import java.util.jar.Attributes;
61 import java.util.jar.JarEntry;
62 import java.util.jar.JarInputStream;
63 import java.util.jar.JarOutputStream;
64 import java.util.jar.Manifest;
65 import java.util.zip.Deflater;
66
67 import aQute.bnd.osgi.Analyzer;
68 import aQute.bnd.osgi.Jar;
69
70 /** Repackages existing jar files into OSGi bundles in an A2 repository. */
71 public class Repackage {
72 final static Logger logger = System.getLogger(Repackage.class.getName());
73
74 /**
75 * Environment variable on whether sources should be packaged separately or
76 * integrated in the bundles.
77 */
78 final static String ENV_SOURCE_BUNDLES = "SOURCE_BUNDLES";
79 /** Environment variable on whether operations should be parallelised. */
80 final static String ENV_ARGEO_BUILD_SEQUENTIAL = "ARGEO_BUILD_SEQUENTIAL";
81
82 /** Whether repackaging should run in parallel (default) or sequentially. */
83 final static boolean sequential = Boolean.parseBoolean(System.getenv(ENV_ARGEO_BUILD_SEQUENTIAL));
84
85 /** Main entry point. */
86 public static void main(String[] args) {
87 if (sequential)
88 logger.log(INFO, "Build will be sequential");
89 if (args.length < 2) {
90 System.err.println("Usage: <path to a2 output dir> <category1> <category2> ...");
91 System.exit(1);
92 }
93 Path a2Base = Paths.get(args[0]).toAbsolutePath().normalize();
94 Path descriptorsBase = Paths.get(".").toAbsolutePath().normalize();
95 Repackage factory = new Repackage(a2Base, descriptorsBase);
96
97 List<CompletableFuture<Void>> toDos = new ArrayList<>();
98 for (int i = 1; i < args.length; i++) {
99 Path p = Paths.get(args[i]);
100 if (sequential)
101 factory.processCategory(p);
102 else
103 toDos.add(CompletableFuture.runAsync(() -> factory.processCategory(p)));
104 }
105 if (!sequential)
106 CompletableFuture.allOf(toDos.toArray(new CompletableFuture[toDos.size()])).join();
107
108 // Summary
109 StringBuilder sb = new StringBuilder();
110 for (String licenseId : licensesUsed.keySet())
111 for (String name : licensesUsed.get(licenseId))
112 sb.append((licenseId.equals("") ? "Proprietary" : licenseId) + "\t\t" + name + "\n");
113 logger.log(INFO, "# License summary:\n" + sb);
114 }
115
116 /** MANIFEST headers. */
117 enum ManifestHeader {
118 // OSGi
119 /** OSGi bundle symbolic name. */
120 BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), //
121 /** OSGi bundle version. */
122 BUNDLE_VERSION("Bundle-Version"), //
123 /** OSGi bundle license. */
124 BUNDLE_LICENSE("Bundle-License"), //
125 /** OSGi exported packages list. */
126 EXPORT_PACKAGE("Export-Package"), //
127 /** OSGi imported packages list. */
128 IMPORT_PACKAGE("Import-Package"), //
129 /** OSGi path to embedded jar. */
130 BUNDLE_CLASSPATH("Bundle-Classpath"), //
131 // Java
132 /** Java module name. */
133 AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), //
134 // Eclipse
135 /** Eclipse source bundle. */
136 ECLIPSE_SOURCE_BUNDLE("Eclipse-SourceBundle"), //
137 // SPDX
138 /**
139 * SPDX license identifier.
140 *
141 * @see https://spdx.org/licenses/
142 */
143 SPDX_LICENSE_IDENTIFIER("SPDX-License-Identifier"), //
144 // Argeo Origin
145 /**
146 * Maven coordinates of the origin, possibly partial when using common.bnd or
147 * merge.bnd.
148 */
149 ARGEO_ORIGIN_M2("Argeo-Origin-M2"), //
150 /** List of Maven coordinates to merge. */
151 ARGEO_ORIGIN_M2_MERGE("Argeo-Origin-M2-Merge"), //
152 /** Maven repository, if not the default one. */
153 ARGEO_ORIGIN_M2_REPO("Argeo-Origin-M2-Repo"), //
154 /**
155 * Do not perform BND analysis of the origin component. Typically IMport_package
156 * and Export-Package will be kept untouched.
157 */
158 ARGEO_ORIGIN_MANIFEST_NOT_MODIFIED("Argeo-Origin-ManifestNotModified"), //
159 /**
160 * Embed the original jar without modifying it (may be required by some
161 * proprietary licenses, such as JCR Day License).
162 */
163 ARGEO_ORIGIN_EMBED("Argeo-Origin-Embed"), //
164 /**
165 * Do not modify original jar (may be required by some proprietary licenses,
166 * such as JCR Day License).
167 */
168 ARGEO_DO_NOT_MODIFY("Argeo-Origin-Do-Not-Modify"), //
169 /**
170 * Origin (non-Maven) URI of the component. It may be anything (jar, archive,
171 * etc.).
172 */
173 ARGEO_ORIGIN_URI("Argeo-Origin-URI"), //
174 ;
175
176 final String value;
177
178 private ManifestHeader(String value) {
179 this.value = value;
180 }
181
182 @Override
183 public String toString() {
184 return value;
185 }
186 }
187
188 /** Name of the file centralising information for multiple M2 artifacts. */
189 final static String COMMON_BND = "common.bnd";
190 /** Name of the file centralising information for mergin M2 artifacts. */
191 final static String MERGE_BND = "merge.bnd";
192 /**
193 * Subdirectory of the jar file where origin informations (changes, legal
194 * notices etc. are stored)
195 */
196 final static String A2_ORIGIN = "A2-ORIGIN";
197 /** File detailing modifications to the original component. */
198 final static String CHANGES = A2_ORIGIN + "/changes";
199 /**
200 * Name of the file at the root of the repackaged jar, which prominently
201 * notifies that the component has be repackaged.
202 */
203 final static String README_REPACKAGED = "README.repackaged";
204
205 // cache
206 /** Summary of all license seen during the repackaging. */
207 final static Map<String, Set<String>> licensesUsed = new TreeMap<>();
208
209 /** Directory where to download archives */
210 final Path originBase;
211 /** Directory where to download Maven artifacts */
212 final Path mavenBase;
213
214 /** A2 repository base for binary bundles */
215 final Path a2Base;
216 /** A2 repository base for source bundles */
217 final Path a2SrcBase;
218 /** A2 base for native components */
219 final Path a2LibBase;
220 /** Location of the descriptors driving the packaging */
221 final Path descriptorsBase;
222 /** URIs of archives to download */
223 final Properties uris = new Properties();
224 /** Mirrors for archive download. Key is URI prefix, value list of base URLs */
225 final Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
226
227 /** Whether sources should be packaged separately */
228 final boolean sourceBundles;
229
230 /** Constructor initialises the various variables */
231 public Repackage(Path a2Base, Path descriptorsBase) {
232 sourceBundles = Boolean.parseBoolean(System.getenv(ENV_SOURCE_BUNDLES));
233 if (sourceBundles)
234 logger.log(INFO, "Sources will be packaged separately");
235
236 Objects.requireNonNull(a2Base);
237 Objects.requireNonNull(descriptorsBase);
238 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/build/origin");
239 this.mavenBase = Paths.get(System.getProperty("user.home"), ".m2", "repository");
240
241 // TODO define and use a build base
242 this.a2Base = a2Base;
243 this.a2SrcBase = a2Base.getParent().resolve(a2Base.getFileName() + ".src");
244 this.a2LibBase = a2Base.resolve("lib");
245 this.descriptorsBase = descriptorsBase;
246 if (!Files.exists(this.descriptorsBase))
247 throw new IllegalArgumentException(this.descriptorsBase + " does not exist");
248
249 // URIs mapping
250 Path urisPath = this.descriptorsBase.resolve("uris.properties");
251 if (Files.exists(urisPath)) {
252 try (InputStream in = Files.newInputStream(urisPath)) {
253 uris.load(in);
254 } catch (IOException e) {
255 throw new IllegalStateException("Cannot load " + urisPath, e);
256 }
257 }
258
259 // Eclipse mirrors
260 Path eclipseMirrorsPath = this.descriptorsBase.resolve("eclipse.mirrors.txt");
261 List<String> eclipseMirrors = new ArrayList<>();
262 if (Files.exists(eclipseMirrorsPath)) {
263 try {
264 eclipseMirrors = Files.readAllLines(eclipseMirrorsPath, StandardCharsets.UTF_8);
265 } catch (IOException e) {
266 throw new IllegalStateException("Cannot load " + eclipseMirrorsPath, e);
267 }
268 for (Iterator<String> it = eclipseMirrors.iterator(); it.hasNext();) {
269 String value = it.next();
270 if (value.strip().equals(""))
271 it.remove();
272 }
273 }
274 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
275 }
276
277 /*
278 * MAVEN ORIGIN
279 */
280 /** Process a whole category/group id. */
281 void processCategory(Path categoryRelativePath) {
282 try {
283 Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath);
284 DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
285 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
286 && !p.getFileName().toString().equals(MERGE_BND));
287 for (Path p : bnds) {
288 processSingleM2ArtifactDistributionUnit(p);
289 }
290
291 DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
292 for (Path duDir : dus) {
293 if (duDir.getFileName().toString().startsWith("eclipse-")) {
294 processEclipseArchive(duDir);
295 } else {
296 processM2BasedDistributionUnit(duDir);
297 }
298 }
299 } catch (IOException e) {
300 throw new RuntimeException("Cannot process category " + categoryRelativePath, e);
301 }
302 }
303
304 /** Process a standalone Maven artifact. */
305 void processSingleM2ArtifactDistributionUnit(Path bndFile) {
306 try {
307 Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent());
308 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
309
310 Properties fileProps = new Properties();
311 try (InputStream in = Files.newInputStream(bndFile)) {
312 fileProps.load(in);
313 }
314 String repoStr = fileProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
315 ? fileProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
316 : null;
317
318 // use file name as symbolic name
319 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
320 String symbolicName = bndFile.getFileName().toString();
321 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
322 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
323 }
324
325 String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
326 if (m2Coordinates == null)
327 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
328 M2Artifact artifact = new M2Artifact(m2Coordinates);
329 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
330 Path downloaded = downloadMaven(url, artifact);
331
332 // some proprietary artifacts do not allow any modification
333 // when releasing (with separate sources) we just copy it
334 boolean doNotModify = Boolean.parseBoolean(
335 fileProps.getOrDefault(ManifestHeader.ARGEO_DO_NOT_MODIFY.toString(), "false").toString());
336 if (doNotModify && sourceBundles) {
337 Path unmodifiedTarget = targetCategoryBase.resolve(
338 fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch() + ".jar");
339 Files.copy(downloaded, unmodifiedTarget, StandardCopyOption.REPLACE_EXISTING);
340 Path bundleDir = targetCategoryBase
341 .resolve(fileProps.getProperty(BUNDLE_SYMBOLICNAME.toString()) + "." + artifact.getBranch());
342 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
343 Manifest manifest;
344 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(unmodifiedTarget))) {
345 manifest = jarIn.getManifest();
346 }
347 createSourceJar(bundleDir, manifest);
348 return;
349 }
350
351 // regular processing
352 A2Origin origin = new A2Origin();
353 Path bundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact, origin);
354 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, false);
355 createJar(bundleDir, origin);
356 } catch (Exception e) {
357 throw new RuntimeException("Cannot process " + bndFile, e);
358 }
359 }
360
361 /**
362 * Process multiple Maven artifacts coming from a same project and therefore
363 * with information in common (typically the version), generating single bundles
364 * or merging them if necessary.
365 *
366 * @see #COMMON_BND
367 * @see #MERGE_BND
368 */
369 void processM2BasedDistributionUnit(Path duDir) {
370 try {
371 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
372 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
373
374 Path mergeBnd = duDir.resolve(MERGE_BND);
375 if (Files.exists(mergeBnd)) // merge
376 mergeM2Artifacts(mergeBnd);
377
378 Path commonBnd = duDir.resolve(COMMON_BND);
379 if (!Files.exists(commonBnd))
380 return;
381
382 Properties commonProps = new Properties();
383 try (InputStream in = Files.newInputStream(commonBnd)) {
384 commonProps.load(in);
385 }
386
387 String m2Version = commonProps.getProperty(ARGEO_ORIGIN_M2.toString());
388 if (m2Version == null) {
389 logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
390 return;// ignore, this is probably an Eclipse archive
391 }
392 if (!m2Version.startsWith(":")) {
393 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
394 }
395 m2Version = m2Version.substring(1);
396
397 DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
398 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
399 && !p.getFileName().toString().equals(MERGE_BND));
400 for (Path p : ds) {
401 Properties fileProps = new Properties();
402 try (InputStream in = Files.newInputStream(p)) {
403 fileProps.load(in);
404 }
405 String m2Coordinates = fileProps.getProperty(ARGEO_ORIGIN_M2.toString());
406 M2Artifact artifact = new M2Artifact(m2Coordinates);
407 artifact.setVersion(m2Version);
408
409 // prepare manifest entries
410 Properties mergeProps = new Properties();
411 mergeProps.putAll(commonProps);
412
413 fileEntries: for (Object key : fileProps.keySet()) {
414 if (ARGEO_ORIGIN_M2.toString().equals(key))
415 continue fileEntries;
416 String value = fileProps.getProperty(key.toString());
417 Object previousValue = mergeProps.put(key.toString(), value);
418 if (previousValue != null) {
419 logger.log(WARNING,
420 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
421 }
422 }
423 mergeProps.put(ARGEO_ORIGIN_M2.toString(), artifact.toM2Coordinates());
424 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
425 // use file name as symbolic name
426 String symbolicName = p.getFileName().toString();
427 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
428 mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
429 }
430
431 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
432 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
433 : null;
434
435 // download
436 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
437 Path downloaded = downloadMaven(url, artifact);
438
439 A2Origin origin = new A2Origin();
440 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact, origin);
441 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir, false);
442 createJar(targetBundleDir, origin);
443 }
444 } catch (IOException e) {
445 throw new RuntimeException("Cannot process " + duDir, e);
446 }
447 }
448
449 /** Merge multiple Maven artifacts. */
450 void mergeM2Artifacts(Path mergeBnd) throws IOException {
451 Path duDir = mergeBnd.getParent();
452 String category = duDir.getParent().getFileName().toString();
453 Path targetCategoryBase = a2Base.resolve(category);
454
455 Properties mergeProps = new Properties();
456 try (InputStream in = Files.newInputStream(mergeBnd)) {
457 mergeProps.load(in);
458 }
459
460 String m2Version = mergeProps.getProperty(ARGEO_ORIGIN_M2.toString());
461 if (m2Version == null) {
462 logger.log(WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
463 return;// ignore, this is probably an Eclipse archive
464 }
465 if (!m2Version.startsWith(":")) {
466 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
467 }
468 m2Version = m2Version.substring(1);
469 mergeProps.put(BUNDLE_VERSION.toString(), m2Version);
470
471 String artifactsStr = mergeProps.getProperty(ARGEO_ORIGIN_M2_MERGE.toString());
472 if (artifactsStr == null)
473 throw new IllegalArgumentException(mergeBnd + ": " + ARGEO_ORIGIN_M2_MERGE + " must be set");
474
475 String repoStr = mergeProps.containsKey(ARGEO_ORIGIN_M2_REPO.toString())
476 ? mergeProps.getProperty(ARGEO_ORIGIN_M2_REPO.toString())
477 : null;
478
479 String bundleSymbolicName = mergeProps.getProperty(BUNDLE_SYMBOLICNAME.toString());
480 if (bundleSymbolicName == null)
481 throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
482 CategoryNameVersion nameVersion = new M2Artifact(category + ":" + bundleSymbolicName + ":" + m2Version);
483
484 A2Origin origin = new A2Origin();
485 Path bundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
486
487 StringJoiner originDesc = new StringJoiner(",");
488 String[] artifacts = artifactsStr.split(",");
489 artifacts: for (String str : artifacts) {
490 String m2Coordinates = str.trim();
491 if ("".equals(m2Coordinates))
492 continue artifacts;
493 M2Artifact artifact = new M2Artifact(m2Coordinates.trim());
494 if (artifact.getVersion() == null)
495 artifact.setVersion(m2Version);
496 originDesc.add(artifact.toString());
497 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
498 Path downloaded = downloadMaven(url, artifact);
499 JarEntry entry;
500 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
501 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
502 if (entry.isDirectory())
503 continue entries;
504 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
505 || entry.getName().endsWith(".SF")) {
506 origin.deleted.add("cryptographic signatures from " + artifact);
507 continue entries;
508 }
509 if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
510 origin.deleted.add("Java module information (module-info.class) from " + artifact);
511 continue entries;
512 }
513 if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
514 origin.deleted.add("additional Java versions (META-INF/versions) from " + artifact);
515 continue entries;
516 }
517 if (entry.getName().startsWith("META-INF/maven/")) {
518 origin.deleted.add("Maven information (META-INF/maven) from " + artifact);
519 continue entries;
520 }
521 if (entry.getName().startsWith(".cache/")) { // Apache SSHD
522 origin.deleted.add("cache directory (.cache) from " + artifact);
523 continue entries;
524 }
525 if (entry.getName().equals("META-INF/DEPENDENCIES")) {
526 origin.deleted.add("Dependencies (META-INF/DEPENDENCIES) from " + artifact);
527 continue entries;
528 }
529 if (entry.getName().equals("META-INF/MANIFEST.MF")) {
530 Path originalManifest = bundleDir.resolve(A2_ORIGIN).resolve(artifact.getGroupId())
531 .resolve(artifact.getArtifactId()).resolve("MANIFEST.MF");
532 Files.createDirectories(originalManifest.getParent());
533 try (OutputStream out = Files.newOutputStream(originalManifest)) {
534 Files.copy(jarIn, originalManifest);
535 }
536 origin.added.add(
537 "original MANIFEST (" + bundleDir.relativize(originalManifest) + ") from " + artifact);
538 continue entries;
539 }
540
541 if (entry.getName().endsWith("NOTICE") || entry.getName().endsWith("NOTICE.txt")
542 || entry.getName().endsWith("LICENSE") || entry.getName().endsWith("LICENSE.md")
543 || entry.getName().endsWith("LICENSE-notice.md") || entry.getName().endsWith("COPYING")
544 || entry.getName().endsWith("COPYING.LESSER")) {
545 Path artifactOriginDir = bundleDir.resolve(A2_ORIGIN).resolve(artifact.getGroupId())
546 .resolve(artifact.getArtifactId());
547 Path target = artifactOriginDir.resolve(entry.getName());
548 Files.createDirectories(target.getParent());
549 Files.copy(jarIn, target);
550 origin.moved.add(entry.getName() + " in " + artifact + " to " + bundleDir.relativize(target));
551 continue entries;
552 }
553 Path target = bundleDir.resolve(entry.getName());
554 Files.createDirectories(target.getParent());
555 if (!Files.exists(target)) {
556 Files.copy(jarIn, target);
557 } else {
558 if (entry.getName().startsWith("META-INF/services/")) {
559 try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
560 out.write("\n".getBytes());
561 jarIn.transferTo(out);
562 logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName());
563 }
564 origin.modified.add(entry.getName() + ", merging from " + artifact);
565 } else if (entry.getName().startsWith("org/apache/batik/")) {
566 logger.log(TRACE, "Skip " + entry.getName());
567 continue entries;
568 } else {
569 throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
570 }
571 }
572 logger.log(TRACE, () -> "Copied " + target);
573 }
574 }
575 origin.added.add("binary content of " + artifact);
576
577 // process sources
578 downloadAndProcessM2Sources(repoStr, artifact, bundleDir, true);
579 }
580
581 // additional service files
582 Path servicesDir = duDir.resolve("services");
583 if (Files.exists(servicesDir)) {
584 for (Path p : Files.newDirectoryStream(servicesDir)) {
585 Path target = bundleDir.resolve("META-INF/services/").resolve(p.getFileName());
586 try (InputStream in = Files.newInputStream(p);
587 OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) {
588 out.write("\n".getBytes());
589 in.transferTo(out);
590 logger.log(DEBUG, "Appended " + p);
591 }
592 origin.added.add(bundleDir.relativize(target).toString());
593 }
594 }
595
596 // BND analysis
597 Map<String, String> entries = new TreeMap<>();
598 try (Analyzer bndAnalyzer = new Analyzer()) {
599 bndAnalyzer.setProperties(mergeProps);
600 Jar jar = new Jar(bundleDir.toFile());
601 bndAnalyzer.setJar(jar);
602 Manifest manifest = bndAnalyzer.calcManifest();
603
604 keys: for (Object key : manifest.getMainAttributes().keySet()) {
605 Object value = manifest.getMainAttributes().get(key);
606
607 switch (key.toString()) {
608 case "Tool":
609 case "Bnd-LastModified":
610 case "Created-By":
611 continue keys;
612 }
613 if ("Require-Capability".equals(key.toString())
614 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
615 continue keys;// hack for very old classes
616 entries.put(key.toString(), value.toString());
617 }
618 } catch (Exception e) {
619 throw new RuntimeException("Cannot process " + mergeBnd, e);
620 }
621
622 Manifest manifest = new Manifest();
623 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
624 Files.createDirectories(manifestPath.getParent());
625 for (String key : entries.keySet()) {
626 String value = entries.get(key);
627 manifest.getMainAttributes().putValue(key, value);
628 }
629 manifest.getMainAttributes().putValue(ARGEO_ORIGIN_M2.toString(), originDesc.toString());
630
631 processLicense(bundleDir, manifest);
632
633 // write MANIFEST
634 try (OutputStream out = Files.newOutputStream(manifestPath)) {
635 manifest.write(out);
636 }
637 createJar(bundleDir, origin);
638 }
639
640 /** Generates MANIFEST using BND. */
641 Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, M2Artifact artifact,
642 A2Origin origin) {
643 try {
644 Map<String, String> additionalEntries = new TreeMap<>();
645 boolean doNotModifyManifest = Boolean.parseBoolean(
646 fileProps.getOrDefault(ARGEO_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString());
647
648 // Note: we always force the symbolic name
649 if (doNotModifyManifest) {
650 fileEntries: for (Object key : fileProps.keySet()) {
651 if (ARGEO_ORIGIN_M2.toString().equals(key))
652 continue fileEntries;
653 String value = fileProps.getProperty(key.toString());
654 additionalEntries.put(key.toString(), value);
655 }
656 } else {
657 if (artifact != null) {
658 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
659 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
660 }
661 if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
662 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
663 }
664 }
665
666 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
667 fileProps.put(EXPORT_PACKAGE.toString(),
668 "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
669 }
670
671 // BND analysis
672 try (Analyzer bndAnalyzer = new Analyzer()) {
673 bndAnalyzer.setProperties(fileProps);
674 Jar jar = new Jar(downloaded.toFile());
675 bndAnalyzer.setJar(jar);
676 Manifest manifest = bndAnalyzer.calcManifest();
677
678 keys: for (Object key : manifest.getMainAttributes().keySet()) {
679 Object value = manifest.getMainAttributes().get(key);
680
681 switch (key.toString()) {
682 case "Tool":
683 case "Bnd-LastModified":
684 case "Created-By":
685 continue keys;
686 }
687 if ("Require-Capability".equals(key.toString())
688 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\"")) {
689 origin.deleted.add("MANIFEST header " + key);
690 continue keys;// !! hack for very old classes
691 }
692 additionalEntries.put(key.toString(), value.toString());
693 }
694 }
695 }
696 Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries, origin);
697 logger.log(DEBUG, () -> "Processed " + downloaded);
698 return targetBundleDir;
699 } catch (Exception e) {
700 throw new RuntimeException("Cannot BND process " + downloaded, e);
701 }
702
703 }
704
705 /** Download and integrates sources for a single Maven artifact. */
706 void downloadAndProcessM2Sources(String repoStr, M2Artifact artifact, Path targetBundleDir, boolean merging)
707 throws IOException {
708 try {
709 M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources");
710 URL sourcesUrl = M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
711 Path sourcesDownloaded = downloadMaven(sourcesUrl, artifact, true);
712 processM2SourceJar(sourcesDownloaded, targetBundleDir, merging ? artifact : null);
713 logger.log(TRACE, () -> "Processed source " + sourcesDownloaded);
714 } catch (Exception e) {
715 logger.log(ERROR, () -> "Cannot download source for " + artifact);
716 }
717
718 }
719
720 /** Integrate sources from a downloaded jar file. */
721 void processM2SourceJar(Path file, Path bundleDir, M2Artifact mergingFrom) throws IOException {
722 A2Origin origin = new A2Origin();
723 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
724 : bundleDir.resolve("OSGI-OPT/src");
725 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
726
727 String mergingMsg = "";
728 if (mergingFrom != null)
729 mergingMsg = " of " + mergingFrom;
730
731 Files.createDirectories(sourceDir);
732 JarEntry entry;
733 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
734 if (entry.isDirectory())
735 continue entries;
736 if (entry.getName().startsWith("META-INF")) {// skip META-INF entries
737 origin.deleted.add("META-INF directory from the sources" + mergingMsg);
738 continue entries;
739 }
740 if (entry.getName().startsWith("module-info.java")) {// skip Java module information
741 origin.deleted.add("Java module information from the sources (module-info.java)" + mergingMsg);
742 continue entries;
743 }
744 if (entry.getName().startsWith("/")) { // absolute paths
745 // TODO does it really happen?
746 logger.log(WARNING, entry.getName() + " has an absolute path");
747 origin.deleted.add(entry.getName() + " from the sources" + mergingMsg);
748 continue entries;
749 }
750 Path target = sourceDir.resolve(entry.getName());
751 Files.createDirectories(target.getParent());
752 if (!Files.exists(target)) {
753 Files.copy(jarIn, target);
754 logger.log(TRACE, () -> "Copied source " + target);
755 } else {
756 logger.log(TRACE, () -> target + " already exists, skipping...");
757 }
758 }
759 }
760 // write the changes
761 if (sourceBundles) {
762 origin.appendChanges(sourceDir);
763 } else {
764 origin.added.add("source code under OSGI-OPT/src");
765 origin.appendChanges(bundleDir);
766 }
767 }
768
769 /** Download a Maven artifact. */
770 Path downloadMaven(URL url, M2Artifact artifact) throws IOException {
771 return downloadMaven(url, artifact, false);
772 }
773
774 /** Download a Maven artifact. */
775 Path downloadMaven(URL url, M2Artifact artifact, boolean sources) throws IOException {
776 return download(url, mavenBase, artifact.getGroupId().replace(".", "/") //
777 + '/' + artifact.getArtifactId() + '/' + artifact.getVersion() //
778 + '/' + artifact.getArtifactId() + "-" + artifact.getVersion() + (sources ? "-sources" : "") + ".jar");
779 }
780
781 /*
782 * ECLIPSE ORIGIN
783 */
784 /** Process an archive in Eclipse format. */
785 void processEclipseArchive(Path duDir) {
786 try {
787 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
788 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
789 Files.createDirectories(targetCategoryBase);
790 // first delete all directories from previous builds
791 for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)))
792 deleteDirectory(dir);
793
794 Files.createDirectories(originBase);
795
796 Path commonBnd = duDir.resolve(COMMON_BND);
797 Properties commonProps = new Properties();
798 try (InputStream in = Files.newInputStream(commonBnd)) {
799 commonProps.load(in);
800 }
801 String url = commonProps.getProperty(ARGEO_ORIGIN_URI.toString());
802 if (url == null) {
803 url = uris.getProperty(duDir.getFileName().toString());
804 if (url == null)
805 throw new IllegalStateException("No url available for " + duDir);
806 commonProps.put(ARGEO_ORIGIN_URI.toString(), url);
807 }
808 Path downloaded = tryDownloadArchive(url, originBase);
809
810 FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
811
812 // filters
813 List<PathMatcher> includeMatchers = new ArrayList<>();
814 Properties includes = new Properties();
815 try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
816 includes.load(in);
817 }
818 for (Object pattern : includes.keySet()) {
819 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
820 includeMatchers.add(pathMatcher);
821 }
822
823 List<PathMatcher> excludeMatchers = new ArrayList<>();
824 Path excludeFile = duDir.resolve("excludes.properties");
825 if (Files.exists(excludeFile)) {
826 Properties excludes = new Properties();
827 try (InputStream in = Files.newInputStream(excludeFile)) {
828 excludes.load(in);
829 }
830 for (Object pattern : excludes.keySet()) {
831 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
832 excludeMatchers.add(pathMatcher);
833 }
834 }
835
836 // keys are the bundle directories
837 Map<Path, A2Origin> origins = new HashMap<>();
838 Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
839
840 @Override
841 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
842 includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
843 if (includeMatcher.matches(file)) {
844 for (PathMatcher excludeMatcher : excludeMatchers) {
845 if (excludeMatcher.matches(file)) {
846 logger.log(TRACE, "Skipping excluded " + file);
847 return FileVisitResult.CONTINUE;
848 }
849 }
850 if (file.getFileName().toString().contains(".source_")) {
851 processEclipseSourceJar(file, targetCategoryBase);
852 logger.log(DEBUG, () -> "Processed source " + file);
853 } else {
854 Map<String, String> map = new HashMap<>();
855 for (Object key : commonProps.keySet())
856 map.put(key.toString(), commonProps.getProperty(key.toString()));
857 A2Origin origin = new A2Origin();
858 Path bundleDir = processBundleJar(file, targetCategoryBase, map, origin);
859 origins.put(bundleDir, origin);
860 logger.log(DEBUG, () -> "Processed " + file);
861 }
862 break includeMatchers;
863 }
864 }
865 return FileVisitResult.CONTINUE;
866 }
867 });
868
869 DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p)
870 && p.getFileName().toString().indexOf('.') >= 0 && !p.getFileName().toString().endsWith(".src"));
871 for (Path bundleDir : dirs) {
872 A2Origin origin = origins.get(bundleDir);
873 Objects.requireNonNull(origin, "No A2 origin found for " + bundleDir);
874 createJar(bundleDir, origin);
875 }
876 } catch (IOException e) {
877 throw new RuntimeException("Cannot process " + duDir, e);
878 }
879
880 }
881
882 /** Process sources in Eclipse format. */
883 void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
884 try {
885 A2Origin origin = new A2Origin();
886 Path bundleDir;
887 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
888 Manifest manifest = jarIn.getManifest();
889
890 String[] relatedBundle = manifest.getMainAttributes().getValue(ECLIPSE_SOURCE_BUNDLE.toString())
891 .split(";");
892 String version = relatedBundle[1].substring("version=\"".length());
893 version = version.substring(0, version.length() - 1);
894 NameVersion nameVersion = new NameVersion(relatedBundle[0], version);
895 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
896
897 Path sourceDir = sourceBundles ? bundleDir.getParent().resolve(bundleDir.toString() + ".src")
898 : bundleDir.resolve("OSGI-OPT/src");
899
900 Files.createDirectories(sourceDir);
901 JarEntry entry;
902 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
903 if (entry.isDirectory())
904 continue entries;
905 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
906 continue entries;
907 Path target = sourceDir.resolve(entry.getName());
908 Files.createDirectories(target.getParent());
909 Files.copy(jarIn, target);
910 logger.log(TRACE, () -> "Copied source " + target);
911 }
912
913 // write the changes
914 if (sourceBundles) {
915 origin.appendChanges(sourceDir);
916 } else {
917 origin.added.add("source code under OSGI-OPT/src");
918 origin.appendChanges(bundleDir);
919 }
920 }
921 } catch (IOException e) {
922 throw new IllegalStateException("Cannot process " + file, e);
923 }
924 }
925
926 /*
927 * COMMON PROCESSING
928 */
929 /** Normalise a single (that is, non-merged) bundle. */
930 Path processBundleJar(Path file, Path targetBase, Map<String, String> entries, A2Origin origin) throws IOException {
931 boolean embed = Boolean.parseBoolean(entries.getOrDefault(ARGEO_ORIGIN_EMBED.toString(), "false").toString());
932 NameVersion nameVersion;
933 Path bundleDir;
934 // singleton
935 boolean isSingleton = false;
936 Manifest manifest;
937 Manifest sourceManifest;
938 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
939 sourceManifest = jarIn.getManifest();
940 manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
941
942 String rawSourceSymbolicName = manifest.getMainAttributes().getValue(BUNDLE_SYMBOLICNAME.toString());
943 if (rawSourceSymbolicName != null) {
944 // make sure there is no directive
945 String[] arr = rawSourceSymbolicName.split(";");
946 for (int i = 1; i < arr.length; i++) {
947 if (arr[i].trim().equals("singleton:=true"))
948 isSingleton = true;
949 logger.log(DEBUG, file.getFileName() + " is a singleton");
950 }
951 }
952 // remove problematic entries in MANIFEST
953 manifest.getEntries().clear();
954
955 String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString());
956 String ourVersion = entries.get(BUNDLE_VERSION.toString());
957
958 if (ourSymbolicName != null && ourVersion != null) {
959 nameVersion = new NameVersion(ourSymbolicName, ourVersion);
960 } else {
961 nameVersion = nameVersionFromManifest(manifest);
962 if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) {
963 logger.log(WARNING,
964 "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion);
965 entries.put(BUNDLE_VERSION.toString(), ourVersion);
966 }
967 if (ourSymbolicName != null) {
968 // we always force our symbolic name
969 nameVersion.setName(ourSymbolicName);
970 }
971 }
972 bundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
973
974 if (sourceManifest != null) {// copy original MANIFEST
975 Path originalManifest = bundleDir.resolve(A2_ORIGIN).resolve("MANIFEST.MF");
976 Files.createDirectories(originalManifest.getParent());
977 try (OutputStream out = Files.newOutputStream(originalManifest)) {
978 sourceManifest.write(out);
979 }
980 origin.moved.add("original MANIFEST (" + bundleDir.relativize(originalManifest) + ")");
981 }
982
983 // force Java 9 module name
984 entries.put(ManifestHeader.AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName());
985
986 boolean isNative = false;
987 String os = null;
988 String arch = null;
989 if (bundleDir.startsWith(a2LibBase)) {
990 isNative = true;
991 Path libRelativePath = a2LibBase.relativize(bundleDir);
992 os = libRelativePath.getName(0).toString();
993 arch = libRelativePath.getName(1).toString();
994 }
995
996 if (!embed) {
997 // copy entries
998 JarEntry entry;
999 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
1000 if (entry.isDirectory())
1001 continue entries;
1002 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".DSA")
1003 || entry.getName().endsWith(".SF")) {
1004 origin.deleted.add("cryptographic signatures");
1005 continue entries;
1006 }
1007 if (entry.getName().endsWith("module-info.class")) { // skip Java 9 module info
1008 origin.deleted.add("Java module information (module-info.class)");
1009 continue entries;
1010 }
1011 if (entry.getName().startsWith("META-INF/versions/")) { // skip multi-version
1012 origin.deleted.add("additional Java versions (META-INF/versions)");
1013 continue entries;
1014 }
1015 if (entry.getName().startsWith("META-INF/maven/")) {
1016 origin.deleted.add("Maven information (META-INF/maven)");
1017 continue entries;
1018 }
1019 // skip file system providers as they cause issues with native image
1020 if (entry.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider")) {
1021 origin.deleted
1022 .add("file system providers (META-INF/services/java.nio.file.spi.FileSystemProvider)");
1023 continue entries;
1024 }
1025 if (entry.getName().startsWith("OSGI-OPT/src/")) { // skip embedded sources
1026 origin.deleted.add("embedded sources");
1027 continue entries;
1028 }
1029 Path target = bundleDir.resolve(entry.getName());
1030 Files.createDirectories(target.getParent());
1031 Files.copy(jarIn, target);
1032
1033 // native libraries
1034 if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll")
1035 || entry.getName().endsWith(".jnilib"))) {
1036 Path categoryDir = bundleDir.getParent();
1037 boolean copyDll = false;
1038 Path targetDll = categoryDir.resolve(bundleDir.relativize(target));
1039 if (nameVersion.getName().equals("com.sun.jna")) {
1040 if (arch.equals("x86_64"))
1041 arch = "x86-64";
1042 if (os.equals("macosx"))
1043 os = "darwin";
1044 if (target.getParent().getFileName().toString().equals(os + "-" + arch)) {
1045 copyDll = true;
1046 }
1047 targetDll = categoryDir.resolve(target.getFileName());
1048 } else {
1049 copyDll = true;
1050 }
1051 if (copyDll) {
1052 Files.createDirectories(targetDll.getParent());
1053 if (Files.exists(targetDll))
1054 Files.delete(targetDll);
1055 Files.copy(target, targetDll);
1056 }
1057 Files.delete(target);
1058 origin.deleted.add(bundleDir.relativize(target).toString());
1059 }
1060 logger.log(TRACE, () -> "Copied " + target);
1061 }
1062 }
1063 }
1064
1065 // copy MANIFEST
1066 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1067 Files.createDirectories(manifestPath.getParent());
1068
1069 if (isSingleton && entries.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
1070 entries.put(BUNDLE_SYMBOLICNAME.toString(),
1071 entries.get(BUNDLE_SYMBOLICNAME.toString()) + ";singleton:=true");
1072 }
1073
1074 if (embed) {// copy embedded jar
1075 Files.copy(file, bundleDir.resolve(file.getFileName()));
1076 entries.put(ManifestHeader.BUNDLE_CLASSPATH.toString(), file.getFileName().toString());
1077 }
1078
1079 // Final MANIFEST decisions
1080 // We also check the original OSGi metadata and compare with our changes
1081 for (String key : entries.keySet()) {
1082 String value = entries.get(key);
1083 String previousValue = manifest.getMainAttributes().getValue(key);
1084 boolean wasDifferent = previousValue != null && !previousValue.equals(value);
1085 boolean keepPrevious = false;
1086 if (wasDifferent) {
1087 if (SPDX_LICENSE_IDENTIFIER.toString().equals(key) && previousValue != null)
1088 keepPrevious = true;
1089 else if (BUNDLE_VERSION.toString().equals(key) && wasDifferent)
1090 if (previousValue.equals(value + ".0")) // typically a Maven first release
1091 keepPrevious = true;
1092
1093 if (keepPrevious) {
1094 if (logger.isLoggable(DEBUG))
1095 logger.log(DEBUG, file.getFileName() + ": " + key + " was NOT modified, value kept is "
1096 + previousValue + ", not overriden with " + value);
1097 value = previousValue;
1098 }
1099 }
1100
1101 manifest.getMainAttributes().putValue(key, value);
1102 if (wasDifferent && !keepPrevious) {
1103 if (IMPORT_PACKAGE.toString().equals(key) || EXPORT_PACKAGE.toString().equals(key))
1104 logger.log(TRACE, () -> file.getFileName() + ": " + key + " was modified");
1105 else
1106 logger.log(WARNING,
1107 file.getFileName() + ": " + key + " was " + previousValue + ", overridden with " + value);
1108 origin.modified.add("MANIFEST header " + key);
1109 }
1110
1111 // !! hack to remove unresolvable
1112 if (key.equals("Provide-Capability") || key.equals("Require-Capability"))
1113 if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) {
1114 manifest.getMainAttributes().remove(key);
1115 origin.deleted.add("MANIFEST header " + key);
1116 }
1117 }
1118
1119 // de-pollute MANIFEST
1120 for (Iterator<Map.Entry<Object, Object>> manifestEntries = manifest.getMainAttributes().entrySet()
1121 .iterator(); manifestEntries.hasNext();) {
1122 Map.Entry<Object, Object> manifestEntry = manifestEntries.next();
1123 switch (manifestEntry.getKey().toString()) {
1124 case "Archiver-Version":
1125 case "Build-By":
1126 case "Created-By":
1127 case "Originally-Created-By":
1128 case "Tool":
1129 case "Bnd-LastModified":
1130 manifestEntries.remove();
1131 origin.deleted.add("MANIFEST header " + manifestEntry.getKey());
1132 break;
1133 default:
1134 if (sourceManifest != null && !sourceManifest.getMainAttributes().containsKey(manifestEntry.getKey()))
1135 origin.added.add("MANIFEST header " + manifestEntry.getKey());
1136 }
1137 }
1138
1139 processLicense(bundleDir, manifest);
1140
1141 origin.modified.add("jar MANIFEST (META-INF/MANIFEST.MF)");
1142 // write the MANIFEST
1143 try (OutputStream out = Files.newOutputStream(manifestPath)) {
1144 manifest.write(out);
1145 }
1146 return bundleDir;
1147 }
1148
1149 /** Process SPDX license identifier. */
1150 void processLicense(Path bundleDir, Manifest manifest) {
1151 String spdxLicenceId = manifest.getMainAttributes().getValue(SPDX_LICENSE_IDENTIFIER.toString());
1152 String bundleLicense = manifest.getMainAttributes().getValue(BUNDLE_LICENSE.toString());
1153 if (spdxLicenceId == null) {
1154 logger.log(ERROR, bundleDir.getFileName() + ": " + SPDX_LICENSE_IDENTIFIER + " not available, "
1155 + BUNDLE_LICENSE + " is " + bundleLicense);
1156 } else {
1157 // only use the first licensing option
1158 int orIndex = spdxLicenceId.indexOf(" OR ");
1159 if (orIndex >= 0)
1160 spdxLicenceId = spdxLicenceId.substring(0, orIndex).trim();
1161
1162 // bundles starting with org.apache MUST be licensed with Apache-2.0
1163 // even if we say otherwise (typically because coming from an Eclipse archive)
1164 if (bundleDir.getFileName().startsWith("org.apache."))
1165 spdxLicenceId = "Apache-2.0";
1166
1167 manifest.getMainAttributes().putValue(SPDX_LICENSE_IDENTIFIER.toString(), spdxLicenceId);
1168 if (!licensesUsed.containsKey(spdxLicenceId))
1169 licensesUsed.put(spdxLicenceId, new TreeSet<>());
1170 licensesUsed.get(spdxLicenceId).add(bundleDir.getParent().getFileName() + "/" + bundleDir.getFileName());
1171 }
1172 }
1173
1174 /*
1175 * UTILITIES
1176 */
1177 /** Recursively deletes a directory. */
1178 static void deleteDirectory(Path path) throws IOException {
1179 if (!Files.exists(path))
1180 return;
1181 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
1182 @Override
1183 public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
1184 if (e != null)
1185 throw e;
1186 Files.delete(directory);
1187 return CONTINUE;
1188 }
1189
1190 @Override
1191 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1192 Files.delete(file);
1193 return CONTINUE;
1194 }
1195 });
1196 }
1197
1198 /** Extract name/version from a MANIFEST. */
1199 NameVersion nameVersionFromManifest(Manifest manifest) {
1200 Attributes attrs = manifest.getMainAttributes();
1201 // symbolic name
1202 String symbolicName = attrs.getValue(ManifestHeader.BUNDLE_SYMBOLICNAME.toString());
1203 if (symbolicName == null)
1204 return null;
1205 // make sure there is no directive
1206 symbolicName = symbolicName.split(";")[0];
1207
1208 String version = attrs.getValue(ManifestHeader.BUNDLE_VERSION.toString());
1209 return new NameVersion(symbolicName, version);
1210 }
1211
1212 /** Try to download from an URI. */
1213 Path tryDownloadArchive(String uri, Path dir) throws IOException {
1214 // find mirror
1215 List<String> urlBases = null;
1216 String uriPrefix = null;
1217 uriPrefixes: for (String uriPref : mirrors.keySet()) {
1218 if (uri.startsWith(uriPref)) {
1219 if (mirrors.get(uriPref).size() > 0) {
1220 urlBases = mirrors.get(uriPref);
1221 uriPrefix = uriPref;
1222 break uriPrefixes;
1223 }
1224 }
1225 }
1226 if (urlBases == null)
1227 try {
1228 return downloadArchive(new URL(uri), dir);
1229 } catch (FileNotFoundException e) {
1230 throw new FileNotFoundException("Cannot find " + uri);
1231 }
1232
1233 // try to download
1234 for (String urlBase : urlBases) {
1235 String relativePath = uri.substring(uriPrefix.length());
1236 URL url = new URL(urlBase + relativePath);
1237 try {
1238 return downloadArchive(url, dir);
1239 } catch (FileNotFoundException e) {
1240 logger.log(WARNING, "Cannot download " + url + ", trying another mirror");
1241 }
1242 }
1243 throw new FileNotFoundException("Cannot find " + uri);
1244 }
1245
1246 /**
1247 * Effectively download. Synchronised in order to avoid downloading twice in
1248 * parallel.
1249 */
1250 synchronized Path downloadArchive(URL url, Path dir) throws IOException {
1251 return download(url, dir, (String) null);
1252 }
1253
1254 /** Effectively download. */
1255 Path download(URL url, Path dir, String name) throws IOException {
1256
1257 Path dest;
1258 if (name == null) {
1259 // We use also use parent directory in case the archive itself has a fixed name
1260 String[] segments = url.getPath().split("/");
1261 name = segments.length > 1 ? segments[segments.length - 2] + '-' + segments[segments.length - 1]
1262 : segments[segments.length - 1];
1263 }
1264
1265 dest = dir.resolve(name);
1266 if (Files.exists(dest)) {
1267 logger.log(TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
1268 return dest;
1269 } else {
1270 Files.createDirectories(dest.getParent());
1271 }
1272
1273 try (InputStream in = url.openStream()) {
1274 Files.copy(in, dest);
1275 logger.log(DEBUG, () -> "Downloaded " + dest + " from " + url);
1276 }
1277 return dest;
1278 }
1279
1280 /** Create a JAR file from a directory. */
1281 Path createJar(Path bundleDir, A2Origin origin) throws IOException {
1282 // write changes
1283 origin.appendChanges(bundleDir);
1284
1285 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
1286 Manifest manifest;
1287 try (InputStream in = Files.newInputStream(manifestPath)) {
1288 manifest = new Manifest(in);
1289 }
1290 createReadMe(bundleDir, manifest);
1291 // create the jar
1292 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
1293 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
1294 jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
1295 Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
1296
1297 @Override
1298 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1299 if (file.getFileName().toString().equals("MANIFEST.MF"))
1300 return super.visitFile(file, attrs);
1301 JarEntry entry = new JarEntry(
1302 bundleDir.relativize(file).toString().replace(File.separatorChar, '/'));
1303 jarOut.putNextEntry(entry);
1304 Files.copy(file, jarOut);
1305 return super.visitFile(file, attrs);
1306 }
1307
1308 });
1309 }
1310 deleteDirectory(bundleDir);
1311
1312 if (sourceBundles)
1313 createSourceJar(bundleDir, manifest);
1314
1315 return jarPath;
1316 }
1317
1318 /** Package sources separately, in the Eclipse-SourceBundle format. */
1319 void createSourceJar(Path bundleDir, Manifest manifest) throws IOException {
1320 Path bundleCategoryDir = bundleDir.getParent();
1321 Path sourceDir = bundleCategoryDir.resolve(bundleDir.toString() + ".src");
1322 if (!Files.exists(sourceDir)) {
1323 logger.log(WARNING, sourceDir + " does not exist, skipping...");
1324 return;
1325 }
1326 createReadMe(sourceDir, manifest);
1327
1328 Path relPath = a2Base.relativize(bundleCategoryDir);
1329 Path srcCategoryDir = a2SrcBase.resolve(relPath);
1330 Path srcJarP = srcCategoryDir.resolve(sourceDir.getFileName() + ".jar");
1331 Files.createDirectories(srcJarP.getParent());
1332
1333 String bundleSymbolicName = manifest.getMainAttributes().getValue("Bundle-SymbolicName").toString();
1334 // in case there are additional directives
1335 bundleSymbolicName = bundleSymbolicName.split(";")[0];
1336 Manifest srcManifest = new Manifest();
1337 srcManifest.getMainAttributes().put(MANIFEST_VERSION, "1.0");
1338 srcManifest.getMainAttributes().putValue(BUNDLE_SYMBOLICNAME.toString(), bundleSymbolicName + ".src");
1339 srcManifest.getMainAttributes().putValue(BUNDLE_VERSION.toString(),
1340 manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()).toString());
1341 srcManifest.getMainAttributes().putValue(ECLIPSE_SOURCE_BUNDLE.toString(), bundleSymbolicName + ";version=\""
1342 + manifest.getMainAttributes().getValue(BUNDLE_VERSION.toString()) + "\"");
1343
1344 try (JarOutputStream srcJarOut = new JarOutputStream(Files.newOutputStream(srcJarP), srcManifest)) {
1345 srcJarOut.setLevel(Deflater.BEST_COMPRESSION);
1346 Files.walkFileTree(sourceDir, new SimpleFileVisitor<Path>() {
1347
1348 @Override
1349 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
1350 if (file.getFileName().toString().equals("MANIFEST.MF"))
1351 return super.visitFile(file, attrs);
1352 JarEntry entry = new JarEntry(
1353 sourceDir.relativize(file).toString().replace(File.separatorChar, '/'));
1354 srcJarOut.putNextEntry(entry);
1355 Files.copy(file, srcJarOut);
1356 return super.visitFile(file, attrs);
1357 }
1358
1359 });
1360 }
1361 deleteDirectory(sourceDir);
1362 }
1363
1364 void createReadMe(Path jarDir, Manifest manifest) throws IOException {
1365 // write repackaged README
1366 try (BufferedWriter writer = Files.newBufferedWriter(jarDir.resolve(README_REPACKAGED))) {
1367 boolean merged = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2_MERGE.toString()) != null;
1368 if (merged)
1369 writer.append("This component is a merging of third party components"
1370 + " in order to comply with A2 packaging standards.\n");
1371 else
1372 writer.append("This component is a repackaging of a third party component"
1373 + " in order to comply with A2 packaging standards.\n");
1374
1375 String m2Repo = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2_REPO.toString());
1376 String originDesc = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_M2.toString());
1377 if (originDesc != null)
1378 writer.append("The original component has Maven coordinates " + originDesc
1379 + (m2Repo != null ? " in M2 repository" + m2Repo : "") + ".\n");
1380 else
1381 originDesc = manifest.getMainAttributes().getValue(ARGEO_ORIGIN_URI.toString());
1382 if (originDesc != null)
1383 writer.append("The original component comes from " + originDesc + ".\n");
1384 else
1385 logger.log(ERROR, "Cannot find origin information in " + jarDir);
1386
1387 writer.append("A detailed list of changes is available under " + CHANGES + "\n");
1388 if (!jarDir.getFileName().endsWith(".src")) {// binary archive
1389 if (sourceBundles)
1390 writer.append("Corresponding sources are available in the related archive named "
1391 + jarDir.toString() + ".src.jar.\n");
1392 else
1393 writer.append("Corresponding sources are available under OSGI-OPT/src.\n");
1394 }
1395 }
1396
1397 }
1398
1399 /**
1400 * Gathers modifications performed on the original binaries and sources,
1401 * especially in order to comply with their license requirements.
1402 */
1403 class A2Origin {
1404 A2Origin() {
1405
1406 }
1407
1408 Set<String> modified = new TreeSet<>();
1409 Set<String> deleted = new TreeSet<>();
1410 Set<String> added = new TreeSet<>();
1411 Set<String> moved = new TreeSet<>();
1412
1413 /** Append changes to the A2-ORIGIN/changes file. */
1414 void appendChanges(Path baseDirectory) throws IOException {
1415 Path changesFile = baseDirectory.resolve(CHANGES);
1416 Files.createDirectories(changesFile.getParent());
1417 try (BufferedWriter writer = Files.newBufferedWriter(changesFile, APPEND, CREATE)) {
1418 for (String msg : added)
1419 writer.write("- Added " + msg + ".\n");
1420 for (String msg : modified)
1421 writer.write("- Modified " + msg + ".\n");
1422 for (String msg : moved)
1423 writer.write("- Moved " + msg + ".\n");
1424 for (String msg : deleted)
1425 writer.write("- Deleted " + msg + ".\n");
1426 }
1427 }
1428 }
1429 }
1430
1431 /** Simple representation of an M2 artifact. */
1432 class M2Artifact extends CategoryNameVersion {
1433 private String classifier;
1434
1435 M2Artifact(String m2coordinates) {
1436 this(m2coordinates, null);
1437 }
1438
1439 M2Artifact(String m2coordinates, String classifier) {
1440 String[] parts = m2coordinates.split(":");
1441 setCategory(parts[0]);
1442 setName(parts[1]);
1443 if (parts.length > 2) {
1444 setVersion(parts[2]);
1445 }
1446 this.classifier = classifier;
1447 }
1448
1449 String getGroupId() {
1450 return super.getCategory();
1451 }
1452
1453 String getArtifactId() {
1454 return super.getName();
1455 }
1456
1457 String toM2Coordinates() {
1458 return getCategory() + ":" + getName() + (getVersion() != null ? ":" + getVersion() : "");
1459 }
1460
1461 String getClassifier() {
1462 return classifier != null ? classifier : "";
1463 }
1464
1465 String getExtension() {
1466 return "jar";
1467 }
1468 }
1469
1470 /** Utilities around Maven (conventions based). */
1471 class M2ConventionsUtils {
1472 final static String MAVEN_CENTRAL_BASE_URL = "https://repo1.maven.org/maven2/";
1473
1474 /** The file name of this artifact when stored */
1475 static String artifactFileName(M2Artifact artifact) {
1476 return artifact.getArtifactId() + '-' + artifact.getVersion()
1477 + (artifact.getClassifier().equals("") ? "" : '-' + artifact.getClassifier()) + '.'
1478 + artifact.getExtension();
1479 }
1480
1481 /** Absolute path to the file */
1482 static String artifactPath(String artifactBasePath, M2Artifact artifact) {
1483 return artifactParentPath(artifactBasePath, artifact) + '/' + artifactFileName(artifact);
1484 }
1485
1486 /** Absolute path to the file */
1487 static String artifactUrl(String repoUrl, M2Artifact artifact) {
1488 if (repoUrl.endsWith("/"))
1489 return repoUrl + artifactPath("/", artifact).substring(1);
1490 else
1491 return repoUrl + artifactPath("/", artifact);
1492 }
1493
1494 /** Absolute path to the file */
1495 static URL mavenRepoUrl(String repoBase, M2Artifact artifact) {
1496 String url = artifactUrl(repoBase == null ? MAVEN_CENTRAL_BASE_URL : repoBase, artifact);
1497 try {
1498 return new URL(url);
1499 } catch (MalformedURLException e) {
1500 // it should not happen
1501 throw new IllegalStateException(e);
1502 }
1503 }
1504
1505 /** Absolute path to the directories where the files will be stored */
1506 static String artifactParentPath(String artifactBasePath, M2Artifact artifact) {
1507 return artifactBasePath + (artifactBasePath.endsWith("/") ? "" : "/") + artifactParentPath(artifact);
1508 }
1509
1510 /** Relative path to the directories where the files will be stored */
1511 static String artifactParentPath(M2Artifact artifact) {
1512 return artifact.getGroupId().replace('.', '/') + '/' + artifact.getArtifactId() + '/' + artifact.getVersion();
1513 }
1514
1515 /** Singleton */
1516 private M2ConventionsUtils() {
1517 }
1518 }
1519
1520 /** Combination of a category, a name and a version. */
1521 class CategoryNameVersion extends NameVersion {
1522 private String category;
1523
1524 CategoryNameVersion() {
1525 }
1526
1527 CategoryNameVersion(String category, String name, String version) {
1528 super(name, version);
1529 this.category = category;
1530 }
1531
1532 CategoryNameVersion(String category, NameVersion nameVersion) {
1533 super(nameVersion);
1534 this.category = category;
1535 }
1536
1537 String getCategory() {
1538 return category;
1539 }
1540
1541 void setCategory(String category) {
1542 this.category = category;
1543 }
1544
1545 @Override
1546 public String toString() {
1547 return category + ":" + super.toString();
1548 }
1549
1550 }
1551
1552 /** Combination of a name and a version. */
1553 class NameVersion implements Comparable<NameVersion> {
1554 private String name;
1555 private String version;
1556
1557 NameVersion() {
1558 }
1559
1560 /** Interprets string in OSGi-like format my.module.name;version=0.0.0 */
1561 NameVersion(String nameVersion) {
1562 int index = nameVersion.indexOf(";version=");
1563 if (index < 0) {
1564 setName(nameVersion);
1565 setVersion(null);
1566 } else {
1567 setName(nameVersion.substring(0, index));
1568 setVersion(nameVersion.substring(index + ";version=".length()));
1569 }
1570 }
1571
1572 NameVersion(String name, String version) {
1573 this.name = name;
1574 this.version = version;
1575 }
1576
1577 NameVersion(NameVersion nameVersion) {
1578 this.name = nameVersion.getName();
1579 this.version = nameVersion.getVersion();
1580 }
1581
1582 String getName() {
1583 return name;
1584 }
1585
1586 void setName(String name) {
1587 this.name = name;
1588 }
1589
1590 String getVersion() {
1591 return version;
1592 }
1593
1594 void setVersion(String version) {
1595 this.version = version;
1596 }
1597
1598 String getBranch() {
1599 String[] parts = getVersion().split("\\.");
1600 if (parts.length < 2)
1601 throw new IllegalStateException("Version " + getVersion() + " cannot be interpreted as branch.");
1602 return parts[0] + "." + parts[1];
1603 }
1604
1605 @Override
1606 public boolean equals(Object obj) {
1607 if (obj instanceof NameVersion) {
1608 NameVersion nameVersion = (NameVersion) obj;
1609 return name.equals(nameVersion.getName()) && version.equals(nameVersion.getVersion());
1610 } else
1611 return false;
1612 }
1613
1614 @Override
1615 public int hashCode() {
1616 return name.hashCode();
1617 }
1618
1619 @Override
1620 public String toString() {
1621 return name + ":" + version;
1622 }
1623
1624 public int compareTo(NameVersion o) {
1625 if (o.getName().equals(name))
1626 return version.compareTo(o.getVersion());
1627 else
1628 return name.compareTo(o.getName());
1629 }
1630 }
Argeo Build - Build scripts and configuration for Argeo projects