]> git.argeo.org Git - cc0/argeo-build.git/blob - Repackage.java
df630055aec75973df0dbbf5046d57324bdcdabb
[cc0/argeo-build.git] / Repackage.java
1 package org.argeo.build;
2
3 import static java.lang.System.Logger.Level.DEBUG;
4 import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_SYMBOLICNAME;
5 import static org.argeo.build.Repackage.ManifestConstants.BUNDLE_VERSION;
6 import static org.argeo.build.Repackage.ManifestConstants.EXPORT_PACKAGE;
7 import static org.argeo.build.Repackage.ManifestConstants.SLC_ORIGIN_M2;
8 import static org.argeo.build.Repackage.ManifestConstants.SLC_ORIGIN_M2_REPO;
9
10 import java.io.FileNotFoundException;
11 import java.io.IOException;
12 import java.io.InputStream;
13 import java.io.OutputStream;
14 import java.lang.System.Logger;
15 import java.lang.System.Logger.Level;
16 import java.net.MalformedURLException;
17 import java.net.URL;
18 import java.nio.file.DirectoryStream;
19 import java.nio.file.FileSystem;
20 import java.nio.file.FileSystems;
21 import java.nio.file.FileVisitResult;
22 import java.nio.file.Files;
23 import java.nio.file.Path;
24 import java.nio.file.PathMatcher;
25 import java.nio.file.Paths;
26 import java.nio.file.SimpleFileVisitor;
27 import java.nio.file.StandardOpenOption;
28 import java.nio.file.attribute.BasicFileAttributes;
29 import java.util.ArrayList;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Objects;
34 import java.util.Properties;
35 import java.util.TreeMap;
36 import java.util.jar.Attributes;
37 import java.util.jar.JarEntry;
38 import java.util.jar.JarInputStream;
39 import java.util.jar.JarOutputStream;
40 import java.util.jar.Manifest;
41 import java.util.zip.Deflater;
42
43 import aQute.bnd.osgi.Analyzer;
44 import aQute.bnd.osgi.Jar;
45
46 /** The central class for A2 packaging. */
47 public class Repackage {
48 private final static Logger logger = System.getLogger(Repackage.class.getName());
49
50 /** Main entry point. */
51 public static void main(String[] args) {
52 if (args.length < 2) {
53 System.err.println("Usage: <path to a2 output dir> <category1> <category2> ...");
54 System.exit(1);
55 }
56 Path a2Base = Paths.get(args[0]).toAbsolutePath().normalize();
57 Path descriptorsBase = Paths.get(".").toAbsolutePath().normalize();
58 Repackage factory = new Repackage(a2Base, descriptorsBase, true);
59
60 for (int i = 1; i < args.length; i++) {
61 Path p = Paths.get(args[i]);
62 factory.processCategory(p);
63 }
64 }
65
66 private final static String COMMON_BND = "common.bnd";
67 private final static String MERGE_BND = "merge.bnd";
68
69 private Path originBase;
70 private Path a2Base;
71 private Path a2LibBase;
72 private Path descriptorsBase;
73
74 private Properties uris = new Properties();
75
76 private boolean includeSources = true;
77
78 /** key is URI prefix, value list of base URLs */
79 private Map<String, List<String>> mirrors = new HashMap<String, List<String>>();
80
81 public Repackage(Path a2Base, Path descriptorsBase, boolean includeSources) {
82 Objects.requireNonNull(a2Base);
83 Objects.requireNonNull(descriptorsBase);
84 this.originBase = Paths.get(System.getProperty("user.home"), ".cache", "argeo/build/origin");
85 // TODO define and use a build base
86 this.a2Base = a2Base;
87 this.a2LibBase = a2Base.resolve("lib");
88 this.descriptorsBase = descriptorsBase;
89 if (!Files.exists(this.descriptorsBase))
90 throw new IllegalArgumentException(this.descriptorsBase + " does not exist");
91 this.includeSources = includeSources;
92
93 Path urisPath = this.descriptorsBase.resolve("uris.properties");
94 if (Files.exists(urisPath)) {
95 try (InputStream in = Files.newInputStream(urisPath)) {
96 uris.load(in);
97 } catch (IOException e) {
98 throw new IllegalStateException("Cannot load " + urisPath, e);
99 }
100 }
101
102 // TODO make it configurable
103 List<String> eclipseMirrors = new ArrayList<>();
104 eclipseMirrors.add("https://archive.eclipse.org/");
105 eclipseMirrors.add("http://ftp-stud.hs-esslingen.de/Mirrors/eclipse/");
106 eclipseMirrors.add("http://ftp.fau.de/eclipse/");
107
108 mirrors.put("http://www.eclipse.org/downloads", eclipseMirrors);
109 }
110
111 /*
112 * MAVEN ORIGIN
113 */
114
115 /** Process a whole category/group id. */
116 public void processCategory(Path categoryRelativePath) {
117 try {
118 Path targetCategoryBase = descriptorsBase.resolve(categoryRelativePath);
119 DirectoryStream<Path> bnds = Files.newDirectoryStream(targetCategoryBase,
120 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
121 && !p.getFileName().toString().equals(MERGE_BND));
122 for (Path p : bnds) {
123 processSingleM2ArtifactDistributionUnit(p);
124 }
125
126 DirectoryStream<Path> dus = Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p));
127 for (Path duDir : dus) {
128 if (duDir.getFileName().toString().startsWith("eclipse-")) {
129 processEclipseArchive(duDir);
130 } else {
131 processM2BasedDistributionUnit(duDir);
132 }
133 }
134 } catch (IOException e) {
135 throw new RuntimeException("Cannot process category " + categoryRelativePath, e);
136 }
137 }
138
139 /** Process a standalone Maven artifact. */
140 public void processSingleM2ArtifactDistributionUnit(Path bndFile) {
141 try {
142 // String category = bndFile.getParent().getFileName().toString();
143 Path categoryRelativePath = descriptorsBase.relativize(bndFile.getParent());
144 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
145
146 Properties fileProps = new Properties();
147 try (InputStream in = Files.newInputStream(bndFile)) {
148 fileProps.load(in);
149 }
150 String repoStr = fileProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
151 ? fileProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
152 : null;
153
154 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
155 // use file name as symbolic name
156 String symbolicName = bndFile.getFileName().toString();
157 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
158 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
159 }
160
161 String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
162 if (m2Coordinates == null)
163 throw new IllegalArgumentException("No M2 coordinates available for " + bndFile);
164 M2Artifact artifact = new M2Artifact(m2Coordinates);
165 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
166 Path downloaded = download(url, originBase, artifact);
167
168 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, fileProps, artifact);
169
170 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
171
172 createJar(targetBundleDir);
173 } catch (Exception e) {
174 throw new RuntimeException("Cannot process " + bndFile, e);
175 }
176 }
177
178 /** Process multiple Maven artifacts. */
179 public void processM2BasedDistributionUnit(Path duDir) {
180 try {
181 // String category = duDir.getParent().getFileName().toString();
182 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
183 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
184
185 // merge
186 Path mergeBnd = duDir.resolve(MERGE_BND);
187 if (Files.exists(mergeBnd)) {
188 mergeM2Artifacts(mergeBnd);
189 // return;
190 }
191
192 Path commonBnd = duDir.resolve(COMMON_BND);
193 if (!Files.exists(commonBnd)) {
194 return;
195 }
196 Properties commonProps = new Properties();
197 try (InputStream in = Files.newInputStream(commonBnd)) {
198 commonProps.load(in);
199 }
200
201 String m2Version = commonProps.getProperty(SLC_ORIGIN_M2.toString());
202 if (m2Version == null) {
203 logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
204 return;// ignore, this is probably an Eclipse archive
205 }
206 if (!m2Version.startsWith(":")) {
207 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
208 }
209 m2Version = m2Version.substring(1);
210
211 DirectoryStream<Path> ds = Files.newDirectoryStream(duDir,
212 (p) -> p.getFileName().toString().endsWith(".bnd") && !p.getFileName().toString().equals(COMMON_BND)
213 && !p.getFileName().toString().equals(MERGE_BND));
214 for (Path p : ds) {
215 Properties fileProps = new Properties();
216 try (InputStream in = Files.newInputStream(p)) {
217 fileProps.load(in);
218 }
219 String m2Coordinates = fileProps.getProperty(SLC_ORIGIN_M2.toString());
220 M2Artifact artifact = new M2Artifact(m2Coordinates);
221
222 artifact.setVersion(m2Version);
223
224 // prepare manifest entries
225 Properties mergeProps = new Properties();
226 mergeProps.putAll(commonProps);
227
228 fileEntries: for (Object key : fileProps.keySet()) {
229 if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
230 continue fileEntries;
231 String value = fileProps.getProperty(key.toString());
232 Object previousValue = mergeProps.put(key.toString(), value);
233 if (previousValue != null) {
234 logger.log(Level.WARNING,
235 commonBnd + ": " + key + " was " + previousValue + ", overridden with " + value);
236 }
237 }
238 mergeProps.put(ManifestConstants.SLC_ORIGIN_M2.toString(), artifact.toM2Coordinates());
239 if (!mergeProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
240 // use file name as symbolic name
241 String symbolicName = p.getFileName().toString();
242 symbolicName = symbolicName.substring(0, symbolicName.length() - ".bnd".length());
243 mergeProps.put(BUNDLE_SYMBOLICNAME.toString(), symbolicName);
244 }
245
246 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
247 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
248 : null;
249
250 // download
251 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
252 Path downloaded = download(url, originBase, artifact);
253
254 Path targetBundleDir = processBndJar(downloaded, targetCategoryBase, mergeProps, artifact);
255 // logger.log(Level.DEBUG, () -> "Processed " + downloaded);
256
257 // sources
258 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
259
260 createJar(targetBundleDir);
261 }
262 } catch (IOException e) {
263 throw new RuntimeException("Cannot process " + duDir, e);
264 }
265
266 }
267
268 /** Merge multiple Maven artifacts. */
269 protected void mergeM2Artifacts(Path mergeBnd) throws IOException {
270 Path duDir = mergeBnd.getParent();
271 String category = duDir.getParent().getFileName().toString();
272 Path targetCategoryBase = a2Base.resolve(category);
273
274 Properties mergeProps = new Properties();
275 try (InputStream in = Files.newInputStream(mergeBnd)) {
276 mergeProps.load(in);
277 }
278
279 // Version
280 String m2Version = mergeProps.getProperty(SLC_ORIGIN_M2.toString());
281 if (m2Version == null) {
282 logger.log(Level.WARNING, "Ignoring " + duDir + " as it is not an M2-based distribution unit");
283 return;// ignore, this is probably an Eclipse archive
284 }
285 if (!m2Version.startsWith(":")) {
286 throw new IllegalStateException("Only the M2 version can be specified: " + m2Version);
287 }
288 m2Version = m2Version.substring(1);
289 mergeProps.put(ManifestConstants.BUNDLE_VERSION.toString(), m2Version);
290
291 String artifactsStr = mergeProps.getProperty(ManifestConstants.SLC_ORIGIN_M2_MERGE.toString());
292 String repoStr = mergeProps.containsKey(SLC_ORIGIN_M2_REPO.toString())
293 ? mergeProps.getProperty(SLC_ORIGIN_M2_REPO.toString())
294 : null;
295
296 String bundleSymbolicName = mergeProps.getProperty(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
297 if (bundleSymbolicName == null)
298 throw new IllegalArgumentException("Bundle-SymbolicName must be set in " + mergeBnd);
299 CategoryNameVersion nameVersion = new M2Artifact(category + ":" + bundleSymbolicName + ":" + m2Version);
300 Path targetBundleDir = targetCategoryBase.resolve(bundleSymbolicName + "." + nameVersion.getBranch());
301
302 String[] artifacts = artifactsStr.split(",");
303 artifacts: for (String str : artifacts) {
304 String m2Coordinates = str.trim();
305 if ("".equals(m2Coordinates))
306 continue artifacts;
307 M2Artifact artifact = new M2Artifact(m2Coordinates.trim());
308 if (artifact.getVersion() == null)
309 artifact.setVersion(m2Version);
310 URL url = M2ConventionsUtils.mavenRepoUrl(repoStr, artifact);
311 Path downloaded = download(url, originBase, artifact);
312 JarEntry entry;
313 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(downloaded), false)) {
314 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
315 if (entry.isDirectory())
316 continue entries;
317 else if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
318 continue entries;
319 else if (entry.getName().startsWith("META-INF/versions/"))
320 continue entries;
321 else if (entry.getName().startsWith("META-INF/maven/"))
322 continue entries;
323 else if (entry.getName().equals("module-info.class"))
324 continue entries;
325 else if (entry.getName().equals("META-INF/NOTICE"))
326 continue entries;
327 else if (entry.getName().equals("META-INF/NOTICE.txt"))
328 continue entries;
329 else if (entry.getName().equals("META-INF/LICENSE"))
330 continue entries;
331 else if (entry.getName().equals("META-INF/LICENSE.md"))
332 continue entries;
333 else if (entry.getName().equals("META-INF/LICENSE-notice.md"))
334 continue entries;
335 else if (entry.getName().equals("META-INF/DEPENDENCIES"))
336 continue entries;
337 if (entry.getName().startsWith(".cache/")) // Apache SSHD
338 continue entries;
339 Path target = targetBundleDir.resolve(entry.getName());
340 Files.createDirectories(target.getParent());
341 if (!Files.exists(target)) {
342 Files.copy(jarIn, target);
343 } else {
344 if (entry.getName().startsWith("META-INF/services/")) {
345 try (OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND)) {
346 out.write("\n".getBytes());
347 jarIn.transferTo(out);
348 if (logger.isLoggable(DEBUG))
349 logger.log(DEBUG, artifact.getArtifactId() + " - Appended " + entry.getName());
350 }
351 } else if (entry.getName().startsWith("org/apache/batik/")) {
352 logger.log(Level.WARNING, "Skip " + entry.getName());
353 continue entries;
354 } else {
355 throw new IllegalStateException("File " + target + " from " + artifact + " already exists");
356 }
357 }
358 logger.log(Level.TRACE, () -> "Copied " + target);
359 }
360
361 }
362 downloadAndProcessM2Sources(repoStr, artifact, targetBundleDir);
363 }
364
365 // additional service files
366 Path servicesDir = duDir.resolve("services");
367 if (Files.exists(servicesDir)) {
368 for (Path p : Files.newDirectoryStream(servicesDir)) {
369 Path target = targetBundleDir.resolve("META-INF/services/").resolve(p.getFileName());
370 try (InputStream in = Files.newInputStream(p);
371 OutputStream out = Files.newOutputStream(target, StandardOpenOption.APPEND);) {
372 out.write("\n".getBytes());
373 in.transferTo(out);
374 if (logger.isLoggable(DEBUG))
375 logger.log(DEBUG, "Appended " + p);
376 }
377 }
378 }
379
380 Map<String, String> entries = new TreeMap<>();
381 try (Analyzer bndAnalyzer = new Analyzer()) {
382 bndAnalyzer.setProperties(mergeProps);
383 Jar jar = new Jar(targetBundleDir.toFile());
384 bndAnalyzer.setJar(jar);
385 Manifest manifest = bndAnalyzer.calcManifest();
386
387 keys: for (Object key : manifest.getMainAttributes().keySet()) {
388 Object value = manifest.getMainAttributes().get(key);
389
390 switch (key.toString()) {
391 case "Tool":
392 case "Bnd-LastModified":
393 case "Created-By":
394 continue keys;
395 }
396 if ("Require-Capability".equals(key.toString())
397 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
398 continue keys;// hack for very old classes
399 entries.put(key.toString(), value.toString());
400 // logger.log(DEBUG, () -> key + "=" + value);
401
402 }
403 } catch (Exception e) {
404 throw new RuntimeException("Cannot process " + mergeBnd, e);
405 }
406
407 Manifest manifest = new Manifest();
408 Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
409 Files.createDirectories(manifestPath.getParent());
410 for (String key : entries.keySet()) {
411 String value = entries.get(key);
412 manifest.getMainAttributes().putValue(key, value);
413 }
414
415 try (OutputStream out = Files.newOutputStream(manifestPath)) {
416 manifest.write(out);
417 }
418 createJar(targetBundleDir);
419 }
420
421 /** Generate MANIFEST using BND. */
422 protected Path processBndJar(Path downloaded, Path targetCategoryBase, Properties fileProps, M2Artifact artifact) {
423
424 try {
425 Map<String, String> additionalEntries = new TreeMap<>();
426 boolean doNotModify = Boolean.parseBoolean(fileProps
427 .getOrDefault(ManifestConstants.SLC_ORIGIN_MANIFEST_NOT_MODIFIED.toString(), "false").toString());
428
429 // we always force the symbolic name
430
431 if (doNotModify) {
432 fileEntries: for (Object key : fileProps.keySet()) {
433 if (ManifestConstants.SLC_ORIGIN_M2.toString().equals(key))
434 continue fileEntries;
435 String value = fileProps.getProperty(key.toString());
436 additionalEntries.put(key.toString(), value);
437 }
438 } else {
439 if (artifact != null) {
440 if (!fileProps.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
441 fileProps.put(BUNDLE_SYMBOLICNAME.toString(), artifact.getName());
442 }
443 if (!fileProps.containsKey(BUNDLE_VERSION.toString())) {
444 fileProps.put(BUNDLE_VERSION.toString(), artifact.getVersion());
445 }
446 }
447
448 if (!fileProps.containsKey(EXPORT_PACKAGE.toString())) {
449 fileProps.put(EXPORT_PACKAGE.toString(),
450 "*;version=\"" + fileProps.getProperty(BUNDLE_VERSION.toString()) + "\"");
451 }
452
453 try (Analyzer bndAnalyzer = new Analyzer()) {
454 bndAnalyzer.setProperties(fileProps);
455 Jar jar = new Jar(downloaded.toFile());
456 bndAnalyzer.setJar(jar);
457 Manifest manifest = bndAnalyzer.calcManifest();
458
459 keys: for (Object key : manifest.getMainAttributes().keySet()) {
460 Object value = manifest.getMainAttributes().get(key);
461
462 switch (key.toString()) {
463 case "Tool":
464 case "Bnd-LastModified":
465 case "Created-By":
466 continue keys;
467 }
468 if ("Require-Capability".equals(key.toString())
469 && value.toString().equals("osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.1))\""))
470 continue keys;// hack for very old classes
471 additionalEntries.put(key.toString(), value.toString());
472 // logger.log(DEBUG, () -> key + "=" + value);
473
474 }
475 }
476 }
477 Path targetBundleDir = processBundleJar(downloaded, targetCategoryBase, additionalEntries);
478 logger.log(Level.DEBUG, () -> "Processed " + downloaded);
479 return targetBundleDir;
480 } catch (Exception e) {
481 throw new RuntimeException("Cannot BND process " + downloaded, e);
482 }
483
484 }
485
486 /** Download and integrates sources for a single Maven artifact. */
487 protected void downloadAndProcessM2Sources(String repoStr, M2Artifact artifact, Path targetBundleDir)
488 throws IOException {
489 if (!includeSources)
490 return;
491 M2Artifact sourcesArtifact = new M2Artifact(artifact.toM2Coordinates(), "sources");
492 URL sourcesUrl = M2ConventionsUtils.mavenRepoUrl(repoStr, sourcesArtifact);
493 Path sourcesDownloaded = download(sourcesUrl, originBase, artifact, true);
494 processM2SourceJar(sourcesDownloaded, targetBundleDir);
495 logger.log(Level.TRACE, () -> "Processed source " + sourcesDownloaded);
496
497 }
498
499 /** Integrate sources from a downloaded jar file. */
500 protected void processM2SourceJar(Path file, Path targetBundleDir) throws IOException {
501 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
502 Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
503
504 // TODO make it less dangerous?
505 if (Files.exists(targetSourceDir)) {
506 // deleteDirectory(targetSourceDir);
507 } else {
508 Files.createDirectories(targetSourceDir);
509 }
510
511 // copy entries
512 JarEntry entry;
513 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
514 if (entry.isDirectory())
515 continue entries;
516 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
517 continue entries;
518 if (entry.getName().startsWith("module-info.java"))// skip META-INF entries
519 continue entries;
520 if (entry.getName().startsWith("/")) // absolute paths
521 continue entries;
522 Path target = targetSourceDir.resolve(entry.getName());
523 Files.createDirectories(target.getParent());
524 if (!Files.exists(target)) {
525 Files.copy(jarIn, target);
526 logger.log(Level.TRACE, () -> "Copied source " + target);
527 } else {
528 logger.log(Level.WARNING, () -> target + " already exists, skipping...");
529 }
530 }
531 }
532
533 }
534
535 /** Download a Maven artifact. */
536 protected Path download(URL url, Path dir, M2Artifact artifact) throws IOException {
537 return download(url, dir, artifact, false);
538 }
539
540 /** Download a Maven artifact. */
541 protected Path download(URL url, Path dir, M2Artifact artifact, boolean sources) throws IOException {
542 return download(url, dir, artifact.getGroupId() + '/' + artifact.getArtifactId() + "-" + artifact.getVersion()
543 + (sources ? "-sources" : "") + ".jar");
544 }
545
546 /*
547 * ECLIPSE ORIGIN
548 */
549
550 /** Process an archive in Eclipse format. */
551 public void processEclipseArchive(Path duDir) {
552 try {
553 Path categoryRelativePath = descriptorsBase.relativize(duDir.getParent());
554 // String category = categoryRelativePath.getFileName().toString();
555 Path targetCategoryBase = a2Base.resolve(categoryRelativePath);
556 Files.createDirectories(targetCategoryBase);
557 // first delete all directories from previous builds
558 for (Path dir : Files.newDirectoryStream(targetCategoryBase, (p) -> Files.isDirectory(p))) {
559 deleteDirectory(dir);
560 }
561
562 Files.createDirectories(originBase);
563
564 Path commonBnd = duDir.resolve(COMMON_BND);
565 Properties commonProps = new Properties();
566 try (InputStream in = Files.newInputStream(commonBnd)) {
567 commonProps.load(in);
568 }
569 String url = commonProps.getProperty(ManifestConstants.SLC_ORIGIN_URI.toString());
570 if (url == null) {
571 url = uris.getProperty(duDir.getFileName().toString());
572 if (url == null)
573 throw new IllegalStateException("No url available for " + duDir);
574 commonProps.put(ManifestConstants.SLC_ORIGIN_URI.toString(), url);
575 }
576 Path downloaded = tryDownload(url, originBase);
577
578 FileSystem zipFs = FileSystems.newFileSystem(downloaded, (ClassLoader) null);
579
580 // filters
581 List<PathMatcher> includeMatchers = new ArrayList<>();
582 Properties includes = new Properties();
583 try (InputStream in = Files.newInputStream(duDir.resolve("includes.properties"))) {
584 includes.load(in);
585 }
586 for (Object pattern : includes.keySet()) {
587 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
588 includeMatchers.add(pathMatcher);
589 }
590
591 List<PathMatcher> excludeMatchers = new ArrayList<>();
592 Path excludeFile = duDir.resolve("excludes.properties");
593 if (Files.exists(excludeFile)) {
594 Properties excludes = new Properties();
595 try (InputStream in = Files.newInputStream(excludeFile)) {
596 excludes.load(in);
597 }
598 for (Object pattern : excludes.keySet()) {
599 PathMatcher pathMatcher = zipFs.getPathMatcher("glob:/" + pattern);
600 excludeMatchers.add(pathMatcher);
601 }
602 }
603
604 Files.walkFileTree(zipFs.getRootDirectories().iterator().next(), new SimpleFileVisitor<Path>() {
605
606 @Override
607 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
608 includeMatchers: for (PathMatcher includeMatcher : includeMatchers) {
609 if (includeMatcher.matches(file)) {
610 for (PathMatcher excludeMatcher : excludeMatchers) {
611 if (excludeMatcher.matches(file)) {
612 logger.log(Level.WARNING, "Skipping excluded " + file);
613 return FileVisitResult.CONTINUE;
614 }
615 }
616 if (includeSources && file.getFileName().toString().contains(".source_")) {
617 processEclipseSourceJar(file, targetCategoryBase);
618 logger.log(Level.DEBUG, () -> "Processed source " + file);
619
620 } else {
621 Map<String, String> map = new HashMap<>();
622 for (Object key : commonProps.keySet())
623 map.put(key.toString(), commonProps.getProperty(key.toString()));
624 processBundleJar(file, targetCategoryBase, map);
625 logger.log(Level.DEBUG, () -> "Processed " + file);
626 }
627 break includeMatchers;
628 }
629 }
630 return FileVisitResult.CONTINUE;
631 }
632 });
633
634 DirectoryStream<Path> dirs = Files.newDirectoryStream(targetCategoryBase,
635 (p) -> Files.isDirectory(p) && p.getFileName().toString().indexOf('.') >= 0);
636 for (Path dir : dirs) {
637 createJar(dir);
638 }
639 } catch (IOException e) {
640 throw new RuntimeException("Cannot process " + duDir, e);
641 }
642
643 }
644
645 /** Process sources in Eclipse format. */
646 protected void processEclipseSourceJar(Path file, Path targetBase) throws IOException {
647 try {
648 Path targetBundleDir;
649 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
650 Manifest manifest = jarIn.getManifest();
651
652 String[] relatedBundle = manifest.getMainAttributes().getValue("Eclipse-SourceBundle").split(";");
653 String version = relatedBundle[1].substring("version=\"".length());
654 version = version.substring(0, version.length() - 1);
655 NameVersion nameVersion = new NameVersion(relatedBundle[0], version);
656 targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
657
658 Path targetSourceDir = targetBundleDir.resolve("OSGI-OPT/src");
659
660 // TODO make it less dangerous?
661 if (Files.exists(targetSourceDir)) {
662 // deleteDirectory(targetSourceDir);
663 } else {
664 Files.createDirectories(targetSourceDir);
665 }
666
667 // copy entries
668 JarEntry entry;
669 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
670 if (entry.isDirectory())
671 continue entries;
672 if (entry.getName().startsWith("META-INF"))// skip META-INF entries
673 continue entries;
674 Path target = targetSourceDir.resolve(entry.getName());
675 Files.createDirectories(target.getParent());
676 Files.copy(jarIn, target);
677 logger.log(Level.TRACE, () -> "Copied source " + target);
678 }
679
680 // copy MANIFEST
681 }
682 } catch (IOException e) {
683 throw new IllegalStateException("Cannot process " + file, e);
684 }
685
686 }
687
688 /*
689 * COMMON PROCESSING
690 */
691 /** Normalise a bundle. */
692 protected Path processBundleJar(Path file, Path targetBase, Map<String, String> entries) throws IOException {
693 NameVersion nameVersion;
694 Path targetBundleDir;
695 try (JarInputStream jarIn = new JarInputStream(Files.newInputStream(file), false)) {
696 Manifest sourceManifest = jarIn.getManifest();
697 Manifest manifest = sourceManifest != null ? new Manifest(sourceManifest) : new Manifest();
698
699 // singleton
700 boolean isSingleton = false;
701 String rawSourceSymbolicName = manifest.getMainAttributes()
702 .getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
703 if (rawSourceSymbolicName != null) {
704
705 // make sure there is no directive
706 String[] arr = rawSourceSymbolicName.split(";");
707 for (int i = 1; i < arr.length; i++) {
708 if (arr[i].trim().equals("singleton:=true"))
709 isSingleton = true;
710 logger.log(DEBUG, file.getFileName() + " is a singleton");
711 }
712 }
713
714 // remove problematic entries in MANIFEST
715 manifest.getEntries().clear();
716
717 String ourSymbolicName = entries.get(BUNDLE_SYMBOLICNAME.toString());
718 String ourVersion = entries.get(BUNDLE_VERSION.toString());
719
720 if (ourSymbolicName != null && ourVersion != null) {
721 nameVersion = new NameVersion(ourSymbolicName, ourVersion);
722 } else {
723 nameVersion = nameVersionFromManifest(manifest);
724 if (ourVersion != null && !nameVersion.getVersion().equals(ourVersion)) {
725 logger.log(Level.WARNING,
726 "Original version is " + nameVersion.getVersion() + " while new version is " + ourVersion);
727 entries.put(BUNDLE_VERSION.toString(), ourVersion);
728 }
729 if (ourSymbolicName != null) {
730 // we always force our symbolic name
731 nameVersion.setName(ourSymbolicName);
732 }
733 }
734 targetBundleDir = targetBase.resolve(nameVersion.getName() + "." + nameVersion.getBranch());
735
736 // force Java 9 module name
737 entries.put(ManifestConstants.AUTOMATIC_MODULE_NAME.toString(), nameVersion.getName());
738
739 boolean isNative = false;
740 String os = null;
741 String arch = null;
742 if (targetBundleDir.startsWith(a2LibBase)) {
743 isNative = true;
744 Path libRelativePath = a2LibBase.relativize(targetBundleDir);
745 os = libRelativePath.getName(0).toString();
746 arch = libRelativePath.getName(1).toString();
747 }
748
749 // copy entries
750 JarEntry entry;
751 entries: while ((entry = jarIn.getNextJarEntry()) != null) {
752 if (entry.isDirectory())
753 continue entries;
754 if (entry.getName().endsWith(".RSA") || entry.getName().endsWith(".SF"))
755 continue entries;
756 if (entry.getName().endsWith("module-info.class")) // skip Java 9 module info
757 continue entries;
758 if (entry.getName().startsWith("META-INF/versions/")) // skip multi-version
759 continue entries;
760 // skip file system providers as they cause issues with native image
761 if (entry.getName().startsWith("META-INF/services/java.nio.file.spi.FileSystemProvider"))
762 continue entries;
763 if (entry.getName().startsWith("OSGI-OPT/src/")) // skip embedded sources
764 continue entries;
765 Path target = targetBundleDir.resolve(entry.getName());
766 Files.createDirectories(target.getParent());
767 Files.copy(jarIn, target);
768
769 // native libraries
770 if (isNative && (entry.getName().endsWith(".so") || entry.getName().endsWith(".dll")
771 || entry.getName().endsWith(".jnilib"))) {
772 Path categoryDir = targetBundleDir.getParent();
773 // String[] segments = categoryDir.getFileName().toString().split("\\.");
774 // String arch = segments[segments.length - 1];
775 // String os = segments[segments.length - 2];
776 boolean copyDll = false;
777 Path targetDll = categoryDir.resolve(targetBundleDir.relativize(target));
778 if (nameVersion.getName().equals("com.sun.jna")) {
779 if (arch.equals("x86_64"))
780 arch = "x86-64";
781 if (os.equals("macosx"))
782 os = "darwin";
783 if (target.getParent().getFileName().toString().equals(os + "-" + arch)) {
784 copyDll = true;
785 }
786 targetDll = categoryDir.resolve(target.getFileName());
787 } else {
788 copyDll = true;
789 }
790 if (copyDll) {
791 Files.createDirectories(targetDll.getParent());
792 if (Files.exists(targetDll))
793 Files.delete(targetDll);
794 Files.copy(target, targetDll);
795 }
796 Files.delete(target);
797 }
798 logger.log(Level.TRACE, () -> "Copied " + target);
799 }
800
801 // copy MANIFEST
802 Path manifestPath = targetBundleDir.resolve("META-INF/MANIFEST.MF");
803 Files.createDirectories(manifestPath.getParent());
804
805 if (isSingleton && entries.containsKey(BUNDLE_SYMBOLICNAME.toString())) {
806 entries.put(BUNDLE_SYMBOLICNAME.toString(),
807 entries.get(BUNDLE_SYMBOLICNAME.toString()) + ";singleton:=true");
808 }
809
810 for (String key : entries.keySet()) {
811 String value = entries.get(key);
812 Object previousValue = manifest.getMainAttributes().putValue(key, value);
813 if (previousValue != null && !previousValue.equals(value)) {
814 if (ManifestConstants.IMPORT_PACKAGE.toString().equals(key)
815 || ManifestConstants.EXPORT_PACKAGE.toString().equals(key))
816 logger.log(Level.TRACE, file.getFileName() + ": " + key + " was modified");
817 else
818 logger.log(Level.WARNING, file.getFileName() + ": " + key + " was " + previousValue
819 + ", overridden with " + value);
820 }
821
822 // hack to remove unresolvable
823 if (key.equals("Provide-Capability") || key.equals("Require-Capability"))
824 if (nameVersion.getName().equals("osgi.core") || nameVersion.getName().equals("osgi.cmpn")) {
825 manifest.getMainAttributes().remove(key);
826 }
827 }
828 try (OutputStream out = Files.newOutputStream(manifestPath)) {
829 manifest.write(out);
830 }
831 }
832 return targetBundleDir;
833 }
834
835 /*
836 * UTILITIES
837 */
838
839 /** Recursively deletes a directory. */
840 private static void deleteDirectory(Path path) throws IOException {
841 if (!Files.exists(path))
842 return;
843 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
844 @Override
845 public FileVisitResult postVisitDirectory(Path directory, IOException e) throws IOException {
846 if (e != null)
847 throw e;
848 Files.delete(directory);
849 return FileVisitResult.CONTINUE;
850 }
851
852 @Override
853 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
854 Files.delete(file);
855 return FileVisitResult.CONTINUE;
856 }
857 });
858 }
859
860 /** Extract name/version from a MANIFEST. */
861 protected NameVersion nameVersionFromManifest(Manifest manifest) {
862 Attributes attrs = manifest.getMainAttributes();
863 // symbolic name
864 String symbolicName = attrs.getValue(ManifestConstants.BUNDLE_SYMBOLICNAME.toString());
865 if (symbolicName == null)
866 return null;
867 // make sure there is no directive
868 symbolicName = symbolicName.split(";")[0];
869
870 String version = attrs.getValue(ManifestConstants.BUNDLE_VERSION.toString());
871 return new NameVersion(symbolicName, version);
872 }
873
874 /** Try to download from an URI. */
875 protected Path tryDownload(String uri, Path dir) throws IOException {
876 // find mirror
877 List<String> urlBases = null;
878 String uriPrefix = null;
879 uriPrefixes: for (String uriPref : mirrors.keySet()) {
880 if (uri.startsWith(uriPref)) {
881 if (mirrors.get(uriPref).size() > 0) {
882 urlBases = mirrors.get(uriPref);
883 uriPrefix = uriPref;
884 break uriPrefixes;
885 }
886 }
887 }
888 if (urlBases == null)
889 try {
890 return download(new URL(uri), dir);
891 } catch (FileNotFoundException e) {
892 throw new FileNotFoundException("Cannot find " + uri);
893 }
894
895 // try to download
896 for (String urlBase : urlBases) {
897 String relativePath = uri.substring(uriPrefix.length());
898 URL url = new URL(urlBase + relativePath);
899 try {
900 return download(url, dir);
901 } catch (FileNotFoundException e) {
902 logger.log(Level.WARNING, "Cannot download " + url + ", trying another mirror");
903 }
904 }
905 throw new FileNotFoundException("Cannot find " + uri);
906 }
907
908 /** Effectively download. */
909 protected Path download(URL url, Path dir) throws IOException {
910 return download(url, dir, (String) null);
911 }
912
913 /** Effectively download. */
914 protected Path download(URL url, Path dir, String name) throws IOException {
915
916 Path dest;
917 if (name == null) {
918 name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1);
919 }
920
921 dest = dir.resolve(name);
922 if (Files.exists(dest)) {
923 logger.log(Level.TRACE, () -> "File " + dest + " already exists for " + url + ", not downloading again");
924 return dest;
925 } else {
926 Files.createDirectories(dest.getParent());
927 }
928
929 try (InputStream in = url.openStream()) {
930 Files.copy(in, dest);
931 logger.log(Level.DEBUG, () -> "Downloaded " + dest + " from " + url);
932 }
933 return dest;
934 }
935
936 /** Create a JAR file from a directory. */
937 protected Path createJar(Path bundleDir) throws IOException {
938 // Create the jar
939 Path jarPath = bundleDir.getParent().resolve(bundleDir.getFileName() + ".jar");
940 Path manifestPath = bundleDir.resolve("META-INF/MANIFEST.MF");
941 Manifest manifest;
942 try (InputStream in = Files.newInputStream(manifestPath)) {
943 manifest = new Manifest(in);
944 }
945 try (JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(jarPath), manifest)) {
946 jarOut.setLevel(Deflater.DEFAULT_COMPRESSION);
947 Files.walkFileTree(bundleDir, new SimpleFileVisitor<Path>() {
948
949 @Override
950 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
951 if (file.getFileName().toString().equals("MANIFEST.MF"))
952 return super.visitFile(file, attrs);
953 JarEntry entry = new JarEntry(bundleDir.relativize(file).toString());
954 jarOut.putNextEntry(entry);
955 Files.copy(file, jarOut);
956 return super.visitFile(file, attrs);
957 }
958
959 });
960 }
961 deleteDirectory(bundleDir);
962 return jarPath;
963 }
964
965 enum ManifestConstants {
966 // OSGi
967 BUNDLE_SYMBOLICNAME("Bundle-SymbolicName"), //
968 BUNDLE_VERSION("Bundle-Version"), //
969 BUNDLE_LICENSE("Bundle-License"), //
970 EXPORT_PACKAGE("Export-Package"), //
971 IMPORT_PACKAGE("Import-Package"), //
972 // JAVA
973 AUTOMATIC_MODULE_NAME("Automatic-Module-Name"), //
974 // SLC
975 SLC_CATEGORY("SLC-Category"), //
976 SLC_ORIGIN_M2("SLC-Origin-M2"), //
977 SLC_ORIGIN_M2_MERGE("SLC-Origin-M2-Merge"), //
978 SLC_ORIGIN_M2_REPO("SLC-Origin-M2-Repo"), //
979 SLC_ORIGIN_MANIFEST_NOT_MODIFIED("SLC-Origin-ManifestNotModified"), //
980 SLC_ORIGIN_URI("SLC-Origin-URI"),//
981 ;
982
983 final String value;
984
985 private ManifestConstants(String value) {
986 this.value = value;
987 }
988
989 @Override
990 public String toString() {
991 return value;
992 }
993
994 }
995
996 }
997
998 /** Simple representation of an M2 artifact. */
999 class M2Artifact extends CategoryNameVersion {
1000 private String classifier;
1001
1002 M2Artifact(String m2coordinates) {
1003 this(m2coordinates, null);
1004 }
1005
1006 M2Artifact(String m2coordinates, String classifier) {
1007 String[] parts = m2coordinates.split(":");
1008 setCategory(parts[0]);
1009 setName(parts[1]);
1010 if (parts.length > 2) {
1011 setVersion(parts[2]);
1012 }
1013 this.classifier = classifier;
1014 }
1015
1016 String getGroupId() {
1017 return super.getCategory();
1018 }
1019
1020 String getArtifactId() {
1021 return super.getName();
1022 }
1023
1024 String toM2Coordinates() {
1025 return getCategory() + ":" + getName() + (getVersion() != null ? ":" + getVersion() : "");
1026 }
1027
1028 String getClassifier() {
1029 return classifier != null ? classifier : "";
1030 }
1031
1032 String getExtension() {
1033 return "jar";
1034 }
1035 }
1036
1037 /** Utilities around Maven (conventions based). */
1038 class M2ConventionsUtils {
1039 final static String MAVEN_CENTRAL_BASE_URL = "https://repo1.maven.org/maven2/";
1040
1041 /** The file name of this artifact when stored */
1042 static String artifactFileName(M2Artifact artifact) {
1043 return artifact.getArtifactId() + '-' + artifact.getVersion()
1044 + (artifact.getClassifier().equals("") ? "" : '-' + artifact.getClassifier()) + '.'
1045 + artifact.getExtension();
1046 }
1047
1048 /** Absolute path to the file */
1049 static String artifactPath(String artifactBasePath, M2Artifact artifact) {
1050 return artifactParentPath(artifactBasePath, artifact) + '/' + artifactFileName(artifact);
1051 }
1052
1053 /** Absolute path to the file */
1054 static String artifactUrl(String repoUrl, M2Artifact artifact) {
1055 if (repoUrl.endsWith("/"))
1056 return repoUrl + artifactPath("/", artifact).substring(1);
1057 else
1058 return repoUrl + artifactPath("/", artifact);
1059 }
1060
1061 /** Absolute path to the file */
1062 static URL mavenRepoUrl(String repoBase, M2Artifact artifact) {
1063 String url = artifactUrl(repoBase == null ? MAVEN_CENTRAL_BASE_URL : repoBase, artifact);
1064 try {
1065 return new URL(url);
1066 } catch (MalformedURLException e) {
1067 // it should not happen
1068 throw new IllegalStateException(e);
1069 }
1070 }
1071
1072 /** Absolute path to the directories where the files will be stored */
1073 static String artifactParentPath(String artifactBasePath, M2Artifact artifact) {
1074 return artifactBasePath + (artifactBasePath.endsWith("/") ? "" : "/") + artifactParentPath(artifact);
1075 }
1076
1077 /** Relative path to the directories where the files will be stored */
1078 static String artifactParentPath(M2Artifact artifact) {
1079 return artifact.getGroupId().replace('.', '/') + '/' + artifact.getArtifactId() + '/' + artifact.getVersion();
1080 }
1081
1082 /** Singleton */
1083 private M2ConventionsUtils() {
1084 }
1085 }
1086
1087 class CategoryNameVersion extends NameVersion {
1088 private String category;
1089
1090 CategoryNameVersion() {
1091 }
1092
1093 CategoryNameVersion(String category, String name, String version) {
1094 super(name, version);
1095 this.category = category;
1096 }
1097
1098 CategoryNameVersion(String category, NameVersion nameVersion) {
1099 super(nameVersion);
1100 this.category = category;
1101 }
1102
1103 String getCategory() {
1104 return category;
1105 }
1106
1107 void setCategory(String category) {
1108 this.category = category;
1109 }
1110
1111 @Override
1112 public String toString() {
1113 return category + ":" + super.toString();
1114 }
1115
1116 }
1117
1118 class NameVersion implements Comparable<NameVersion> {
1119 private String name;
1120 private String version;
1121
1122 NameVersion() {
1123 }
1124
1125 /** Interprets string in OSGi-like format my.module.name;version=0.0.0 */
1126 NameVersion(String nameVersion) {
1127 int index = nameVersion.indexOf(";version=");
1128 if (index < 0) {
1129 setName(nameVersion);
1130 setVersion(null);
1131 } else {
1132 setName(nameVersion.substring(0, index));
1133 setVersion(nameVersion.substring(index + ";version=".length()));
1134 }
1135 }
1136
1137 NameVersion(String name, String version) {
1138 this.name = name;
1139 this.version = version;
1140 }
1141
1142 NameVersion(NameVersion nameVersion) {
1143 this.name = nameVersion.getName();
1144 this.version = nameVersion.getVersion();
1145 }
1146
1147 String getName() {
1148 return name;
1149 }
1150
1151 void setName(String name) {
1152 this.name = name;
1153 }
1154
1155 String getVersion() {
1156 return version;
1157 }
1158
1159 void setVersion(String version) {
1160 this.version = version;
1161 }
1162
1163 String getBranch() {
1164 String[] parts = getVersion().split("\\.");
1165 if (parts.length < 2)
1166 throw new IllegalStateException("Version " + getVersion() + " cannot be interpreted as branch.");
1167 return parts[0] + "." + parts[1];
1168 }
1169
1170 @Override
1171 public boolean equals(Object obj) {
1172 if (obj instanceof NameVersion) {
1173 NameVersion nameVersion = (NameVersion) obj;
1174 return name.equals(nameVersion.getName()) && version.equals(nameVersion.getVersion());
1175 } else
1176 return false;
1177 }
1178
1179 @Override
1180 public int hashCode() {
1181 return name.hashCode();
1182 }
1183
1184 @Override
1185 public String toString() {
1186 return name + ":" + version;
1187 }
1188
1189 public int compareTo(NameVersion o) {
1190 if (o.getName().equals(name))
1191 return version.compareTo(o.getVersion());
1192 else
1193 return name.compareTo(o.getName());
1194 }
1195 }