]> git.argeo.org Git - gpl/argeo-slc.git/blob - runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoUtils.java
Main free licences
[gpl/argeo-slc.git] / runtime / org.argeo.slc.repo / src / main / java / org / argeo / slc / repo / RepoUtils.java
1 /*
2 * Copyright (C) 2007-2012 Argeo GmbH
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package org.argeo.slc.repo;
17
18 import java.io.ByteArrayOutputStream;
19 import java.io.File;
20 import java.io.FileInputStream;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.util.Enumeration;
25 import java.util.Iterator;
26 import java.util.Set;
27 import java.util.StringTokenizer;
28 import java.util.TreeSet;
29 import java.util.jar.Attributes;
30 import java.util.jar.JarEntry;
31 import java.util.jar.JarFile;
32 import java.util.jar.JarInputStream;
33 import java.util.jar.JarOutputStream;
34 import java.util.jar.Manifest;
35 import java.util.zip.ZipInputStream;
36
37 import javax.jcr.Credentials;
38 import javax.jcr.GuestCredentials;
39 import javax.jcr.Node;
40 import javax.jcr.NodeIterator;
41 import javax.jcr.Property;
42 import javax.jcr.PropertyIterator;
43 import javax.jcr.Repository;
44 import javax.jcr.RepositoryException;
45 import javax.jcr.RepositoryFactory;
46 import javax.jcr.Session;
47 import javax.jcr.SimpleCredentials;
48 import javax.jcr.nodetype.NodeType;
49
50 import org.apache.commons.io.FilenameUtils;
51 import org.apache.commons.io.IOUtils;
52 import org.apache.commons.logging.Log;
53 import org.apache.commons.logging.LogFactory;
54 import org.argeo.ArgeoMonitor;
55 import org.argeo.jcr.ArgeoJcrUtils;
56 import org.argeo.jcr.ArgeoNames;
57 import org.argeo.jcr.ArgeoTypes;
58 import org.argeo.jcr.JcrUtils;
59 import org.argeo.slc.DefaultNameVersion;
60 import org.argeo.slc.NameVersion;
61 import org.argeo.slc.SlcException;
62 import org.argeo.slc.aether.ArtifactIdComparator;
63 import org.argeo.slc.jcr.SlcNames;
64 import org.argeo.slc.jcr.SlcTypes;
65 import org.argeo.slc.repo.maven.MavenConventionsUtils;
66 import org.argeo.util.security.Keyring;
67 import org.osgi.framework.Constants;
68 import org.sonatype.aether.artifact.Artifact;
69 import org.sonatype.aether.util.artifact.DefaultArtifact;
70
71 /** Utilities around repo */
72 public class RepoUtils implements ArgeoNames, SlcNames {
73 private final static Log log = LogFactory.getLog(RepoUtils.class);
74
75 /** Packages a regular sources jar as PDE source. */
76 public static void packagesAsPdeSource(File sourceFile,
77 NameVersion nameVersion, OutputStream out) throws IOException {
78 if (isAlreadyPdeSource(sourceFile)) {
79 FileInputStream in = new FileInputStream(sourceFile);
80 IOUtils.copy(in, out);
81 IOUtils.closeQuietly(in);
82 } else {
83 String sourceSymbolicName = nameVersion.getName() + ".source";
84
85 Manifest sourceManifest = null;
86 sourceManifest = new Manifest();
87 sourceManifest.getMainAttributes().put(
88 Attributes.Name.MANIFEST_VERSION, "1.0");
89 sourceManifest.getMainAttributes().putValue("Bundle-SymbolicName",
90 sourceSymbolicName);
91 sourceManifest.getMainAttributes().putValue("Bundle-Version",
92 nameVersion.getVersion());
93 sourceManifest.getMainAttributes().putValue(
94 "Eclipse-SourceBundle",
95 nameVersion.getName() + ";version="
96 + nameVersion.getVersion());
97 copyJar(sourceFile, out, sourceManifest);
98 }
99 }
100
101 public static byte[] packageAsPdeSource(InputStream sourceJar,
102 NameVersion nameVersion) {
103 String sourceSymbolicName = nameVersion.getName() + ".source";
104
105 Manifest sourceManifest = null;
106 sourceManifest = new Manifest();
107 sourceManifest.getMainAttributes().put(
108 Attributes.Name.MANIFEST_VERSION, "1.0");
109 sourceManifest.getMainAttributes().putValue("Bundle-SymbolicName",
110 sourceSymbolicName);
111 sourceManifest.getMainAttributes().putValue("Bundle-Version",
112 nameVersion.getVersion());
113 sourceManifest.getMainAttributes().putValue("Eclipse-SourceBundle",
114 nameVersion.getName() + ";version=" + nameVersion.getVersion());
115
116 return modifyManifest(sourceJar, sourceManifest);
117 }
118
119 /**
120 * Check whether the file as already been packaged as PDE source, in order
121 * not to mess with Jar signing
122 */
123 private static boolean isAlreadyPdeSource(File sourceFile) {
124 JarInputStream jarInputStream = null;
125
126 try {
127 jarInputStream = new JarInputStream(new FileInputStream(sourceFile));
128
129 Manifest manifest = jarInputStream.getManifest();
130 Iterator<?> it = manifest.getMainAttributes().keySet().iterator();
131 boolean res = false;
132 // containsKey() does not work, iterating...
133 while (it.hasNext())
134 if (it.next().toString().equals("Eclipse-SourceBundle")) {
135 res = true;
136 break;
137 }
138 // boolean res = manifest.getMainAttributes().get(
139 // "Eclipse-SourceBundle") != null;
140 if (res)
141 log.info(sourceFile + " is already a PDE source");
142 return res;
143 } catch (Exception e) {
144 // probably not a jar, skipping
145 if (log.isDebugEnabled())
146 log.debug("Skipping " + sourceFile + " because of "
147 + e.getMessage());
148 return false;
149 } finally {
150 IOUtils.closeQuietly(jarInputStream);
151 }
152 }
153
154 /**
155 * Copy a jar, replacing its manifest with the provided one
156 *
157 * @param manifest
158 * can be null
159 */
160 private static void copyJar(File source, OutputStream out, Manifest manifest)
161 throws IOException {
162 JarFile sourceJar = null;
163 JarOutputStream output = null;
164 try {
165 output = manifest != null ? new JarOutputStream(out, manifest)
166 : new JarOutputStream(out);
167 sourceJar = new JarFile(source);
168
169 entries: for (Enumeration<?> entries = sourceJar.entries(); entries
170 .hasMoreElements();) {
171 JarEntry entry = (JarEntry) entries.nextElement();
172 if (manifest != null
173 && entry.getName().equals("META-INF/MANIFEST.MF"))
174 continue entries;
175
176 InputStream entryStream = sourceJar.getInputStream(entry);
177 JarEntry newEntry = new JarEntry(entry.getName());
178 // newEntry.setMethod(JarEntry.DEFLATED);
179 output.putNextEntry(newEntry);
180 IOUtils.copy(entryStream, output);
181 }
182 } finally {
183 IOUtils.closeQuietly(output);
184 try {
185 if (sourceJar != null)
186 sourceJar.close();
187 } catch (IOException e) {
188 // silent
189 }
190 }
191 }
192
193 /** Copy a jar changing onlythe manifest */
194 public static void copyJar(InputStream in, OutputStream out,
195 Manifest manifest) {
196 JarInputStream jarIn = null;
197 JarOutputStream jarOut = null;
198 try {
199 jarIn = new JarInputStream(in);
200 jarOut = new JarOutputStream(out, manifest);
201 JarEntry jarEntry = null;
202 while ((jarEntry = jarIn.getNextJarEntry()) != null) {
203 jarOut.putNextEntry(jarEntry);
204 IOUtils.copy(jarIn, jarOut);
205 jarIn.closeEntry();
206 jarOut.closeEntry();
207 }
208 } catch (IOException e) {
209 throw new SlcException("Could not copy jar with MANIFEST "
210 + manifest.getMainAttributes(), e);
211 } finally {
212 if (!(in instanceof ZipInputStream))
213 IOUtils.closeQuietly(jarIn);
214 IOUtils.closeQuietly(jarOut);
215 }
216 }
217
218 /** Reads a jar file, modify its manifest */
219 public static byte[] modifyManifest(InputStream in, Manifest manifest) {
220 ByteArrayOutputStream out = new ByteArrayOutputStream(200 * 1024);
221 try {
222 copyJar(in, out, manifest);
223 return out.toByteArray();
224 } finally {
225 IOUtils.closeQuietly(out);
226 }
227 }
228
229 /** Read the OSGi {@link NameVersion} */
230 public static NameVersion readNameVersion(Artifact artifact) {
231 File artifactFile = artifact.getFile();
232 if (artifact.getExtension().equals("pom")) {
233 // hack to process jars which weirdly appear as POMs
234 File jarFile = new File(artifactFile.getParentFile(),
235 FilenameUtils.getBaseName(artifactFile.getPath()) + ".jar");
236 if (jarFile.exists()) {
237 log.warn("Use " + jarFile + " instead of " + artifactFile
238 + " for " + artifact);
239 artifactFile = jarFile;
240 }
241 }
242 return readNameVersion(artifactFile);
243 }
244
245 /** Read the OSGi {@link NameVersion} */
246 public static NameVersion readNameVersion(File artifactFile) {
247 try {
248 return readNameVersion(new FileInputStream(artifactFile));
249 } catch (Exception e) {
250 // probably not a jar, skipping
251 if (log.isDebugEnabled()) {
252 log.debug("Skipping " + artifactFile + " because of " + e);
253 // e.printStackTrace();
254 }
255 }
256 return null;
257 }
258
259 /** Read the OSGi {@link NameVersion} */
260 public static NameVersion readNameVersion(InputStream in) {
261 JarInputStream jarInputStream = null;
262 try {
263 jarInputStream = new JarInputStream(in);
264 return readNameVersion(jarInputStream.getManifest());
265 } catch (Exception e) {
266 // probably not a jar, skipping
267 if (log.isDebugEnabled()) {
268 log.debug("Skipping because of " + e);
269 e.printStackTrace();
270 }
271 } finally {
272 IOUtils.closeQuietly(jarInputStream);
273 }
274 return null;
275 }
276
277 /** Read the OSGi {@link NameVersion} */
278 public static NameVersion readNameVersion(Manifest manifest) {
279 DefaultNameVersion nameVersion = new DefaultNameVersion();
280 nameVersion.setName(manifest.getMainAttributes().getValue(
281 Constants.BUNDLE_SYMBOLICNAME));
282
283 // Skip additional specs such as
284 // ; singleton:=true
285 if (nameVersion.getName().indexOf(';') > -1) {
286 nameVersion
287 .setName(new StringTokenizer(nameVersion.getName(), " ;")
288 .nextToken());
289 }
290
291 nameVersion.setVersion(manifest.getMainAttributes().getValue(
292 Constants.BUNDLE_VERSION));
293
294 return nameVersion;
295 }
296
297 /*
298 * DATA MODEL
299 */
300 /** The artifact described by this node */
301 public static Artifact asArtifact(Node node) throws RepositoryException {
302 if (node.isNodeType(SlcTypes.SLC_ARTIFACT_VERSION_BASE)) {
303 // FIXME update data model to store packaging at this level
304 String extension = "jar";
305 return new DefaultArtifact(node.getProperty(SLC_GROUP_ID)
306 .getString(),
307 node.getProperty(SLC_ARTIFACT_ID).getString(), extension,
308 node.getProperty(SLC_ARTIFACT_VERSION).getString());
309 } else if (node.isNodeType(SlcTypes.SLC_ARTIFACT)) {
310 return new DefaultArtifact(node.getProperty(SLC_GROUP_ID)
311 .getString(),
312 node.getProperty(SLC_ARTIFACT_ID).getString(), node
313 .getProperty(SLC_ARTIFACT_CLASSIFIER).getString(),
314 node.getProperty(SLC_ARTIFACT_EXTENSION).getString(), node
315 .getProperty(SLC_ARTIFACT_VERSION).getString());
316 } else {
317 throw new SlcException("Unsupported node type for " + node);
318 }
319 }
320
321 /**
322 * The path to the PDE source related to this artifact (or artifact version
323 * base). There may or there may not be a node at this location (the
324 * returned path will typically be used to test whether PDE sources are
325 * attached to this artifact).
326 */
327 public static String relatedPdeSourcePath(String artifactBasePath,
328 Node artifactNode) throws RepositoryException {
329 Artifact artifact = asArtifact(artifactNode);
330 Artifact pdeSourceArtifact = new DefaultArtifact(artifact.getGroupId(),
331 artifact.getArtifactId() + ".source", artifact.getExtension(),
332 artifact.getVersion());
333 return MavenConventionsUtils.artifactPath(artifactBasePath,
334 pdeSourceArtifact);
335 }
336
337 /**
338 * Copy this bytes array as an artifact, relative to the root of the
339 * repository (typically the workspace root node)
340 */
341 public static Node copyBytesAsArtifact(Node artifactsBase,
342 Artifact artifact, byte[] bytes) throws RepositoryException {
343 String parentPath = MavenConventionsUtils.artifactParentPath(
344 artifactsBase.getPath(), artifact);
345 Node folderNode = JcrUtils.mkfolders(artifactsBase.getSession(),
346 parentPath);
347 return JcrUtils.copyBytesAsFile(folderNode,
348 MavenConventionsUtils.artifactFileName(artifact), bytes);
349 }
350
351 private RepoUtils() {
352 }
353
354 /** If a source return the base bundle name, does not change otherwise */
355 public static String extractBundleNameFromSourceName(String sourceBundleName) {
356 if (sourceBundleName.endsWith(".source"))
357 return sourceBundleName.substring(0, sourceBundleName.length()
358 - ".source".length());
359 else
360 return sourceBundleName;
361 }
362
363 /*
364 * SOFTWARE REPOSITORIES
365 */
366
367 /** Retrieve repository based on information in the repo node */
368 public static Repository getRepository(RepositoryFactory repositoryFactory,
369 Keyring keyring, Node repoNode) {
370 try {
371 Repository repository;
372 if (repoNode.isNodeType(ArgeoTypes.ARGEO_REMOTE_REPOSITORY)) {
373 String uri = repoNode.getProperty(ARGEO_URI).getString();
374 if (uri.startsWith("http")) {// http, https
375 repository = ArgeoJcrUtils.getRepositoryByUri(
376 repositoryFactory, uri);
377 } else if (uri.startsWith("vm:")) {// alias
378 repository = ArgeoJcrUtils.getRepositoryByUri(
379 repositoryFactory, uri);
380 } else {
381 throw new SlcException("Unsupported repository uri " + uri);
382 }
383 return repository;
384 } else {
385 throw new SlcException("Unsupported node type " + repoNode);
386 }
387 } catch (RepositoryException e) {
388 throw new SlcException("Cannot connect to repository " + repoNode,
389 e);
390 }
391 }
392
393 /**
394 * Reads credentials from node, using keyring if there is a password. Can
395 * return null if no credentials needed (local repo) at all, but returns
396 * {@link GuestCredentials} if user id is 'anonymous' .
397 */
398 public static Credentials getRepositoryCredentials(Keyring keyring,
399 Node repoNode) {
400 try {
401 if (repoNode.isNodeType(ArgeoTypes.ARGEO_REMOTE_REPOSITORY)) {
402 if (!repoNode.hasProperty(ARGEO_USER_ID))
403 return null;
404
405 String userId = repoNode.getProperty(ARGEO_USER_ID).getString();
406 if (userId.equals("anonymous"))// FIXME hardcoded userId
407 return new GuestCredentials();
408 char[] password = keyring.getAsChars(repoNode.getPath() + '/'
409 + ARGEO_PASSWORD);
410 Credentials credentials = new SimpleCredentials(userId,
411 password);
412 return credentials;
413 } else {
414 throw new SlcException("Unsupported node type " + repoNode);
415 }
416 } catch (RepositoryException e) {
417 throw new SlcException("Cannot connect to repository " + repoNode,
418 e);
419 }
420 }
421
422 /**
423 * Shortcut to retrieve a session given variable information: Handle the
424 * case where we only have an URI of the repository, that we want to connect
425 * as anonymous or the case of a identified connection to a local or remote
426 * repository.
427 *
428 * Callers must close the session once it has been used
429 */
430 public static Session getRemoteSession(RepositoryFactory repositoryFactory,
431 Keyring keyring, Node repoNode, String uri, String workspaceName) {
432 try {
433 if (repoNode == null && uri == null)
434 throw new SlcException(
435 "At least one of repoNode and uri must be defined");
436 Repository currRepo = null;
437 Credentials credentials = null;
438 // Anonymous URI only workspace
439 if (repoNode == null)
440 // Anonymous
441 currRepo = ArgeoJcrUtils.getRepositoryByUri(repositoryFactory,
442 uri);
443 else {
444 currRepo = RepoUtils.getRepository(repositoryFactory, keyring,
445 repoNode);
446 credentials = RepoUtils.getRepositoryCredentials(keyring,
447 repoNode);
448 }
449 return currRepo.login(credentials, workspaceName);
450 } catch (RepositoryException e) {
451 throw new SlcException("Cannot connect to workspace "
452 + workspaceName + " of repository " + repoNode
453 + " with URI " + uri, e);
454 }
455 }
456
457 /**
458 * Shortcut to retrieve a session on a remote Jrc Repository from
459 * information stored in a local argeo node or from an URI: Handle the case
460 * where we only have an URI of the repository, that we want to connect as
461 * anonymous or the case of a identified connection to a local or remote
462 * repository.
463 *
464 * Callers must close the session once it has been used
465 */
466 public static Session getRemoteSession(RepositoryFactory repositoryFactory,
467 Keyring keyring, Repository localRepository, String repoNodePath,
468 String uri, String workspaceName) {
469 Session localSession = null;
470 Node repoNode = null;
471 try {
472 localSession = localRepository.login();
473 if (repoNodePath != null && localSession.nodeExists(repoNodePath))
474 repoNode = localSession.getNode(repoNodePath);
475
476 return RepoUtils.getRemoteSession(repositoryFactory, keyring,
477 repoNode, uri, workspaceName);
478 } catch (RepositoryException e) {
479 throw new SlcException("Cannot log to workspace " + workspaceName
480 + " for repo defined in " + repoNodePath, e);
481 } finally {
482 JcrUtils.logoutQuietly(localSession);
483 }
484 }
485
486 /**
487 * Write group indexes: 'binaries' lists all bundles and their versions,
488 * 'sources' list their sources, and 'sdk' aggregates both.
489 */
490 public static void writeGroupIndexes(Session session,
491 String artifactBasePath, String groupId, String version,
492 Set<Artifact> binaries, Set<Artifact> sources) {
493 try {
494 Set<Artifact> indexes = new TreeSet<Artifact>(
495 new ArtifactIdComparator());
496 Artifact binariesArtifact = writeIndex(session, artifactBasePath,
497 groupId, RepoConstants.BINARIES_ARTIFACT_ID, version,
498 binaries);
499 indexes.add(binariesArtifact);
500 if (sources != null) {
501 Artifact sourcesArtifact = writeIndex(session,
502 artifactBasePath, groupId,
503 RepoConstants.SOURCES_ARTIFACT_ID, version, sources);
504 indexes.add(sourcesArtifact);
505 }
506 // sdk
507 writeIndex(session, artifactBasePath, groupId,
508 RepoConstants.SDK_ARTIFACT_ID, version, indexes);
509 session.save();
510 } catch (RepositoryException e) {
511 throw new SlcException("Cannot write indexes for group " + groupId,
512 e);
513 }
514 }
515
516 /** Write a group index. */
517 private static Artifact writeIndex(Session session,
518 String artifactBasePath, String groupId, String artifactId,
519 String version, Set<Artifact> artifacts) throws RepositoryException {
520 Artifact artifact = new DefaultArtifact(groupId, artifactId, "pom",
521 version);
522 String pom = MavenConventionsUtils.artifactsAsDependencyPom(artifact,
523 artifacts, null);
524 Node node = RepoUtils.copyBytesAsArtifact(
525 session.getNode(artifactBasePath), artifact, pom.getBytes());
526 addMavenChecksums(node);
527 return artifact;
528 }
529
530 /** Add files containing the SHA-1 and MD5 checksums. */
531 public static void addMavenChecksums(Node node) throws RepositoryException {
532 // TODO optimize
533 String sha = JcrUtils.checksumFile(node, "SHA-1");
534 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".sha1",
535 sha.getBytes());
536 String md5 = JcrUtils.checksumFile(node, "MD5");
537 JcrUtils.copyBytesAsFile(node.getParent(), node.getName() + ".md5",
538 md5.getBytes());
539 }
540
541 /**
542 * Custom copy since the one in commons does not fit the needs when copying
543 * a workspace completely.
544 */
545 public static void copy(Node fromNode, Node toNode) {
546 copy(fromNode, toNode, null);
547 }
548
549 public static void copy(Node fromNode, Node toNode, ArgeoMonitor monitor) {
550 try {
551 String fromPath = fromNode.getPath();
552 if (monitor != null)
553 monitor.subTask("copying node :" + fromPath);
554 if (log.isDebugEnabled())
555 log.debug("copy node :" + fromPath);
556
557 // FIXME : small hack to enable specific workspace copy
558 if (fromNode.isNodeType("rep:ACL")
559 || fromNode.isNodeType("rep:system")) {
560 if (log.isTraceEnabled())
561 log.trace("node " + fromNode + " skipped");
562 return;
563 }
564
565 // add mixins
566 for (NodeType mixinType : fromNode.getMixinNodeTypes()) {
567 toNode.addMixin(mixinType.getName());
568 }
569
570 // Double check
571 for (NodeType mixinType : toNode.getMixinNodeTypes()) {
572 if (log.isDebugEnabled())
573 log.debug(mixinType.getName());
574 }
575
576 // process properties
577 PropertyIterator pit = fromNode.getProperties();
578 properties: while (pit.hasNext()) {
579 Property fromProperty = pit.nextProperty();
580 String propName = fromProperty.getName();
581 try {
582 String propertyName = fromProperty.getName();
583 if (toNode.hasProperty(propertyName)
584 && toNode.getProperty(propertyName).getDefinition()
585 .isProtected())
586 continue properties;
587
588 if (fromProperty.getDefinition().isProtected())
589 continue properties;
590
591 if (propertyName.equals("jcr:created")
592 || propertyName.equals("jcr:createdBy")
593 || propertyName.equals("jcr:lastModified")
594 || propertyName.equals("jcr:lastModifiedBy"))
595 continue properties;
596
597 if (fromProperty.isMultiple()) {
598 toNode.setProperty(propertyName,
599 fromProperty.getValues());
600 } else {
601 toNode.setProperty(propertyName,
602 fromProperty.getValue());
603 }
604 } catch (RepositoryException e) {
605 throw new SlcException("Cannot property " + propName, e);
606 }
607 }
608
609 // recursively process children nodes
610 NodeIterator nit = fromNode.getNodes();
611 while (nit.hasNext()) {
612 Node fromChild = nit.nextNode();
613 Integer index = fromChild.getIndex();
614 String nodeRelPath = fromChild.getName() + "[" + index + "]";
615 Node toChild;
616 if (toNode.hasNode(nodeRelPath))
617 toChild = toNode.getNode(nodeRelPath);
618 else
619 toChild = toNode.addNode(fromChild.getName(), fromChild
620 .getPrimaryNodeType().getName());
621 copy(fromChild, toChild);
622 }
623
624 // update jcr:lastModified and jcr:lastModifiedBy in toNode in
625 // case
626 // they existed
627 if (!toNode.getDefinition().isProtected()
628 && toNode.isNodeType(NodeType.MIX_LAST_MODIFIED))
629 JcrUtils.updateLastModified(toNode);
630
631 // Workaround to reduce session size: artifact is a saveable
632 // unity
633 if (toNode.isNodeType(SlcTypes.SLC_ARTIFACT))
634 toNode.getSession().save();
635
636 if (monitor != null)
637 monitor.worked(1);
638
639 } catch (RepositoryException e) {
640 throw new SlcException("Cannot copy " + fromNode + " to " + toNode,
641 e);
642 }
643 }
644
645 }