]> git.argeo.org Git - gpl/argeo-slc.git/blob - org.argeo.slc.repo/src/org/argeo/slc/repo/RepoSync.java
Adapt to changes in Argeo Commons
[gpl/argeo-slc.git] / org.argeo.slc.repo / src / org / argeo / slc / repo / RepoSync.java
1 package org.argeo.slc.repo;
2
3 import java.io.InputStream;
4 import java.util.Arrays;
5 import java.util.Calendar;
6 import java.util.GregorianCalendar;
7 import java.util.HashMap;
8 import java.util.List;
9 import java.util.Map;
10 import java.util.TimeZone;
11
12 import javax.jcr.Binary;
13 import javax.jcr.Credentials;
14 import javax.jcr.NoSuchWorkspaceException;
15 import javax.jcr.Node;
16 import javax.jcr.NodeIterator;
17 import javax.jcr.Property;
18 import javax.jcr.PropertyIterator;
19 import javax.jcr.PropertyType;
20 import javax.jcr.Repository;
21 import javax.jcr.RepositoryException;
22 import javax.jcr.RepositoryFactory;
23 import javax.jcr.Session;
24 import javax.jcr.SimpleCredentials;
25 import javax.jcr.nodetype.NodeType;
26 import javax.jcr.query.Query;
27 import javax.jcr.query.QueryResult;
28
29 import org.apache.commons.io.IOUtils;
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.argeo.jcr.JcrMonitor;
33 import org.argeo.jcr.JcrUtils;
34 import org.argeo.api.NodeUtils;
35 import org.argeo.slc.SlcException;
36 import org.xml.sax.SAXException;
37
38 /**
39 * Synchronise workspaces from a remote software repository to the local
40 * repository (Synchronisation in the other direction does not work).
41 *
42 * Workspaces are retrieved by name given a map that links the source with a
43 * target name. If a target workspace does not exist, it is created. Otherwise
44 * we copy the content of the source workspace into the target one.
45 */
46 public class RepoSync implements Runnable {
47 private final static Log log = LogFactory.getLog(RepoSync.class);
48
49 // Centralizes definition of workspaces that must be ignored by the sync.
50 private final static List<String> IGNORED_WKSP_LIST = Arrays.asList("security", "localrepo");
51
52 private final Calendar zero;
53 private Session sourceDefaultSession = null;
54 private Session targetDefaultSession = null;
55
56 private Repository sourceRepository;
57 private Credentials sourceCredentials;
58 private Repository targetRepository;
59 private Credentials targetCredentials;
60
61 // if Repository and Credentials objects are not explicitly set
62 private String sourceRepoUri;
63 private String sourceUsername;
64 private char[] sourcePassword;
65 private String targetRepoUri;
66 private String targetUsername;
67 private char[] targetPassword;
68
69 private RepositoryFactory repositoryFactory;
70
71 private JcrMonitor monitor;
72 private Map<String, String> workspaceMap;
73
74 // TODO fix monitor
75 private Boolean filesOnly = false;
76
77 public RepoSync() {
78 zero = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
79 zero.setTimeInMillis(0);
80 }
81
82 /**
83 *
84 * Shortcut to instantiate a RepoSync with already known repositories and
85 * credentials.
86 *
87 * @param sourceRepository
88 * @param sourceCredentials
89 * @param targetRepository
90 * @param targetCredentials
91 */
92 public RepoSync(Repository sourceRepository, Credentials sourceCredentials, Repository targetRepository,
93 Credentials targetCredentials) {
94 this();
95 this.sourceRepository = sourceRepository;
96 this.sourceCredentials = sourceCredentials;
97 this.targetRepository = targetRepository;
98 this.targetCredentials = targetCredentials;
99 }
100
101 public void run() {
102 try {
103 long begin = System.currentTimeMillis();
104
105 // Setup
106 if (sourceRepository == null)
107 sourceRepository = NodeUtils.getRepositoryByUri(repositoryFactory, sourceRepoUri);
108 if (sourceCredentials == null && sourceUsername != null)
109 sourceCredentials = new SimpleCredentials(sourceUsername, sourcePassword);
110 // FIXME make it more generic
111 sourceDefaultSession = sourceRepository.login(sourceCredentials, RepoConstants.DEFAULT_DEFAULT_WORKSPACE);
112
113 if (targetRepository == null)
114 targetRepository = NodeUtils.getRepositoryByUri(repositoryFactory, targetRepoUri);
115 if (targetCredentials == null && targetUsername != null)
116 targetCredentials = new SimpleCredentials(targetUsername, targetPassword);
117 targetDefaultSession = targetRepository.login(targetCredentials);
118
119 Map<String, Exception> errors = new HashMap<String, Exception>();
120 for (String sourceWorkspaceName : sourceDefaultSession.getWorkspace().getAccessibleWorkspaceNames()) {
121 if (monitor != null && monitor.isCanceled())
122 break;
123
124 if (workspaceMap != null && !workspaceMap.containsKey(sourceWorkspaceName))
125 continue;
126 if (IGNORED_WKSP_LIST.contains(sourceWorkspaceName))
127 continue;
128
129 Session sourceSession = null;
130 Session targetSession = null;
131 String targetWorkspaceName = workspaceMap.get(sourceWorkspaceName);
132 try {
133 try {
134 targetSession = targetRepository.login(targetCredentials, targetWorkspaceName);
135 } catch (NoSuchWorkspaceException e) {
136 targetDefaultSession.getWorkspace().createWorkspace(targetWorkspaceName);
137 targetSession = targetRepository.login(targetCredentials, targetWorkspaceName);
138 }
139 sourceSession = sourceRepository.login(sourceCredentials, sourceWorkspaceName);
140 syncWorkspace(sourceSession, targetSession);
141 } catch (Exception e) {
142 errors.put("Could not sync workspace " + sourceWorkspaceName, e);
143 if (log.isErrorEnabled())
144 e.printStackTrace();
145
146 } finally {
147 JcrUtils.logoutQuietly(sourceSession);
148 JcrUtils.logoutQuietly(targetSession);
149 }
150 }
151
152 if (monitor != null && monitor.isCanceled())
153 log.info("Sync has been canceled by user");
154
155 long duration = (System.currentTimeMillis() - begin) / 1000;// s
156 log.info("Sync " + sourceRepoUri + " to " + targetRepoUri + " in " + (duration / 60)
157
158 + "min " + (duration % 60) + "s");
159
160 if (errors.size() > 0) {
161 throw new SlcException("Sync failed " + errors);
162 }
163 } catch (RepositoryException e) {
164 throw new SlcException("Cannot sync " + sourceRepoUri + " to " + targetRepoUri, e);
165 } finally {
166 JcrUtils.logoutQuietly(sourceDefaultSession);
167 JcrUtils.logoutQuietly(targetDefaultSession);
168 }
169 }
170
171 private long getNodesNumber(Session session) {
172 if (IGNORED_WKSP_LIST.contains(session.getWorkspace().getName()))
173 return 0l;
174 try {
175 Query countQuery = session.getWorkspace().getQueryManager().createQuery(
176 "select file from [" + (true ? NodeType.NT_FILE : NodeType.NT_BASE) + "] as file", Query.JCR_SQL2);
177
178 QueryResult result = countQuery.execute();
179 Long expectedCount = result.getNodes().getSize();
180 return expectedCount;
181 } catch (RepositoryException e) {
182 throw new SlcException("Unexpected error while computing " + "the size of the fetch for workspace "
183 + session.getWorkspace().getName(), e);
184 }
185 }
186
187 protected void syncWorkspace(Session sourceSession, Session targetSession) {
188 if (monitor != null) {
189 monitor.beginTask("Computing fetch size...", -1);
190 Long totalAmount = getNodesNumber(sourceSession);
191 monitor.beginTask("Fetch", totalAmount.intValue());
192 }
193
194 try {
195 String msg = "Synchronizing workspace: " + sourceSession.getWorkspace().getName();
196 if (monitor != null)
197 monitor.setTaskName(msg);
198 if (log.isDebugEnabled())
199 log.debug(msg);
200
201 for (NodeIterator it = sourceSession.getRootNode().getNodes(); it.hasNext();) {
202 Node node = it.nextNode();
203 if (node.getName().contains(":"))
204 continue;
205 if (node.getName().equals("download"))
206 continue;
207 if (!node.isNodeType(NodeType.NT_HIERARCHY_NODE))
208 continue;
209 syncNode(node, targetSession);
210 }
211 // if (filesOnly) {
212 // JcrUtils.copyFiles(sourceSession.getRootNode(), targetSession.getRootNode(),
213 // true, monitor);
214 // } else {
215 // for (NodeIterator it = sourceSession.getRootNode().getNodes(); it.hasNext();)
216 // {
217 // Node node = it.nextNode();
218 // if (node.getName().equals("jcr:system"))
219 // continue;
220 // syncNode(node, targetSession);
221 // }
222 // }
223 if (log.isDebugEnabled())
224 log.debug("Synced " + sourceSession.getWorkspace().getName());
225 } catch (Exception e) {
226 e.printStackTrace();
227 throw new SlcException("Cannot sync " + sourceSession.getWorkspace().getName() + " to "
228 + targetSession.getWorkspace().getName(), e);
229 }
230 }
231
232 /** factorizes monitor management */
233 private void updateMonitor(String msg) {
234 updateMonitor(msg, false);
235 }
236
237 protected void syncNode(Node sourceNode, Session targetSession) throws RepositoryException, SAXException {
238 if (filesOnly) {
239 Node targetNode;
240 if (targetSession.itemExists(sourceNode.getPath()))
241 targetNode = targetSession.getNode(sourceNode.getPath());
242 else
243 targetNode = JcrUtils.mkdirs(targetSession, sourceNode.getPath(), NodeType.NT_FOLDER);
244 JcrUtils.copyFiles(sourceNode, targetNode, true, monitor, true);
245 return;
246 }
247 // Boolean singleLevel = singleLevel(sourceNode);
248 try {
249 if (monitor != null && monitor.isCanceled()) {
250 updateMonitor("Fetched has been canceled, " + "process is terminating");
251 return;
252 }
253
254 Node targetParentNode = targetSession.getNode(sourceNode.getParent().getPath());
255 Node targetNode;
256 if (monitor != null && sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE))
257 monitor.subTask("Process " + sourceNode.getPath());
258
259 final Boolean isNew;
260 if (!targetSession.itemExists(sourceNode.getPath())) {
261 isNew = true;
262 targetNode = targetParentNode.addNode(sourceNode.getName(), sourceNode.getPrimaryNodeType().getName());
263 } else {
264 isNew = false;
265 targetNode = targetSession.getNode(sourceNode.getPath());
266 if (!targetNode.getPrimaryNodeType().getName().equals(sourceNode.getPrimaryNodeType().getName()))
267 targetNode.setPrimaryType(sourceNode.getPrimaryNodeType().getName());
268 }
269
270 // export
271 // sourceNode.getSession().exportSystemView(sourceNode.getPath(),
272 // contentHandler, false, singleLevel);
273
274 // if (singleLevel) {
275 // if (targetSession.hasPendingChanges()) {
276 // // updateMonitor(
277 // // (isNew ? "Added " : "Updated ") + targetNode.getPath(),
278 // // true);
279 // if (doSave)
280 // targetSession.save();
281 // } else {
282 // // updateMonitor("Checked " + targetNode.getPath(), false);
283 // }
284 // }
285
286 // mixin and properties
287 for (NodeType nt : sourceNode.getMixinNodeTypes()) {
288 if (!targetNode.isNodeType(nt.getName()) && targetNode.canAddMixin(nt.getName()))
289 targetNode.addMixin(nt.getName());
290 }
291 copyProperties(sourceNode, targetNode);
292
293 // next level
294 NodeIterator ni = sourceNode.getNodes();
295 while (ni != null && ni.hasNext()) {
296 Node sourceChild = ni.nextNode();
297 syncNode(sourceChild, targetSession);
298 }
299
300 copyTimestamps(sourceNode, targetNode);
301
302 if (sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE)) {
303 if (targetSession.hasPendingChanges()) {
304 if (sourceNode.isNodeType(NodeType.NT_FILE))
305 updateMonitor((isNew ? "Added " : "Updated ") + targetNode.getPath(), true);
306 // if (doSave)
307 targetSession.save();
308 } else {
309 if (sourceNode.isNodeType(NodeType.NT_FILE))
310 updateMonitor("Checked " + targetNode.getPath(), false);
311 }
312 }
313 } catch (RepositoryException e) {
314 throw new SlcException("Cannot sync source node " + sourceNode, e);
315 }
316 }
317
318 private void copyTimestamps(Node sourceNode, Node targetNode) throws RepositoryException {
319 if (sourceNode.getDefinition().isProtected())
320 return;
321 if (targetNode.getDefinition().isProtected())
322 return;
323 copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED);
324 copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED_BY);
325 copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED);
326 copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED_BY);
327 }
328
329 private void copyTimestamp(Node sourceNode, Node targetNode, String property) throws RepositoryException {
330 if (sourceNode.hasProperty(property)) {
331 Property p = sourceNode.getProperty(property);
332 if (p.getDefinition().isProtected())
333 return;
334 if (targetNode.hasProperty(property)
335 && targetNode.getProperty(property).getValue().equals(sourceNode.getProperty(property).getValue()))
336 return;
337 targetNode.setProperty(property, sourceNode.getProperty(property).getValue());
338 }
339 }
340
341 private void copyProperties(Node sourceNode, Node targetNode) throws RepositoryException {
342 properties: for (PropertyIterator pi = sourceNode.getProperties(); pi.hasNext();) {
343 Property p = pi.nextProperty();
344 if (p.getDefinition().isProtected())
345 continue properties;
346 if (p.getName().equals(Property.JCR_CREATED) || p.getName().equals(Property.JCR_CREATED_BY)
347 || p.getName().equals(Property.JCR_LAST_MODIFIED)
348 || p.getName().equals(Property.JCR_LAST_MODIFIED_BY))
349 continue properties;
350
351 if (p.getType() == PropertyType.BINARY) {
352 copyBinary(p, targetNode);
353 } else {
354
355 if (p.isMultiple()) {
356 if (!targetNode.hasProperty(p.getName())
357 || !Arrays.equals(targetNode.getProperty(p.getName()).getValues(), p.getValues()))
358 targetNode.setProperty(p.getName(), p.getValues());
359 } else {
360 if (!targetNode.hasProperty(p.getName())
361 || !targetNode.getProperty(p.getName()).getValue().equals(p.getValue()))
362 targetNode.setProperty(p.getName(), p.getValue());
363 }
364 }
365 }
366 }
367
368 private static void copyBinary(Property p, Node targetNode) throws RepositoryException {
369 InputStream in = null;
370 Binary sourceBinary = null;
371 Binary targetBinary = null;
372 try {
373 sourceBinary = p.getBinary();
374 if (targetNode.hasProperty(p.getName()))
375 targetBinary = targetNode.getProperty(p.getName()).getBinary();
376
377 // optim FIXME make it more configurable
378 if (targetBinary != null)
379 if (sourceBinary.getSize() == targetBinary.getSize()) {
380 if (log.isTraceEnabled())
381 log.trace("Skipped " + p.getPath());
382 return;
383 }
384
385 in = sourceBinary.getStream();
386 targetBinary = targetNode.getSession().getValueFactory().createBinary(in);
387 targetNode.setProperty(p.getName(), targetBinary);
388 } catch (Exception e) {
389 throw new SlcException("Could not transfer " + p, e);
390 } finally {
391 IOUtils.closeQuietly(in);
392 JcrUtils.closeQuietly(sourceBinary);
393 JcrUtils.closeQuietly(targetBinary);
394 }
395 }
396
397 /** factorizes monitor management */
398 private void updateMonitor(String msg, Boolean doLog) {
399 if (doLog && log.isDebugEnabled())
400 log.debug(msg);
401 if (monitor != null) {
402 monitor.worked(1);
403 monitor.subTask(msg);
404 }
405 }
406
407 // private void syncNode_old(Node sourceNode, Node targetParentNode)
408 // throws RepositoryException, SAXException {
409 //
410 // // enable cancelation of the current fetch process
411 // // fxme insure the repository stays in a stable state
412 // if (monitor != null && monitor.isCanceled()) {
413 // updateMonitor("Fetched has been canceled, "
414 // + "process is terminating");
415 // return;
416 // }
417 //
418 // Boolean noRecurse = singleLevel(sourceNode);
419 // Calendar sourceLastModified = null;
420 // if (sourceNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
421 // sourceLastModified = sourceNode.getProperty(
422 // Property.JCR_LAST_MODIFIED).getDate();
423 // }
424 //
425 // if (sourceNode.getDefinition().isProtected())
426 // log.warn(sourceNode + " is protected.");
427 //
428 // if (!targetParentNode.hasNode(sourceNode.getName())) {
429 // String msg = "Adding " + sourceNode.getPath();
430 // updateMonitor(msg);
431 // if (log.isDebugEnabled())
432 // log.debug(msg);
433 // ContentHandler contentHandler = targetParentNode
434 // .getSession()
435 // .getWorkspace()
436 // .getImportContentHandler(targetParentNode.getPath(),
437 // ImportUUIDBehavior.IMPORT_UUID_COLLISION_THROW);
438 // sourceNode.getSession().exportSystemView(sourceNode.getPath(),
439 // contentHandler, false, noRecurse);
440 // } else {
441 // Node targetNode = targetParentNode.getNode(sourceNode.getName());
442 // if (sourceLastModified != null) {
443 // Calendar targetLastModified = null;
444 // if (targetNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
445 // targetLastModified = targetNode.getProperty(
446 // Property.JCR_LAST_MODIFIED).getDate();
447 // }
448 //
449 // if (targetLastModified == null
450 // || targetLastModified.before(sourceLastModified)) {
451 // String msg = "Updating " + targetNode.getPath();
452 // updateMonitor(msg);
453 // if (log.isDebugEnabled())
454 // log.debug(msg);
455 // ContentHandler contentHandler = targetParentNode
456 // .getSession()
457 // .getWorkspace()
458 // .getImportContentHandler(
459 // targetParentNode.getPath(),
460 // ImportUUIDBehavior.IMPORT_UUID_COLLISION_REMOVE_EXISTING);
461 // sourceNode.getSession().exportSystemView(
462 // sourceNode.getPath(), contentHandler, false,
463 // noRecurse);
464 // } else {
465 // String msg = "Skipped up to date " + targetNode.getPath();
466 // updateMonitor(msg);
467 // if (log.isDebugEnabled())
468 // log.debug(msg);
469 // return;
470 // }
471 // }
472 // }
473 //
474 // if (noRecurse) {
475 // // recurse
476 // Node targetNode = targetParentNode.getNode(sourceNode.getName());
477 // if (sourceLastModified != null) {
478 // Calendar zero = new GregorianCalendar();
479 // zero.setTimeInMillis(0);
480 // targetNode.setProperty(Property.JCR_LAST_MODIFIED, zero);
481 // targetNode.getSession().save();
482 // }
483 //
484 // for (NodeIterator it = sourceNode.getNodes(); it.hasNext();) {
485 // syncNode_old(it.nextNode(), targetNode);
486 // }
487 //
488 // if (sourceLastModified != null) {
489 // targetNode.setProperty(Property.JCR_LAST_MODIFIED,
490 // sourceLastModified);
491 // targetNode.getSession().save();
492 // }
493 // }
494 // }
495
496 protected Boolean singleLevel(Node sourceNode) throws RepositoryException {
497 if (sourceNode.isNodeType(NodeType.NT_FILE))
498 return false;
499 return true;
500 }
501
502 /**
503 * Synchronises only one workspace, retrieved by name without changing its name.
504 */
505 public void setSourceWksp(String sourceWksp) {
506 if (sourceWksp != null && !sourceWksp.trim().equals("")) {
507 Map<String, String> map = new HashMap<String, String>();
508 map.put(sourceWksp, sourceWksp);
509 setWkspMap(map);
510 }
511 }
512
513 /**
514 * Synchronises a map of workspaces that will be retrieved by name. If the
515 * target name is not defined (eg null or an empty string) for a given source
516 * workspace, we use the source name as target name.
517 */
518 public void setWkspMap(Map<String, String> workspaceMap) {
519 // clean the list to ease later use
520 this.workspaceMap = new HashMap<String, String>();
521 if (workspaceMap != null) {
522 workspaceNames: for (String srcName : workspaceMap.keySet()) {
523 String targetName = workspaceMap.get(srcName);
524
525 // Sanity check
526 if (srcName.trim().equals(""))
527 continue workspaceNames;
528 if (targetName == null || "".equals(targetName.trim()))
529 targetName = srcName;
530 this.workspaceMap.put(srcName, targetName);
531 }
532 }
533 // clean the map to ease later use
534 if (this.workspaceMap.size() == 0)
535 this.workspaceMap = null;
536 }
537
538 public void setMonitor(JcrMonitor monitor) {
539 this.monitor = monitor;
540 }
541
542 public void setRepositoryFactory(RepositoryFactory repositoryFactory) {
543 this.repositoryFactory = repositoryFactory;
544 }
545
546 public void setSourceRepoUri(String sourceRepoUri) {
547 this.sourceRepoUri = sourceRepoUri;
548 }
549
550 public void setSourceUsername(String sourceUsername) {
551 this.sourceUsername = sourceUsername;
552 }
553
554 public void setSourcePassword(char[] sourcePassword) {
555 this.sourcePassword = sourcePassword;
556 }
557
558 public void setTargetRepoUri(String targetRepoUri) {
559 this.targetRepoUri = targetRepoUri;
560 }
561
562 public void setTargetUsername(String targetUsername) {
563 this.targetUsername = targetUsername;
564 }
565
566 public void setTargetPassword(char[] targetPassword) {
567 this.targetPassword = targetPassword;
568 }
569
570 public void setSourceRepository(Repository sourceRepository) {
571 this.sourceRepository = sourceRepository;
572 }
573
574 public void setSourceCredentials(Credentials sourceCredentials) {
575 this.sourceCredentials = sourceCredentials;
576 }
577
578 public void setTargetRepository(Repository targetRepository) {
579 this.targetRepository = targetRepository;
580 }
581
582 public void setTargetCredentials(Credentials targetCredentials) {
583 this.targetCredentials = targetCredentials;
584 }
585
586 public void setFilesOnly(Boolean filesOnly) {
587 this.filesOnly = filesOnly;
588 }
589
590 }