]> git.argeo.org Git - gpl/argeo-slc.git/blob - RepoSync.java
567ea36d502ccb37341a89df1972afb573437cf1
[gpl/argeo-slc.git] / RepoSync.java
1 package org.argeo.slc.repo;
2
3 import java.io.InputStream;
4 import java.util.Arrays;
5 import java.util.Calendar;
6 import java.util.GregorianCalendar;
7 import java.util.HashMap;
8 import java.util.List;
9 import java.util.Map;
10 import java.util.TimeZone;
11
12 import javax.jcr.Binary;
13 import javax.jcr.Credentials;
14 import javax.jcr.NoSuchWorkspaceException;
15 import javax.jcr.Node;
16 import javax.jcr.NodeIterator;
17 import javax.jcr.Property;
18 import javax.jcr.PropertyIterator;
19 import javax.jcr.PropertyType;
20 import javax.jcr.Repository;
21 import javax.jcr.RepositoryException;
22 import javax.jcr.RepositoryFactory;
23 import javax.jcr.Session;
24 import javax.jcr.SimpleCredentials;
25 import javax.jcr.nodetype.NodeType;
26 import javax.jcr.query.Query;
27 import javax.jcr.query.QueryResult;
28
29 import org.apache.commons.io.IOUtils;
30 import org.argeo.api.cms.CmsLog;
31 import org.argeo.cms.jcr.CmsJcrUtils;
32 import org.argeo.jcr.JcrMonitor;
33 import org.argeo.jcr.JcrUtils;
34 import org.argeo.slc.SlcException;
35 import org.xml.sax.SAXException;
36
37 /**
38 * Synchronise workspaces from a remote software repository to the local
39 * repository (Synchronisation in the other direction does not work).
40 *
41 * Workspaces are retrieved by name given a map that links the source with a
42 * target name. If a target workspace does not exist, it is created. Otherwise
43 * we copy the content of the source workspace into the target one.
44 */
45 public class RepoSync implements Runnable {
46 private final static CmsLog log = CmsLog.getLog(RepoSync.class);
47
48 // Centralizes definition of workspaces that must be ignored by the sync.
49 private final static List<String> IGNORED_WKSP_LIST = Arrays.asList("security", "localrepo");
50
51 private final Calendar zero;
52 private Session sourceDefaultSession = null;
53 private Session targetDefaultSession = null;
54
55 private Repository sourceRepository;
56 private Credentials sourceCredentials;
57 private Repository targetRepository;
58 private Credentials targetCredentials;
59
60 // if Repository and Credentials objects are not explicitly set
61 private String sourceRepoUri;
62 private String sourceUsername;
63 private char[] sourcePassword;
64 private String targetRepoUri;
65 private String targetUsername;
66 private char[] targetPassword;
67
68 private RepositoryFactory repositoryFactory;
69
70 private JcrMonitor monitor;
71 private Map<String, String> workspaceMap;
72
73 // TODO fix monitor
74 private Boolean filesOnly = false;
75
76 public RepoSync() {
77 zero = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
78 zero.setTimeInMillis(0);
79 }
80
81 /**
82 *
83 * Shortcut to instantiate a RepoSync with already known repositories and
84 * credentials.
85 *
86 * @param sourceRepository
87 * @param sourceCredentials
88 * @param targetRepository
89 * @param targetCredentials
90 */
91 public RepoSync(Repository sourceRepository, Credentials sourceCredentials, Repository targetRepository,
92 Credentials targetCredentials) {
93 this();
94 this.sourceRepository = sourceRepository;
95 this.sourceCredentials = sourceCredentials;
96 this.targetRepository = targetRepository;
97 this.targetCredentials = targetCredentials;
98 }
99
100 public void run() {
101 try {
102 long begin = System.currentTimeMillis();
103
104 // Setup
105 if (sourceRepository == null)
106 sourceRepository = CmsJcrUtils.getRepositoryByUri(repositoryFactory, sourceRepoUri);
107 if (sourceCredentials == null && sourceUsername != null)
108 sourceCredentials = new SimpleCredentials(sourceUsername, sourcePassword);
109 // FIXME make it more generic
110 sourceDefaultSession = sourceRepository.login(sourceCredentials, RepoConstants.DEFAULT_DEFAULT_WORKSPACE);
111
112 if (targetRepository == null)
113 targetRepository = CmsJcrUtils.getRepositoryByUri(repositoryFactory, targetRepoUri);
114 if (targetCredentials == null && targetUsername != null)
115 targetCredentials = new SimpleCredentials(targetUsername, targetPassword);
116 targetDefaultSession = targetRepository.login(targetCredentials);
117
118 Map<String, Exception> errors = new HashMap<String, Exception>();
119 for (String sourceWorkspaceName : sourceDefaultSession.getWorkspace().getAccessibleWorkspaceNames()) {
120 if (monitor != null && monitor.isCanceled())
121 break;
122
123 if (workspaceMap != null && !workspaceMap.containsKey(sourceWorkspaceName))
124 continue;
125 if (IGNORED_WKSP_LIST.contains(sourceWorkspaceName))
126 continue;
127
128 Session sourceSession = null;
129 Session targetSession = null;
130 String targetWorkspaceName = workspaceMap.get(sourceWorkspaceName);
131 try {
132 try {
133 targetSession = targetRepository.login(targetCredentials, targetWorkspaceName);
134 } catch (NoSuchWorkspaceException e) {
135 targetDefaultSession.getWorkspace().createWorkspace(targetWorkspaceName);
136 targetSession = targetRepository.login(targetCredentials, targetWorkspaceName);
137 }
138 sourceSession = sourceRepository.login(sourceCredentials, sourceWorkspaceName);
139 syncWorkspace(sourceSession, targetSession);
140 } catch (Exception e) {
141 errors.put("Could not sync workspace " + sourceWorkspaceName, e);
142 if (log.isErrorEnabled())
143 e.printStackTrace();
144
145 } finally {
146 JcrUtils.logoutQuietly(sourceSession);
147 JcrUtils.logoutQuietly(targetSession);
148 }
149 }
150
151 if (monitor != null && monitor.isCanceled())
152 log.info("Sync has been canceled by user");
153
154 long duration = (System.currentTimeMillis() - begin) / 1000;// s
155 log.info("Sync " + sourceRepoUri + " to " + targetRepoUri + " in " + (duration / 60)
156
157 + "min " + (duration % 60) + "s");
158
159 if (errors.size() > 0) {
160 throw new SlcException("Sync failed " + errors);
161 }
162 } catch (RepositoryException e) {
163 throw new SlcException("Cannot sync " + sourceRepoUri + " to " + targetRepoUri, e);
164 } finally {
165 JcrUtils.logoutQuietly(sourceDefaultSession);
166 JcrUtils.logoutQuietly(targetDefaultSession);
167 }
168 }
169
170 private long getNodesNumber(Session session) {
171 if (IGNORED_WKSP_LIST.contains(session.getWorkspace().getName()))
172 return 0l;
173 try {
174 Query countQuery = session.getWorkspace().getQueryManager().createQuery(
175 "select file from [" + (true ? NodeType.NT_FILE : NodeType.NT_BASE) + "] as file", Query.JCR_SQL2);
176
177 QueryResult result = countQuery.execute();
178 Long expectedCount = result.getNodes().getSize();
179 return expectedCount;
180 } catch (RepositoryException e) {
181 throw new SlcException("Unexpected error while computing " + "the size of the fetch for workspace "
182 + session.getWorkspace().getName(), e);
183 }
184 }
185
186 protected void syncWorkspace(Session sourceSession, Session targetSession) {
187 if (monitor != null) {
188 monitor.beginTask("Computing fetch size...", -1);
189 Long totalAmount = getNodesNumber(sourceSession);
190 monitor.beginTask("Fetch", totalAmount.intValue());
191 }
192
193 try {
194 String msg = "Synchronizing workspace: " + sourceSession.getWorkspace().getName();
195 if (monitor != null)
196 monitor.setTaskName(msg);
197 if (log.isDebugEnabled())
198 log.debug(msg);
199
200 for (NodeIterator it = sourceSession.getRootNode().getNodes(); it.hasNext();) {
201 Node node = it.nextNode();
202 if (node.getName().contains(":"))
203 continue;
204 if (node.getName().equals("download"))
205 continue;
206 if (!node.isNodeType(NodeType.NT_HIERARCHY_NODE))
207 continue;
208 syncNode(node, targetSession);
209 }
210 // if (filesOnly) {
211 // JcrUtils.copyFiles(sourceSession.getRootNode(), targetSession.getRootNode(),
212 // true, monitor);
213 // } else {
214 // for (NodeIterator it = sourceSession.getRootNode().getNodes(); it.hasNext();)
215 // {
216 // Node node = it.nextNode();
217 // if (node.getName().equals("jcr:system"))
218 // continue;
219 // syncNode(node, targetSession);
220 // }
221 // }
222 if (log.isDebugEnabled())
223 log.debug("Synced " + sourceSession.getWorkspace().getName());
224 } catch (Exception e) {
225 e.printStackTrace();
226 throw new SlcException("Cannot sync " + sourceSession.getWorkspace().getName() + " to "
227 + targetSession.getWorkspace().getName(), e);
228 }
229 }
230
231 /** factorizes monitor management */
232 private void updateMonitor(String msg) {
233 updateMonitor(msg, false);
234 }
235
236 protected void syncNode(Node sourceNode, Session targetSession) throws RepositoryException, SAXException {
237 if (filesOnly) {
238 Node targetNode;
239 if (targetSession.itemExists(sourceNode.getPath()))
240 targetNode = targetSession.getNode(sourceNode.getPath());
241 else
242 targetNode = JcrUtils.mkdirs(targetSession, sourceNode.getPath(), NodeType.NT_FOLDER);
243 JcrUtils.copyFiles(sourceNode, targetNode, true, monitor, true);
244 return;
245 }
246 // Boolean singleLevel = singleLevel(sourceNode);
247 try {
248 if (monitor != null && monitor.isCanceled()) {
249 updateMonitor("Fetched has been canceled, " + "process is terminating");
250 return;
251 }
252
253 Node targetParentNode = targetSession.getNode(sourceNode.getParent().getPath());
254 Node targetNode;
255 if (monitor != null && sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE))
256 monitor.subTask("Process " + sourceNode.getPath());
257
258 final Boolean isNew;
259 if (!targetSession.itemExists(sourceNode.getPath())) {
260 isNew = true;
261 targetNode = targetParentNode.addNode(sourceNode.getName(), sourceNode.getPrimaryNodeType().getName());
262 } else {
263 isNew = false;
264 targetNode = targetSession.getNode(sourceNode.getPath());
265 if (!targetNode.getPrimaryNodeType().getName().equals(sourceNode.getPrimaryNodeType().getName()))
266 targetNode.setPrimaryType(sourceNode.getPrimaryNodeType().getName());
267 }
268
269 // export
270 // sourceNode.getSession().exportSystemView(sourceNode.getPath(),
271 // contentHandler, false, singleLevel);
272
273 // if (singleLevel) {
274 // if (targetSession.hasPendingChanges()) {
275 // // updateMonitor(
276 // // (isNew ? "Added " : "Updated ") + targetNode.getPath(),
277 // // true);
278 // if (doSave)
279 // targetSession.save();
280 // } else {
281 // // updateMonitor("Checked " + targetNode.getPath(), false);
282 // }
283 // }
284
285 // mixin and properties
286 for (NodeType nt : sourceNode.getMixinNodeTypes()) {
287 if (!targetNode.isNodeType(nt.getName()) && targetNode.canAddMixin(nt.getName()))
288 targetNode.addMixin(nt.getName());
289 }
290 copyProperties(sourceNode, targetNode);
291
292 // next level
293 NodeIterator ni = sourceNode.getNodes();
294 while (ni != null && ni.hasNext()) {
295 Node sourceChild = ni.nextNode();
296 syncNode(sourceChild, targetSession);
297 }
298
299 copyTimestamps(sourceNode, targetNode);
300
301 if (sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE)) {
302 if (targetSession.hasPendingChanges()) {
303 if (sourceNode.isNodeType(NodeType.NT_FILE))
304 updateMonitor((isNew ? "Added " : "Updated ") + targetNode.getPath(), true);
305 // if (doSave)
306 targetSession.save();
307 } else {
308 if (sourceNode.isNodeType(NodeType.NT_FILE))
309 updateMonitor("Checked " + targetNode.getPath(), false);
310 }
311 }
312 } catch (RepositoryException e) {
313 throw new SlcException("Cannot sync source node " + sourceNode, e);
314 }
315 }
316
317 private void copyTimestamps(Node sourceNode, Node targetNode) throws RepositoryException {
318 if (sourceNode.getDefinition().isProtected())
319 return;
320 if (targetNode.getDefinition().isProtected())
321 return;
322 copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED);
323 copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED_BY);
324 copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED);
325 copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED_BY);
326 }
327
328 private void copyTimestamp(Node sourceNode, Node targetNode, String property) throws RepositoryException {
329 if (sourceNode.hasProperty(property)) {
330 Property p = sourceNode.getProperty(property);
331 if (p.getDefinition().isProtected())
332 return;
333 if (targetNode.hasProperty(property)
334 && targetNode.getProperty(property).getValue().equals(sourceNode.getProperty(property).getValue()))
335 return;
336 targetNode.setProperty(property, sourceNode.getProperty(property).getValue());
337 }
338 }
339
340 private void copyProperties(Node sourceNode, Node targetNode) throws RepositoryException {
341 properties: for (PropertyIterator pi = sourceNode.getProperties(); pi.hasNext();) {
342 Property p = pi.nextProperty();
343 if (p.getDefinition().isProtected())
344 continue properties;
345 if (p.getName().equals(Property.JCR_CREATED) || p.getName().equals(Property.JCR_CREATED_BY)
346 || p.getName().equals(Property.JCR_LAST_MODIFIED)
347 || p.getName().equals(Property.JCR_LAST_MODIFIED_BY))
348 continue properties;
349
350 if (p.getType() == PropertyType.BINARY) {
351 copyBinary(p, targetNode);
352 } else {
353
354 if (p.isMultiple()) {
355 if (!targetNode.hasProperty(p.getName())
356 || !Arrays.equals(targetNode.getProperty(p.getName()).getValues(), p.getValues()))
357 targetNode.setProperty(p.getName(), p.getValues());
358 } else {
359 if (!targetNode.hasProperty(p.getName())
360 || !targetNode.getProperty(p.getName()).getValue().equals(p.getValue()))
361 targetNode.setProperty(p.getName(), p.getValue());
362 }
363 }
364 }
365 }
366
367 private static void copyBinary(Property p, Node targetNode) throws RepositoryException {
368 InputStream in = null;
369 Binary sourceBinary = null;
370 Binary targetBinary = null;
371 try {
372 sourceBinary = p.getBinary();
373 if (targetNode.hasProperty(p.getName()))
374 targetBinary = targetNode.getProperty(p.getName()).getBinary();
375
376 // optim FIXME make it more configurable
377 if (targetBinary != null)
378 if (sourceBinary.getSize() == targetBinary.getSize()) {
379 if (log.isTraceEnabled())
380 log.trace("Skipped " + p.getPath());
381 return;
382 }
383
384 in = sourceBinary.getStream();
385 targetBinary = targetNode.getSession().getValueFactory().createBinary(in);
386 targetNode.setProperty(p.getName(), targetBinary);
387 } catch (Exception e) {
388 throw new SlcException("Could not transfer " + p, e);
389 } finally {
390 IOUtils.closeQuietly(in);
391 JcrUtils.closeQuietly(sourceBinary);
392 JcrUtils.closeQuietly(targetBinary);
393 }
394 }
395
396 /** factorizes monitor management */
397 private void updateMonitor(String msg, Boolean doLog) {
398 if (doLog && log.isDebugEnabled())
399 log.debug(msg);
400 if (monitor != null) {
401 monitor.worked(1);
402 monitor.subTask(msg);
403 }
404 }
405
406 // private void syncNode_old(Node sourceNode, Node targetParentNode)
407 // throws RepositoryException, SAXException {
408 //
409 // // enable cancelation of the current fetch process
410 // // fxme insure the repository stays in a stable state
411 // if (monitor != null && monitor.isCanceled()) {
412 // updateMonitor("Fetched has been canceled, "
413 // + "process is terminating");
414 // return;
415 // }
416 //
417 // Boolean noRecurse = singleLevel(sourceNode);
418 // Calendar sourceLastModified = null;
419 // if (sourceNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
420 // sourceLastModified = sourceNode.getProperty(
421 // Property.JCR_LAST_MODIFIED).getDate();
422 // }
423 //
424 // if (sourceNode.getDefinition().isProtected())
425 // log.warn(sourceNode + " is protected.");
426 //
427 // if (!targetParentNode.hasNode(sourceNode.getName())) {
428 // String msg = "Adding " + sourceNode.getPath();
429 // updateMonitor(msg);
430 // if (log.isDebugEnabled())
431 // log.debug(msg);
432 // ContentHandler contentHandler = targetParentNode
433 // .getSession()
434 // .getWorkspace()
435 // .getImportContentHandler(targetParentNode.getPath(),
436 // ImportUUIDBehavior.IMPORT_UUID_COLLISION_THROW);
437 // sourceNode.getSession().exportSystemView(sourceNode.getPath(),
438 // contentHandler, false, noRecurse);
439 // } else {
440 // Node targetNode = targetParentNode.getNode(sourceNode.getName());
441 // if (sourceLastModified != null) {
442 // Calendar targetLastModified = null;
443 // if (targetNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
444 // targetLastModified = targetNode.getProperty(
445 // Property.JCR_LAST_MODIFIED).getDate();
446 // }
447 //
448 // if (targetLastModified == null
449 // || targetLastModified.before(sourceLastModified)) {
450 // String msg = "Updating " + targetNode.getPath();
451 // updateMonitor(msg);
452 // if (log.isDebugEnabled())
453 // log.debug(msg);
454 // ContentHandler contentHandler = targetParentNode
455 // .getSession()
456 // .getWorkspace()
457 // .getImportContentHandler(
458 // targetParentNode.getPath(),
459 // ImportUUIDBehavior.IMPORT_UUID_COLLISION_REMOVE_EXISTING);
460 // sourceNode.getSession().exportSystemView(
461 // sourceNode.getPath(), contentHandler, false,
462 // noRecurse);
463 // } else {
464 // String msg = "Skipped up to date " + targetNode.getPath();
465 // updateMonitor(msg);
466 // if (log.isDebugEnabled())
467 // log.debug(msg);
468 // return;
469 // }
470 // }
471 // }
472 //
473 // if (noRecurse) {
474 // // recurse
475 // Node targetNode = targetParentNode.getNode(sourceNode.getName());
476 // if (sourceLastModified != null) {
477 // Calendar zero = new GregorianCalendar();
478 // zero.setTimeInMillis(0);
479 // targetNode.setProperty(Property.JCR_LAST_MODIFIED, zero);
480 // targetNode.getSession().save();
481 // }
482 //
483 // for (NodeIterator it = sourceNode.getNodes(); it.hasNext();) {
484 // syncNode_old(it.nextNode(), targetNode);
485 // }
486 //
487 // if (sourceLastModified != null) {
488 // targetNode.setProperty(Property.JCR_LAST_MODIFIED,
489 // sourceLastModified);
490 // targetNode.getSession().save();
491 // }
492 // }
493 // }
494
495 protected Boolean singleLevel(Node sourceNode) throws RepositoryException {
496 if (sourceNode.isNodeType(NodeType.NT_FILE))
497 return false;
498 return true;
499 }
500
501 /**
502 * Synchronises only one workspace, retrieved by name without changing its name.
503 */
504 public void setSourceWksp(String sourceWksp) {
505 if (sourceWksp != null && !sourceWksp.trim().equals("")) {
506 Map<String, String> map = new HashMap<String, String>();
507 map.put(sourceWksp, sourceWksp);
508 setWkspMap(map);
509 }
510 }
511
512 /**
513 * Synchronises a map of workspaces that will be retrieved by name. If the
514 * target name is not defined (eg null or an empty string) for a given source
515 * workspace, we use the source name as target name.
516 */
517 public void setWkspMap(Map<String, String> workspaceMap) {
518 // clean the list to ease later use
519 this.workspaceMap = new HashMap<String, String>();
520 if (workspaceMap != null) {
521 workspaceNames: for (String srcName : workspaceMap.keySet()) {
522 String targetName = workspaceMap.get(srcName);
523
524 // Sanity check
525 if (srcName.trim().equals(""))
526 continue workspaceNames;
527 if (targetName == null || "".equals(targetName.trim()))
528 targetName = srcName;
529 this.workspaceMap.put(srcName, targetName);
530 }
531 }
532 // clean the map to ease later use
533 if (this.workspaceMap.size() == 0)
534 this.workspaceMap = null;
535 }
536
537 public void setMonitor(JcrMonitor monitor) {
538 this.monitor = monitor;
539 }
540
541 public void setRepositoryFactory(RepositoryFactory repositoryFactory) {
542 this.repositoryFactory = repositoryFactory;
543 }
544
545 public void setSourceRepoUri(String sourceRepoUri) {
546 this.sourceRepoUri = sourceRepoUri;
547 }
548
549 public void setSourceUsername(String sourceUsername) {
550 this.sourceUsername = sourceUsername;
551 }
552
553 public void setSourcePassword(char[] sourcePassword) {
554 this.sourcePassword = sourcePassword;
555 }
556
557 public void setTargetRepoUri(String targetRepoUri) {
558 this.targetRepoUri = targetRepoUri;
559 }
560
561 public void setTargetUsername(String targetUsername) {
562 this.targetUsername = targetUsername;
563 }
564
565 public void setTargetPassword(char[] targetPassword) {
566 this.targetPassword = targetPassword;
567 }
568
569 public void setSourceRepository(Repository sourceRepository) {
570 this.sourceRepository = sourceRepository;
571 }
572
573 public void setSourceCredentials(Credentials sourceCredentials) {
574 this.sourceCredentials = sourceCredentials;
575 }
576
577 public void setTargetRepository(Repository targetRepository) {
578 this.targetRepository = targetRepository;
579 }
580
581 public void setTargetCredentials(Credentials targetCredentials) {
582 this.targetCredentials = targetCredentials;
583 }
584
585 public void setFilesOnly(Boolean filesOnly) {
586 this.filesOnly = filesOnly;
587 }
588
589 }