]> git.argeo.org Git - gpl/argeo-slc.git/blob - runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoSync.java
Improve OSGi factory
[gpl/argeo-slc.git] / runtime / org.argeo.slc.repo / src / main / java / org / argeo / slc / repo / RepoSync.java
1 /*
2 * Copyright (C) 2007-2012 Argeo GmbH
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package org.argeo.slc.repo;
17
18 import java.io.InputStream;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.Calendar;
22 import java.util.GregorianCalendar;
23 import java.util.HashMap;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.TimeZone;
27
28 import javax.jcr.Binary;
29 import javax.jcr.Credentials;
30 import javax.jcr.NoSuchWorkspaceException;
31 import javax.jcr.Node;
32 import javax.jcr.NodeIterator;
33 import javax.jcr.Property;
34 import javax.jcr.PropertyIterator;
35 import javax.jcr.PropertyType;
36 import javax.jcr.Repository;
37 import javax.jcr.RepositoryException;
38 import javax.jcr.RepositoryFactory;
39 import javax.jcr.Session;
40 import javax.jcr.SimpleCredentials;
41 import javax.jcr.nodetype.NodeType;
42 import javax.jcr.query.Query;
43 import javax.jcr.query.QueryResult;
44
45 import org.apache.commons.io.IOUtils;
46 import org.apache.commons.logging.Log;
47 import org.apache.commons.logging.LogFactory;
48 import org.argeo.ArgeoMonitor;
49 import org.argeo.jcr.ArgeoJcrUtils;
50 import org.argeo.jcr.JcrUtils;
51 import org.argeo.slc.SlcException;
52 import org.xml.sax.SAXException;
53
54 /** Sync to from software repositories */
55 public class RepoSync implements Runnable {
56 private final static Log log = LogFactory.getLog(RepoSync.class);
57
58 // Centralizes definition of workspaces that must be ignored by the sync.
59 private final static List<String> IGNORED_WSKP_LIST = Arrays.asList(
60 "security", "localrepo");
61
62 private final Calendar zero;
63 private Session sourceDefaultSession = null;
64 private Session targetDefaultSession = null;
65
66 private Repository sourceRepository;
67 private Credentials sourceCredentials;
68 private Repository targetRepository;
69 private Credentials targetCredentials;
70
71 // if Repository and Credentials objects are not explicitly set
72 private String sourceRepoUri;
73 private String sourceUsername;
74 private char[] sourcePassword;
75 private String targetRepoUri;
76 private String targetUsername;
77 private char[] targetPassword;
78
79 private RepositoryFactory repositoryFactory;
80
81 private ArgeoMonitor monitor;
82 private List<String> sourceWkspList;
83
84 // TODO fix monitor
85 private Boolean filesOnly = false;
86
87 public RepoSync() {
88 zero = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
89 zero.setTimeInMillis(0);
90 }
91
92 /**
93 *
94 * Shortcut to instantiate a RepoSync with already known repositories and
95 * credentials.
96 *
97 * @param sourceRepository
98 * @param sourceCredentials
99 * @param targetRepository
100 * @param targetCredentials
101 */
102 public RepoSync(Repository sourceRepository, Credentials sourceCredentials,
103 Repository targetRepository, Credentials targetCredentials) {
104 this();
105 this.sourceRepository = sourceRepository;
106 this.sourceCredentials = sourceCredentials;
107 this.targetRepository = targetRepository;
108 this.targetCredentials = targetCredentials;
109 }
110
111 public void run() {
112 try {
113 long begin = System.currentTimeMillis();
114
115 // Setup
116 if (sourceRepository == null)
117 sourceRepository = ArgeoJcrUtils.getRepositoryByUri(
118 repositoryFactory, sourceRepoUri);
119 if (sourceCredentials == null && sourceUsername != null)
120 sourceCredentials = new SimpleCredentials(sourceUsername,
121 sourcePassword);
122 sourceDefaultSession = sourceRepository.login(sourceCredentials);
123
124 if (targetRepository == null)
125 targetRepository = ArgeoJcrUtils.getRepositoryByUri(
126 repositoryFactory, targetRepoUri);
127 if (targetCredentials == null && targetUsername != null)
128 targetCredentials = new SimpleCredentials(targetUsername,
129 targetPassword);
130 targetDefaultSession = targetRepository.login(targetCredentials);
131
132 Map<String, Exception> errors = new HashMap<String, Exception>();
133 for (String sourceWorkspaceName : sourceDefaultSession
134 .getWorkspace().getAccessibleWorkspaceNames()) {
135 if (monitor != null && monitor.isCanceled())
136 break;
137
138 if (sourceWkspList != null
139 && !sourceWkspList.contains(sourceWorkspaceName))
140 continue;
141 if (IGNORED_WSKP_LIST.contains(sourceWorkspaceName))
142 continue;
143
144 Session sourceSession = null;
145 Session targetSession = null;
146 try {
147 try {
148 targetSession = targetRepository.login(
149 targetCredentials, sourceWorkspaceName);
150 } catch (NoSuchWorkspaceException e) {
151 targetDefaultSession.getWorkspace().createWorkspace(
152 sourceWorkspaceName);
153 targetSession = targetRepository.login(
154 targetCredentials, sourceWorkspaceName);
155 }
156 sourceSession = sourceRepository.login(sourceCredentials,
157 sourceWorkspaceName);
158 syncWorkspace(sourceSession, targetSession);
159 } catch (Exception e) {
160 errors.put("Could not sync workspace "
161 + sourceWorkspaceName, e);
162 if (log.isDebugEnabled())
163 e.printStackTrace();
164 } finally {
165 JcrUtils.logoutQuietly(sourceSession);
166 JcrUtils.logoutQuietly(targetSession);
167 }
168 }
169
170 if (monitor != null && monitor.isCanceled())
171 log.info("Sync has been canceled by user");
172
173 long duration = (System.currentTimeMillis() - begin) / 1000;// s
174 log.info("Sync " + sourceRepoUri + " to " + targetRepoUri + " in "
175 + (duration / 60)
176
177 + "min " + (duration % 60) + "s");
178
179 if (errors.size() > 0) {
180 throw new SlcException("Sync failed " + errors);
181 }
182 } catch (RepositoryException e) {
183 throw new SlcException("Cannot sync " + sourceRepoUri + " to "
184 + targetRepoUri, e);
185 } finally {
186 JcrUtils.logoutQuietly(sourceDefaultSession);
187 JcrUtils.logoutQuietly(targetDefaultSession);
188 }
189 }
190
191 private long getNodesNumber(Session session) {
192 if (IGNORED_WSKP_LIST.contains(session.getWorkspace().getName()))
193 return 0l;
194 Session sourceSession = null;
195 try {
196 Query countQuery = session
197 .getWorkspace()
198 .getQueryManager()
199 .createQuery(
200 "select file from ["
201 + (true ? "nt:file" : "nt:base")
202 + "] as file", Query.JCR_SQL2);
203 QueryResult result = countQuery.execute();
204 Long expectedCount = result.getNodes().getSize();
205 return expectedCount;
206 } catch (RepositoryException e) {
207 throw new SlcException("Unexpected error while computing "
208 + "the size of the fetch for workspace "
209 + session.getWorkspace().getName(), e);
210 } finally {
211 JcrUtils.logoutQuietly(sourceSession);
212 }
213 }
214
215 protected void syncWorkspace(Session sourceSession, Session targetSession) {
216 if (monitor != null) {
217 monitor.beginTask("Computing fetch size...", -1);
218 Long totalAmount = getNodesNumber(sourceSession);
219 // if (sourceWkspList != null) {
220 // for (String wkspName : sourceWkspList) {
221 // totalAmount += getNodesNumber(wkspName);
222 // }
223 // } else
224 // for (String sourceWorkspaceName : sourceDefaultSession
225 // .getWorkspace().getAccessibleWorkspaceNames()) {
226 // totalAmount += getNodesNumber(sourceWorkspaceName);
227 // }
228 monitor.beginTask("Fetch", totalAmount.intValue());
229
230 // if (log.isDebugEnabled())
231 // log.debug("Nb of nodes to sync: " + totalAmount.intValue());
232 }
233
234 try {
235 String msg = "Synchronizing workspace: "
236 + sourceSession.getWorkspace().getName();
237 if (monitor != null)
238 monitor.setTaskName(msg);
239 if (log.isDebugEnabled())
240 log.debug(msg);
241
242 if (filesOnly) {
243 JcrUtils.copyFiles(sourceSession.getRootNode(),
244 targetSession.getRootNode(), true, monitor);
245 } else {
246 for (NodeIterator it = sourceSession.getRootNode().getNodes(); it
247 .hasNext();) {
248 Node node = it.nextNode();
249 if (node.getName().equals("jcr:system"))
250 continue;
251 syncNode(node, targetSession);
252 }
253 }
254 if (log.isDebugEnabled())
255 log.debug("Synced " + sourceSession.getWorkspace().getName());
256 } catch (Exception e) {
257 throw new SlcException("Cannot sync "
258 + sourceSession.getWorkspace().getName() + " to "
259 + targetSession.getWorkspace().getName(), e);
260 }
261 }
262
263 /** factorizes monitor management */
264 private void updateMonitor(String msg) {
265 updateMonitor(msg, false);
266 }
267
268 protected void syncNode(Node sourceNode, Session targetSession)
269 throws RepositoryException, SAXException {
270 // Boolean singleLevel = singleLevel(sourceNode);
271
272 if (monitor != null && monitor.isCanceled()) {
273 updateMonitor("Fetched has been canceled, "
274 + "process is terminating");
275 return;
276 }
277
278 Node targetParentNode = targetSession.getNode(sourceNode.getParent()
279 .getPath());
280 Node targetNode;
281 if (monitor != null
282 && sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE))
283 monitor.subTask("Process " + sourceNode.getPath());
284
285 final Boolean isNew;
286 if (!targetSession.itemExists(sourceNode.getPath())) {
287 isNew = true;
288 targetNode = targetParentNode.addNode(sourceNode.getName(),
289 sourceNode.getPrimaryNodeType().getName());
290 } else {
291 isNew = false;
292 targetNode = targetSession.getNode(sourceNode.getPath());
293 if (!targetNode.getPrimaryNodeType().getName()
294 .equals(sourceNode.getPrimaryNodeType().getName()))
295 targetNode.setPrimaryType(sourceNode.getPrimaryNodeType()
296 .getName());
297 }
298
299 // export
300 // sourceNode.getSession().exportSystemView(sourceNode.getPath(),
301 // contentHandler, false, singleLevel);
302
303 // if (singleLevel) {
304 // if (targetSession.hasPendingChanges()) {
305 // // updateMonitor(
306 // // (isNew ? "Added " : "Updated ") + targetNode.getPath(),
307 // // true);
308 // if (doSave)
309 // targetSession.save();
310 // } else {
311 // // updateMonitor("Checked " + targetNode.getPath(), false);
312 // }
313 // }
314
315 // mixin and properties
316 for (NodeType nt : sourceNode.getMixinNodeTypes()) {
317 if (!targetNode.isNodeType(nt.getName())
318 && targetNode.canAddMixin(nt.getName()))
319 targetNode.addMixin(nt.getName());
320 }
321 copyProperties(sourceNode, targetNode);
322
323 // next level
324 for (NodeIterator ni = sourceNode.getNodes(); ni.hasNext();) {
325 Node sourceChild = ni.nextNode();
326 syncNode(sourceChild, targetSession);
327 }
328
329 copyTimestamps(sourceNode, targetNode);
330
331 if (sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE)) {
332 if (targetSession.hasPendingChanges()) {
333 if (sourceNode.isNodeType(NodeType.NT_FILE))
334 updateMonitor(
335 (isNew ? "Added " : "Updated ")
336 + targetNode.getPath(), true);
337 // if (doSave)
338 targetSession.save();
339 } else {
340 if (sourceNode.isNodeType(NodeType.NT_FILE))
341 updateMonitor("Checked " + targetNode.getPath(), false);
342 }
343 }
344 }
345
346 private void copyTimestamps(Node sourceNode, Node targetNode)
347 throws RepositoryException {
348 if (sourceNode.getDefinition().isProtected())
349 return;
350 if (targetNode.getDefinition().isProtected())
351 return;
352 copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED);
353 copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED_BY);
354 copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED);
355 copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED_BY);
356 }
357
358 private void copyTimestamp(Node sourceNode, Node targetNode, String property)
359 throws RepositoryException {
360 if (sourceNode.hasProperty(property)) {
361 Property p = sourceNode.getProperty(property);
362 if (p.getDefinition().isProtected())
363 return;
364 if (targetNode.hasProperty(property)
365 && targetNode
366 .getProperty(property)
367 .getValue()
368 .equals(sourceNode.getProperty(property).getValue()))
369 return;
370 targetNode.setProperty(property, sourceNode.getProperty(property)
371 .getValue());
372 }
373 }
374
375 private void copyProperties(Node sourceNode, Node targetNode)
376 throws RepositoryException {
377 properties: for (PropertyIterator pi = sourceNode.getProperties(); pi
378 .hasNext();) {
379 Property p = pi.nextProperty();
380 if (p.getDefinition().isProtected())
381 continue properties;
382 if (p.getName().equals(Property.JCR_CREATED)
383 || p.getName().equals(Property.JCR_CREATED_BY)
384 || p.getName().equals(Property.JCR_LAST_MODIFIED)
385 || p.getName().equals(Property.JCR_LAST_MODIFIED_BY))
386 continue properties;
387
388 if (p.getType() == PropertyType.BINARY) {
389 copyBinary(p, targetNode);
390 } else {
391
392 if (p.isMultiple()) {
393 if (!targetNode.hasProperty(p.getName())
394 || !Arrays.equals(
395 targetNode.getProperty(p.getName())
396 .getValues(), p.getValues()))
397 targetNode.setProperty(p.getName(), p.getValues());
398 } else {
399 if (!targetNode.hasProperty(p.getName())
400 || !targetNode.getProperty(p.getName()).getValue()
401 .equals(p.getValue()))
402 targetNode.setProperty(p.getName(), p.getValue());
403 }
404 }
405 }
406 }
407
408 private static void copyBinary(Property p, Node targetNode)
409 throws RepositoryException {
410 InputStream in = null;
411 Binary sourceBinary = null;
412 Binary targetBinary = null;
413 try {
414 sourceBinary = p.getBinary();
415 if (targetNode.hasProperty(p.getName()))
416 targetBinary = targetNode.getProperty(p.getName()).getBinary();
417
418 // optim FIXME make it more configurable
419 if (targetBinary != null)
420 if (sourceBinary.getSize() == targetBinary.getSize()) {
421 if (log.isTraceEnabled())
422 log.trace("Skipped " + p.getPath());
423 return;
424 }
425
426 in = sourceBinary.getStream();
427 targetBinary = targetNode.getSession().getValueFactory()
428 .createBinary(in);
429 targetNode.setProperty(p.getName(), targetBinary);
430 } catch (Exception e) {
431 throw new SlcException("Could not transfer " + p, e);
432 } finally {
433 IOUtils.closeQuietly(in);
434 JcrUtils.closeQuietly(sourceBinary);
435 JcrUtils.closeQuietly(targetBinary);
436 }
437 }
438
439 /** factorizes monitor management */
440 private void updateMonitor(String msg, Boolean doLog) {
441 if (doLog && log.isDebugEnabled())
442 log.debug(msg);
443 if (monitor != null) {
444 monitor.worked(1);
445 monitor.subTask(msg);
446 }
447 }
448
449 // private void syncNode_old(Node sourceNode, Node targetParentNode)
450 // throws RepositoryException, SAXException {
451 //
452 // // enable cancelation of the current fetch process
453 // // FIXME insure the repository stays in a stable state
454 // if (monitor != null && monitor.isCanceled()) {
455 // updateMonitor("Fetched has been canceled, "
456 // + "process is terminating");
457 // return;
458 // }
459 //
460 // Boolean noRecurse = singleLevel(sourceNode);
461 // Calendar sourceLastModified = null;
462 // if (sourceNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
463 // sourceLastModified = sourceNode.getProperty(
464 // Property.JCR_LAST_MODIFIED).getDate();
465 // }
466 //
467 // if (sourceNode.getDefinition().isProtected())
468 // log.warn(sourceNode + " is protected.");
469 //
470 // if (!targetParentNode.hasNode(sourceNode.getName())) {
471 // String msg = "Adding " + sourceNode.getPath();
472 // updateMonitor(msg);
473 // if (log.isDebugEnabled())
474 // log.debug(msg);
475 // ContentHandler contentHandler = targetParentNode
476 // .getSession()
477 // .getWorkspace()
478 // .getImportContentHandler(targetParentNode.getPath(),
479 // ImportUUIDBehavior.IMPORT_UUID_COLLISION_THROW);
480 // sourceNode.getSession().exportSystemView(sourceNode.getPath(),
481 // contentHandler, false, noRecurse);
482 // } else {
483 // Node targetNode = targetParentNode.getNode(sourceNode.getName());
484 // if (sourceLastModified != null) {
485 // Calendar targetLastModified = null;
486 // if (targetNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
487 // targetLastModified = targetNode.getProperty(
488 // Property.JCR_LAST_MODIFIED).getDate();
489 // }
490 //
491 // if (targetLastModified == null
492 // || targetLastModified.before(sourceLastModified)) {
493 // String msg = "Updating " + targetNode.getPath();
494 // updateMonitor(msg);
495 // if (log.isDebugEnabled())
496 // log.debug(msg);
497 // ContentHandler contentHandler = targetParentNode
498 // .getSession()
499 // .getWorkspace()
500 // .getImportContentHandler(
501 // targetParentNode.getPath(),
502 // ImportUUIDBehavior.IMPORT_UUID_COLLISION_REMOVE_EXISTING);
503 // sourceNode.getSession().exportSystemView(
504 // sourceNode.getPath(), contentHandler, false,
505 // noRecurse);
506 // } else {
507 // String msg = "Skipped up to date " + targetNode.getPath();
508 // updateMonitor(msg);
509 // if (log.isDebugEnabled())
510 // log.debug(msg);
511 // return;
512 // }
513 // }
514 // }
515 //
516 // if (noRecurse) {
517 // // recurse
518 // Node targetNode = targetParentNode.getNode(sourceNode.getName());
519 // if (sourceLastModified != null) {
520 // Calendar zero = new GregorianCalendar();
521 // zero.setTimeInMillis(0);
522 // targetNode.setProperty(Property.JCR_LAST_MODIFIED, zero);
523 // targetNode.getSession().save();
524 // }
525 //
526 // for (NodeIterator it = sourceNode.getNodes(); it.hasNext();) {
527 // syncNode_old(it.nextNode(), targetNode);
528 // }
529 //
530 // if (sourceLastModified != null) {
531 // targetNode.setProperty(Property.JCR_LAST_MODIFIED,
532 // sourceLastModified);
533 // targetNode.getSession().save();
534 // }
535 // }
536 // }
537
538 protected Boolean singleLevel(Node sourceNode) throws RepositoryException {
539 if (sourceNode.isNodeType(NodeType.NT_FILE))
540 return false;
541 return true;
542 }
543
544 /** synchronise only one workspace retrieved by name */
545 public void setSourceWksp(String sourceWksp) {
546 if (sourceWksp != null && !sourceWksp.trim().equals("")) {
547 List<String> list = new ArrayList<String>();
548 list.add(sourceWksp);
549 setSourceWkspList(list);
550 }
551 }
552
553 /** synchronise a list workspace that will be retrieved by name */
554 public void setSourceWkspList(List<String> sourceWkspList) {
555 // clean the list to ease later use
556 this.sourceWkspList = null;
557 if (sourceWkspList != null) {
558 for (String wkspName : sourceWkspList) {
559 if (!wkspName.trim().equals("")) {
560 // only instantiate if needed
561 if (this.sourceWkspList == null)
562 this.sourceWkspList = new ArrayList<String>();
563 this.sourceWkspList.add(wkspName);
564 }
565 }
566 }
567 }
568
569 public void setMonitor(ArgeoMonitor monitor) {
570 this.monitor = monitor;
571 }
572
573 public void setRepositoryFactory(RepositoryFactory repositoryFactory) {
574 this.repositoryFactory = repositoryFactory;
575 }
576
577 public void setSourceRepoUri(String sourceRepoUri) {
578 this.sourceRepoUri = sourceRepoUri;
579 }
580
581 public void setSourceUsername(String sourceUsername) {
582 this.sourceUsername = sourceUsername;
583 }
584
585 public void setSourcePassword(char[] sourcePassword) {
586 this.sourcePassword = sourcePassword;
587 }
588
589 public void setTargetRepoUri(String targetRepoUri) {
590 this.targetRepoUri = targetRepoUri;
591 }
592
593 public void setTargetUsername(String targetUsername) {
594 this.targetUsername = targetUsername;
595 }
596
597 public void setTargetPassword(char[] targetPassword) {
598 this.targetPassword = targetPassword;
599 }
600
601 public void setSourceRepository(Repository sourceRepository) {
602 this.sourceRepository = sourceRepository;
603 }
604
605 public void setSourceCredentials(Credentials sourceCredentials) {
606 this.sourceCredentials = sourceCredentials;
607 }
608
609 public void setTargetRepository(Repository targetRepository) {
610 this.targetRepository = targetRepository;
611 }
612
613 public void setTargetCredentials(Credentials targetCredentials) {
614 this.targetCredentials = targetCredentials;
615 }
616
617 public void setFilesOnly(Boolean filesOnly) {
618 this.filesOnly = filesOnly;
619 }
620
621 }