RepoSync.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.repo;
  17
  18 import java.io.InputStream;
  19 import java.util.Arrays;
  20 import java.util.Calendar;
  21 import java.util.GregorianCalendar;
  22 import java.util.HashMap;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.TimeZone;
  26
  27 import javax.jcr.Binary;
  28 import javax.jcr.Credentials;
  29 import javax.jcr.NoSuchWorkspaceException;
  30 import javax.jcr.Node;
  31 import javax.jcr.NodeIterator;
  32 import javax.jcr.Property;
  33 import javax.jcr.PropertyIterator;
  34 import javax.jcr.PropertyType;
  35 import javax.jcr.Repository;
  36 import javax.jcr.RepositoryException;
  37 import javax.jcr.RepositoryFactory;
  38 import javax.jcr.Session;
  39 import javax.jcr.SimpleCredentials;
  40 import javax.jcr.nodetype.NodeType;
  41 import javax.jcr.query.Query;
  42 import javax.jcr.query.QueryResult;
  43
  44 import org.apache.commons.io.IOUtils;
  45 import org.apache.commons.logging.Log;
  46 import org.apache.commons.logging.LogFactory;
  47 import org.argeo.jcr.JcrMonitor;
  48 import org.argeo.jcr.JcrUtils;
  49 import org.argeo.node.NodeUtils;
  50 import org.argeo.slc.SlcException;
  51 import org.xml.sax.SAXException;
  52
  53 /**
  54  * Synchronise workspaces from a remote software repository to the local
  55  * repository (Synchronisation in the other direction does not work).
  56  *
  57  * Workspaces are retrieved by name given a map that links the source with a
  58  * target name. If a target workspace does not exist, it is created. Otherwise
  59  * we copy the content of the source workspace into the target one.
  60  */
  61 public class RepoSync implements Runnable {
  62         private final static Log log = LogFactory.getLog(RepoSync.class);
  63
  64         // Centralizes definition of workspaces that must be ignored by the sync.
  65         private final static List<String> IGNORED_WKSP_LIST = Arrays.asList("security", "localrepo");
  66
  67         private final Calendar zero;
  68         private Session sourceDefaultSession = null;
  69         private Session targetDefaultSession = null;
  70
  71         private Repository sourceRepository;
  72         private Credentials sourceCredentials;
  73         private Repository targetRepository;
  74         private Credentials targetCredentials;
  75
  76         // if Repository and Credentials objects are not explicitly set
  77         private String sourceRepoUri;
  78         private String sourceUsername;
  79         private char[] sourcePassword;
  80         private String targetRepoUri;
  81         private String targetUsername;
  82         private char[] targetPassword;
  83
  84         private RepositoryFactory repositoryFactory;
  85
  86         private JcrMonitor monitor;
  87         private Map<String, String> workspaceMap;
  88
  89         // TODO fix monitor
  90         private Boolean filesOnly = false;
  91
  92         public RepoSync() {
  93                 zero = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
  94                 zero.setTimeInMillis(0);
  95         }
  96
  97         /**
  98          *
  99          * Shortcut to instantiate a RepoSync with already known repositories and
 100          * credentials.
 101          *
 102          * @param sourceRepository
 103          * @param sourceCredentials
 104          * @param targetRepository
 105          * @param targetCredentials
 106          */
 107         public RepoSync(Repository sourceRepository, Credentials sourceCredentials, Repository targetRepository,
 108                         Credentials targetCredentials) {
 109                 this();
 110                 this.sourceRepository = sourceRepository;
 111                 this.sourceCredentials = sourceCredentials;
 112                 this.targetRepository = targetRepository;
 113                 this.targetCredentials = targetCredentials;
 114         }
 115
 116         public void run() {
 117                 try {
 118                         long begin = System.currentTimeMillis();
 119
 120                         // Setup
 121                         if (sourceRepository == null)
 122                                 sourceRepository = NodeUtils.getRepositoryByUri(repositoryFactory, sourceRepoUri);
 123                         if (sourceCredentials == null && sourceUsername != null)
 124                                 sourceCredentials = new SimpleCredentials(sourceUsername, sourcePassword);
 125                         // FIXME make it more generic
 126                         sourceDefaultSession = sourceRepository.login(sourceCredentials, RepoConstants.DEFAULT_DEFAULT_WORKSPACE);
 127
 128                         if (targetRepository == null)
 129                                 targetRepository = NodeUtils.getRepositoryByUri(repositoryFactory, targetRepoUri);
 130                         if (targetCredentials == null && targetUsername != null)
 131                                 targetCredentials = new SimpleCredentials(targetUsername, targetPassword);
 132                         targetDefaultSession = targetRepository.login(targetCredentials);
 133
 134                         Map<String, Exception> errors = new HashMap<String, Exception>();
 135                         for (String sourceWorkspaceName : sourceDefaultSession.getWorkspace().getAccessibleWorkspaceNames()) {
 136                                 if (monitor != null && monitor.isCanceled())
 137                                         break;
 138
 139                                 if (workspaceMap != null && !workspaceMap.containsKey(sourceWorkspaceName))
 140                                         continue;
 141                                 if (IGNORED_WKSP_LIST.contains(sourceWorkspaceName))
 142                                         continue;
 143
 144                                 Session sourceSession = null;
 145                                 Session targetSession = null;
 146                                 String targetWorkspaceName = workspaceMap.get(sourceWorkspaceName);
 147                                 try {
 148                                         try {
 149                                                 targetSession = targetRepository.login(targetCredentials, targetWorkspaceName);
 150                                         } catch (NoSuchWorkspaceException e) {
 151                                                 targetDefaultSession.getWorkspace().createWorkspace(targetWorkspaceName);
 152                                                 targetSession = targetRepository.login(targetCredentials, targetWorkspaceName);
 153                                         }
 154                                         sourceSession = sourceRepository.login(sourceCredentials, sourceWorkspaceName);
 155                                         syncWorkspace(sourceSession, targetSession);
 156                                 } catch (Exception e) {
 157                                         errors.put("Could not sync workspace " + sourceWorkspaceName, e);
 158                                         if (log.isErrorEnabled())
 159                                                 e.printStackTrace();
 160
 161                                 } finally {
 162                                         JcrUtils.logoutQuietly(sourceSession);
 163                                         JcrUtils.logoutQuietly(targetSession);
 164                                 }
 165                         }
 166
 167                         if (monitor != null && monitor.isCanceled())
 168                                 log.info("Sync has been canceled by user");
 169
 170                         long duration = (System.currentTimeMillis() - begin) / 1000;// s
 171                         log.info("Sync " + sourceRepoUri + " to " + targetRepoUri + " in " + (duration / 60)
 172
 173                                         + "min " + (duration % 60) + "s");
 174
 175                         if (errors.size() > 0) {
 176                                 throw new SlcException("Sync failed " + errors);
 177                         }
 178                 } catch (RepositoryException e) {
 179                         throw new SlcException("Cannot sync " + sourceRepoUri + " to " + targetRepoUri, e);
 180                 } finally {
 181                         JcrUtils.logoutQuietly(sourceDefaultSession);
 182                         JcrUtils.logoutQuietly(targetDefaultSession);
 183                 }
 184         }
 185
 186         private long getNodesNumber(Session session) {
 187                 if (IGNORED_WKSP_LIST.contains(session.getWorkspace().getName()))
 188                         return 0l;
 189                 try {
 190                         Query countQuery = session.getWorkspace().getQueryManager().createQuery(
 191                                         "select file from [" + (true ? NodeType.NT_FILE : NodeType.NT_BASE) + "] as file", Query.JCR_SQL2);
 192
 193                         QueryResult result = countQuery.execute();
 194                         Long expectedCount = result.getNodes().getSize();
 195                         return expectedCount;
 196                 } catch (RepositoryException e) {
 197                         throw new SlcException("Unexpected error while computing " + "the size of the fetch for workspace "
 198                                         + session.getWorkspace().getName(), e);
 199                 }
 200         }
 201
 202         protected void syncWorkspace(Session sourceSession, Session targetSession) {
 203                 if (monitor != null) {
 204                         monitor.beginTask("Computing fetch size...", -1);
 205                         Long totalAmount = getNodesNumber(sourceSession);
 206                         monitor.beginTask("Fetch", totalAmount.intValue());
 207                 }
 208
 209                 try {
 210                         String msg = "Synchronizing workspace: " + sourceSession.getWorkspace().getName();
 211                         if (monitor != null)
 212                                 monitor.setTaskName(msg);
 213                         if (log.isDebugEnabled())
 214                                 log.debug(msg);
 215
 216                         if (filesOnly) {
 217                                 JcrUtils.copyFiles(sourceSession.getRootNode(), targetSession.getRootNode(), true, monitor);
 218                         } else {
 219                                 for (NodeIterator it = sourceSession.getRootNode().getNodes(); it.hasNext();) {
 220                                         Node node = it.nextNode();
 221                                         if (node.getName().equals("jcr:system"))
 222                                                 continue;
 223                                         syncNode(node, targetSession);
 224                                 }
 225                         }
 226                         if (log.isDebugEnabled())
 227                                 log.debug("Synced " + sourceSession.getWorkspace().getName());
 228                 } catch (Exception e) {
 229                         e.printStackTrace();
 230                         throw new SlcException("Cannot sync " + sourceSession.getWorkspace().getName() + " to "
 231                                         + targetSession.getWorkspace().getName(), e);
 232                 }
 233         }
 234
 235         /** factorizes monitor management */
 236         private void updateMonitor(String msg) {
 237                 updateMonitor(msg, false);
 238         }
 239
 240         protected void syncNode(Node sourceNode, Session targetSession) throws RepositoryException, SAXException {
 241                 // Boolean singleLevel = singleLevel(sourceNode);
 242                 try {
 243                         if (monitor != null && monitor.isCanceled()) {
 244                                 updateMonitor("Fetched has been canceled, " + "process is terminating");
 245                                 return;
 246                         }
 247
 248                         Node targetParentNode = targetSession.getNode(sourceNode.getParent().getPath());
 249                         Node targetNode;
 250                         if (monitor != null && sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE))
 251                                 monitor.subTask("Process " + sourceNode.getPath());
 252
 253                         final Boolean isNew;
 254                         if (!targetSession.itemExists(sourceNode.getPath())) {
 255                                 isNew = true;
 256                                 targetNode = targetParentNode.addNode(sourceNode.getName(), sourceNode.getPrimaryNodeType().getName());
 257                         } else {
 258                                 isNew = false;
 259                                 targetNode = targetSession.getNode(sourceNode.getPath());
 260                                 if (!targetNode.getPrimaryNodeType().getName().equals(sourceNode.getPrimaryNodeType().getName()))
 261                                         targetNode.setPrimaryType(sourceNode.getPrimaryNodeType().getName());
 262                         }
 263
 264                         // export
 265                         // sourceNode.getSession().exportSystemView(sourceNode.getPath(),
 266                         // contentHandler, false, singleLevel);
 267
 268                         // if (singleLevel) {
 269                         // if (targetSession.hasPendingChanges()) {
 270                         // // updateMonitor(
 271                         // // (isNew ? "Added " : "Updated ") + targetNode.getPath(),
 272                         // // true);
 273                         // if (doSave)
 274                         // targetSession.save();
 275                         // } else {
 276                         // // updateMonitor("Checked " + targetNode.getPath(), false);
 277                         // }
 278                         // }
 279
 280                         // mixin and properties
 281                         for (NodeType nt : sourceNode.getMixinNodeTypes()) {
 282                                 if (!targetNode.isNodeType(nt.getName()) && targetNode.canAddMixin(nt.getName()))
 283                                         targetNode.addMixin(nt.getName());
 284                         }
 285                         copyProperties(sourceNode, targetNode);
 286
 287                         // next level
 288                         NodeIterator ni = sourceNode.getNodes();
 289                         while (ni != null && ni.hasNext()) {
 290                                 Node sourceChild = ni.nextNode();
 291                                 syncNode(sourceChild, targetSession);
 292                         }
 293
 294                         copyTimestamps(sourceNode, targetNode);
 295
 296                         if (sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE)) {
 297                                 if (targetSession.hasPendingChanges()) {
 298                                         if (sourceNode.isNodeType(NodeType.NT_FILE))
 299                                                 updateMonitor((isNew ? "Added " : "Updated ") + targetNode.getPath(), true);
 300                                         // if (doSave)
 301                                         targetSession.save();
 302                                 } else {
 303                                         if (sourceNode.isNodeType(NodeType.NT_FILE))
 304                                                 updateMonitor("Checked " + targetNode.getPath(), false);
 305                                 }
 306                         }
 307                 } catch (RepositoryException e) {
 308                         throw new SlcException("Cannot sync source node " + sourceNode, e);
 309                 }
 310         }
 311
 312         private void copyTimestamps(Node sourceNode, Node targetNode) throws RepositoryException {
 313                 if (sourceNode.getDefinition().isProtected())
 314                         return;
 315                 if (targetNode.getDefinition().isProtected())
 316                         return;
 317                 copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED);
 318                 copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED_BY);
 319                 copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED);
 320                 copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED_BY);
 321         }
 322
 323         private void copyTimestamp(Node sourceNode, Node targetNode, String property) throws RepositoryException {
 324                 if (sourceNode.hasProperty(property)) {
 325                         Property p = sourceNode.getProperty(property);
 326                         if (p.getDefinition().isProtected())
 327                                 return;
 328                         if (targetNode.hasProperty(property)
 329                                         && targetNode.getProperty(property).getValue().equals(sourceNode.getProperty(property).getValue()))
 330                                 return;
 331                         targetNode.setProperty(property, sourceNode.getProperty(property).getValue());
 332                 }
 333         }
 334
 335         private void copyProperties(Node sourceNode, Node targetNode) throws RepositoryException {
 336                 properties: for (PropertyIterator pi = sourceNode.getProperties(); pi.hasNext();) {
 337                         Property p = pi.nextProperty();
 338                         if (p.getDefinition().isProtected())
 339                                 continue properties;
 340                         if (p.getName().equals(Property.JCR_CREATED) || p.getName().equals(Property.JCR_CREATED_BY)
 341                                         || p.getName().equals(Property.JCR_LAST_MODIFIED)
 342                                         || p.getName().equals(Property.JCR_LAST_MODIFIED_BY))
 343                                 continue properties;
 344
 345                         if (p.getType() == PropertyType.BINARY) {
 346                                 copyBinary(p, targetNode);
 347                         } else {
 348
 349                                 if (p.isMultiple()) {
 350                                         if (!targetNode.hasProperty(p.getName())
 351                                                         || !Arrays.equals(targetNode.getProperty(p.getName()).getValues(), p.getValues()))
 352                                                 targetNode.setProperty(p.getName(), p.getValues());
 353                                 } else {
 354                                         if (!targetNode.hasProperty(p.getName())
 355                                                         || !targetNode.getProperty(p.getName()).getValue().equals(p.getValue()))
 356                                                 targetNode.setProperty(p.getName(), p.getValue());
 357                                 }
 358                         }
 359                 }
 360         }
 361
 362         private static void copyBinary(Property p, Node targetNode) throws RepositoryException {
 363                 InputStream in = null;
 364                 Binary sourceBinary = null;
 365                 Binary targetBinary = null;
 366                 try {
 367                         sourceBinary = p.getBinary();
 368                         if (targetNode.hasProperty(p.getName()))
 369                                 targetBinary = targetNode.getProperty(p.getName()).getBinary();
 370
 371                         // optim FIXME make it more configurable
 372                         if (targetBinary != null)
 373                                 if (sourceBinary.getSize() == targetBinary.getSize()) {
 374                                         if (log.isTraceEnabled())
 375                                                 log.trace("Skipped " + p.getPath());
 376                                         return;
 377                                 }
 378
 379                         in = sourceBinary.getStream();
 380                         targetBinary = targetNode.getSession().getValueFactory().createBinary(in);
 381                         targetNode.setProperty(p.getName(), targetBinary);
 382                 } catch (Exception e) {
 383                         throw new SlcException("Could not transfer " + p, e);
 384                 } finally {
 385                         IOUtils.closeQuietly(in);
 386                         JcrUtils.closeQuietly(sourceBinary);
 387                         JcrUtils.closeQuietly(targetBinary);
 388                 }
 389         }
 390
 391         /** factorizes monitor management */
 392         private void updateMonitor(String msg, Boolean doLog) {
 393                 if (doLog && log.isDebugEnabled())
 394                         log.debug(msg);
 395                 if (monitor != null) {
 396                         monitor.worked(1);
 397                         monitor.subTask(msg);
 398                 }
 399         }
 400
 401         // private void syncNode_old(Node sourceNode, Node targetParentNode)
 402         // throws RepositoryException, SAXException {
 403         //
 404         // // enable cancelation of the current fetch process
 405         // // fxme insure the repository stays in a stable state
 406         // if (monitor != null && monitor.isCanceled()) {
 407         // updateMonitor("Fetched has been canceled, "
 408         // + "process is terminating");
 409         // return;
 410         // }
 411         //
 412         // Boolean noRecurse = singleLevel(sourceNode);
 413         // Calendar sourceLastModified = null;
 414         // if (sourceNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
 415         // sourceLastModified = sourceNode.getProperty(
 416         // Property.JCR_LAST_MODIFIED).getDate();
 417         // }
 418         //
 419         // if (sourceNode.getDefinition().isProtected())
 420         // log.warn(sourceNode + " is protected.");
 421         //
 422         // if (!targetParentNode.hasNode(sourceNode.getName())) {
 423         // String msg = "Adding " + sourceNode.getPath();
 424         // updateMonitor(msg);
 425         // if (log.isDebugEnabled())
 426         // log.debug(msg);
 427         // ContentHandler contentHandler = targetParentNode
 428         // .getSession()
 429         // .getWorkspace()
 430         // .getImportContentHandler(targetParentNode.getPath(),
 431         // ImportUUIDBehavior.IMPORT_UUID_COLLISION_THROW);
 432         // sourceNode.getSession().exportSystemView(sourceNode.getPath(),
 433         // contentHandler, false, noRecurse);
 434         // } else {
 435         // Node targetNode = targetParentNode.getNode(sourceNode.getName());
 436         // if (sourceLastModified != null) {
 437         // Calendar targetLastModified = null;
 438         // if (targetNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
 439         // targetLastModified = targetNode.getProperty(
 440         // Property.JCR_LAST_MODIFIED).getDate();
 441         // }
 442         //
 443         // if (targetLastModified == null
 444         // || targetLastModified.before(sourceLastModified)) {
 445         // String msg = "Updating " + targetNode.getPath();
 446         // updateMonitor(msg);
 447         // if (log.isDebugEnabled())
 448         // log.debug(msg);
 449         // ContentHandler contentHandler = targetParentNode
 450         // .getSession()
 451         // .getWorkspace()
 452         // .getImportContentHandler(
 453         // targetParentNode.getPath(),
 454         // ImportUUIDBehavior.IMPORT_UUID_COLLISION_REMOVE_EXISTING);
 455         // sourceNode.getSession().exportSystemView(
 456         // sourceNode.getPath(), contentHandler, false,
 457         // noRecurse);
 458         // } else {
 459         // String msg = "Skipped up to date " + targetNode.getPath();
 460         // updateMonitor(msg);
 461         // if (log.isDebugEnabled())
 462         // log.debug(msg);
 463         // return;
 464         // }
 465         // }
 466         // }
 467         //
 468         // if (noRecurse) {
 469         // // recurse
 470         // Node targetNode = targetParentNode.getNode(sourceNode.getName());
 471         // if (sourceLastModified != null) {
 472         // Calendar zero = new GregorianCalendar();
 473         // zero.setTimeInMillis(0);
 474         // targetNode.setProperty(Property.JCR_LAST_MODIFIED, zero);
 475         // targetNode.getSession().save();
 476         // }
 477         //
 478         // for (NodeIterator it = sourceNode.getNodes(); it.hasNext();) {
 479         // syncNode_old(it.nextNode(), targetNode);
 480         // }
 481         //
 482         // if (sourceLastModified != null) {
 483         // targetNode.setProperty(Property.JCR_LAST_MODIFIED,
 484         // sourceLastModified);
 485         // targetNode.getSession().save();
 486         // }
 487         // }
 488         // }
 489
 490         protected Boolean singleLevel(Node sourceNode) throws RepositoryException {
 491                 if (sourceNode.isNodeType(NodeType.NT_FILE))
 492                         return false;
 493                 return true;
 494         }
 495
 496         /**
 497          * Synchronises only one workspace, retrieved by name without changing its
 498          * name.
 499          */
 500         public void setSourceWksp(String sourceWksp) {
 501                 if (sourceWksp != null && !sourceWksp.trim().equals("")) {
 502                         Map<String, String> map = new HashMap<String, String>();
 503                         map.put(sourceWksp, sourceWksp);
 504                         setWkspMap(map);
 505                 }
 506         }
 507
 508         /**
 509          * Synchronises a map of workspaces that will be retrieved by name. If the
 510          * target name is not defined (eg null or an empty string) for a given
 511          * source workspace, we use the source name as target name.
 512          */
 513         public void setWkspMap(Map<String, String> workspaceMap) {
 514                 // clean the list to ease later use
 515                 this.workspaceMap = new HashMap<String, String>();
 516                 if (workspaceMap != null) {
 517                         workspaceNames: for (String srcName : workspaceMap.keySet()) {
 518                                 String targetName = workspaceMap.get(srcName);
 519
 520                                 // Sanity check
 521                                 if (srcName.trim().equals(""))
 522                                         continue workspaceNames;
 523                                 if (targetName == null || "".equals(targetName.trim()))
 524                                         targetName = srcName;
 525                                 this.workspaceMap.put(srcName, targetName);
 526                         }
 527                 }
 528                 // clean the map to ease later use
 529                 if (this.workspaceMap.size() == 0)
 530                         this.workspaceMap = null;
 531         }
 532
 533         public void setMonitor(JcrMonitor monitor) {
 534                 this.monitor = monitor;
 535         }
 536
 537         public void setRepositoryFactory(RepositoryFactory repositoryFactory) {
 538                 this.repositoryFactory = repositoryFactory;
 539         }
 540
 541         public void setSourceRepoUri(String sourceRepoUri) {
 542                 this.sourceRepoUri = sourceRepoUri;
 543         }
 544
 545         public void setSourceUsername(String sourceUsername) {
 546                 this.sourceUsername = sourceUsername;
 547         }
 548
 549         public void setSourcePassword(char[] sourcePassword) {
 550                 this.sourcePassword = sourcePassword;
 551         }
 552
 553         public void setTargetRepoUri(String targetRepoUri) {
 554                 this.targetRepoUri = targetRepoUri;
 555         }
 556
 557         public void setTargetUsername(String targetUsername) {
 558                 this.targetUsername = targetUsername;
 559         }
 560
 561         public void setTargetPassword(char[] targetPassword) {
 562                 this.targetPassword = targetPassword;
 563         }
 564
 565         public void setSourceRepository(Repository sourceRepository) {
 566                 this.sourceRepository = sourceRepository;
 567         }
 568
 569         public void setSourceCredentials(Credentials sourceCredentials) {
 570                 this.sourceCredentials = sourceCredentials;
 571         }
 572
 573         public void setTargetRepository(Repository targetRepository) {
 574                 this.targetRepository = targetRepository;
 575         }
 576
 577         public void setTargetCredentials(Credentials targetCredentials) {
 578                 this.targetCredentials = targetCredentials;
 579         }
 580
 581         public void setFilesOnly(Boolean filesOnly) {
 582                 this.filesOnly = filesOnly;
 583         }
 584
 585 }