X-Git-Url: http://git.argeo.org/?a=blobdiff_plain;f=runtime%2Forg.argeo.slc.repo%2Fsrc%2Fmain%2Fjava%2Forg%2Fargeo%2Fslc%2Frepo%2FRepoSync.java;h=7f1e1e32d2f70b7cd4e4f6424dbaf6f2dc755445;hb=04ef2e4533e909122a560a5cb6499fa62bac82ec;hp=49a59b060163b86cb5f7b30a2e8e91c1cc73e91e;hpb=7a08af1e692c666180b26a8bca35010e9e80c4e8;p=gpl%2Fargeo-slc.git diff --git a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoSync.java b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoSync.java index 49a59b060..7f1e1e32d 100644 --- a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoSync.java +++ b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoSync.java @@ -15,7 +15,7 @@ */ package org.argeo.slc.repo; -import java.util.ArrayList; +import java.io.InputStream; import java.util.Arrays; import java.util.Calendar; import java.util.GregorianCalendar; @@ -24,12 +24,14 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; +import javax.jcr.Binary; import javax.jcr.Credentials; -import javax.jcr.ImportUUIDBehavior; import javax.jcr.NoSuchWorkspaceException; import javax.jcr.Node; import javax.jcr.NodeIterator; import javax.jcr.Property; +import javax.jcr.PropertyIterator; +import javax.jcr.PropertyType; import javax.jcr.Repository; import javax.jcr.RepositoryException; import javax.jcr.RepositoryFactory; @@ -39,21 +41,28 @@ import javax.jcr.nodetype.NodeType; import javax.jcr.query.Query; import javax.jcr.query.QueryResult; +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.argeo.ArgeoMonitor; import org.argeo.jcr.ArgeoJcrUtils; import org.argeo.jcr.JcrUtils; import org.argeo.slc.SlcException; -import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; -/** Sync to from software repositories */ +/** + * Synchronise workspaces from a remote software repository to the local + * repository (Synchronisation in the other direction does not work). + * + * Workspaces are retrieved by name given a map that links the source with a + * target name. If a target workspace does not exist, it is created. Otherwise + * we copy the content of the source workspace into the target one. + */ public class RepoSync implements Runnable { private final static Log log = LogFactory.getLog(RepoSync.class); // Centralizes definition of workspaces that must be ignored by the sync. - private final static List IGNORED_WSKP_LIST = Arrays.asList( + private final static List IGNORED_WKSP_LIST = Arrays.asList( "security", "localrepo"); private final Calendar zero; @@ -76,7 +85,10 @@ public class RepoSync implements Runnable { private RepositoryFactory repositoryFactory; private ArgeoMonitor monitor; - private List sourceWkspList; + private Map workspaceMap; + + // TODO fix monitor + private Boolean filesOnly = false; public RepoSync() { zero = new GregorianCalendar(TimeZone.getTimeZone("UTC")); @@ -123,47 +135,31 @@ public class RepoSync implements Runnable { targetPassword); targetDefaultSession = targetRepository.login(targetCredentials); - // FIXME implement a cleaner way to compute job size. - // Compute job size - if (monitor != null) { - monitor.beginTask("Computing fetch size...", -1); - Long totalAmount = 0l; - if (sourceWkspList != null) { - for (String wkspName : sourceWkspList) { - totalAmount += getNodesNumber(wkspName); - } - } else - for (String sourceWorkspaceName : sourceDefaultSession - .getWorkspace().getAccessibleWorkspaceNames()) { - totalAmount += getNodesNumber(sourceWorkspaceName); - } - monitor.beginTask("Fetch", totalAmount.intValue()); - - if (log.isDebugEnabled()) - log.debug("Nb of nodes to sync: " + totalAmount.intValue()); - } - Map errors = new HashMap(); for (String sourceWorkspaceName : sourceDefaultSession .getWorkspace().getAccessibleWorkspaceNames()) { + if (monitor != null && monitor.isCanceled()) + break; - if (sourceWkspList != null - && !sourceWkspList.contains(sourceWorkspaceName)) + if (workspaceMap != null + && !workspaceMap.containsKey(sourceWorkspaceName)) continue; - if (IGNORED_WSKP_LIST.contains(sourceWorkspaceName)) + if (IGNORED_WKSP_LIST.contains(sourceWorkspaceName)) continue; Session sourceSession = null; Session targetSession = null; + String targetWorkspaceName = workspaceMap + .get(sourceWorkspaceName); try { try { targetSession = targetRepository.login( - targetCredentials, sourceWorkspaceName); + targetCredentials, targetWorkspaceName); } catch (NoSuchWorkspaceException e) { targetDefaultSession.getWorkspace().createWorkspace( - sourceWorkspaceName); + targetWorkspaceName); targetSession = targetRepository.login( - targetCredentials, sourceWorkspaceName); + targetCredentials, targetWorkspaceName); } sourceSession = sourceRepository.login(sourceCredentials, sourceWorkspaceName); @@ -171,8 +167,9 @@ public class RepoSync implements Runnable { } catch (Exception e) { errors.put("Could not sync workspace " + sourceWorkspaceName, e); - if (log.isDebugEnabled()) + if (log.isErrorEnabled()) e.printStackTrace(); + } finally { JcrUtils.logoutQuietly(sourceSession); JcrUtils.logoutQuietly(targetSession); @@ -200,29 +197,36 @@ public class RepoSync implements Runnable { } } - private long getNodesNumber(String wkspName) { - if (IGNORED_WSKP_LIST.contains(wkspName)) + private long getNodesNumber(Session session) { + if (IGNORED_WKSP_LIST.contains(session.getWorkspace().getName())) return 0l; - Session sourceSession = null; try { - sourceSession = sourceRepository.login(sourceCredentials, wkspName); - Query countQuery = sourceDefaultSession + Query countQuery = session .getWorkspace() .getQueryManager() - .createQuery("select file from [nt:base] as file", + .createQuery( + "select file from [" + + (true ? NodeType.NT_FILE + : NodeType.NT_BASE) + "] as file", Query.JCR_SQL2); + QueryResult result = countQuery.execute(); Long expectedCount = result.getNodes().getSize(); return expectedCount; } catch (RepositoryException e) { throw new SlcException("Unexpected error while computing " - + "the size of the fetch for workspace " + wkspName, e); - } finally { - JcrUtils.logoutQuietly(sourceSession); + + "the size of the fetch for workspace " + + session.getWorkspace().getName(), e); } } protected void syncWorkspace(Session sourceSession, Session targetSession) { + if (monitor != null) { + monitor.beginTask("Computing fetch size...", -1); + Long totalAmount = getNodesNumber(sourceSession); + monitor.beginTask("Fetch", totalAmount.intValue()); + } + try { String msg = "Synchronizing workspace: " + sourceSession.getWorkspace().getName(); @@ -230,16 +234,23 @@ public class RepoSync implements Runnable { monitor.setTaskName(msg); if (log.isDebugEnabled()) log.debug(msg); - for (NodeIterator it = sourceSession.getRootNode().getNodes(); it - .hasNext();) { - Node node = it.nextNode(); - if (node.getName().equals("jcr:system")) - continue; - syncNode(node, targetSession.getRootNode()); + + if (filesOnly) { + JcrUtils.copyFiles(sourceSession.getRootNode(), + targetSession.getRootNode(), true, monitor); + } else { + for (NodeIterator it = sourceSession.getRootNode().getNodes(); it + .hasNext();) { + Node node = it.nextNode(); + if (node.getName().equals("jcr:system")) + continue; + syncNode(node, targetSession); + } } if (log.isDebugEnabled()) log.debug("Synced " + sourceSession.getWorkspace().getName()); } catch (Exception e) { + e.printStackTrace(); throw new SlcException("Cannot sync " + sourceSession.getWorkspace().getName() + " to " + targetSession.getWorkspace().getName(), e); @@ -248,127 +259,323 @@ public class RepoSync implements Runnable { /** factorizes monitor management */ private void updateMonitor(String msg) { - if (monitor != null) { - monitor.worked(1); - monitor.subTask(msg); - } + updateMonitor(msg, false); } - protected void syncNode(Node sourceNode, Node targetParentNode) + protected void syncNode(Node sourceNode, Session targetSession) throws RepositoryException, SAXException { + // Boolean singleLevel = singleLevel(sourceNode); + try { + if (monitor != null && monitor.isCanceled()) { + updateMonitor("Fetched has been canceled, " + + "process is terminating"); + return; + } - // enable cancelation of the current fetch process - // FIXME insure the repository stays in a stable state - if (monitor != null && monitor.isCanceled()) { - updateMonitor("Fetched has been canceled, " - + "process is terminating"); - return; - } + Node targetParentNode = targetSession.getNode(sourceNode + .getParent().getPath()); + Node targetNode; + if (monitor != null + && sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE)) + monitor.subTask("Process " + sourceNode.getPath()); + + final Boolean isNew; + if (!targetSession.itemExists(sourceNode.getPath())) { + isNew = true; + targetNode = targetParentNode.addNode(sourceNode.getName(), + sourceNode.getPrimaryNodeType().getName()); + } else { + isNew = false; + targetNode = targetSession.getNode(sourceNode.getPath()); + if (!targetNode.getPrimaryNodeType().getName() + .equals(sourceNode.getPrimaryNodeType().getName())) + targetNode.setPrimaryType(sourceNode.getPrimaryNodeType() + .getName()); + } - Boolean noRecurse = noRecurse(sourceNode); - Calendar sourceLastModified = null; - if (sourceNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) { - sourceLastModified = sourceNode.getProperty( - Property.JCR_LAST_MODIFIED).getDate(); - } + // export + // sourceNode.getSession().exportSystemView(sourceNode.getPath(), + // contentHandler, false, singleLevel); + + // if (singleLevel) { + // if (targetSession.hasPendingChanges()) { + // // updateMonitor( + // // (isNew ? "Added " : "Updated ") + targetNode.getPath(), + // // true); + // if (doSave) + // targetSession.save(); + // } else { + // // updateMonitor("Checked " + targetNode.getPath(), false); + // } + // } + + // mixin and properties + for (NodeType nt : sourceNode.getMixinNodeTypes()) { + if (!targetNode.isNodeType(nt.getName()) + && targetNode.canAddMixin(nt.getName())) + targetNode.addMixin(nt.getName()); + } + copyProperties(sourceNode, targetNode); - if (!targetParentNode.hasNode(sourceNode.getName())) { - String msg = "Adding " + sourceNode.getPath(); - updateMonitor(msg); - if (log.isDebugEnabled()) - log.debug(msg); - ContentHandler contentHandler = targetParentNode - .getSession() - .getWorkspace() - .getImportContentHandler(targetParentNode.getPath(), - ImportUUIDBehavior.IMPORT_UUID_COLLISION_THROW); - sourceNode.getSession().exportSystemView(sourceNode.getPath(), - contentHandler, false, noRecurse); - } else { - Node targetNode = targetParentNode.getNode(sourceNode.getName()); - if (sourceLastModified != null) { - Calendar targetLastModified = null; - if (targetNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) { - targetLastModified = targetNode.getProperty( - Property.JCR_LAST_MODIFIED).getDate(); - } + // next level + NodeIterator ni = sourceNode.getNodes(); + while (ni != null && ni.hasNext()) { + Node sourceChild = ni.nextNode(); + syncNode(sourceChild, targetSession); + } + + copyTimestamps(sourceNode, targetNode); - if (targetLastModified == null - || targetLastModified.before(sourceLastModified)) { - String msg = "Updating " + targetNode.getPath(); - updateMonitor(msg); - if (log.isDebugEnabled()) - log.debug(msg); - ContentHandler contentHandler = targetParentNode - .getSession() - .getWorkspace() - .getImportContentHandler( - targetParentNode.getPath(), - ImportUUIDBehavior.IMPORT_UUID_COLLISION_REMOVE_EXISTING); - sourceNode.getSession().exportSystemView( - sourceNode.getPath(), contentHandler, false, - noRecurse); + if (sourceNode.isNodeType(NodeType.NT_HIERARCHY_NODE)) { + if (targetSession.hasPendingChanges()) { + if (sourceNode.isNodeType(NodeType.NT_FILE)) + updateMonitor((isNew ? "Added " : "Updated ") + + targetNode.getPath(), true); + // if (doSave) + targetSession.save(); } else { - String msg = "Skipped up to date " + targetNode.getPath(); - updateMonitor(msg); - if (log.isDebugEnabled()) - log.debug(msg); - return; + if (sourceNode.isNodeType(NodeType.NT_FILE)) + updateMonitor("Checked " + targetNode.getPath(), false); } } + } catch (RepositoryException e) { + throw new SlcException("Cannot sync source node " + sourceNode, e); } + } - if (noRecurse) { - // recurse - Node targetNode = targetParentNode.getNode(sourceNode.getName()); - if (sourceLastModified != null) { - Calendar zero = new GregorianCalendar(); - zero.setTimeInMillis(0); - targetNode.setProperty(Property.JCR_LAST_MODIFIED, zero); - targetNode.getSession().save(); - } + private void copyTimestamps(Node sourceNode, Node targetNode) + throws RepositoryException { + if (sourceNode.getDefinition().isProtected()) + return; + if (targetNode.getDefinition().isProtected()) + return; + copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED); + copyTimestamp(sourceNode, targetNode, Property.JCR_CREATED_BY); + copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED); + copyTimestamp(sourceNode, targetNode, Property.JCR_LAST_MODIFIED_BY); + } - for (NodeIterator it = sourceNode.getNodes(); it.hasNext();) { - syncNode(it.nextNode(), targetNode); - } + private void copyTimestamp(Node sourceNode, Node targetNode, String property) + throws RepositoryException { + if (sourceNode.hasProperty(property)) { + Property p = sourceNode.getProperty(property); + if (p.getDefinition().isProtected()) + return; + if (targetNode.hasProperty(property) + && targetNode + .getProperty(property) + .getValue() + .equals(sourceNode.getProperty(property).getValue())) + return; + targetNode.setProperty(property, sourceNode.getProperty(property) + .getValue()); + } + } - if (sourceLastModified != null) { - targetNode.setProperty(Property.JCR_LAST_MODIFIED, - sourceLastModified); - targetNode.getSession().save(); + private void copyProperties(Node sourceNode, Node targetNode) + throws RepositoryException { + properties: for (PropertyIterator pi = sourceNode.getProperties(); pi + .hasNext();) { + Property p = pi.nextProperty(); + if (p.getDefinition().isProtected()) + continue properties; + if (p.getName().equals(Property.JCR_CREATED) + || p.getName().equals(Property.JCR_CREATED_BY) + || p.getName().equals(Property.JCR_LAST_MODIFIED) + || p.getName().equals(Property.JCR_LAST_MODIFIED_BY)) + continue properties; + + if (p.getType() == PropertyType.BINARY) { + copyBinary(p, targetNode); + } else { + + if (p.isMultiple()) { + if (!targetNode.hasProperty(p.getName()) + || !Arrays.equals( + targetNode.getProperty(p.getName()) + .getValues(), p.getValues())) + targetNode.setProperty(p.getName(), p.getValues()); + } else { + if (!targetNode.hasProperty(p.getName()) + || !targetNode.getProperty(p.getName()).getValue() + .equals(p.getValue())) + targetNode.setProperty(p.getName(), p.getValue()); + } } } } - protected Boolean noRecurse(Node sourceNode) throws RepositoryException { + private static void copyBinary(Property p, Node targetNode) + throws RepositoryException { + InputStream in = null; + Binary sourceBinary = null; + Binary targetBinary = null; + try { + sourceBinary = p.getBinary(); + if (targetNode.hasProperty(p.getName())) + targetBinary = targetNode.getProperty(p.getName()).getBinary(); + + // optim FIXME make it more configurable + if (targetBinary != null) + if (sourceBinary.getSize() == targetBinary.getSize()) { + if (log.isTraceEnabled()) + log.trace("Skipped " + p.getPath()); + return; + } + + in = sourceBinary.getStream(); + targetBinary = targetNode.getSession().getValueFactory() + .createBinary(in); + targetNode.setProperty(p.getName(), targetBinary); + } catch (Exception e) { + throw new SlcException("Could not transfer " + p, e); + } finally { + IOUtils.closeQuietly(in); + JcrUtils.closeQuietly(sourceBinary); + JcrUtils.closeQuietly(targetBinary); + } + } + + /** factorizes monitor management */ + private void updateMonitor(String msg, Boolean doLog) { + if (doLog && log.isDebugEnabled()) + log.debug(msg); + if (monitor != null) { + monitor.worked(1); + monitor.subTask(msg); + } + } + + // private void syncNode_old(Node sourceNode, Node targetParentNode) + // throws RepositoryException, SAXException { + // + // // enable cancelation of the current fetch process + // // FIXME insure the repository stays in a stable state + // if (monitor != null && monitor.isCanceled()) { + // updateMonitor("Fetched has been canceled, " + // + "process is terminating"); + // return; + // } + // + // Boolean noRecurse = singleLevel(sourceNode); + // Calendar sourceLastModified = null; + // if (sourceNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) { + // sourceLastModified = sourceNode.getProperty( + // Property.JCR_LAST_MODIFIED).getDate(); + // } + // + // if (sourceNode.getDefinition().isProtected()) + // log.warn(sourceNode + " is protected."); + // + // if (!targetParentNode.hasNode(sourceNode.getName())) { + // String msg = "Adding " + sourceNode.getPath(); + // updateMonitor(msg); + // if (log.isDebugEnabled()) + // log.debug(msg); + // ContentHandler contentHandler = targetParentNode + // .getSession() + // .getWorkspace() + // .getImportContentHandler(targetParentNode.getPath(), + // ImportUUIDBehavior.IMPORT_UUID_COLLISION_THROW); + // sourceNode.getSession().exportSystemView(sourceNode.getPath(), + // contentHandler, false, noRecurse); + // } else { + // Node targetNode = targetParentNode.getNode(sourceNode.getName()); + // if (sourceLastModified != null) { + // Calendar targetLastModified = null; + // if (targetNode.isNodeType(NodeType.MIX_LAST_MODIFIED)) { + // targetLastModified = targetNode.getProperty( + // Property.JCR_LAST_MODIFIED).getDate(); + // } + // + // if (targetLastModified == null + // || targetLastModified.before(sourceLastModified)) { + // String msg = "Updating " + targetNode.getPath(); + // updateMonitor(msg); + // if (log.isDebugEnabled()) + // log.debug(msg); + // ContentHandler contentHandler = targetParentNode + // .getSession() + // .getWorkspace() + // .getImportContentHandler( + // targetParentNode.getPath(), + // ImportUUIDBehavior.IMPORT_UUID_COLLISION_REMOVE_EXISTING); + // sourceNode.getSession().exportSystemView( + // sourceNode.getPath(), contentHandler, false, + // noRecurse); + // } else { + // String msg = "Skipped up to date " + targetNode.getPath(); + // updateMonitor(msg); + // if (log.isDebugEnabled()) + // log.debug(msg); + // return; + // } + // } + // } + // + // if (noRecurse) { + // // recurse + // Node targetNode = targetParentNode.getNode(sourceNode.getName()); + // if (sourceLastModified != null) { + // Calendar zero = new GregorianCalendar(); + // zero.setTimeInMillis(0); + // targetNode.setProperty(Property.JCR_LAST_MODIFIED, zero); + // targetNode.getSession().save(); + // } + // + // for (NodeIterator it = sourceNode.getNodes(); it.hasNext();) { + // syncNode_old(it.nextNode(), targetNode); + // } + // + // if (sourceLastModified != null) { + // targetNode.setProperty(Property.JCR_LAST_MODIFIED, + // sourceLastModified); + // targetNode.getSession().save(); + // } + // } + // } + + protected Boolean singleLevel(Node sourceNode) throws RepositoryException { if (sourceNode.isNodeType(NodeType.NT_FILE)) return false; return true; } - /** synchronise only one workspace retrieved by name */ + /** + * Synchronises only one workspace, retrieved by name without changing its + * name. + */ public void setSourceWksp(String sourceWksp) { if (sourceWksp != null && !sourceWksp.trim().equals("")) { - List list = new ArrayList(); - list.add(sourceWksp); - setSourceWkspList(list); + Map map = new HashMap(); + map.put(sourceWksp, sourceWksp); + setWkspMap(map); } } - /** synchronise a list workspace that will be retrieved by name */ - public void setSourceWkspList(List sourceWkspList) { + /** + * Synchronises a map of workspaces that will be retrieved by name. If the + * target name is not defined (eg null or an empty string) for a given + * source workspace, we use the source name as target name. + */ + public void setWkspMap(Map workspaceMap) { // clean the list to ease later use - this.sourceWkspList = null; - if (sourceWkspList != null) { - for (String wkspName : sourceWkspList) { - if (!wkspName.trim().equals("")) { - // only instantiate if needed - if (this.sourceWkspList == null) - this.sourceWkspList = new ArrayList(); - this.sourceWkspList.add(wkspName); - } + this.workspaceMap = new HashMap(); + if (workspaceMap != null) { + workspaceNames: for (String srcName : workspaceMap.keySet()) { + String targetName = workspaceMap.get(srcName); + + // Sanity check + if (srcName.trim().equals("")) + continue workspaceNames; + if (targetName == null || "".equals(targetName.trim())) + targetName = srcName; + this.workspaceMap.put(srcName, targetName); } } + // clean the map to ease later use + if (this.workspaceMap.size() == 0) + this.workspaceMap = null; } public void setMonitor(ArgeoMonitor monitor) { @@ -418,4 +625,9 @@ public class RepoSync implements Runnable { public void setTargetCredentials(Credentials targetCredentials) { this.targetCredentials = targetCredentials; } + + public void setFilesOnly(Boolean filesOnly) { + this.filesOnly = filesOnly; + } + } \ No newline at end of file