From: Mathieu Baudier Date: Wed, 20 Jan 2021 05:59:04 +0000 (+0100) Subject: Work on logical backups. X-Git-Tag: argeo-commons-2.1.91~21 X-Git-Url: https://git.argeo.org/?p=lgpl%2Fargeo-commons.git;a=commitdiff_plain;h=54440e81b1446dc13897f4add75ea6607f30a8de Work on logical backups. --- diff --git a/org.argeo.jcr/src/org/argeo/jcr/Jcr.java b/org.argeo.jcr/src/org/argeo/jcr/Jcr.java index 4b8bfc3ea..20325cfe0 100644 --- a/org.argeo.jcr/src/org/argeo/jcr/Jcr.java +++ b/org.argeo.jcr/src/org/argeo/jcr/Jcr.java @@ -132,6 +132,14 @@ public class Jcr { } } + /** + * @see Node#getParent() + * @throws JcrException caused by {@link RepositoryException} + */ + public static String getParentPath(Node node) { + return getPath(getParent(node)); + } + /** * Whether this node is the root node. * diff --git a/org.argeo.maintenance/src/org/argeo/maintenance/backup/BackupContentHandler.java b/org.argeo.maintenance/src/org/argeo/maintenance/backup/BackupContentHandler.java index e29483e97..ef83c1ff9 100644 --- a/org.argeo.maintenance/src/org/argeo/maintenance/backup/BackupContentHandler.java +++ b/org.argeo.maintenance/src/org/argeo/maintenance/backup/BackupContentHandler.java @@ -3,15 +3,18 @@ package org.argeo.maintenance.backup; import java.io.IOException; import java.io.InputStream; import java.io.Writer; +import java.util.Arrays; import java.util.Base64; import java.util.Set; import java.util.TreeSet; import javax.jcr.Binary; +import javax.jcr.Node; import javax.jcr.RepositoryException; import javax.jcr.Session; import org.apache.commons.io.IOUtils; +import org.argeo.jcr.Jcr; import org.argeo.jcr.JcrException; import org.xml.sax.Attributes; import org.xml.sax.SAXException; @@ -39,12 +42,17 @@ public class BackupContentHandler extends DefaultHandler { private Session session; private Set contentPaths = new TreeSet<>(); + boolean prettyPrint = true; + + private final String parentPath; + // private boolean inSystem = false; - public BackupContentHandler(Writer out, Session session) { + public BackupContentHandler(Writer out, Node node) { super(); this.out = out; - this.session = session; + this.session = Jcr.getSession(node); + parentPath = Jcr.getParentPath(node); } private int currentDepth = -1; @@ -87,6 +95,19 @@ public class BackupContentHandler extends DefaultHandler { if (SV_NAMESPACE_URI.equals(uri)) try { + if (prettyPrint) { + if (isNode) { + out.write(spaces()); + out.write("\n"); + out.write(spaces()); + } else if (isProperty) + out.write(spaces()); + else if (currentPropertyIsMultiple) + out.write(spaces()); + } + out.write("<"); out.write(SV_PREFIX + ":" + localName); if (isProperty) @@ -108,9 +129,9 @@ public class BackupContentHandler extends DefaultHandler { else if (TYPE.equals(attrName)) { if (BINARY.equals(attrValue)) { if (JCR_CONTENT.equals(getCurrentName())) { - contentPaths.add(getCurrentPath()); + contentPaths.add(getCurrentJcrPath()); } else { - Binary binary = session.getNode(getCurrentPath()).getProperty(attrName) + Binary binary = session.getNode(getCurrentJcrPath()).getProperty(attrName) .getBinary(); try (InputStream in = binary.getStream()) { currentEncoded = base64encore.encodeToString(IOUtils.toByteArray(in)); @@ -128,10 +149,12 @@ public class BackupContentHandler extends DefaultHandler { out.write(" xmlns:" + SV_PREFIX + "=\"" + SV_NAMESPACE_URI + "\""); } out.write(">"); - if (isNode) - out.write("\n"); - else if (isProperty && currentPropertyIsMultiple) - out.write("\n"); + + if (prettyPrint) + if (isNode) + out.write("\n"); + else if (isProperty && currentPropertyIsMultiple) + out.write("\n"); } catch (IOException e) { throw new RuntimeException(e); } catch (RepositoryException e) { @@ -141,12 +164,21 @@ public class BackupContentHandler extends DefaultHandler { @Override public void endElement(String uri, String localName, String qName) throws SAXException { - if (localName.equals(NODE)) { + boolean isNode = localName.equals(NODE); + boolean isValue = localName.equals(VALUE); + if (prettyPrint) + if (!isValue) + try { + if (isNode || currentPropertyIsMultiple) + out.write(spaces()); + } catch (IOException e1) { + throw new RuntimeException(e1); + } + if (isNode) { // System.out.println("endElement " + getCurrentPath() + " , depth=" + currentDepth); // if (currentDepth > 0) currentPath[currentDepth] = null; currentDepth = currentDepth - 1; - assert currentDepth >= 0; // if (inSystem) { // // System.out.println("Skip " + getCurrentPath()+" , // // currentDepth="+currentDepth); @@ -158,7 +190,6 @@ public class BackupContentHandler extends DefaultHandler { } // if (inSystem) // return; - boolean isValue = localName.equals(VALUE); if (SV_NAMESPACE_URI.equals(uri)) try { if (isValue && currentEncoded != null) { @@ -168,15 +199,25 @@ public class BackupContentHandler extends DefaultHandler { out.write(""); - if (!isValue) - out.write("\n"); - else { - if (currentPropertyIsMultiple) + if (prettyPrint) + if (!isValue) out.write("\n"); - } + else { + if (currentPropertyIsMultiple) + out.write("\n"); + } + if (currentDepth == 0) + out.flush(); } catch (IOException e) { throw new RuntimeException(e); } + + } + + private char[] spaces() { + char[] arr = new char[currentDepth]; + Arrays.fill(arr, ' '); + return arr; } @Override @@ -197,13 +238,13 @@ public class BackupContentHandler extends DefaultHandler { return currentPath[currentDepth]; } - protected String getCurrentPath() { + protected String getCurrentJcrPath() { // if (currentDepth == 0) // return "/"; - StringBuilder sb = new StringBuilder(); + StringBuilder sb = new StringBuilder(parentPath.equals("/") ? "" : parentPath); for (int i = 0; i <= currentDepth; i++) { // if (i != 0) - sb.append('/'); + sb.append('/'); sb.append(currentPath[i]); } return sb.toString(); diff --git a/org.argeo.maintenance/src/org/argeo/maintenance/backup/LogicalBackup.java b/org.argeo.maintenance/src/org/argeo/maintenance/backup/LogicalBackup.java index 4f7a2cfea..60e8f8e5d 100644 --- a/org.argeo.maintenance/src/org/argeo/maintenance/backup/LogicalBackup.java +++ b/org.argeo.maintenance/src/org/argeo/maintenance/backup/LogicalBackup.java @@ -18,6 +18,7 @@ import java.util.Enumeration; import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -42,6 +43,8 @@ import javax.jcr.Session; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.jackrabbit.api.JackrabbitSession; +import org.apache.jackrabbit.api.JackrabbitValue; import org.argeo.api.NodeConstants; import org.argeo.api.NodeUtils; import org.argeo.jackrabbit.client.ClientDavexRepositoryFactory; @@ -50,7 +53,6 @@ import org.argeo.jcr.JcrException; import org.argeo.jcr.JcrUtils; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; -import org.xml.sax.SAXException; /** * Performs a backup of the data based only on programmatic interfaces. Useful @@ -64,6 +66,10 @@ public class LogicalBackup implements Runnable { public final static String WORKSPACES_BASE = "workspaces/"; public final static String FILES_BASE = "files/"; public final static String OSGI_BASE = "share/osgi/"; + + public final static String JCR_SYSTEM = "jcr:system"; + public final static String JCR_VERSION_STORAGE_PATH = "/jcr:system/jcr:versionStorage"; + private final Repository repository; private String defaultWorkspace; private final BundleContext bundleContext; @@ -73,13 +79,19 @@ public class LogicalBackup implements Runnable { private ExecutorService executorService; + private boolean performSoftwareBackup = false; + + private Map checksums = new TreeMap<>(); + + private int threadCount = 5; + + private boolean backupFailed = false; + public LogicalBackup(BundleContext bundleContext, Repository repository, Path basePath) { this.repository = repository; this.zout = null; this.basePath = basePath; this.bundleContext = bundleContext; - - executorService = Executors.newFixedThreadPool(3); } @Override @@ -95,9 +107,12 @@ public class LogicalBackup implements Runnable { } public void perform() throws RepositoryException, IOException { + if (executorService != null && !executorService.isTerminated()) + throw new IllegalStateException("Another backup is running"); + executorService = Executors.newFixedThreadPool(threadCount); long begin = System.currentTimeMillis(); // software backup - if (bundleContext != null) + if (bundleContext != null && performSoftwareBackup) executorService.submit(() -> performSoftwareBackup(bundleContext)); // data backup @@ -117,18 +132,37 @@ public class LogicalBackup implements Runnable { executorService.awaitTermination(24, TimeUnit.HOURS); } catch (InterruptedException e) { // silent + throw new IllegalStateException("Backup was interrupted before completion", e); + } + } + // versions + executorService = Executors.newFixedThreadPool(threadCount); + try { + performVersionsBackup(); + } finally { + executorService.shutdown(); + try { + executorService.awaitTermination(24, TimeUnit.HOURS); + } catch (InterruptedException e) { + // silent + throw new IllegalStateException("Backup was interrupted before completion", e); } } long duration = System.currentTimeMillis() - begin; - log.info("System logical backup completed in " + (duration / 60000) + "min " + (duration / 1000) + "s"); + if (isBackupFailed()) + log.info("System logical backup failed after " + (duration / 60000) + "min " + (duration / 1000) + "s"); + else + log.info("System logical backup completed in " + (duration / 60000) + "min " + (duration / 1000) + "s"); } protected void performDataBackup(String workspaceName) throws RepositoryException, IOException { Session session = login(workspaceName); try { nodes: for (NodeIterator nit = session.getRootNode().getNodes(); nit.hasNext();) { + if (isBackupFailed()) + return; Node nodeToExport = nit.nextNode(); - if ("jcr:system".equals(nodeToExport.getName()) && !workspaceName.equals(defaultWorkspace)) + if (JCR_SYSTEM.equals(nodeToExport.getName())) continue nodes; String nodePath = nodeToExport.getPath(); Future> contentPathsFuture = executorService @@ -140,33 +174,51 @@ public class LogicalBackup implements Runnable { } } + protected void performVersionsBackup() throws RepositoryException, IOException { + Session session = login(defaultWorkspace); + Node versionStorageNode = session.getNode(JCR_VERSION_STORAGE_PATH); + try { + for (NodeIterator nit = versionStorageNode.getNodes(); nit.hasNext();) { + Node nodeToExport = nit.nextNode(); + String nodePath = nodeToExport.getPath(); + if (isBackupFailed()) + return; + Future> contentPathsFuture = executorService + .submit(() -> performNodeBackup(defaultWorkspace, nodePath)); + executorService.submit(() -> performFilesBackup(defaultWorkspace, contentPathsFuture)); + } + } finally { + Jcr.logout(session); + } + + } + protected Set performNodeBackup(String workspaceName, String nodePath) { Session session = login(workspaceName); try { Node nodeToExport = session.getNode(nodePath); - String nodeName = nodeToExport.getName(); +// String nodeName = nodeToExport.getName(); // if (nodeName.startsWith("jcr:") || nodeName.startsWith("rep:")) // continue nodes; // // TODO make it more robust / configurable // if (nodeName.equals("user")) // continue nodes; - String relativePath = WORKSPACES_BASE + workspaceName + "/" + nodeName + ".xml"; + String relativePath = WORKSPACES_BASE + workspaceName + nodePath + ".xml"; OutputStream xmlOut = openOutputStream(relativePath); BackupContentHandler contentHandler; try (Writer writer = new BufferedWriter(new OutputStreamWriter(xmlOut, StandardCharsets.UTF_8))) { - contentHandler = new BackupContentHandler(writer, session); + contentHandler = new BackupContentHandler(writer, nodeToExport); session.exportSystemView(nodeToExport.getPath(), contentHandler, true, false); if (log.isDebugEnabled()) - log.debug(workspaceName + ":/" + nodeName + " metadata exported to " + relativePath); + log.debug(workspaceName + ":" + nodePath + " metadata exported to " + relativePath); } // Files Set contentPaths = contentHandler.getContentPaths(); return contentPaths; - } catch (IOException | SAXException e) { - throw new RuntimeException("Cannot backup node " + workspaceName + ":" + nodePath, e); - } catch (RepositoryException e) { - throw new JcrException("Cannot backup node " + workspaceName + ":" + nodePath, e); + } catch (Exception e) { + markBackupFailed("Cannot backup node " + workspaceName + ":" + nodePath, e); + throw new ThreadDeath(); } finally { Jcr.logout(session); } @@ -177,7 +229,8 @@ public class LogicalBackup implements Runnable { try { contentPaths = contentPathsFuture.get(24, TimeUnit.HOURS); } catch (InterruptedException | ExecutionException | TimeoutException e1) { - throw new RuntimeException("Cannot retrieve content paths for workspace " + workspaceName); + markBackupFailed("Cannot retrieve content paths for workspace " + workspaceName, e1); + return; } if (contentPaths == null || contentPaths.size() == 0) return; @@ -185,23 +238,55 @@ public class LogicalBackup implements Runnable { try { String workspacesFilesBasePath = FILES_BASE + workspaceName; for (String path : contentPaths) { + if (isBackupFailed()) + return; Node contentNode = session.getNode(path); - Binary binary = contentNode.getProperty(Property.JCR_DATA).getBinary(); - String fileRelativePath = workspacesFilesBasePath + contentNode.getParent().getPath(); - try (InputStream in = binary.getStream(); OutputStream out = openOutputStream(fileRelativePath)) { - IOUtils.copy(in, out); - if (log.isTraceEnabled()) - log.trace("Workspace " + workspaceName + ": file content exported to " + fileRelativePath); + Binary binary = null; + try { + binary = contentNode.getProperty(Property.JCR_DATA).getBinary(); + String fileRelativePath = workspacesFilesBasePath + contentNode.getParent().getPath(); + + // checksum + boolean skip = false; + String checksum = null; + if (session instanceof JackrabbitSession) { + JackrabbitValue value = (JackrabbitValue) contentNode.getProperty(Property.JCR_DATA).getValue(); +// ReferenceBinary referenceBinary = (ReferenceBinary) binary; + checksum = value.getContentIdentity(); + } + if (checksum != null) { + if (!checksums.containsKey(checksum)) { + checksums.put(checksum, fileRelativePath); + } else { + skip = true; + String sourcePath = checksums.get(checksum); + if (log.isTraceEnabled()) + log.trace(fileRelativePath + " : already " + sourcePath + " with checksum " + checksum); + createLink(sourcePath, fileRelativePath); + try (Writer writerSum = new OutputStreamWriter( + openOutputStream(fileRelativePath + ".sha256"), StandardCharsets.UTF_8)) { + writerSum.write(checksum); + } + } + } + + // copy file + if (!skip) + try (InputStream in = binary.getStream(); + OutputStream out = openOutputStream(fileRelativePath)) { + IOUtils.copy(in, out); + if (log.isTraceEnabled()) + log.trace("Workspace " + workspaceName + ": file content exported to " + + fileRelativePath); + } } finally { JcrUtils.closeQuietly(binary); } } if (log.isDebugEnabled()) log.debug(workspaceName + ":" + contentPaths.size() + " files exported to " + workspacesFilesBasePath); - } catch (RepositoryException e) { - throw new JcrException("Cannot backup files from " + workspaceName + ":", e); - } catch (IOException e) { - throw new RuntimeException("Cannot backup files from " + workspaceName + ":", e); + } catch (Exception e) { + markBackupFailed("Cannot backup files from " + workspaceName + ":", e); } finally { Jcr.logout(session); } @@ -221,6 +306,21 @@ public class LogicalBackup implements Runnable { } } + protected void createLink(String source, String target) throws IOException { + if (zout != null) { + // TODO implement for zip + throw new UnsupportedOperationException(); + } else if (basePath != null) { + Path sourcePath = basePath.resolve(Paths.get(source)); + Path targetPath = basePath.resolve(Paths.get(target)); + Path relativeSource = targetPath.getParent().relativize(sourcePath); + Files.createDirectories(targetPath.getParent()); + Files.createSymbolicLink(targetPath, relativeSource); + } else { + throw new UnsupportedOperationException(); + } + } + protected void closeOutputStream(String relativePath, OutputStream out) throws IOException { if (zout != null) { zout.closeEntry(); @@ -335,4 +435,15 @@ public class LogicalBackup implements Runnable { } + protected synchronized void markBackupFailed(Object message, Exception e) { + log.error(message, e); + backupFailed = true; + notifyAll(); + if (executorService != null) + executorService.shutdownNow(); + } + + protected boolean isBackupFailed() { + return backupFailed; + } } diff --git a/org.argeo.maintenance/src/org/argeo/maintenance/backup/LogicalRestore.java b/org.argeo.maintenance/src/org/argeo/maintenance/backup/LogicalRestore.java index 145c5bb3d..a12bb41c9 100644 --- a/org.argeo.maintenance/src/org/argeo/maintenance/backup/LogicalRestore.java +++ b/org.argeo.maintenance/src/org/argeo/maintenance/backup/LogicalRestore.java @@ -13,6 +13,8 @@ import javax.jcr.Session; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.argeo.api.NodeConstants; +import org.argeo.api.NodeUtils; import org.argeo.jcr.Jcr; import org.argeo.jcr.JcrException; import org.argeo.jcr.JcrUtils; @@ -37,27 +39,22 @@ public class LogicalRestore implements Runnable { Path workspaces = basePath.resolve(LogicalBackup.WORKSPACES_BASE); try { // import jcr:system first - try (DirectoryStream workspaceDirs = Files.newDirectoryStream(workspaces)) { - dirs: for (Path workspacePath : workspaceDirs) { - String workspaceName = workspacePath.getFileName().toString(); - try (DirectoryStream xmls = Files.newDirectoryStream(workspacePath, "*.xml")) { - for (Path xml : xmls) { - if (xml.getFileName().toString().equals("jcr:system.xml")) { - Session session = JcrUtils.loginOrCreateWorkspace(repository, workspaceName); - try (InputStream in = Files.newInputStream(xml)) { - session.getWorkspace().importXML("/", in, - ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING); - if (log.isDebugEnabled()) - log.debug("Restored " + xml + " to workspace " + workspaceName); - break dirs; - } finally { - Jcr.logout(session); - } - } - } - } - } - } +// Session defaultSession = NodeUtils.openDataAdminSession(repository, null); +// try (DirectoryStream xmls = Files.newDirectoryStream( +// workspaces.resolve(NodeConstants.SYS_WORKSPACE + LogicalBackup.JCR_VERSION_STORAGE_PATH), +// "*.xml")) { +// for (Path xml : xmls) { +// try (InputStream in = Files.newInputStream(xml)) { +// defaultSession.getWorkspace().importXML(LogicalBackup.JCR_VERSION_STORAGE_PATH, in, +// ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING); +// if (log.isDebugEnabled()) +// log.debug("Restored " + xml + " to " + defaultSession.getWorkspace().getName() + ":"); +// } +// } +// } finally { +// Jcr.logout(defaultSession); +// } + // non-system content try (DirectoryStream workspaceDirs = Files.newDirectoryStream(workspaces)) { for (Path workspacePath : workspaceDirs) { @@ -65,7 +62,7 @@ public class LogicalRestore implements Runnable { Session session = JcrUtils.loginOrCreateWorkspace(repository, workspaceName); try (DirectoryStream xmls = Files.newDirectoryStream(workspacePath, "*.xml")) { xmls: for (Path xml : xmls) { - if (xml.getFileName().toString().equals("jcr:system.xml")) + if (xml.getFileName().toString().startsWith("rep:")) continue xmls; try (InputStream in = Files.newInputStream(xml)) { session.getWorkspace().importXML("/", in,