org.argeo.maintenance/src/org/argeo/maintenance/backup/LogicalBackup.java

   1 package org.argeo.maintenance.backup;
   2
   3 import java.io.BufferedWriter;
   4 import java.io.FileNotFoundException;
   5 import java.io.IOException;
   6 import java.io.InputStream;
   7 import java.io.OutputStream;
   8 import java.io.OutputStreamWriter;
   9 import java.io.Writer;
  10 import java.net.URI;
  11 import java.net.URL;
  12 import java.nio.charset.StandardCharsets;
  13 import java.nio.file.Files;
  14 import java.nio.file.Path;
  15 import java.nio.file.Paths;
  16 import java.util.Dictionary;
  17 import java.util.Enumeration;
  18 import java.util.HashMap;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.TreeMap;
  22 import java.util.concurrent.ExecutionException;
  23 import java.util.concurrent.ExecutorService;
  24 import java.util.concurrent.Executors;
  25 import java.util.concurrent.Future;
  26 import java.util.concurrent.TimeUnit;
  27 import java.util.concurrent.TimeoutException;
  28 import java.util.jar.JarOutputStream;
  29 import java.util.jar.Manifest;
  30 import java.util.zip.ZipEntry;
  31 import java.util.zip.ZipException;
  32 import java.util.zip.ZipOutputStream;
  33
  34 import javax.jcr.Binary;
  35 import javax.jcr.Node;
  36 import javax.jcr.NodeIterator;
  37 import javax.jcr.Property;
  38 import javax.jcr.Repository;
  39 import javax.jcr.RepositoryException;
  40 import javax.jcr.RepositoryFactory;
  41 import javax.jcr.Session;
  42
  43 import org.apache.commons.io.IOUtils;
  44 import org.apache.commons.logging.Log;
  45 import org.apache.commons.logging.LogFactory;
  46 import org.apache.jackrabbit.api.JackrabbitSession;
  47 import org.apache.jackrabbit.api.JackrabbitValue;
  48 import org.argeo.api.NodeConstants;
  49 import org.argeo.api.NodeUtils;
  50 import org.argeo.jackrabbit.client.ClientDavexRepositoryFactory;
  51 import org.argeo.jcr.Jcr;
  52 import org.argeo.jcr.JcrException;
  53 import org.argeo.jcr.JcrUtils;
  54 import org.osgi.framework.Bundle;
  55 import org.osgi.framework.BundleContext;
  56
  57 /**
  58  * Performs a backup of the data based only on programmatic interfaces. Useful
  59  * for migration or live backup. Physical backups of the underlying file
  60  * systems, databases, LDAP servers, etc. should be performed for disaster
  61  * recovery.
  62  */
  63 public class LogicalBackup implements Runnable {
  64         private final static Log log = LogFactory.getLog(LogicalBackup.class);
  65
  66         public final static String WORKSPACES_BASE = "workspaces/";
  67         public final static String FILES_BASE = "files/";
  68         public final static String OSGI_BASE = "share/osgi/";
  69
  70         public final static String JCR_SYSTEM = "jcr:system";
  71         public final static String JCR_VERSION_STORAGE_PATH = "/jcr:system/jcr:versionStorage";
  72
  73         private final Repository repository;
  74         private String defaultWorkspace;
  75         private final BundleContext bundleContext;
  76
  77         private final ZipOutputStream zout;
  78         private final Path basePath;
  79
  80         private ExecutorService executorService;
  81
  82         private boolean performSoftwareBackup = false;
  83
  84         private Map<String, String> checksums = new TreeMap<>();
  85
  86         private int threadCount = 5;
  87
  88         private boolean backupFailed = false;
  89
  90         public LogicalBackup(BundleContext bundleContext, Repository repository, Path basePath) {
  91                 this.repository = repository;
  92                 this.zout = null;
  93                 this.basePath = basePath;
  94                 this.bundleContext = bundleContext;
  95         }
  96
  97         @Override
  98         public void run() {
  99                 try {
 100                         log.info("Start logical backup to " + basePath);
 101                         perform();
 102                 } catch (Exception e) {
 103                         log.error("Unexpected exception when performing logical backup", e);
 104                         throw new IllegalStateException("Logical backup failed", e);
 105                 }
 106
 107         }
 108
 109         public void perform() throws RepositoryException, IOException {
 110                 if (executorService != null && !executorService.isTerminated())
 111                         throw new IllegalStateException("Another backup is running");
 112                 executorService = Executors.newFixedThreadPool(threadCount);
 113                 long begin = System.currentTimeMillis();
 114                 // software backup
 115                 if (bundleContext != null && performSoftwareBackup)
 116                         executorService.submit(() -> performSoftwareBackup(bundleContext));
 117
 118                 // data backup
 119                 Session defaultSession = login(null);
 120                 defaultWorkspace = defaultSession.getWorkspace().getName();
 121                 try {
 122                         String[] workspaceNames = defaultSession.getWorkspace().getAccessibleWorkspaceNames();
 123                         workspaces: for (String workspaceName : workspaceNames) {
 124                                 if ("security".equals(workspaceName))
 125                                         continue workspaces;
 126                                 performDataBackup(workspaceName);
 127                         }
 128                 } finally {
 129                         JcrUtils.logoutQuietly(defaultSession);
 130                         executorService.shutdown();
 131                         try {
 132                                 executorService.awaitTermination(24, TimeUnit.HOURS);
 133                         } catch (InterruptedException e) {
 134                                 // silent
 135                                 throw new IllegalStateException("Backup was interrupted before completion", e);
 136                         }
 137                 }
 138                 // versions
 139                 executorService = Executors.newFixedThreadPool(threadCount);
 140                 try {
 141                         performVersionsBackup();
 142                 } finally {
 143                         executorService.shutdown();
 144                         try {
 145                                 executorService.awaitTermination(24, TimeUnit.HOURS);
 146                         } catch (InterruptedException e) {
 147                                 // silent
 148                                 throw new IllegalStateException("Backup was interrupted before completion", e);
 149                         }
 150                 }
 151                 long duration = System.currentTimeMillis() - begin;
 152                 if (isBackupFailed())
 153                         log.info("System logical backup failed after " + (duration / 60000) + "min " + (duration / 1000) + "s");
 154                 else
 155                         log.info("System logical backup completed in " + (duration / 60000) + "min " + (duration / 1000) + "s");
 156         }
 157
 158         protected void performDataBackup(String workspaceName) throws RepositoryException, IOException {
 159                 Session session = login(workspaceName);
 160                 try {
 161                         nodes: for (NodeIterator nit = session.getRootNode().getNodes(); nit.hasNext();) {
 162                                 if (isBackupFailed())
 163                                         return;
 164                                 Node nodeToExport = nit.nextNode();
 165                                 if (JCR_SYSTEM.equals(nodeToExport.getName()))
 166                                         continue nodes;
 167                                 String nodePath = nodeToExport.getPath();
 168                                 Future<Set<String>> contentPathsFuture = executorService
 169                                                 .submit(() -> performNodeBackup(workspaceName, nodePath));
 170                                 executorService.submit(() -> performFilesBackup(workspaceName, contentPathsFuture));
 171                         }
 172                 } finally {
 173                         Jcr.logout(session);
 174                 }
 175         }
 176
 177         protected void performVersionsBackup() throws RepositoryException, IOException {
 178                 Session session = login(defaultWorkspace);
 179                 Node versionStorageNode = session.getNode(JCR_VERSION_STORAGE_PATH);
 180                 try {
 181                         for (NodeIterator nit = versionStorageNode.getNodes(); nit.hasNext();) {
 182                                 Node nodeToExport = nit.nextNode();
 183                                 String nodePath = nodeToExport.getPath();
 184                                 if (isBackupFailed())
 185                                         return;
 186                                 Future<Set<String>> contentPathsFuture = executorService
 187                                                 .submit(() -> performNodeBackup(defaultWorkspace, nodePath));
 188                                 executorService.submit(() -> performFilesBackup(defaultWorkspace, contentPathsFuture));
 189                         }
 190                 } finally {
 191                         Jcr.logout(session);
 192                 }
 193
 194         }
 195
 196         protected Set<String> performNodeBackup(String workspaceName, String nodePath) {
 197                 Session session = login(workspaceName);
 198                 try {
 199                         Node nodeToExport = session.getNode(nodePath);
 200 //                      String nodeName = nodeToExport.getName();
 201 //              if (nodeName.startsWith("jcr:") || nodeName.startsWith("rep:"))
 202 //                      continue nodes;
 203 //              // TODO make it more robust / configurable
 204 //              if (nodeName.equals("user"))
 205 //                      continue nodes;
 206                         String relativePath = WORKSPACES_BASE + workspaceName + nodePath + ".xml";
 207                         OutputStream xmlOut = openOutputStream(relativePath);
 208                         BackupContentHandler contentHandler;
 209                         try (Writer writer = new BufferedWriter(new OutputStreamWriter(xmlOut, StandardCharsets.UTF_8))) {
 210                                 contentHandler = new BackupContentHandler(writer, nodeToExport);
 211                                 session.exportSystemView(nodeToExport.getPath(), contentHandler, true, false);
 212                                 if (log.isDebugEnabled())
 213                                         log.debug(workspaceName + ":" + nodePath + " metadata exported to " + relativePath);
 214                         }
 215
 216                         // Files
 217                         Set<String> contentPaths = contentHandler.getContentPaths();
 218                         return contentPaths;
 219                 } catch (Exception e) {
 220                         markBackupFailed("Cannot backup node " + workspaceName + ":" + nodePath, e);
 221                         throw new ThreadDeath();
 222                 } finally {
 223                         Jcr.logout(session);
 224                 }
 225         }
 226
 227         protected void performFilesBackup(String workspaceName, Future<Set<String>> contentPathsFuture) {
 228                 Set<String> contentPaths;
 229                 try {
 230                         contentPaths = contentPathsFuture.get(24, TimeUnit.HOURS);
 231                 } catch (InterruptedException | ExecutionException | TimeoutException e1) {
 232                         markBackupFailed("Cannot retrieve content paths for workspace " + workspaceName, e1);
 233                         return;
 234                 }
 235                 if (contentPaths == null || contentPaths.size() == 0)
 236                         return;
 237                 Session session = login(workspaceName);
 238                 try {
 239                         String workspacesFilesBasePath = FILES_BASE + workspaceName;
 240                         for (String path : contentPaths) {
 241                                 if (isBackupFailed())
 242                                         return;
 243                                 Node contentNode = session.getNode(path);
 244                                 Binary binary = null;
 245                                 try {
 246                                         binary = contentNode.getProperty(Property.JCR_DATA).getBinary();
 247                                         String fileRelativePath = workspacesFilesBasePath + contentNode.getParent().getPath();
 248
 249                                         // checksum
 250                                         boolean skip = false;
 251                                         String checksum = null;
 252                                         if (session instanceof JackrabbitSession) {
 253                                                 JackrabbitValue value = (JackrabbitValue) contentNode.getProperty(Property.JCR_DATA).getValue();
 254 //                                      ReferenceBinary referenceBinary = (ReferenceBinary) binary;
 255                                                 checksum = value.getContentIdentity();
 256                                         }
 257                                         if (checksum != null) {
 258                                                 if (!checksums.containsKey(checksum)) {
 259                                                         checksums.put(checksum, fileRelativePath);
 260                                                 } else {
 261                                                         skip = true;
 262                                                         String sourcePath = checksums.get(checksum);
 263                                                         if (log.isTraceEnabled())
 264                                                                 log.trace(fileRelativePath + " : already " + sourcePath + " with checksum " + checksum);
 265                                                         createLink(sourcePath, fileRelativePath);
 266                                                         try (Writer writerSum = new OutputStreamWriter(
 267                                                                         openOutputStream(fileRelativePath + ".sha256"), StandardCharsets.UTF_8)) {
 268                                                                 writerSum.write(checksum);
 269                                                         }
 270                                                 }
 271                                         }
 272
 273                                         // copy file
 274                                         if (!skip)
 275                                                 try (InputStream in = binary.getStream();
 276                                                                 OutputStream out = openOutputStream(fileRelativePath)) {
 277                                                         IOUtils.copy(in, out);
 278                                                         if (log.isTraceEnabled())
 279                                                                 log.trace("Workspace " + workspaceName + ": file content exported to "
 280                                                                                 + fileRelativePath);
 281                                                 }
 282                                 } finally {
 283                                         JcrUtils.closeQuietly(binary);
 284                                 }
 285                         }
 286                         if (log.isDebugEnabled())
 287                                 log.debug(workspaceName + ":" + contentPaths.size() + " files exported to " + workspacesFilesBasePath);
 288                 } catch (Exception e) {
 289                         markBackupFailed("Cannot backup files from " + workspaceName + ":", e);
 290                 } finally {
 291                         Jcr.logout(session);
 292                 }
 293         }
 294
 295         protected OutputStream openOutputStream(String relativePath) throws IOException {
 296                 if (zout != null) {
 297                         ZipEntry entry = new ZipEntry(relativePath);
 298                         zout.putNextEntry(entry);
 299                         return zout;
 300                 } else if (basePath != null) {
 301                         Path targetPath = basePath.resolve(Paths.get(relativePath));
 302                         Files.createDirectories(targetPath.getParent());
 303                         return Files.newOutputStream(targetPath);
 304                 } else {
 305                         throw new UnsupportedOperationException();
 306                 }
 307         }
 308
 309         protected void createLink(String source, String target) throws IOException {
 310                 if (zout != null) {
 311                         // TODO implement for zip
 312                         throw new UnsupportedOperationException();
 313                 } else if (basePath != null) {
 314                         Path sourcePath = basePath.resolve(Paths.get(source));
 315                         Path targetPath = basePath.resolve(Paths.get(target));
 316                         Path relativeSource = targetPath.getParent().relativize(sourcePath);
 317                         Files.createDirectories(targetPath.getParent());
 318                         Files.createSymbolicLink(targetPath, relativeSource);
 319                 } else {
 320                         throw new UnsupportedOperationException();
 321                 }
 322         }
 323
 324         protected void closeOutputStream(String relativePath, OutputStream out) throws IOException {
 325                 if (zout != null) {
 326                         zout.closeEntry();
 327                 } else if (basePath != null) {
 328                         out.close();
 329                 } else {
 330                         throw new UnsupportedOperationException();
 331                 }
 332         }
 333
 334         protected Session login(String workspaceName) {
 335                 if (bundleContext != null) {// local
 336                         return NodeUtils.openDataAdminSession(repository, workspaceName);
 337                 } else {// remote
 338                         try {
 339                                 return repository.login(workspaceName);
 340                         } catch (RepositoryException e) {
 341                                 throw new JcrException(e);
 342                         }
 343                 }
 344         }
 345
 346         public final static void main(String[] args) throws Exception {
 347                 if (args.length == 0) {
 348                         printUsage("No argument");
 349                         System.exit(1);
 350                 }
 351                 URI uri = new URI(args[0]);
 352                 Repository repository = createRemoteRepository(uri);
 353                 Path basePath = args.length > 1 ? Paths.get(args[1]) : Paths.get(System.getProperty("user.dir"));
 354                 if (!Files.exists(basePath))
 355                         Files.createDirectories(basePath);
 356                 LogicalBackup backup = new LogicalBackup(null, repository, basePath);
 357                 backup.run();
 358         }
 359
 360         private static void printUsage(String errorMessage) {
 361                 if (errorMessage != null)
 362                         System.err.println(errorMessage);
 363                 System.out.println("Usage: LogicalBackup <remote URL> [<target directory>]");
 364
 365         }
 366
 367         protected static Repository createRemoteRepository(URI uri) throws RepositoryException {
 368                 RepositoryFactory repositoryFactory = new ClientDavexRepositoryFactory();
 369                 Map<String, String> params = new HashMap<String, String>();
 370                 params.put(ClientDavexRepositoryFactory.JACKRABBIT_DAVEX_URI, uri.toString());
 371                 // TODO make it configurable
 372                 params.put(ClientDavexRepositoryFactory.JACKRABBIT_REMOTE_DEFAULT_WORKSPACE, NodeConstants.SYS_WORKSPACE);
 373                 return repositoryFactory.getRepository(params);
 374         }
 375
 376         public void performSoftwareBackup(BundleContext bundleContext) {
 377                 String bootBasePath = OSGI_BASE + "boot";
 378                 Bundle[] bundles = bundleContext.getBundles();
 379                 for (Bundle bundle : bundles) {
 380                         String relativePath = bootBasePath + "/" + bundle.getSymbolicName() + ".jar";
 381                         Dictionary<String, String> headers = bundle.getHeaders();
 382                         Manifest manifest = new Manifest();
 383                         Enumeration<String> headerKeys = headers.keys();
 384                         while (headerKeys.hasMoreElements()) {
 385                                 String headerKey = headerKeys.nextElement();
 386                                 String headerValue = headers.get(headerKey);
 387                                 manifest.getMainAttributes().putValue(headerKey, headerValue);
 388                         }
 389                         try (JarOutputStream jarOut = new JarOutputStream(openOutputStream(relativePath), manifest)) {
 390                                 Enumeration<URL> resourcePaths = bundle.findEntries("/", "*", true);
 391                                 resources: while (resourcePaths.hasMoreElements()) {
 392                                         URL entryUrl = resourcePaths.nextElement();
 393                                         String entryPath = entryUrl.getPath();
 394                                         if (entryPath.equals(""))
 395                                                 continue resources;
 396                                         if (entryPath.endsWith("/"))
 397                                                 continue resources;
 398                                         String entryName = entryPath.substring(1);// remove first '/'
 399                                         if (entryUrl.getPath().equals("/META-INF/"))
 400                                                 continue resources;
 401                                         if (entryUrl.getPath().equals("/META-INF/MANIFEST.MF"))
 402                                                 continue resources;
 403                                         // dev
 404                                         if (entryUrl.getPath().startsWith("/target"))
 405                                                 continue resources;
 406                                         if (entryUrl.getPath().startsWith("/src"))
 407                                                 continue resources;
 408                                         if (entryUrl.getPath().startsWith("/ext"))
 409                                                 continue resources;
 410
 411                                         if (entryName.startsWith("bin/")) {// dev
 412                                                 entryName = entryName.substring("bin/".length());
 413                                         }
 414
 415                                         ZipEntry entry = new ZipEntry(entryName);
 416                                         try (InputStream in = entryUrl.openStream()) {
 417                                                 try {
 418                                                         jarOut.putNextEntry(entry);
 419                                                 } catch (ZipException e) {// duplicate
 420                                                         continue resources;
 421                                                 }
 422                                                 IOUtils.copy(in, jarOut);
 423                                                 jarOut.closeEntry();
 424 //                                              log.info(entryUrl);
 425                                         } catch (FileNotFoundException e) {
 426                                                 log.warn(entryUrl + ": " + e.getMessage());
 427                                         }
 428                                 }
 429                         } catch (IOException e1) {
 430                                 throw new RuntimeException("Cannot export bundle " + bundle, e1);
 431                         }
 432                 }
 433                 if (log.isDebugEnabled())
 434                         log.debug(bundles.length + " OSGi bundles exported to " + bootBasePath);
 435
 436         }
 437
 438         protected synchronized void markBackupFailed(Object message, Exception e) {
 439                 log.error(message, e);
 440                 backupFailed = true;
 441                 notifyAll();
 442                 if (executorService != null)
 443                         executorService.shutdownNow();
 444         }
 445
 446         protected boolean isBackupFailed() {
 447                 return backupFailed;
 448         }
 449 }