org.argeo.cms.jcr/src/org/argeo/maintenance/backup/LogicalBackup.java

   1 package org.argeo.maintenance.backup;
   2
   3 import java.io.BufferedWriter;
   4 import java.io.FileNotFoundException;
   5 import java.io.IOException;
   6 import java.io.InputStream;
   7 import java.io.OutputStream;
   8 import java.io.OutputStreamWriter;
   9 import java.io.Writer;
  10 import java.net.URI;
  11 import java.net.URL;
  12 import java.nio.charset.StandardCharsets;
  13 import java.nio.file.Files;
  14 import java.nio.file.Path;
  15 import java.nio.file.Paths;
  16 import java.util.Dictionary;
  17 import java.util.Enumeration;
  18 import java.util.HashMap;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.TreeMap;
  22 import java.util.concurrent.ExecutionException;
  23 import java.util.concurrent.ExecutorService;
  24 import java.util.concurrent.Executors;
  25 import java.util.concurrent.Future;
  26 import java.util.concurrent.TimeUnit;
  27 import java.util.concurrent.TimeoutException;
  28 import java.util.jar.JarOutputStream;
  29 import java.util.jar.Manifest;
  30 import java.util.zip.ZipEntry;
  31 import java.util.zip.ZipException;
  32 import java.util.zip.ZipOutputStream;
  33
  34 import javax.jcr.Binary;
  35 import javax.jcr.Node;
  36 import javax.jcr.NodeIterator;
  37 import javax.jcr.Property;
  38 import javax.jcr.Repository;
  39 import javax.jcr.RepositoryException;
  40 import javax.jcr.RepositoryFactory;
  41 import javax.jcr.Session;
  42
  43 import org.apache.commons.io.IOUtils;
  44 import org.apache.jackrabbit.api.JackrabbitSession;
  45 import org.apache.jackrabbit.api.JackrabbitValue;
  46 import org.argeo.api.cms.CmsConstants;
  47 import org.argeo.api.cms.CmsLog;
  48 import org.argeo.cms.jcr.CmsJcrUtils;
  49 import org.argeo.jackrabbit.client.ClientDavexRepositoryFactory;
  50 import org.argeo.jcr.Jcr;
  51 import org.argeo.jcr.JcrException;
  52 import org.argeo.jcr.JcrUtils;
  53 import org.osgi.framework.Bundle;
  54 import org.osgi.framework.BundleContext;
  55
  56 /**
  57  * Performs a backup of the data based only on programmatic interfaces. Useful
  58  * for migration or live backup. Physical backups of the underlying file
  59  * systems, databases, LDAP servers, etc. should be performed for disaster
  60  * recovery.
  61  */
  62 public class LogicalBackup implements Runnable {
  63         private final static CmsLog log = CmsLog.getLog(LogicalBackup.class);
  64
  65         public final static String WORKSPACES_BASE = "workspaces/";
  66         public final static String FILES_BASE = "files/";
  67         public final static String OSGI_BASE = "share/osgi/";
  68
  69         public final static String JCR_SYSTEM = "jcr:system";
  70         public final static String JCR_VERSION_STORAGE_PATH = "/jcr:system/jcr:versionStorage";
  71
  72         private final Repository repository;
  73         private String defaultWorkspace;
  74         private final BundleContext bundleContext;
  75
  76         private final ZipOutputStream zout;
  77         private final Path basePath;
  78
  79         private ExecutorService executorService;
  80
  81         private boolean performSoftwareBackup = false;
  82
  83         private Map<String, String> checksums = new TreeMap<>();
  84
  85         private int threadCount = 5;
  86
  87         private boolean backupFailed = false;
  88
  89         public LogicalBackup(BundleContext bundleContext, Repository repository, Path basePath) {
  90                 this.repository = repository;
  91                 this.zout = null;
  92                 this.basePath = basePath;
  93                 this.bundleContext = bundleContext;
  94         }
  95
  96         @Override
  97         public void run() {
  98                 try {
  99                         log.info("Start logical backup to " + basePath);
 100                         perform();
 101                 } catch (Exception e) {
 102                         log.error("Unexpected exception when performing logical backup", e);
 103                         throw new IllegalStateException("Logical backup failed", e);
 104                 }
 105
 106         }
 107
 108         public void perform() throws RepositoryException, IOException {
 109                 if (executorService != null && !executorService.isTerminated())
 110                         throw new IllegalStateException("Another backup is running");
 111                 executorService = Executors.newFixedThreadPool(threadCount);
 112                 long begin = System.currentTimeMillis();
 113                 // software backup
 114                 if (bundleContext != null && performSoftwareBackup)
 115                         executorService.submit(() -> performSoftwareBackup(bundleContext));
 116
 117                 // data backup
 118                 Session defaultSession = login(null);
 119                 defaultWorkspace = defaultSession.getWorkspace().getName();
 120                 try {
 121                         String[] workspaceNames = defaultSession.getWorkspace().getAccessibleWorkspaceNames();
 122                         workspaces: for (String workspaceName : workspaceNames) {
 123                                 if ("security".equals(workspaceName))
 124                                         continue workspaces;
 125                                 performDataBackup(workspaceName);
 126                         }
 127                 } finally {
 128                         JcrUtils.logoutQuietly(defaultSession);
 129                         executorService.shutdown();
 130                         try {
 131                                 executorService.awaitTermination(24, TimeUnit.HOURS);
 132                         } catch (InterruptedException e) {
 133                                 // silent
 134                                 throw new IllegalStateException("Backup was interrupted before completion", e);
 135                         }
 136                 }
 137                 // versions
 138                 executorService = Executors.newFixedThreadPool(threadCount);
 139                 try {
 140                         performVersionsBackup();
 141                 } finally {
 142                         executorService.shutdown();
 143                         try {
 144                                 executorService.awaitTermination(24, TimeUnit.HOURS);
 145                         } catch (InterruptedException e) {
 146                                 // silent
 147                                 throw new IllegalStateException("Backup was interrupted before completion", e);
 148                         }
 149                 }
 150                 long duration = System.currentTimeMillis() - begin;
 151                 if (isBackupFailed())
 152                         log.info("System logical backup failed after " + (duration / 60000) + "min " + (duration / 1000) + "s");
 153                 else
 154                         log.info("System logical backup completed in " + (duration / 60000) + "min " + (duration / 1000) + "s");
 155         }
 156
 157         protected void performDataBackup(String workspaceName) throws RepositoryException, IOException {
 158                 Session session = login(workspaceName);
 159                 try {
 160                         nodes: for (NodeIterator nit = session.getRootNode().getNodes(); nit.hasNext();) {
 161                                 if (isBackupFailed())
 162                                         return;
 163                                 Node nodeToExport = nit.nextNode();
 164                                 if (JCR_SYSTEM.equals(nodeToExport.getName()))
 165                                         continue nodes;
 166                                 String nodePath = nodeToExport.getPath();
 167                                 Future<Set<String>> contentPathsFuture = executorService
 168                                                 .submit(() -> performNodeBackup(workspaceName, nodePath));
 169                                 executorService.submit(() -> performFilesBackup(workspaceName, contentPathsFuture));
 170                         }
 171                 } finally {
 172                         Jcr.logout(session);
 173                 }
 174         }
 175
 176         protected void performVersionsBackup() throws RepositoryException, IOException {
 177                 Session session = login(defaultWorkspace);
 178                 Node versionStorageNode = session.getNode(JCR_VERSION_STORAGE_PATH);
 179                 try {
 180                         for (NodeIterator nit = versionStorageNode.getNodes(); nit.hasNext();) {
 181                                 Node nodeToExport = nit.nextNode();
 182                                 String nodePath = nodeToExport.getPath();
 183                                 if (isBackupFailed())
 184                                         return;
 185                                 Future<Set<String>> contentPathsFuture = executorService
 186                                                 .submit(() -> performNodeBackup(defaultWorkspace, nodePath));
 187                                 executorService.submit(() -> performFilesBackup(defaultWorkspace, contentPathsFuture));
 188                         }
 189                 } finally {
 190                         Jcr.logout(session);
 191                 }
 192
 193         }
 194
 195         protected Set<String> performNodeBackup(String workspaceName, String nodePath) {
 196                 Session session = login(workspaceName);
 197                 try {
 198                         Node nodeToExport = session.getNode(nodePath);
 199 //                      String nodeName = nodeToExport.getName();
 200 //              if (nodeName.startsWith("jcr:") || nodeName.startsWith("rep:"))
 201 //                      continue nodes;
 202 //              // TODO make it more robust / configurable
 203 //              if (nodeName.equals("user"))
 204 //                      continue nodes;
 205                         String relativePath = WORKSPACES_BASE + workspaceName + nodePath + ".xml";
 206                         OutputStream xmlOut = openOutputStream(relativePath);
 207                         BackupContentHandler contentHandler;
 208                         try (Writer writer = new BufferedWriter(new OutputStreamWriter(xmlOut, StandardCharsets.UTF_8))) {
 209                                 contentHandler = new BackupContentHandler(writer, nodeToExport);
 210                                 session.exportSystemView(nodeToExport.getPath(), contentHandler, true, false);
 211                                 if (log.isDebugEnabled())
 212                                         log.debug(workspaceName + ":" + nodePath + " metadata exported to " + relativePath);
 213                         }
 214
 215                         // Files
 216                         Set<String> contentPaths = contentHandler.getContentPaths();
 217                         return contentPaths;
 218                 } catch (Exception e) {
 219                         markBackupFailed("Cannot backup node " + workspaceName + ":" + nodePath, e);
 220                         throw new ThreadDeath();
 221                 } finally {
 222                         Jcr.logout(session);
 223                 }
 224         }
 225
 226         protected void performFilesBackup(String workspaceName, Future<Set<String>> contentPathsFuture) {
 227                 Set<String> contentPaths;
 228                 try {
 229                         contentPaths = contentPathsFuture.get(24, TimeUnit.HOURS);
 230                 } catch (InterruptedException | ExecutionException | TimeoutException e1) {
 231                         markBackupFailed("Cannot retrieve content paths for workspace " + workspaceName, e1);
 232                         return;
 233                 }
 234                 if (contentPaths == null || contentPaths.size() == 0)
 235                         return;
 236                 Session session = login(workspaceName);
 237                 try {
 238                         String workspacesFilesBasePath = FILES_BASE + workspaceName;
 239                         for (String path : contentPaths) {
 240                                 if (isBackupFailed())
 241                                         return;
 242                                 Node contentNode = session.getNode(path);
 243                                 Binary binary = null;
 244                                 try {
 245                                         binary = contentNode.getProperty(Property.JCR_DATA).getBinary();
 246                                         String fileRelativePath = workspacesFilesBasePath + contentNode.getParent().getPath();
 247
 248                                         // checksum
 249                                         boolean skip = false;
 250                                         String checksum = null;
 251                                         if (session instanceof JackrabbitSession) {
 252                                                 JackrabbitValue value = (JackrabbitValue) contentNode.getProperty(Property.JCR_DATA).getValue();
 253 //                                      ReferenceBinary referenceBinary = (ReferenceBinary) binary;
 254                                                 checksum = value.getContentIdentity();
 255                                         }
 256                                         if (checksum != null) {
 257                                                 if (!checksums.containsKey(checksum)) {
 258                                                         checksums.put(checksum, fileRelativePath);
 259                                                 } else {
 260                                                         skip = true;
 261                                                         String sourcePath = checksums.get(checksum);
 262                                                         if (log.isTraceEnabled())
 263                                                                 log.trace(fileRelativePath + " : already " + sourcePath + " with checksum " + checksum);
 264                                                         createLink(sourcePath, fileRelativePath);
 265                                                         try (Writer writerSum = new OutputStreamWriter(
 266                                                                         openOutputStream(fileRelativePath + ".sha256"), StandardCharsets.UTF_8)) {
 267                                                                 writerSum.write(checksum);
 268                                                         }
 269                                                 }
 270                                         }
 271
 272                                         // copy file
 273                                         if (!skip)
 274                                                 try (InputStream in = binary.getStream();
 275                                                                 OutputStream out = openOutputStream(fileRelativePath)) {
 276                                                         IOUtils.copy(in, out);
 277                                                         if (log.isTraceEnabled())
 278                                                                 log.trace("Workspace " + workspaceName + ": file content exported to "
 279                                                                                 + fileRelativePath);
 280                                                 }
 281                                 } finally {
 282                                         JcrUtils.closeQuietly(binary);
 283                                 }
 284                         }
 285                         if (log.isDebugEnabled())
 286                                 log.debug(workspaceName + ":" + contentPaths.size() + " files exported to " + workspacesFilesBasePath);
 287                 } catch (Exception e) {
 288                         markBackupFailed("Cannot backup files from " + workspaceName + ":", e);
 289                 } finally {
 290                         Jcr.logout(session);
 291                 }
 292         }
 293
 294         protected OutputStream openOutputStream(String relativePath) throws IOException {
 295                 if (zout != null) {
 296                         ZipEntry entry = new ZipEntry(relativePath);
 297                         zout.putNextEntry(entry);
 298                         return zout;
 299                 } else if (basePath != null) {
 300                         Path targetPath = basePath.resolve(Paths.get(relativePath));
 301                         Files.createDirectories(targetPath.getParent());
 302                         return Files.newOutputStream(targetPath);
 303                 } else {
 304                         throw new UnsupportedOperationException();
 305                 }
 306         }
 307
 308         protected void createLink(String source, String target) throws IOException {
 309                 if (zout != null) {
 310                         // TODO implement for zip
 311                         throw new UnsupportedOperationException();
 312                 } else if (basePath != null) {
 313                         Path sourcePath = basePath.resolve(Paths.get(source));
 314                         Path targetPath = basePath.resolve(Paths.get(target));
 315                         Path relativeSource = targetPath.getParent().relativize(sourcePath);
 316                         Files.createDirectories(targetPath.getParent());
 317                         Files.createSymbolicLink(targetPath, relativeSource);
 318                 } else {
 319                         throw new UnsupportedOperationException();
 320                 }
 321         }
 322
 323         protected void closeOutputStream(String relativePath, OutputStream out) throws IOException {
 324                 if (zout != null) {
 325                         zout.closeEntry();
 326                 } else if (basePath != null) {
 327                         out.close();
 328                 } else {
 329                         throw new UnsupportedOperationException();
 330                 }
 331         }
 332
 333         protected Session login(String workspaceName) {
 334                 if (bundleContext != null) {// local
 335                         return CmsJcrUtils.openDataAdminSession(repository, workspaceName);
 336                 } else {// remote
 337                         try {
 338                                 return repository.login(workspaceName);
 339                         } catch (RepositoryException e) {
 340                                 throw new JcrException(e);
 341                         }
 342                 }
 343         }
 344
 345         public final static void main(String[] args) throws Exception {
 346                 if (args.length == 0) {
 347                         printUsage("No argument");
 348                         System.exit(1);
 349                 }
 350                 URI uri = new URI(args[0]);
 351                 Repository repository = createRemoteRepository(uri);
 352                 Path basePath = args.length > 1 ? Paths.get(args[1]) : Paths.get(System.getProperty("user.dir"));
 353                 if (!Files.exists(basePath))
 354                         Files.createDirectories(basePath);
 355                 LogicalBackup backup = new LogicalBackup(null, repository, basePath);
 356                 backup.run();
 357         }
 358
 359         private static void printUsage(String errorMessage) {
 360                 if (errorMessage != null)
 361                         System.err.println(errorMessage);
 362                 System.out.println("Usage: LogicalBackup <remote URL> [<target directory>]");
 363
 364         }
 365
 366         protected static Repository createRemoteRepository(URI uri) throws RepositoryException {
 367                 RepositoryFactory repositoryFactory = new ClientDavexRepositoryFactory();
 368                 Map<String, String> params = new HashMap<String, String>();
 369                 params.put(ClientDavexRepositoryFactory.JACKRABBIT_DAVEX_URI, uri.toString());
 370                 // TODO make it configurable
 371                 params.put(ClientDavexRepositoryFactory.JACKRABBIT_REMOTE_DEFAULT_WORKSPACE, CmsConstants.SYS_WORKSPACE);
 372                 return repositoryFactory.getRepository(params);
 373         }
 374
 375         public void performSoftwareBackup(BundleContext bundleContext) {
 376                 String bootBasePath = OSGI_BASE + "boot";
 377                 Bundle[] bundles = bundleContext.getBundles();
 378                 for (Bundle bundle : bundles) {
 379                         String relativePath = bootBasePath + "/" + bundle.getSymbolicName() + ".jar";
 380                         Dictionary<String, String> headers = bundle.getHeaders();
 381                         Manifest manifest = new Manifest();
 382                         Enumeration<String> headerKeys = headers.keys();
 383                         while (headerKeys.hasMoreElements()) {
 384                                 String headerKey = headerKeys.nextElement();
 385                                 String headerValue = headers.get(headerKey);
 386                                 manifest.getMainAttributes().putValue(headerKey, headerValue);
 387                         }
 388                         try (JarOutputStream jarOut = new JarOutputStream(openOutputStream(relativePath), manifest)) {
 389                                 Enumeration<URL> resourcePaths = bundle.findEntries("/", "*", true);
 390                                 resources: while (resourcePaths.hasMoreElements()) {
 391                                         URL entryUrl = resourcePaths.nextElement();
 392                                         String entryPath = entryUrl.getPath();
 393                                         if (entryPath.equals(""))
 394                                                 continue resources;
 395                                         if (entryPath.endsWith("/"))
 396                                                 continue resources;
 397                                         String entryName = entryPath.substring(1);// remove first '/'
 398                                         if (entryUrl.getPath().equals("/META-INF/"))
 399                                                 continue resources;
 400                                         if (entryUrl.getPath().equals("/META-INF/MANIFEST.MF"))
 401                                                 continue resources;
 402                                         // dev
 403                                         if (entryUrl.getPath().startsWith("/target"))
 404                                                 continue resources;
 405                                         if (entryUrl.getPath().startsWith("/src"))
 406                                                 continue resources;
 407                                         if (entryUrl.getPath().startsWith("/ext"))
 408                                                 continue resources;
 409
 410                                         if (entryName.startsWith("bin/")) {// dev
 411                                                 entryName = entryName.substring("bin/".length());
 412                                         }
 413
 414                                         ZipEntry entry = new ZipEntry(entryName);
 415                                         try (InputStream in = entryUrl.openStream()) {
 416                                                 try {
 417                                                         jarOut.putNextEntry(entry);
 418                                                 } catch (ZipException e) {// duplicate
 419                                                         continue resources;
 420                                                 }
 421                                                 IOUtils.copy(in, jarOut);
 422                                                 jarOut.closeEntry();
 423 //                                              log.info(entryUrl);
 424                                         } catch (FileNotFoundException e) {
 425                                                 log.warn(entryUrl + ": " + e.getMessage());
 426                                         }
 427                                 }
 428                         } catch (IOException e1) {
 429                                 throw new RuntimeException("Cannot export bundle " + bundle, e1);
 430                         }
 431                 }
 432                 if (log.isDebugEnabled())
 433                         log.debug(bundles.length + " OSGi bundles exported to " + bootBasePath);
 434
 435         }
 436
 437         protected synchronized void markBackupFailed(Object message, Exception e) {
 438                 log.error(message, e);
 439                 backupFailed = true;
 440                 notifyAll();
 441                 if (executorService != null)
 442                         executorService.shutdownNow();
 443         }
 444
 445         protected boolean isBackupFailed() {
 446                 return backupFailed;
 447         }
 448 }