Improve CMS workspace indexer.
authorMathieu Baudier <mbaudier@argeo.org>
Mon, 27 Jan 2020 09:04:13 +0000 (10:04 +0100)
committerMathieu Baudier <mbaudier@argeo.org>
Mon, 27 Jan 2020 09:04:13 +0000 (10:04 +0100)
org.argeo.cms/src/org/argeo/cms/internal/kernel/Activator.java
org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsDeployment.java
org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsWorkspaceIndexer.java [new file with mode: 0644]
org.argeo.cms/src/org/argeo/cms/internal/kernel/HomeRepository.java
org.argeo.cms/src/org/argeo/cms/internal/kernel/JackrabbitLocalRepository.java
org.argeo.cms/src/org/argeo/cms/internal/kernel/KernelUtils.java

index d7b953b5389eae6a4cecbb5b4bbe336c3aea6131..5728d4fa56386bbddc938604dc315511dbd25ba1 100644 (file)
@@ -8,6 +8,8 @@ import java.security.AllPermission;
 import java.util.Dictionary;
 import java.util.List;
 import java.util.Locale;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 import javax.security.auth.login.Configuration;
 
@@ -58,6 +60,7 @@ public class Activator implements BundleActivator {
        private CmsInstance nodeInstance;
 
        private ServiceTracker<UserAdmin, NodeUserAdmin> userAdminSt;
+       private ExecutorService internalExecutorService;
 
        @Override
        public void start(BundleContext bundleContext) throws Exception {
@@ -65,6 +68,7 @@ public class Activator implements BundleActivator {
                instance = this;
                this.bc = bundleContext;
                this.logReaderService = getService(LogReaderService.class);
+               this.internalExecutorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
 
                try {
                        initSecurity();
@@ -168,6 +172,7 @@ public class Activator implements BundleActivator {
                        if (userAdminSt != null)
                                userAdminSt.close();
 
+                       internalExecutorService.shutdown();
                        instance = null;
                        this.bc = null;
                        this.logReaderService = null;
@@ -229,6 +234,10 @@ public class Activator implements BundleActivator {
 
        }
 
+       static ExecutorService getInternalExecutorService() {
+               return instance.internalExecutorService;
+       }
+
        // static CmsSecurity getCmsSecurity() {
        // return instance.nodeSecurity;
        // }
index f09a008072104b90050fb64feb221a16ba023766..fba39b6d4e91955be766708db3fc3f0405b87181 100644 (file)
@@ -42,6 +42,7 @@ import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.Constants;
 import org.osgi.framework.FrameworkUtil;
+import org.osgi.framework.InvalidSyntaxException;
 import org.osgi.framework.ServiceReference;
 import org.osgi.framework.wiring.BundleCapability;
 import org.osgi.framework.wiring.BundleWire;
@@ -199,6 +200,15 @@ public class CmsDeployment implements NodeDeployment {
                if (nodeHttp != null)
                        nodeHttp.destroy();
 
+               try {
+                       for (ServiceReference<JackrabbitLocalRepository> sr : bc
+                                       .getServiceReferences(JackrabbitLocalRepository.class, null)) {
+                               bc.getService(sr).destroy();
+                       }
+               } catch (InvalidSyntaxException e1) {
+                       log.error("Cannot sclean repsoitories", e1);
+               }
+
                try {
                        JettyConfigurator.stopServer(KernelConstants.DEFAULT_JETTY_SERVER);
                } catch (Exception e) {
@@ -403,11 +413,16 @@ public class CmsDeployment implements NodeDeployment {
                if (dataModelName.equals(NodeConstants.NODE))
                        properties.put(Constants.SERVICE_RANKING, Integer.MAX_VALUE);
                LocalRepository localRepository;
-               if (repository instanceof RepositoryImpl)
+               String[] classes;
+               if (repository instanceof RepositoryImpl) {
                        localRepository = new JackrabbitLocalRepository((RepositoryImpl) repository, dataModelName);
-               else
+                       classes = new String[] { Repository.class.getName(), LocalRepository.class.getName(),
+                                       JackrabbitLocalRepository.class.getName() };
+               } else {
                        localRepository = new LocalRepository(repository, dataModelName);
-               bc.registerService(Repository.class, localRepository, properties);
+                       classes = new String[] { Repository.class.getName(), LocalRepository.class.getName() };
+               }
+               bc.registerService(classes, localRepository, properties);
                if (log.isTraceEnabled())
                        log.trace("Published data model " + dataModelName);
        }
diff --git a/org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsWorkspaceIndexer.java b/org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsWorkspaceIndexer.java
new file mode 100644 (file)
index 0000000..20ced32
--- /dev/null
@@ -0,0 +1,205 @@
+package org.argeo.cms.internal.kernel;
+
+import java.util.GregorianCalendar;
+
+import javax.jcr.Node;
+import javax.jcr.Property;
+import javax.jcr.PropertyType;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.Value;
+import javax.jcr.nodetype.NodeType;
+import javax.jcr.observation.Event;
+import javax.jcr.observation.EventIterator;
+import javax.jcr.observation.EventListener;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.jackrabbit.api.JackrabbitValue;
+import org.apache.jackrabbit.core.RepositoryImpl;
+import org.argeo.jcr.JcrUtils;
+
+/** Ensure consistency of files, folder and last modified nodes. */
+class CmsWorkspaceIndexer implements EventListener {
+       private final static Log log = LogFactory.getLog(CmsWorkspaceIndexer.class);
+
+       private final static String MIX_ETAG = "mix:etag";
+       private final static String JCR_ETAG = "jcr:etag";
+       private final static String JCR_LAST_MODIFIED = "jcr:lastModified";
+       private final static String JCR_LAST_MODIFIED_BY = "jcr:lastModifiedBy";
+       private final static String JCR_DATA = "jcr:data";
+       String cn;
+       String workspaceName;
+       RepositoryImpl repositoryImpl;
+       Session session;
+
+       public CmsWorkspaceIndexer(RepositoryImpl repositoryImpl, String cn, String workspaceName)
+                       throws RepositoryException {
+               this.cn = cn;
+               this.workspaceName = workspaceName;
+               this.repositoryImpl = repositoryImpl;
+       }
+
+       public void init() {
+               session = KernelUtils.openAdminSession(repositoryImpl, workspaceName);
+               try {
+                       String[] nodeTypes = { NodeType.NT_FILE, NodeType.MIX_LAST_MODIFIED };
+                       session.getWorkspace().getObservationManager().addEventListener(this,
+                                       Event.NODE_ADDED | Event.NODE_REMOVED | Event.PROPERTY_CHANGED, "/", true, null, nodeTypes, true);
+               } catch (RepositoryException e1) {
+                       throw new IllegalStateException(e1);
+               }
+       }
+
+       public void destroy() {
+               try {
+                       session.getWorkspace().getObservationManager().removeEventListener(this);
+               } catch (RepositoryException e) {
+                       log.warn("Cannot unregistered JCR event listener", e);
+               } finally {
+                       JcrUtils.logoutQuietly(session);
+               }
+       }
+
+       private synchronized void processEvents(EventIterator events) {
+               while (events.hasNext()) {
+                       Event event = events.nextEvent();
+                       processEvent(event);
+               }
+               notifyAll();
+       }
+
+       protected synchronized void processEvent(Event event) {
+               try {
+                       if (event.getType() == Event.NODE_ADDED) {
+                               session.refresh(true);
+                               Node node = session.getNode(event.getPath());
+                               if (node.getParent().isNodeType(NodeType.NT_FILE)) {
+                                       if (node.isNodeType(NodeType.NT_UNSTRUCTURED)) {
+                                               if (!node.isNodeType(NodeType.MIX_LAST_MODIFIED))
+                                                       node.addMixin(NodeType.MIX_LAST_MODIFIED);
+                                               Property property = node.getProperty(Property.JCR_DATA);
+                                               String etag = toEtag(property.getValue());
+                                               node.setProperty(JCR_ETAG, etag);
+                                       } else if (node.isNodeType(NodeType.NT_RESOURCE)) {
+                                               if (!node.isNodeType(MIX_ETAG))
+                                                       node.addMixin(MIX_ETAG);
+                                               session.save();
+                                               Property property = node.getProperty(Property.JCR_DATA);
+                                               String etag = toEtag(property.getValue());
+                                               node.setProperty(JCR_ETAG, etag);
+                                               session.save();
+                                       }
+                                       setLastModified(node.getParent(), event);
+                                       session.save();
+                                       if (log.isDebugEnabled())
+                                               log.debug("ETag and last modified added to new " + node);
+                               }
+                       } else if (event.getType() == Event.PROPERTY_CHANGED) {
+                               if (!session.propertyExists(event.getPath()))
+                                       return;
+                               Property property = session.getProperty(event.getPath());
+                               String propertyName = property.getName();
+                               // skip if last modified properties are explicitly set
+                               if (propertyName.equals(JCR_LAST_MODIFIED))
+                                       return;
+                               if (propertyName.equals(JCR_LAST_MODIFIED_BY))
+                                       return;
+                               Node node = property.getParent();
+                               if (property.getType() == PropertyType.BINARY && propertyName.equals(JCR_DATA)
+                                               && node.isNodeType(NodeType.NT_UNSTRUCTURED)) {
+                                       String etag = toEtag(property.getValue());
+                                       node.setProperty(JCR_ETAG, etag);
+                               }
+                               setLastModified(node, event);
+                               session.save();
+                               if (log.isDebugEnabled())
+                                       log.debug("ETag and last modified updated for " + node);
+                       } else if (event.getType() == Event.NODE_REMOVED) {
+                               String removeNodePath = event.getPath();
+                               String parentPath = JcrUtils.parentPath(removeNodePath);
+                               session.refresh(true);
+                               setLastModified(session, parentPath, event);
+                               session.save();
+                               if (log.isDebugEnabled())
+                                       log.debug("Last modified updated for parents of removed " + removeNodePath);
+                       }
+               } catch (Exception e) {
+                       log.warn("Cannot process event " + event, e);
+               } finally {
+                       try {
+                               if (session.hasPendingChanges())
+                                       session.save();
+//                             session.refresh(false);
+                       } catch (RepositoryException e) {
+                               log.warn("Cannot refresh JCR session", e);
+                       }
+               }
+
+       }
+
+       @Override
+       public void onEvent(EventIterator events) {
+               Runnable toRun = new Runnable() {
+
+                       @Override
+                       public void run() {
+                               processEvents(events);
+                       }
+               };
+               Activator.getInternalExecutorService().execute(toRun);
+       }
+
+       static String toEtag(Value v) {
+               if (v instanceof JackrabbitValue) {
+                       JackrabbitValue value = (JackrabbitValue) v;
+                       return '\"' + value.getContentIdentity() + '\"';
+               } else {
+                       return null;
+               }
+
+       }
+
+       /** Recursively set the last updated time on parents. */
+       static void setLastModified(Node node, Event event) throws RepositoryException {
+               GregorianCalendar calendar = new GregorianCalendar();
+               calendar.setTimeInMillis(event.getDate());
+               if (node.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
+                       node.setProperty(Property.JCR_LAST_MODIFIED, calendar);
+                       node.setProperty(Property.JCR_LAST_MODIFIED_BY, event.getUserID());
+               }
+               if (node.isNodeType(NodeType.NT_FOLDER) && !node.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
+                       node.addMixin(NodeType.MIX_LAST_MODIFIED);
+               }
+
+               try {
+                       node.getSession().save();
+               } catch (RepositoryException e) {
+                       // fail silently and keep recursing
+               }
+               if (node.getDepth() == 0)
+                       return;
+               Node parent = node.getParent();
+               setLastModified(parent, event);
+       }
+
+       /**
+        * Recursively set the last updated time on parents. Useful to use paths when
+        * dealing with deletions.
+        */
+       static void setLastModified(Session session, String path, Event event) throws RepositoryException {
+               // root node will always exist, so end condition is delegated to the other
+               // recursive setLastModified method
+               if (session.nodeExists(path)) {
+                       setLastModified(session.getNode(path), event);
+               } else {
+                       setLastModified(session, JcrUtils.parentPath(path), event);
+               }
+       }
+
+       @Override
+       public String toString() {
+               return "Monitor workspace " + workspaceName + " of repository " + cn;
+       }
+
+}
\ No newline at end of file
index e5164b6c3fc6424e7dc14e234c25a33c8f3021fd..aa0f1fc3f53d287e5a1c1b946c6184f8fa84dce9 100644 (file)
@@ -58,7 +58,7 @@ class HomeRepository extends JcrRepositoryWrapper implements KernelConstants {
                                @Override
                                public Void run() {
                                        try {
-                                               Session adminSession = getRepository().login();
+                                               Session adminSession = getDefaultRepository().login();
                                                initJcr(adminSession);
                                        } catch (RepositoryException e) {
                                                throw new CmsException("Cannot init JCR home", e);
@@ -80,7 +80,7 @@ class HomeRepository extends JcrRepositoryWrapper implements KernelConstants {
 
                if (checkedUsers.contains(username))
                        return;
-               Session adminSession = KernelUtils.openAdminSession(getRepository(), session.getWorkspace().getName());
+               Session adminSession = KernelUtils.openAdminSession(getRepository(workspaceName), workspaceName);
                try {
                        syncJcr(adminSession, username, workspaceName);
                        checkedUsers.add(username);
index dc47e6e1367af92de2347451b0ff2c2adad5526b..572ab1fcc2376aec1df201ec3ae8aaf0e999dc37 100644 (file)
@@ -1,34 +1,21 @@
 package org.argeo.cms.internal.kernel;
 
-import java.util.GregorianCalendar;
 import java.util.Map;
 import java.util.TreeMap;
 
-import javax.jcr.ItemNotFoundException;
-import javax.jcr.Node;
-import javax.jcr.PathNotFoundException;
-import javax.jcr.Property;
-import javax.jcr.PropertyType;
 import javax.jcr.RepositoryException;
 import javax.jcr.Session;
-import javax.jcr.Value;
-import javax.jcr.nodetype.NodeType;
-import javax.jcr.observation.Event;
-import javax.jcr.observation.EventIterator;
-import javax.jcr.observation.EventListener;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.jackrabbit.api.JackrabbitValue;
 import org.apache.jackrabbit.core.RepositoryImpl;
 import org.argeo.jcr.JcrUtils;
 
 class JackrabbitLocalRepository extends LocalRepository {
        private final static Log log = LogFactory.getLog(JackrabbitLocalRepository.class);
-       private final static String MIX_ETAG = "mix:etag";
-       private final static String JCR_ETAG = "jcr:etag";
+       final String SECURITY_WORKSPACE = "security";
 
-       private Map<String, WorkspaceMonitor> workspaceMonitors = new TreeMap<>();
+       private Map<String, CmsWorkspaceIndexer> workspaceMonitors = new TreeMap<>();
 
        public JackrabbitLocalRepository(RepositoryImpl repository, String cn) {
                super(repository, cn);
@@ -55,14 +42,16 @@ class JackrabbitLocalRepository extends LocalRepository {
        }
 
        private void addMonitor(String realWorkspaceName) {
+               if (realWorkspaceName.equals(SECURITY_WORKSPACE))
+                       return;
                if (!workspaceMonitors.containsKey(realWorkspaceName)) {
                        try {
-                               WorkspaceMonitor workspaceMonitor = new WorkspaceMonitor(getJackrabbitrepository(realWorkspaceName),
-                                               getCn(), realWorkspaceName);
+                               CmsWorkspaceIndexer workspaceMonitor = new CmsWorkspaceIndexer(
+                                               getJackrabbitrepository(realWorkspaceName), getCn(), realWorkspaceName);
                                workspaceMonitors.put(realWorkspaceName, workspaceMonitor);
-                               workspaceMonitor.start();
+                               workspaceMonitor.init();
                                if (log.isDebugEnabled())
-                                       log.debug("Registered " + workspaceMonitor.getName());
+                                       log.debug("Registered " + workspaceMonitor);
                        } catch (RepositoryException e) {
                                // TODO Auto-generated catch block
                                e.printStackTrace();
@@ -70,132 +59,10 @@ class JackrabbitLocalRepository extends LocalRepository {
                }
        }
 
-       static String toEtag(Value v) {
-               JackrabbitValue value = (JackrabbitValue) v;
-               return '\"' + value.getContentIdentity() + '\"';
-
-       }
-
-       /** recursive */
-       static void setLastModified(Node node, Event event) throws RepositoryException {
-               GregorianCalendar calendar = new GregorianCalendar();
-               calendar.setTimeInMillis(event.getDate());
-               if (node.isNodeType(NodeType.NT_FOLDER)) {
-                       node.addMixin(NodeType.MIX_LAST_MODIFIED);
+       public void destroy() {
+               for (String workspaceName : workspaceMonitors.keySet()) {
+                       workspaceMonitors.get(workspaceName).destroy();
                }
-               if (node.isNodeType(NodeType.MIX_LAST_MODIFIED)) {
-                       node.setProperty(Property.JCR_LAST_MODIFIED, calendar);
-                       node.setProperty(Property.JCR_LAST_MODIFIED_BY, event.getUserID());
-               }
-               if (node.getDepth() == 0)
-                       return;
-               Node parent = node.getParent();
-               setLastModified(parent, event);
        }
 
-       static class WorkspaceMonitor extends Thread implements EventListener {
-               String workspaceName;
-               RepositoryImpl repositoryImpl;
-               Session session;
-
-               public WorkspaceMonitor(RepositoryImpl repositoryImpl, String cn, String workspaceName)
-                               throws RepositoryException {
-                       super("Monitor workspace " + workspaceName + " of repository " + cn);
-                       this.workspaceName = workspaceName;
-                       this.repositoryImpl = repositoryImpl;
-               }
-
-               public void run() {
-
-                       session = KernelUtils.openAdminSession(repositoryImpl, workspaceName);
-                       try {
-                               String[] nodeTypes = { NodeType.NT_FILE, NodeType.MIX_LAST_MODIFIED };
-                               session.getWorkspace().getObservationManager().addEventListener(this,
-                                               Event.NODE_ADDED | Event.NODE_REMOVED | Event.PROPERTY_CHANGED, "/", true, null, nodeTypes,
-                                               true);
-                               while (save()) {
-
-                               }
-
-                       } catch (RepositoryException e1) {
-                               // TODO Auto-generated catch block
-                               e1.printStackTrace();
-                       } finally {
-                               JcrUtils.logoutQuietly(session);
-                       }
-               }
-
-               protected synchronized boolean save() {
-                       try {
-                               wait(100);
-                       } catch (InterruptedException e) {
-                               // silent
-                       }
-                       if (!session.isLive())
-                               return false;
-                       try {
-                               if (session.hasPendingChanges())
-                                       session.save();
-                       } catch (RepositoryException e) {
-                               // TODO Auto-generated catch block
-                               e.printStackTrace();
-                       }
-                       return true;
-               }
-
-               @Override
-               public synchronized void onEvent(EventIterator events) {
-                       events: while (events.hasNext()) {
-                               Event event = events.nextEvent();
-//                             if (log.isDebugEnabled())
-//                                     log.debug(event);
-                               try {
-                                       if (event.getType() == Event.NODE_ADDED) {
-                                               Node node = session.getNode(event.getPath());
-                                               if (node.getParent().isNodeType(NodeType.NT_FILE)) {
-//                                                     Node contentNode = node.getNode(Node.JCR_CONTENT);
-                                                       node.addMixin(NodeType.MIX_LAST_MODIFIED);
-                                                       Property property = node.getProperty(Property.JCR_DATA);
-                                                       String etag = toEtag(property.getValue());
-                                                       node.setProperty(JCR_ETAG, etag);
-                                                       setLastModified(node, event);
-//                                             node.getSession().save();
-                                                       if (log.isDebugEnabled())
-                                                               log.debug("Node " + node.getPath() + ": " + event);
-                                               }
-                                       } else if (event.getType() == Event.NODE_REMOVED) {
-                                               String parentPath = JcrUtils.parentPath(event.getPath());
-                                               try {
-                                                       Node parent = session.getNode(parentPath);
-                                                       setLastModified(parent, event);
-
-                                                       if (log.isDebugEnabled())
-                                                               log.debug("Node removed from " + parent.getPath() + ": " + event);
-                                               } catch (ItemNotFoundException | PathNotFoundException e) {
-                                                       continue events;
-                                               }
-                                       } else if (event.getType() == Event.PROPERTY_CHANGED) {
-                                               Property property = session.getProperty(event.getPath());
-                                               if (property.getName().equals("jcr:lastModified"))
-                                                       continue events;
-                                               if (property.getType() == PropertyType.BINARY && property.getName().equals("jcr:data")
-                                                               && property.getParent().isNodeType(NodeType.NT_UNSTRUCTURED)) {
-                                                       String etag = toEtag(property.getValue());
-                                                       property.getParent().setProperty(JCR_ETAG, etag);
-                                               }
-                                               setLastModified(property.getParent(), event);
-//                                             property.getParent().getSession().save();
-                                               if (log.isDebugEnabled())
-                                                       log.debug("Property " + property.getPath() + ": " + event);
-                                       }
-                               } catch (Exception e) {
-                                       // TODO Auto-generated catch block
-                                       e.printStackTrace();
-                               }
-                       }
-                       notifyAll();
-
-               }
-
-       }
 }
index 1d81409115b9bcba31ae862117c18c2a728df571..86289f0ff648544fffebe643241b123153237db2 100644 (file)
@@ -91,18 +91,6 @@ class KernelUtils implements KernelConstants {
                return safeUri(osgiInstanceBaseUri + (relativePath != null ? relativePath : ""));
        }
 
-       // static String getOsgiInstancePath(String relativePath) {
-       // try {
-       // if (relativePath == null)
-       // return getOsgiInstanceDir().getCanonicalPath();
-       // else
-       // return new File(getOsgiInstanceDir(), relativePath).getCanonicalPath();
-       // } catch (IOException e) {
-       // throw new CmsException("Cannot get instance path for " + relativePath,
-       // e);
-       // }
-       // }
-
        static File getOsgiConfigurationFile(String relativePath) {
                try {
                        return new File(new URI(getBundleContext().getProperty(OSGI_CONFIGURATION_AREA) + relativePath))
@@ -198,14 +186,14 @@ class KernelUtils implements KernelConstants {
                                st.open();
                        }
                };
-               new Thread(run, "Open service tracker " + st).start();
+               Activator.getInternalExecutorService().execute(run);
+//             new Thread(run, "Open service tracker " + st).start();
        }
 
        /**
         * @return the {@link BundleContext} of the {@link Bundle} which provided this
         *         class, never null.
-        * @throws CmsException
-        *             if the related bundle is not active
+        * @throws CmsException if the related bundle is not active
         */
        static BundleContext getBundleContext(Class<?> clzz) {
                Bundle bundle = FrameworkUtil.getBundle(clzz);
@@ -228,8 +216,7 @@ class KernelUtils implements KernelConstants {
                case "false":
                        return false;
                default:
-                       throw new CmsException("Unsupported value for attribute " + DataModelNamespace.ABSTRACT
-                                       + ": " + value);
+                       throw new CmsException("Unsupported value for attribute " + DataModelNamespace.ABSTRACT + ": " + value);
                }
        }