From eb3116df3624b3d32793548b79e137e2dad429cb Mon Sep 17 00:00:00 2001 From: Mathieu Baudier Date: Mon, 27 Jan 2020 10:04:13 +0100 Subject: [PATCH] Improve CMS workspace indexer. --- .../argeo/cms/internal/kernel/Activator.java | 9 + .../cms/internal/kernel/CmsDeployment.java | 21 +- .../internal/kernel/CmsWorkspaceIndexer.java | 205 ++++++++++++++++++ .../cms/internal/kernel/HomeRepository.java | 4 +- .../kernel/JackrabbitLocalRepository.java | 155 +------------ .../cms/internal/kernel/KernelUtils.java | 21 +- 6 files changed, 249 insertions(+), 166 deletions(-) create mode 100644 org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsWorkspaceIndexer.java diff --git a/org.argeo.cms/src/org/argeo/cms/internal/kernel/Activator.java b/org.argeo.cms/src/org/argeo/cms/internal/kernel/Activator.java index d7b953b53..5728d4fa5 100644 --- a/org.argeo.cms/src/org/argeo/cms/internal/kernel/Activator.java +++ b/org.argeo.cms/src/org/argeo/cms/internal/kernel/Activator.java @@ -8,6 +8,8 @@ import java.security.AllPermission; import java.util.Dictionary; import java.util.List; import java.util.Locale; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import javax.security.auth.login.Configuration; @@ -58,6 +60,7 @@ public class Activator implements BundleActivator { private CmsInstance nodeInstance; private ServiceTracker userAdminSt; + private ExecutorService internalExecutorService; @Override public void start(BundleContext bundleContext) throws Exception { @@ -65,6 +68,7 @@ public class Activator implements BundleActivator { instance = this; this.bc = bundleContext; this.logReaderService = getService(LogReaderService.class); + this.internalExecutorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); try { initSecurity(); @@ -168,6 +172,7 @@ public class Activator implements BundleActivator { if (userAdminSt != null) userAdminSt.close(); + internalExecutorService.shutdown(); instance = null; this.bc = null; this.logReaderService = null; @@ -229,6 +234,10 @@ public class Activator implements BundleActivator { } + static ExecutorService getInternalExecutorService() { + return instance.internalExecutorService; + } + // static CmsSecurity getCmsSecurity() { // return instance.nodeSecurity; // } diff --git a/org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsDeployment.java b/org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsDeployment.java index f09a00807..fba39b6d4 100644 --- a/org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsDeployment.java +++ b/org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsDeployment.java @@ -42,6 +42,7 @@ import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.Constants; import org.osgi.framework.FrameworkUtil; +import org.osgi.framework.InvalidSyntaxException; import org.osgi.framework.ServiceReference; import org.osgi.framework.wiring.BundleCapability; import org.osgi.framework.wiring.BundleWire; @@ -199,6 +200,15 @@ public class CmsDeployment implements NodeDeployment { if (nodeHttp != null) nodeHttp.destroy(); + try { + for (ServiceReference sr : bc + .getServiceReferences(JackrabbitLocalRepository.class, null)) { + bc.getService(sr).destroy(); + } + } catch (InvalidSyntaxException e1) { + log.error("Cannot sclean repsoitories", e1); + } + try { JettyConfigurator.stopServer(KernelConstants.DEFAULT_JETTY_SERVER); } catch (Exception e) { @@ -403,11 +413,16 @@ public class CmsDeployment implements NodeDeployment { if (dataModelName.equals(NodeConstants.NODE)) properties.put(Constants.SERVICE_RANKING, Integer.MAX_VALUE); LocalRepository localRepository; - if (repository instanceof RepositoryImpl) + String[] classes; + if (repository instanceof RepositoryImpl) { localRepository = new JackrabbitLocalRepository((RepositoryImpl) repository, dataModelName); - else + classes = new String[] { Repository.class.getName(), LocalRepository.class.getName(), + JackrabbitLocalRepository.class.getName() }; + } else { localRepository = new LocalRepository(repository, dataModelName); - bc.registerService(Repository.class, localRepository, properties); + classes = new String[] { Repository.class.getName(), LocalRepository.class.getName() }; + } + bc.registerService(classes, localRepository, properties); if (log.isTraceEnabled()) log.trace("Published data model " + dataModelName); } diff --git a/org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsWorkspaceIndexer.java b/org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsWorkspaceIndexer.java new file mode 100644 index 000000000..20ced3230 --- /dev/null +++ b/org.argeo.cms/src/org/argeo/cms/internal/kernel/CmsWorkspaceIndexer.java @@ -0,0 +1,205 @@ +package org.argeo.cms.internal.kernel; + +import java.util.GregorianCalendar; + +import javax.jcr.Node; +import javax.jcr.Property; +import javax.jcr.PropertyType; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.Value; +import javax.jcr.nodetype.NodeType; +import javax.jcr.observation.Event; +import javax.jcr.observation.EventIterator; +import javax.jcr.observation.EventListener; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.jackrabbit.api.JackrabbitValue; +import org.apache.jackrabbit.core.RepositoryImpl; +import org.argeo.jcr.JcrUtils; + +/** Ensure consistency of files, folder and last modified nodes. */ +class CmsWorkspaceIndexer implements EventListener { + private final static Log log = LogFactory.getLog(CmsWorkspaceIndexer.class); + + private final static String MIX_ETAG = "mix:etag"; + private final static String JCR_ETAG = "jcr:etag"; + private final static String JCR_LAST_MODIFIED = "jcr:lastModified"; + private final static String JCR_LAST_MODIFIED_BY = "jcr:lastModifiedBy"; + private final static String JCR_DATA = "jcr:data"; + String cn; + String workspaceName; + RepositoryImpl repositoryImpl; + Session session; + + public CmsWorkspaceIndexer(RepositoryImpl repositoryImpl, String cn, String workspaceName) + throws RepositoryException { + this.cn = cn; + this.workspaceName = workspaceName; + this.repositoryImpl = repositoryImpl; + } + + public void init() { + session = KernelUtils.openAdminSession(repositoryImpl, workspaceName); + try { + String[] nodeTypes = { NodeType.NT_FILE, NodeType.MIX_LAST_MODIFIED }; + session.getWorkspace().getObservationManager().addEventListener(this, + Event.NODE_ADDED | Event.NODE_REMOVED | Event.PROPERTY_CHANGED, "/", true, null, nodeTypes, true); + } catch (RepositoryException e1) { + throw new IllegalStateException(e1); + } + } + + public void destroy() { + try { + session.getWorkspace().getObservationManager().removeEventListener(this); + } catch (RepositoryException e) { + log.warn("Cannot unregistered JCR event listener", e); + } finally { + JcrUtils.logoutQuietly(session); + } + } + + private synchronized void processEvents(EventIterator events) { + while (events.hasNext()) { + Event event = events.nextEvent(); + processEvent(event); + } + notifyAll(); + } + + protected synchronized void processEvent(Event event) { + try { + if (event.getType() == Event.NODE_ADDED) { + session.refresh(true); + Node node = session.getNode(event.getPath()); + if (node.getParent().isNodeType(NodeType.NT_FILE)) { + if (node.isNodeType(NodeType.NT_UNSTRUCTURED)) { + if (!node.isNodeType(NodeType.MIX_LAST_MODIFIED)) + node.addMixin(NodeType.MIX_LAST_MODIFIED); + Property property = node.getProperty(Property.JCR_DATA); + String etag = toEtag(property.getValue()); + node.setProperty(JCR_ETAG, etag); + } else if (node.isNodeType(NodeType.NT_RESOURCE)) { + if (!node.isNodeType(MIX_ETAG)) + node.addMixin(MIX_ETAG); + session.save(); + Property property = node.getProperty(Property.JCR_DATA); + String etag = toEtag(property.getValue()); + node.setProperty(JCR_ETAG, etag); + session.save(); + } + setLastModified(node.getParent(), event); + session.save(); + if (log.isDebugEnabled()) + log.debug("ETag and last modified added to new " + node); + } + } else if (event.getType() == Event.PROPERTY_CHANGED) { + if (!session.propertyExists(event.getPath())) + return; + Property property = session.getProperty(event.getPath()); + String propertyName = property.getName(); + // skip if last modified properties are explicitly set + if (propertyName.equals(JCR_LAST_MODIFIED)) + return; + if (propertyName.equals(JCR_LAST_MODIFIED_BY)) + return; + Node node = property.getParent(); + if (property.getType() == PropertyType.BINARY && propertyName.equals(JCR_DATA) + && node.isNodeType(NodeType.NT_UNSTRUCTURED)) { + String etag = toEtag(property.getValue()); + node.setProperty(JCR_ETAG, etag); + } + setLastModified(node, event); + session.save(); + if (log.isDebugEnabled()) + log.debug("ETag and last modified updated for " + node); + } else if (event.getType() == Event.NODE_REMOVED) { + String removeNodePath = event.getPath(); + String parentPath = JcrUtils.parentPath(removeNodePath); + session.refresh(true); + setLastModified(session, parentPath, event); + session.save(); + if (log.isDebugEnabled()) + log.debug("Last modified updated for parents of removed " + removeNodePath); + } + } catch (Exception e) { + log.warn("Cannot process event " + event, e); + } finally { + try { + if (session.hasPendingChanges()) + session.save(); +// session.refresh(false); + } catch (RepositoryException e) { + log.warn("Cannot refresh JCR session", e); + } + } + + } + + @Override + public void onEvent(EventIterator events) { + Runnable toRun = new Runnable() { + + @Override + public void run() { + processEvents(events); + } + }; + Activator.getInternalExecutorService().execute(toRun); + } + + static String toEtag(Value v) { + if (v instanceof JackrabbitValue) { + JackrabbitValue value = (JackrabbitValue) v; + return '\"' + value.getContentIdentity() + '\"'; + } else { + return null; + } + + } + + /** Recursively set the last updated time on parents. */ + static void setLastModified(Node node, Event event) throws RepositoryException { + GregorianCalendar calendar = new GregorianCalendar(); + calendar.setTimeInMillis(event.getDate()); + if (node.isNodeType(NodeType.MIX_LAST_MODIFIED)) { + node.setProperty(Property.JCR_LAST_MODIFIED, calendar); + node.setProperty(Property.JCR_LAST_MODIFIED_BY, event.getUserID()); + } + if (node.isNodeType(NodeType.NT_FOLDER) && !node.isNodeType(NodeType.MIX_LAST_MODIFIED)) { + node.addMixin(NodeType.MIX_LAST_MODIFIED); + } + + try { + node.getSession().save(); + } catch (RepositoryException e) { + // fail silently and keep recursing + } + if (node.getDepth() == 0) + return; + Node parent = node.getParent(); + setLastModified(parent, event); + } + + /** + * Recursively set the last updated time on parents. Useful to use paths when + * dealing with deletions. + */ + static void setLastModified(Session session, String path, Event event) throws RepositoryException { + // root node will always exist, so end condition is delegated to the other + // recursive setLastModified method + if (session.nodeExists(path)) { + setLastModified(session.getNode(path), event); + } else { + setLastModified(session, JcrUtils.parentPath(path), event); + } + } + + @Override + public String toString() { + return "Monitor workspace " + workspaceName + " of repository " + cn; + } + +} \ No newline at end of file diff --git a/org.argeo.cms/src/org/argeo/cms/internal/kernel/HomeRepository.java b/org.argeo.cms/src/org/argeo/cms/internal/kernel/HomeRepository.java index e5164b6c3..aa0f1fc3f 100644 --- a/org.argeo.cms/src/org/argeo/cms/internal/kernel/HomeRepository.java +++ b/org.argeo.cms/src/org/argeo/cms/internal/kernel/HomeRepository.java @@ -58,7 +58,7 @@ class HomeRepository extends JcrRepositoryWrapper implements KernelConstants { @Override public Void run() { try { - Session adminSession = getRepository().login(); + Session adminSession = getDefaultRepository().login(); initJcr(adminSession); } catch (RepositoryException e) { throw new CmsException("Cannot init JCR home", e); @@ -80,7 +80,7 @@ class HomeRepository extends JcrRepositoryWrapper implements KernelConstants { if (checkedUsers.contains(username)) return; - Session adminSession = KernelUtils.openAdminSession(getRepository(), session.getWorkspace().getName()); + Session adminSession = KernelUtils.openAdminSession(getRepository(workspaceName), workspaceName); try { syncJcr(adminSession, username, workspaceName); checkedUsers.add(username); diff --git a/org.argeo.cms/src/org/argeo/cms/internal/kernel/JackrabbitLocalRepository.java b/org.argeo.cms/src/org/argeo/cms/internal/kernel/JackrabbitLocalRepository.java index dc47e6e13..572ab1fcc 100644 --- a/org.argeo.cms/src/org/argeo/cms/internal/kernel/JackrabbitLocalRepository.java +++ b/org.argeo.cms/src/org/argeo/cms/internal/kernel/JackrabbitLocalRepository.java @@ -1,34 +1,21 @@ package org.argeo.cms.internal.kernel; -import java.util.GregorianCalendar; import java.util.Map; import java.util.TreeMap; -import javax.jcr.ItemNotFoundException; -import javax.jcr.Node; -import javax.jcr.PathNotFoundException; -import javax.jcr.Property; -import javax.jcr.PropertyType; import javax.jcr.RepositoryException; import javax.jcr.Session; -import javax.jcr.Value; -import javax.jcr.nodetype.NodeType; -import javax.jcr.observation.Event; -import javax.jcr.observation.EventIterator; -import javax.jcr.observation.EventListener; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.jackrabbit.api.JackrabbitValue; import org.apache.jackrabbit.core.RepositoryImpl; import org.argeo.jcr.JcrUtils; class JackrabbitLocalRepository extends LocalRepository { private final static Log log = LogFactory.getLog(JackrabbitLocalRepository.class); - private final static String MIX_ETAG = "mix:etag"; - private final static String JCR_ETAG = "jcr:etag"; + final String SECURITY_WORKSPACE = "security"; - private Map workspaceMonitors = new TreeMap<>(); + private Map workspaceMonitors = new TreeMap<>(); public JackrabbitLocalRepository(RepositoryImpl repository, String cn) { super(repository, cn); @@ -55,14 +42,16 @@ class JackrabbitLocalRepository extends LocalRepository { } private void addMonitor(String realWorkspaceName) { + if (realWorkspaceName.equals(SECURITY_WORKSPACE)) + return; if (!workspaceMonitors.containsKey(realWorkspaceName)) { try { - WorkspaceMonitor workspaceMonitor = new WorkspaceMonitor(getJackrabbitrepository(realWorkspaceName), - getCn(), realWorkspaceName); + CmsWorkspaceIndexer workspaceMonitor = new CmsWorkspaceIndexer( + getJackrabbitrepository(realWorkspaceName), getCn(), realWorkspaceName); workspaceMonitors.put(realWorkspaceName, workspaceMonitor); - workspaceMonitor.start(); + workspaceMonitor.init(); if (log.isDebugEnabled()) - log.debug("Registered " + workspaceMonitor.getName()); + log.debug("Registered " + workspaceMonitor); } catch (RepositoryException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -70,132 +59,10 @@ class JackrabbitLocalRepository extends LocalRepository { } } - static String toEtag(Value v) { - JackrabbitValue value = (JackrabbitValue) v; - return '\"' + value.getContentIdentity() + '\"'; - - } - - /** recursive */ - static void setLastModified(Node node, Event event) throws RepositoryException { - GregorianCalendar calendar = new GregorianCalendar(); - calendar.setTimeInMillis(event.getDate()); - if (node.isNodeType(NodeType.NT_FOLDER)) { - node.addMixin(NodeType.MIX_LAST_MODIFIED); + public void destroy() { + for (String workspaceName : workspaceMonitors.keySet()) { + workspaceMonitors.get(workspaceName).destroy(); } - if (node.isNodeType(NodeType.MIX_LAST_MODIFIED)) { - node.setProperty(Property.JCR_LAST_MODIFIED, calendar); - node.setProperty(Property.JCR_LAST_MODIFIED_BY, event.getUserID()); - } - if (node.getDepth() == 0) - return; - Node parent = node.getParent(); - setLastModified(parent, event); } - static class WorkspaceMonitor extends Thread implements EventListener { - String workspaceName; - RepositoryImpl repositoryImpl; - Session session; - - public WorkspaceMonitor(RepositoryImpl repositoryImpl, String cn, String workspaceName) - throws RepositoryException { - super("Monitor workspace " + workspaceName + " of repository " + cn); - this.workspaceName = workspaceName; - this.repositoryImpl = repositoryImpl; - } - - public void run() { - - session = KernelUtils.openAdminSession(repositoryImpl, workspaceName); - try { - String[] nodeTypes = { NodeType.NT_FILE, NodeType.MIX_LAST_MODIFIED }; - session.getWorkspace().getObservationManager().addEventListener(this, - Event.NODE_ADDED | Event.NODE_REMOVED | Event.PROPERTY_CHANGED, "/", true, null, nodeTypes, - true); - while (save()) { - - } - - } catch (RepositoryException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } finally { - JcrUtils.logoutQuietly(session); - } - } - - protected synchronized boolean save() { - try { - wait(100); - } catch (InterruptedException e) { - // silent - } - if (!session.isLive()) - return false; - try { - if (session.hasPendingChanges()) - session.save(); - } catch (RepositoryException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return true; - } - - @Override - public synchronized void onEvent(EventIterator events) { - events: while (events.hasNext()) { - Event event = events.nextEvent(); -// if (log.isDebugEnabled()) -// log.debug(event); - try { - if (event.getType() == Event.NODE_ADDED) { - Node node = session.getNode(event.getPath()); - if (node.getParent().isNodeType(NodeType.NT_FILE)) { -// Node contentNode = node.getNode(Node.JCR_CONTENT); - node.addMixin(NodeType.MIX_LAST_MODIFIED); - Property property = node.getProperty(Property.JCR_DATA); - String etag = toEtag(property.getValue()); - node.setProperty(JCR_ETAG, etag); - setLastModified(node, event); -// node.getSession().save(); - if (log.isDebugEnabled()) - log.debug("Node " + node.getPath() + ": " + event); - } - } else if (event.getType() == Event.NODE_REMOVED) { - String parentPath = JcrUtils.parentPath(event.getPath()); - try { - Node parent = session.getNode(parentPath); - setLastModified(parent, event); - - if (log.isDebugEnabled()) - log.debug("Node removed from " + parent.getPath() + ": " + event); - } catch (ItemNotFoundException | PathNotFoundException e) { - continue events; - } - } else if (event.getType() == Event.PROPERTY_CHANGED) { - Property property = session.getProperty(event.getPath()); - if (property.getName().equals("jcr:lastModified")) - continue events; - if (property.getType() == PropertyType.BINARY && property.getName().equals("jcr:data") - && property.getParent().isNodeType(NodeType.NT_UNSTRUCTURED)) { - String etag = toEtag(property.getValue()); - property.getParent().setProperty(JCR_ETAG, etag); - } - setLastModified(property.getParent(), event); -// property.getParent().getSession().save(); - if (log.isDebugEnabled()) - log.debug("Property " + property.getPath() + ": " + event); - } - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - notifyAll(); - - } - - } } diff --git a/org.argeo.cms/src/org/argeo/cms/internal/kernel/KernelUtils.java b/org.argeo.cms/src/org/argeo/cms/internal/kernel/KernelUtils.java index 1d8140911..86289f0ff 100644 --- a/org.argeo.cms/src/org/argeo/cms/internal/kernel/KernelUtils.java +++ b/org.argeo.cms/src/org/argeo/cms/internal/kernel/KernelUtils.java @@ -91,18 +91,6 @@ class KernelUtils implements KernelConstants { return safeUri(osgiInstanceBaseUri + (relativePath != null ? relativePath : "")); } - // static String getOsgiInstancePath(String relativePath) { - // try { - // if (relativePath == null) - // return getOsgiInstanceDir().getCanonicalPath(); - // else - // return new File(getOsgiInstanceDir(), relativePath).getCanonicalPath(); - // } catch (IOException e) { - // throw new CmsException("Cannot get instance path for " + relativePath, - // e); - // } - // } - static File getOsgiConfigurationFile(String relativePath) { try { return new File(new URI(getBundleContext().getProperty(OSGI_CONFIGURATION_AREA) + relativePath)) @@ -198,14 +186,14 @@ class KernelUtils implements KernelConstants { st.open(); } }; - new Thread(run, "Open service tracker " + st).start(); + Activator.getInternalExecutorService().execute(run); +// new Thread(run, "Open service tracker " + st).start(); } /** * @return the {@link BundleContext} of the {@link Bundle} which provided this * class, never null. - * @throws CmsException - * if the related bundle is not active + * @throws CmsException if the related bundle is not active */ static BundleContext getBundleContext(Class clzz) { Bundle bundle = FrameworkUtil.getBundle(clzz); @@ -228,8 +216,7 @@ class KernelUtils implements KernelConstants { case "false": return false; default: - throw new CmsException("Unsupported value for attribute " + DataModelNamespace.ABSTRACT - + ": " + value); + throw new CmsException("Unsupported value for attribute " + DataModelNamespace.ABSTRACT + ": " + value); } } -- 2.30.2