From 9a664d1631d98dbcd75a7200fa21ed85779b4328 Mon Sep 17 00:00:00 2001 From: Mathieu Baudier Date: Tue, 4 Jun 2013 14:42:52 +0000 Subject: [PATCH] Working generic RPM Factory git-svn-id: https://svn.argeo.org/slc/trunk@6329 4cfe0d0a-d680-48aa-b62c-e0a02a3f76cc --- dep/org.argeo.slc.dep.backend/pom.xml | 4 +- .../META-INF/spring/repo-osgi.xml | 4 +- .../META-INF/spring/services.xml | 34 ++-- .../org.argeo.slc.server.repo/repo.properties | 1 + .../argeo/slc/repo/NodeIndexerVisitor.java | 48 ++++++ .../repo/{RepoImpl.java => RepoIndexer.java} | 30 ++-- .../slc/repo/maven/MavenProxyServiceImpl.java | 11 ++ .../resources/org/argeo/slc/repo/repo.cnd | 10 ++ runtime/org.argeo.slc.rpmfactory/pom.xml | 5 + .../slc/rpmfactory/core/BuildInMock.java | 29 +++- .../core/CreateRpmDistribution.java | 4 +- .../slc/rpmfactory/core/RpmDistribution.java | 10 ++ ...mBuildEnvironment.java => RpmFactory.java} | 159 +++++++++++++----- .../argeo/slc/rpmfactory/core/RpmIndexer.java | 105 ++++++++++++ .../rpmfactory/core/RpmProxyServiceImpl.java | 70 ++++++-- .../main/java/org/argeo/slc/SlcConstants.java | 7 + .../main/java/org/argeo/slc/jcr/SlcNames.java | 6 + .../main/java/org/argeo/slc/jcr/SlcTypes.java | 4 +- runtime/org.argeo.slc.support.simple/pom.xml | 5 - 19 files changed, 442 insertions(+), 104 deletions(-) create mode 100644 runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/NodeIndexerVisitor.java rename runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/{RepoImpl.java => RepoIndexer.java} (80%) rename runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/{RpmBuildEnvironment.java => RpmFactory.java} (72%) create mode 100644 runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmIndexer.java create mode 100644 runtime/org.argeo.slc.specs/src/main/java/org/argeo/slc/SlcConstants.java diff --git a/dep/org.argeo.slc.dep.backend/pom.xml b/dep/org.argeo.slc.dep.backend/pom.xml index 896449469..4d65b42fc 100644 --- a/dep/org.argeo.slc.dep.backend/pom.xml +++ b/dep/org.argeo.slc.dep.backend/pom.xml @@ -75,8 +75,8 @@ 1.1.13-SNAPSHOT - org.argeo.slc.rpmfactory - org.argeo.slc.repo + org.argeo.slc + org.argeo.slc.rpmfactory 1.1.13-SNAPSHOT diff --git a/modules/org.argeo.slc.server.repo/META-INF/spring/repo-osgi.xml b/modules/org.argeo.slc.server.repo/META-INF/spring/repo-osgi.xml index d7c53a2f5..50a61ec6d 100644 --- a/modules/org.argeo.slc.server.repo/META-INF/spring/repo-osgi.xml +++ b/modules/org.argeo.slc.server.repo/META-INF/spring/repo-osgi.xml @@ -48,8 +48,8 @@ - + diff --git a/modules/org.argeo.slc.server.repo/META-INF/spring/services.xml b/modules/org.argeo.slc.server.repo/META-INF/spring/services.xml index 06560aebc..308000254 100644 --- a/modules/org.argeo.slc.server.repo/META-INF/spring/services.xml +++ b/modules/org.argeo.slc.server.repo/META-INF/spring/services.xml @@ -15,9 +15,18 @@ - + + + + + + + + + @@ -26,17 +35,22 @@ - - - - - - + + init-method="init" destroy-method="destroy" depends-on="ROLE_SLC"> + + + + + + + + + + \ No newline at end of file diff --git a/modules/org.argeo.slc.server.repo/repo.properties b/modules/org.argeo.slc.server.repo/repo.properties index 61e2fa752..aa14aeb8d 100644 --- a/modules/org.argeo.slc.server.repo/repo.properties +++ b/modules/org.argeo.slc.server.repo/repo.properties @@ -1,5 +1,6 @@ slc.repo.jcr.reposBase=${osgi.instance.area}/repos slc.repo.jcr.forceCndImport=true +slc.repo.jcr.proxyWorkspace=proxy slc.repo.jcr.configuration.java=osgibundle:repository-h2.xml slc.repo.jcr.configuration.dist=osgibundle:repository-h2.xml diff --git a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/NodeIndexerVisitor.java b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/NodeIndexerVisitor.java new file mode 100644 index 000000000..c0f90d146 --- /dev/null +++ b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/NodeIndexerVisitor.java @@ -0,0 +1,48 @@ +package org.argeo.slc.repo; + +import java.util.ArrayList; +import java.util.List; + +import javax.jcr.ItemVisitor; +import javax.jcr.Node; +import javax.jcr.NodeIterator; +import javax.jcr.Property; +import javax.jcr.RepositoryException; + +/** + * Recursively visit a sub tree and apply the list of node indexer on supported + * nodes. + */ +public class NodeIndexerVisitor implements ItemVisitor { + /** order may be important */ + private List nodeIndexers = new ArrayList(); + + public NodeIndexerVisitor() { + } + + /** Convenience constructor */ + public NodeIndexerVisitor(NodeIndexer nodeIndexer) { + nodeIndexers.add(nodeIndexer); + } + + public NodeIndexerVisitor(List nodeIndexers) { + this.nodeIndexers = nodeIndexers; + } + + public void visit(Node node) throws RepositoryException { + for (NodeIndexer nodeIndexer : nodeIndexers) + if (nodeIndexer.support(node.getPath())) + nodeIndexer.index(node); + + for (NodeIterator it = node.getNodes(); it.hasNext();) + visit(it.nextNode()); + } + + public void visit(Property property) throws RepositoryException { + } + + public void setNodeIndexers(List nodeIndexers) { + this.nodeIndexers = nodeIndexers; + } + +} diff --git a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoImpl.java b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoIndexer.java similarity index 80% rename from runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoImpl.java rename to runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoIndexer.java index 265530f89..c18911436 100644 --- a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoImpl.java +++ b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/RepoIndexer.java @@ -16,7 +16,6 @@ package org.argeo.slc.repo; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import javax.jcr.Node; @@ -31,32 +30,27 @@ import org.argeo.jcr.JcrUtils; import org.argeo.slc.SlcException; /** Repository backend, maintain the JCR repository, mainly through listeners */ -public class RepoImpl { +public class RepoIndexer { private Repository jcrRepository; - private Session adminSession; + private String workspace = null; - private ArtifactListener artifactListener; + // Internal + private Session adminSession; + private FilesListener artifactListener; /** order may be important */ private List nodeIndexers = new ArrayList(); public void init() { try { - adminSession = jcrRepository.login(); - artifactListener = new ArtifactListener(); + adminSession = jcrRepository.login(workspace); + artifactListener = new FilesListener(); adminSession .getWorkspace() .getObservationManager() - .addEventListener(artifactListener, Event.NODE_ADDED, - RepoConstants.DEFAULT_ARTIFACTS_BASE_PATH, true, null, - null, true); - // localrepo - String localrepoWorkspace = "localrepo"; - if (!Arrays.asList( - adminSession.getWorkspace().getAccessibleWorkspaceNames()) - .contains(localrepoWorkspace)) - adminSession.getWorkspace().createWorkspace(localrepoWorkspace); + .addEventListener(artifactListener, Event.NODE_ADDED, "/", + true, null, null, true); } catch (RepositoryException e) { throw new SlcException("Cannot initialize repository backend", e); } @@ -74,7 +68,11 @@ public class RepoImpl { this.nodeIndexers = nodeIndexers; } - class ArtifactListener implements EventListener { + public void setWorkspace(String workspace) { + this.workspace = workspace; + } + + class FilesListener implements EventListener { public void onEvent(EventIterator events) { while (events.hasNext()) { diff --git a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/MavenProxyServiceImpl.java b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/MavenProxyServiceImpl.java index 87ee867b7..a3277a8c7 100644 --- a/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/MavenProxyServiceImpl.java +++ b/runtime/org.argeo.slc.repo/src/main/java/org/argeo/slc/repo/maven/MavenProxyServiceImpl.java @@ -23,12 +23,14 @@ import javax.jcr.NodeIterator; import javax.jcr.RepositoryException; import javax.jcr.Session; import javax.jcr.nodetype.NodeType; +import javax.jcr.security.AccessControlException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.argeo.jcr.ArgeoNames; import org.argeo.jcr.JcrUtils; import org.argeo.jcr.proxy.AbstractUrlProxy; +import org.argeo.slc.SlcConstants; import org.argeo.slc.SlcException; import org.argeo.slc.jcr.SlcNames; import org.argeo.slc.jcr.SlcTypes; @@ -48,6 +50,15 @@ public class MavenProxyServiceImpl extends AbstractUrlProxy implements @Override protected void beforeInitSessionSave(Session session) throws RepositoryException { + JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read"); + try { + JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC, + "jcr:all"); + } catch (AccessControlException e) { + if (log.isTraceEnabled()) + log.trace("Cannot give jcr:all privileges to ROLE_SLC"); + } + JcrUtils.mkdirsSafe(session, RepoConstants.DEFAULT_ARTIFACTS_BASE_PATH); Node proxiedRepositories = JcrUtils.mkdirsSafe(session, RepoConstants.PROXIED_REPOSITORIES); diff --git a/runtime/org.argeo.slc.repo/src/main/resources/org/argeo/slc/repo/repo.cnd b/runtime/org.argeo.slc.repo/src/main/resources/org/argeo/slc/repo/repo.cnd index 578beb570..c0bd68171 100644 --- a/runtime/org.argeo.slc.repo/src/main/resources/org/argeo/slc/repo/repo.cnd +++ b/runtime/org.argeo.slc.repo/src/main/resources/org/argeo/slc/repo/repo.cnd @@ -129,3 +129,13 @@ mixin [slc:bundleArtifact] > slc:artifact,slc:jarFile,slc:bundle mixin + +// RPM +[slc:rpm] > mix:referenceable, mix:created, mix:lastModified, mix:title +mixin +- slc:name (STRING) +- slc:version (STRING) +- slc:rpmVersion (STRING) +- slc:rpmRelease (STRING) +- slc:rpmArch (STRING) +- slc:rpmArchivaeSize (STRING) diff --git a/runtime/org.argeo.slc.rpmfactory/pom.xml b/runtime/org.argeo.slc.rpmfactory/pom.xml index d8ef23fe3..cbca152d8 100644 --- a/runtime/org.argeo.slc.rpmfactory/pom.xml +++ b/runtime/org.argeo.slc.rpmfactory/pom.xml @@ -53,5 +53,10 @@ org.argeo.tp slf4j.org.apache.commons.logging + + org.argeo.tp + org.redline_rpm + + \ No newline at end of file diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/BuildInMock.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/BuildInMock.java index 0bdb4c7ce..50daabce3 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/BuildInMock.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/BuildInMock.java @@ -36,16 +36,17 @@ public class BuildInMock implements Runnable { private final static String NOARCH = "noarch"; private String rpmPackage = null; + private String branch = null; private String arch = NOARCH; - private RpmBuildEnvironment buildEnvironment; + private RpmFactory factory; private Executor executor; private String debuginfoDirName = "debuginfo"; public void run() { - String mockCfg = buildEnvironment.getMockConfig(arch); - File mockConfigFile = buildEnvironment.getMockConfigFile(arch); + String mockCfg = factory.getMockConfig(arch); + File mockConfigFile = factory.getMockConfigFile(arch, branch); // prepare mock call SystemCall mock = new SystemCall(); @@ -58,6 +59,10 @@ public class BuildInMock implements Runnable { mock.arg("--arch=" + arch); mock.arg("-r").arg(mockCfg); mock.arg("--scm-enable"); + // mock.arg("--scm-option"); + // mock.arg("git_get='git clone " + (branch != null ? "-b " + branch : + // "") + // + " " + factory.getGitBaseUrl() + "/SCM_PKG SCM_PKG'"); mock.arg("--scm-option").arg("package=" + rpmPackage); mock.setLogCommand(true); @@ -69,7 +74,8 @@ public class BuildInMock implements Runnable { mock.run(); // - File stagingDir = buildEnvironment.getStagingDir(); + File stagingDir = factory + .getWorkspaceDir(factory.getStagingWorkspace()); File srpmDir = new File(stagingDir, "SRPMS"); srpmDir.mkdirs(); File archDir = null; @@ -82,7 +88,7 @@ public class BuildInMock implements Runnable { // copy RPMs Set reposToRecreate = new HashSet(); - File resultDir = buildEnvironment.getResultDir(arch); + File resultDir = factory.getResultDir(arch); rpms: for (File file : resultDir.listFiles()) { if (file.isDirectory()) continue rpms; @@ -97,7 +103,7 @@ public class BuildInMock implements Runnable { targetDirs = new File[] { archDir }; else if (file.getName().contains(".noarch.rpm")) { List dirs = new ArrayList(); - for (String arch : buildEnvironment.getArchs()) + for (String arch : factory.getArchs()) dirs.add(new File(stagingDir, arch)); targetDirs = dirs.toArray(new File[dirs.size()]); } else if (file.getName().contains(".rpm")) @@ -129,6 +135,9 @@ public class BuildInMock implements Runnable { createrepo.run(); log.info("Updated repo " + repoToRecreate); } + + // index staging workspace + factory.indexWorkspace(factory.getStagingWorkspace()); } protected void copyToDirs(File file, File[] dirs) { @@ -151,8 +160,12 @@ public class BuildInMock implements Runnable { this.rpmPackage = rpmPackage; } - public void setBuildEnvironment(RpmBuildEnvironment buildEnvironment) { - this.buildEnvironment = buildEnvironment; + public void setBranch(String branch) { + this.branch = branch; + } + + public void setFactory(RpmFactory env) { + this.factory = env; } public void setExecutor(Executor executor) { diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java index cd893acd9..32d28144a 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java @@ -5,12 +5,10 @@ package org.argeo.slc.rpmfactory.core; * consistent distributable set (typically to be used to generate an ISO). */ public class CreateRpmDistribution implements Runnable { - + // private RpmDistribution rpmDistribution; @Override public void run() { - // TODO Auto-generated method stub } - } diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java index 0556f6a3a..66d87346f 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java @@ -1,6 +1,16 @@ package org.argeo.slc.rpmfactory.core; +import java.util.List; + /** A consistent distributable set of RPM. */ public class RpmDistribution { + private List packages; + + public List getPackages() { + return packages; + } + public void setPackages(List packages) { + this.packages = packages; + } } diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmBuildEnvironment.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmFactory.java similarity index 72% rename from runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmBuildEnvironment.java rename to runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmFactory.java index 61147f34b..8396a9bbe 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmBuildEnvironment.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmFactory.java @@ -29,33 +29,40 @@ import javax.jcr.Session; import org.apache.commons.io.FileUtils; import org.argeo.jcr.JcrUtils; +import org.argeo.slc.SlcConstants; import org.argeo.slc.SlcException; import org.argeo.slc.core.execution.tasks.SystemCall; +import org.argeo.slc.repo.NodeIndexerVisitor; import org.argeo.slc.rpmfactory.RpmRepository; /** * Defines a build environment. This information is typically used by other * components performing the various actions related to RPM build. */ -public class RpmBuildEnvironment { - +public class RpmFactory { private Repository rpmRepository; private Repository distRepository; private String id; - private String stagingBase = "/mnt/slc/repos/rpm"; + private List repositories = new ArrayList(); + private List archs = new ArrayList(); + + private String rpmBase = "/mnt/slc/repos/rpm"; + private String distBase = "/mnt/slc/repos/dist"; private String mockVar = "/var/lib/mock"; private String mockEtc = "/etc/mock"; - private List archs = new ArrayList(); + + private String gitWorkspace = "git"; private String localUrlBase = "http://localhost:7070/"; + private Boolean withTestingRepository = false; + private String yumConfigMainSection = "cachedir=/var/cache/yum\n" + "debuglevel=1\n" + "reposdir=/dev/null\n" + "logfile=/var/log/yum.log\n" + "retries=20\n" + "obsoletes=1\n" + "gpgcheck=0\n" + "assumeyes=1\n" + "syslog_ident=mock\n" - + "syslog_device=\n"; - private List repositories = new ArrayList(); + + "syslog_device=\n" + "http_caching=none\n"; private String defaultMacroFiles = "/usr/lib/rpm/macros:" + "/usr/lib/rpm/ia32e-linux/macros:" @@ -77,24 +84,28 @@ public class RpmBuildEnvironment { private File yumConfDir; public void init() { + // local URL bases proxiedReposBase = localUrlBase + "repo/rpm/"; managedReposBase = localUrlBase + "data/public/rpm/"; - stagingWorkspace = id + "-staging"; - testingWorkspace = id + "-testing"; - stableWorkspace = id; - - // rpmFactoryBaseDir = new File(System.getProperty("osgi.instance.area") - // .substring("file://".length()) + "/rpmfactory"); + // local directories rpmFactoryBaseDir.mkdirs(); mockConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/mock"); mockConfDir.mkdirs(); yumConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/yum"); yumConfDir.mkdirs(); - initDistWorkspace(stagingWorkspace); + // managed repositories + stagingWorkspace = id + "-staging"; + if (withTestingRepository) + testingWorkspace = id + "-testing"; + stableWorkspace = id; + + initDistWorkspace(stableWorkspace); + initGitWorkspace(); initRpmWorkspace(stagingWorkspace); - initRpmWorkspace(testingWorkspace); + if (withTestingRepository) + initRpmWorkspace(testingWorkspace); initRpmWorkspace(stableWorkspace); } @@ -103,18 +114,29 @@ public class RpmBuildEnvironment { try { session = JcrUtils.loginOrCreateWorkspace(rpmRepository, workspace); JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read"); + JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC, + "jcr:all"); for (String arch : archs) { Node archFolder = JcrUtils.mkfolders(session, "/" + arch); + session.save(); if (!archFolder.hasNode("repodata")) { + File workspaceDir = getWorkspaceDir(workspace); + // touch a file in order to make sue this is properly + // mounted. + File touch = new File(workspaceDir, ".touch"); + touch.createNewFile(); + touch.delete(); + SystemCall createrepo = new SystemCall(); createrepo.arg("createrepo"); createrepo.arg("-q"); - File archDir = new File(getStagingDir(), arch); + File archDir = new File(workspaceDir, arch); createrepo.arg(archDir.getAbsolutePath()); + createrepo.run(); } } - } catch (RepositoryException e) { + } catch (Exception e) { throw new SlcException("Cannot initialize workspace " + workspace, e); } finally { @@ -122,6 +144,22 @@ public class RpmBuildEnvironment { } } + protected void initGitWorkspace() { + Session session = null; + try { + session = JcrUtils.loginOrCreateWorkspace(rpmRepository, + gitWorkspace); + JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read"); + JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC, + "jcr:all"); + } catch (Exception e) { + throw new SlcException("Cannot initialize workspace " + + gitWorkspace, e); + } finally { + JcrUtils.logoutQuietly(session); + } + } + protected void initDistWorkspace(String workspace) { Session session = null; try { @@ -140,7 +178,7 @@ public class RpmBuildEnvironment { } - public String generateMockConfigFile(String arch) { + public String generateMockConfigFile(String arch, String branch) { StringBuffer buf = new StringBuffer(); buf.append("config_opts['root'] = '" + getIdWithArch(arch) + "'\n"); @@ -148,15 +186,19 @@ public class RpmBuildEnvironment { buf.append("config_opts['legal_host_arches'] = ('" + arch + "',)\n"); buf.append("config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'\n"); // buf.append("config_opts['dist'] = 'el6'\n"); + buf.append("config_opts['plugin_conf']['yum_cache_enable'] = False\n"); buf.append("config_opts['scm'] = False\n"); buf.append("config_opts['scm_opts']['method'] = 'git'\n"); - buf.append("config_opts['scm_opts']['git_get'] = 'git clone -b elgis6 /home/mbaudier/dev/git/elgis.argeo.org/SCM_PKG SCM_PKG'\n"); buf.append("config_opts['scm_opts']['spec'] = 'SCM_PKG.spec'\n"); - buf.append("config_opts['scm_opts']['ext_src_dir'] = '/home/mbaudier/dev/git/elgis.argeo.org/sources'\n"); + buf.append("config_opts['scm_opts']['ext_src_dir'] = '" + + getSourcesDir().getAbsolutePath() + "'\n"); buf.append("config_opts['scm_opts']['git_timestamps'] = True\n"); + buf.append("config_opts['scm_opts']['git_get'] = 'git clone " + + (branch != null ? "-b " + branch : "") + " " + + getGitBaseUrl() + "/SCM_PKG.git SCM_PKG'\n"); - buf.append("config_opts['yum.conf'] = \"\"\"\n"); + buf.append("\nconfig_opts['yum.conf'] = \"\"\"\n"); buf.append(generateYumConfigFile(arch)).append('\n'); buf.append("\"\"\"\n"); return buf.toString(); @@ -179,24 +221,22 @@ public class RpmBuildEnvironment { } // managed repos - buf.append('[').append(stagingWorkspace).append("]\n"); - buf.append("baseurl=").append(managedReposBase) - .append(stagingWorkspace).append('/').append(arch).append('/') - .append("\n"); - - buf.append('[').append(testingWorkspace).append("]\n"); - buf.append("baseurl=").append(managedReposBase) - .append(testingWorkspace).append('/').append(arch).append('/') - .append("\n"); - - buf.append('[').append(stableWorkspace).append("]\n"); - buf.append("baseurl=").append(managedReposBase).append(stableWorkspace) - .append('/').append(arch).append('/').append("\n"); + addManagedRepository(buf, stagingWorkspace, arch); + if (withTestingRepository) + addManagedRepository(buf, testingWorkspace, arch); + addManagedRepository(buf, stableWorkspace, arch); return buf.toString(); } + protected void addManagedRepository(StringBuffer buf, String workspace, + String arch) { + buf.append('[').append(workspace).append("]\n"); + buf.append("baseurl=").append(managedReposBase).append(workspace) + .append('/').append(arch).append('/').append("\n"); + } + /** Creates a mock config file. */ - public File getMockConfigFile(String arch) { + public File getMockConfigFile(String arch, String branch) { File mockSiteDefaultsFile = new File(mockConfDir, "site-defaults.cfg"); File mockLoggingFile = new File(mockConfDir, "logging.ini"); File mockConfigFile = new File(mockConfDir, getIdWithArch(arch) @@ -209,7 +249,7 @@ public class RpmBuildEnvironment { mockLoggingFile); FileUtils.writeStringToFile(mockConfigFile, - generateMockConfigFile(arch)); + generateMockConfigFile(arch, branch)); return mockConfigFile; } catch (IOException e) { throw new SlcException("Cannot write mock config file to " @@ -234,8 +274,12 @@ public class RpmBuildEnvironment { return new File(mockVar + "/" + getIdWithArch(arch) + "/result"); } - public File getStagingDir() { - return new File(stagingBase + "/" + stagingWorkspace); + public File getWorkspaceDir(String workspace) { + return new File(rpmBase + "/" + workspace); + } + + public File getSourcesDir() { + return new File(distBase + "/" + stagingWorkspace); } public String getMockConfig(String arch) { @@ -246,6 +290,23 @@ public class RpmBuildEnvironment { return id + "-" + arch; } + public String getGitBaseUrl() { + return managedReposBase + gitWorkspace; + } + + public void indexWorkspace(String workspace) { + Session session = null; + try { + session = rpmRepository.login(workspace); + session.getRootNode().accept( + new NodeIndexerVisitor(new RpmIndexer())); + } catch (RepositoryException e) { + throw new SlcException("Cannot index workspace " + workspace, e); + } finally { + JcrUtils.logoutQuietly(session); + } + } + /** Write (topdir)/rpmmacros and (topdir)/rpmrc */ public void writeRpmbuildConfigFiles(File topdir) { writeRpmbuildConfigFiles(topdir, new File(topdir, "rpmmacros"), @@ -297,12 +358,8 @@ public class RpmBuildEnvironment { return archs; } - public String getStagingBase() { - return stagingBase; - } - - public void setStagingBase(String stagingBase) { - this.stagingBase = stagingBase; + public void setRpmBase(String stagingBase) { + this.rpmBase = stagingBase; } public void setId(String id) { @@ -337,4 +394,20 @@ public class RpmBuildEnvironment { this.rpmFactoryBaseDir = rpmFactoryBaseDir; } + public String getStagingWorkspace() { + return stagingWorkspace; + } + + public String getTestingWorkspace() { + return testingWorkspace; + } + + public String getStableWorkspace() { + return stableWorkspace; + } + + public void setWithTestingRepository(Boolean withTestingRepository) { + this.withTestingRepository = withTestingRepository; + } + } diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmIndexer.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmIndexer.java new file mode 100644 index 000000000..ef8b9914b --- /dev/null +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmIndexer.java @@ -0,0 +1,105 @@ +package org.argeo.slc.rpmfactory.core; + +import static org.freecompany.redline.header.Header.HeaderTag.HEADERIMMUTABLE; +import static org.freecompany.redline.header.Signature.SignatureTag.SIGNATURES; + +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; + +import javax.jcr.Node; +import javax.jcr.Property; + +import org.apache.commons.io.FilenameUtils; +import org.argeo.slc.SlcException; +import org.argeo.slc.jcr.SlcNames; +import org.argeo.slc.jcr.SlcTypes; +import org.argeo.slc.repo.NodeIndexer; +import org.freecompany.redline.ChannelWrapper.Key; +import org.freecompany.redline.ReadableChannelWrapper; +import org.freecompany.redline.header.AbstractHeader; +import org.freecompany.redline.header.Format; +import org.freecompany.redline.header.Header; + +/** Indexes an RPM file. */ +public class RpmIndexer implements NodeIndexer, SlcNames { + + @Override + public Boolean support(String path) { + return FilenameUtils.getExtension(path).equals("rpm"); + } + + @Override + public void index(Node node) { + try { + InputStream in = node.getNode(Node.JCR_CONTENT) + .getProperty(Property.JCR_DATA).getBinary().getStream(); + ReadableChannelWrapper channel = new ReadableChannelWrapper( + Channels.newChannel(in)); + Format format = readRpmInfo(channel); + + node.addMixin(SlcTypes.SLC_RPM); + node.setProperty(SLC_NAME, readTag(format, Header.HeaderTag.NAME)); + String rpmVersion = readTag(format, Header.HeaderTag.VERSION); + String rpmRelease = readTag(format, Header.HeaderTag.RELEASE); + node.setProperty(SLC_RPM_VERSION, rpmVersion); + node.setProperty(SLC_RPM_RELEASE, rpmRelease); + node.setProperty(SLC_VERSION, rpmVersion + "-" + rpmRelease); + + String arch = readTag(format, Header.HeaderTag.ARCH); + if (arch != null) + node.setProperty(SLC_RPM_ARCH, arch); + + String archiveSize = readTag(format, Header.HeaderTag.ARCHIVESIZE); + if (archiveSize != null) + node.setProperty(SLC_RPM_ARCHIVE_SIZE, + Long.parseLong(archiveSize)); + + node.getSession().save(); + } catch (Exception e) { + throw new SlcException("Cannot index " + node, e); + } + + } + + @SuppressWarnings("unused") + public Format readRpmInfo(ReadableChannelWrapper channel) throws Exception { + Format format = new Format(); + + Key lead = channel.start(); + format.getLead().read(channel); + // System.out.println( "Lead ended at '" + in.finish( lead) + "'."); + + Key signature = channel.start(); + int count = format.getSignature().read(channel); + int expected = ByteBuffer + .wrap((byte[]) format.getSignature().getEntry(SIGNATURES) + .getValues(), 8, 4).getInt() + / -16; + // System.out.println( "Signature ended at '" + in.finish( signature) + + // "' and contained '" + count + "' headers (expected '" + expected + + // "')."); + + Key header = channel.start(); + count = format.getHeader().read(channel); + expected = ByteBuffer.wrap( + (byte[]) format.getHeader().getEntry(HEADERIMMUTABLE) + .getValues(), 8, 4).getInt() + / -16; + // System.out.println( "Header ended at '" + in.finish( header) + + // " and contained '" + count + "' headers (expected '" + expected + + // "')."); + + return format; + } + + private String readTag(Format format, Header.HeaderTag tag) { + AbstractHeader.Entry entry = format.getHeader().getEntry(tag); + if (entry == null) + return null; + if (entry.getValues() == null) + return null; + Object[] values = (Object[]) entry.getValues(); + return values[0].toString().trim(); + } +} diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmProxyServiceImpl.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmProxyServiceImpl.java index d5b0bc78b..82d9e553b 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmProxyServiceImpl.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmProxyServiceImpl.java @@ -20,15 +20,21 @@ import java.util.Iterator; import java.util.Set; import javax.jcr.Node; +import javax.jcr.RepositoryException; import javax.jcr.Session; +import javax.jcr.nodetype.NodeType; +import javax.jcr.security.AccessControlException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.argeo.jcr.ArgeoNames; import org.argeo.jcr.JcrUtils; import org.argeo.jcr.proxy.AbstractUrlProxy; +import org.argeo.slc.SlcConstants; import org.argeo.slc.SlcException; import org.argeo.slc.jcr.SlcNames; +import org.argeo.slc.jcr.SlcTypes; +import org.argeo.slc.repo.RepoConstants; import org.argeo.slc.rpmfactory.RpmProxyService; import org.argeo.slc.rpmfactory.RpmRepository; @@ -39,6 +45,21 @@ public class RpmProxyServiceImpl extends AbstractUrlProxy implements private Set defaultRepositories = new HashSet(); + @Override + protected void beforeInitSessionSave(Session session) + throws RepositoryException { + JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read"); + try { + JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC, + "jcr:all"); + } catch (AccessControlException e) { + if (log.isTraceEnabled()) + log.trace("Cannot give jcr:all privileges to ROLE_SLC"); + } + + JcrUtils.mkdirsSafe(session, RepoConstants.PROXIED_REPOSITORIES); + } + /** * Retrieve and add this file to the repository */ @@ -46,7 +67,8 @@ public class RpmProxyServiceImpl extends AbstractUrlProxy implements protected Node retrieve(Session session, String path) { StringBuilder relativePathBuilder = new StringBuilder(); String repoId = extractRepoId(path, relativePathBuilder); - String relativePath = relativePathBuilder.toString(); + // remove starting '/' + String relativePath = relativePathBuilder.toString().substring(1); RpmRepository sourceRepo = null; for (Iterator reposIt = defaultRepositories.iterator(); reposIt @@ -62,21 +84,11 @@ public class RpmProxyServiceImpl extends AbstractUrlProxy implements throw new SlcException("No RPM repository found for " + path); try { - // if (session.hasPendingChanges()) - // throw new SlcException("Session has pending changed"); String baseUrl = sourceRepo.getUrl(); String remoteUrl = baseUrl + relativePath; Node node = proxyUrl(session, remoteUrl, path); if (node != null) { - // node.addMixin(SlcTypes.SLC_KNOWN_ORIGIN); - // Node origin; - // if (!node.hasNode(SLC_ORIGIN)) - // origin = node.addNode(SLC_ORIGIN, SlcTypes.SLC_PROXIED); - // else - // origin = node.getNode(SLC_ORIGIN); - // // origin.setProperty(SLC_PROXY, sourceRepo.getId()); - // JcrUtils.urlToAddressProperties(origin, remoteUrl); - + registerSource(sourceRepo, node, remoteUrl); if (log.isDebugEnabled()) log.debug("Imported " + remoteUrl + " to " + node); return node; @@ -88,6 +100,36 @@ public class RpmProxyServiceImpl extends AbstractUrlProxy implements throw new SlcException("No proxy found for " + path); } + protected void registerSource(RpmRepository sourceRepo, Node node, + String remoteUrl) throws RepositoryException { + node.addMixin(SlcTypes.SLC_KNOWN_ORIGIN); + Node origin; + if (!node.hasNode(SLC_ORIGIN)) + origin = node.addNode(SLC_ORIGIN, SlcTypes.SLC_PROXIED); + else + origin = node.getNode(SLC_ORIGIN); + + // proxied repository + Node proxiedRepository; + String proxiedRepositoryPath = RepoConstants.PROXIED_REPOSITORIES + '/' + + sourceRepo.getId(); + Session session = node.getSession(); + if (session.itemExists(proxiedRepositoryPath)) { + proxiedRepository = session.getNode(proxiedRepositoryPath); + } else { + proxiedRepository = session.getNode( + RepoConstants.PROXIED_REPOSITORIES).addNode( + sourceRepo.getId()); + proxiedRepository.addMixin(NodeType.MIX_REFERENCEABLE); + JcrUtils.urlToAddressProperties(proxiedRepository, + sourceRepo.getUrl()); + proxiedRepository.setProperty(SLC_URL, sourceRepo.getUrl()); + } + + origin.setProperty(SLC_PROXY, proxiedRepository); + JcrUtils.urlToAddressProperties(origin, remoteUrl); + } + /** Returns the first token of the path */ protected String extractRepoId(String path, StringBuilder relativePath) { StringBuilder workspace = new StringBuilder(); @@ -104,8 +146,8 @@ public class RpmProxyServiceImpl extends AbstractUrlProxy implements @Override protected Boolean shouldUpdate(Session clientSession, String nodePath) { - if (nodePath.contains("/repodata/")) - return true; + // if (nodePath.contains("/repodata/")) + // return true; return super.shouldUpdate(clientSession, nodePath); } diff --git a/runtime/org.argeo.slc.specs/src/main/java/org/argeo/slc/SlcConstants.java b/runtime/org.argeo.slc.specs/src/main/java/org/argeo/slc/SlcConstants.java new file mode 100644 index 000000000..3cc53e4ee --- /dev/null +++ b/runtime/org.argeo.slc.specs/src/main/java/org/argeo/slc/SlcConstants.java @@ -0,0 +1,7 @@ +package org.argeo.slc; + +/** Constants useful across all SLC components */ +public interface SlcConstants { + /** Read-write role. */ + public final static String ROLE_SLC = "ROLE_SLC"; +} diff --git a/runtime/org.argeo.slc.support.jcr/src/main/java/org/argeo/slc/jcr/SlcNames.java b/runtime/org.argeo.slc.support.jcr/src/main/java/org/argeo/slc/jcr/SlcNames.java index b0f12702f..a728700ed 100644 --- a/runtime/org.argeo.slc.support.jcr/src/main/java/org/argeo/slc/jcr/SlcNames.java +++ b/runtime/org.argeo.slc.support.jcr/src/main/java/org/argeo/slc/jcr/SlcNames.java @@ -115,4 +115,10 @@ public interface SlcNames { // slc:exportedPackage public final static String SLC_USES = "slc:uses"; + // RPM + // slc:rpm + public final static String SLC_RPM_VERSION = "slc:rpmVersion"; + public final static String SLC_RPM_RELEASE = "slc:rpmRelease"; + public final static String SLC_RPM_ARCH = "slc:rpmArch"; + public final static String SLC_RPM_ARCHIVE_SIZE = "slc:rpmArchiveSize"; } diff --git a/runtime/org.argeo.slc.support.jcr/src/main/java/org/argeo/slc/jcr/SlcTypes.java b/runtime/org.argeo.slc.support.jcr/src/main/java/org/argeo/slc/jcr/SlcTypes.java index 47106eb38..d53494cef 100644 --- a/runtime/org.argeo.slc.support.jcr/src/main/java/org/argeo/slc/jcr/SlcTypes.java +++ b/runtime/org.argeo.slc.support.jcr/src/main/java/org/argeo/slc/jcr/SlcTypes.java @@ -35,7 +35,7 @@ public interface SlcTypes { public final static String SLC_CHECK = "slc:check"; public final static String SLC_PROPERTY = "slc:property"; public final static String SLC_DIFF_RESULT = "slc:diffResult"; - + // Node types used for user defined and managed result UI tree public final static String SLC_MY_RESULT_ROOT_FOLDER = "slc:myResultRootFolder"; public final static String SLC_RESULT_FOLDER = "slc:resultFolder"; @@ -69,4 +69,6 @@ public interface SlcTypes { public final static String SLC_KNOWN_ORIGIN = "slc:knownOrigin"; public final static String SLC_PROXIED = "slc:proxied"; + // rpm + public final static String SLC_RPM = "slc:rpm"; } diff --git a/runtime/org.argeo.slc.support.simple/pom.xml b/runtime/org.argeo.slc.support.simple/pom.xml index fb815b341..d65876f51 100644 --- a/runtime/org.argeo.slc.support.simple/pom.xml +++ b/runtime/org.argeo.slc.support.simple/pom.xml @@ -85,11 +85,6 @@ org.apache.commons.exec - - org.argeo.tp - org.redline_rpm - - org.argeo.tp -- 2.39.2