From b301144d5f9dc598da7d75bd1eb515f02f32f5ca Mon Sep 17 00:00:00 2001 From: Mathieu Baudier Date: Mon, 3 Jun 2013 16:24:24 +0000 Subject: [PATCH] Improve RPM Factory git-svn-id: https://svn.argeo.org/slc/trunk@6326 4cfe0d0a-d680-48aa-b62c-e0a02a3f76cc --- dep/org.argeo.slc.dep.backend/pom.xml | 5 + .../META-INF/spring/jcr.xml | 5 - .../META-INF/spring/repo-osgi.xml | 5 + .../slc/core/execution/ProcessThread.java | 28 +- .../slc/rpmfactory/core/BuildInMock.java | 90 ++----- .../core/CreateRpmDistribution.java | 16 ++ .../rpmfactory/core/RpmBuildEnvironment.java | 247 +++++++++++++++++- .../slc/rpmfactory/core/RpmDistribution.java | 6 + .../rpmfactory/core/RpmProxyServiceImpl.java | 17 +- .../rpmfactory/core/StagingRpmRepository.java | 6 + 10 files changed, 323 insertions(+), 102 deletions(-) create mode 100644 runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java create mode 100644 runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java create mode 100644 runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/StagingRpmRepository.java diff --git a/dep/org.argeo.slc.dep.backend/pom.xml b/dep/org.argeo.slc.dep.backend/pom.xml index c622d2afc..896449469 100644 --- a/dep/org.argeo.slc.dep.backend/pom.xml +++ b/dep/org.argeo.slc.dep.backend/pom.xml @@ -74,6 +74,11 @@ org.argeo.slc.repo 1.1.13-SNAPSHOT + + org.argeo.slc.rpmfactory + org.argeo.slc.repo + 1.1.13-SNAPSHOT + org.argeo.slc org.argeo.slc.server.repo diff --git a/modules/org.argeo.slc.server.repo/META-INF/spring/jcr.xml b/modules/org.argeo.slc.server.repo/META-INF/spring/jcr.xml index 7de0d2ffe..e6ed3f895 100644 --- a/modules/org.argeo.slc.server.repo/META-INF/spring/jcr.xml +++ b/modules/org.argeo.slc.server.repo/META-INF/spring/jcr.xml @@ -35,11 +35,6 @@ - - - - + + + + diff --git a/runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/ProcessThread.java b/runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/ProcessThread.java index 5552d664e..b64639a8c 100644 --- a/runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/ProcessThread.java +++ b/runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/ProcessThread.java @@ -65,25 +65,25 @@ public class ProcessThread extends Thread { throw new SlcException("Can only execute authenticated threads"); SecurityContextHolder.getContext().setAuthentication(authentication); - // log.info("\n##\n## SLC Process #" + process.getUuid() + - // " STARTED by " - // + authentication.getName() + "\n##\n"); log.info("\n##\n## SLC Process #" + process.getUuid() + " STARTED\n##\n"); // Start logging new LoggingThread().start(); - // String oldStatus = process.getStatus(); process.setStatus(ExecutionProcess.RUNNING); - // executionModulesManager.dispatchUpdateStatus(process, oldStatus, - // ExecutionProcess.RUNNING); - try { process(); } catch (InterruptedException e) { die(); return; + } catch (Exception e) { + String msg = "Process " + getProcess().getUuid() + + " failed unexpectedly."; + log.error(msg, e); + getProcessThreadGroup().dispatchAddStep( + new ExecutionStep("Process", ExecutionStep.ERROR, msg + " " + + e.getMessage())); } // waits for all execution threads to complete (in case they were @@ -162,13 +162,13 @@ public class ProcessThread extends Thread { return; } -// public void notifyError() { -// hadAnError = true; -// } -// -// public synchronized void flowCompleted() { -// // notifyAll(); -// } + // public void notifyError() { + // hadAnError = true; + // } + // + // public synchronized void flowCompleted() { + // // notifyAll(); + // } public ExecutionProcess getProcess() { return process; diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/BuildInMock.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/BuildInMock.java index a085a9d08..0bdb4c7ce 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/BuildInMock.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/BuildInMock.java @@ -30,30 +30,13 @@ import org.apache.commons.logging.LogFactory; import org.argeo.slc.SlcException; import org.argeo.slc.core.execution.tasks.SystemCall; -/** Rebuild an SRPM in mock. (Historical) Replaces the build-mock.sh script. */ +/** Build an RPM in mock. */ public class BuildInMock implements Runnable { private final static Log log = LogFactory.getLog(BuildInMock.class); - - /** Mock flavour provided by the EPEL repository */ - public final static String EPEL = "EPEL"; - /** Mock flavour provided by CentOS until v5 */ - public final static String CENTOS = "CENTOS"; - - public final static String NOARCH = "noarch"; - - private String mockVar = "/var/lib/mock"; - - private String mockFlavour = EPEL; - private String mockConfig = null; - - private String repository; - private String release = null; - private String level = null; - private String arch = NOARCH; + private final static String NOARCH = "noarch"; private String rpmPackage = null; - - private Boolean mkdirs = true; + private String arch = NOARCH; private RpmBuildEnvironment buildEnvironment; private Executor executor; @@ -61,51 +44,45 @@ public class BuildInMock implements Runnable { private String debuginfoDirName = "debuginfo"; public void run() { - // TODO check if caller is in mock group - - String cfgId = repository + "-" + release + "-" + arch; - String cfg = mockConfig != null ? mockConfig : "slc/" + cfgId; + String mockCfg = buildEnvironment.getMockConfig(arch); + File mockConfigFile = buildEnvironment.getMockConfigFile(arch); // prepare mock call SystemCall mock = new SystemCall(); if (arch != null) mock.arg("setarch").arg(arch); mock.arg("mock"); - if (mockFlavour.equals(EPEL)) - mock.arg("-v"); - else if (mockFlavour.equals(CENTOS)) - mock.arg("--debug"); + mock.arg("-v"); + mock.arg("--configdir=" + mockConfigFile.getAbsoluteFile().getParent()); if (arch != null) mock.arg("--arch=" + arch); - mock.arg("-r").arg(cfg); - + mock.arg("-r").arg(mockCfg); mock.arg("--scm-enable"); mock.arg("--scm-option").arg("package=" + rpmPackage); mock.setLogCommand(true); + mock.setExecutor(executor); + // // mock command execution - mock.setExecutor(executor); + // mock.run(); + // - // File repoDir = new File(buildEnvironment.getStagingBase() + "/" - // + repository + "/" + level + "/" + release); - File repoDir = new File(buildEnvironment.getStagingBase() + "/" - + repository + "-" + release + "-staging"); - File srpmDir = new File(repoDir, "SRPMS"); - if (mkdirs) - srpmDir.mkdirs(); + File stagingDir = buildEnvironment.getStagingDir(); + File srpmDir = new File(stagingDir, "SRPMS"); + srpmDir.mkdirs(); File archDir = null; File debuginfoDir = null; if (!arch.equals(NOARCH)) { - archDir = new File(repoDir, arch); + archDir = new File(stagingDir, arch); debuginfoDir = new File(archDir, debuginfoDirName); debuginfoDir.mkdirs(); } // copy RPMs Set reposToRecreate = new HashSet(); - File resultDir = new File(mockVar + "/" + cfgId + "/result"); + File resultDir = buildEnvironment.getResultDir(arch); rpms: for (File file : resultDir.listFiles()) { if (file.isDirectory()) continue rpms; @@ -121,7 +98,7 @@ public class BuildInMock implements Runnable { else if (file.getName().contains(".noarch.rpm")) { List dirs = new ArrayList(); for (String arch : buildEnvironment.getArchs()) - dirs.add(new File(repoDir, arch)); + dirs.add(new File(stagingDir, arch)); targetDirs = dirs.toArray(new File[dirs.size()]); } else if (file.getName().contains(".rpm")) throw new SlcException("Don't know where to copy " + file); @@ -166,26 +143,6 @@ public class BuildInMock implements Runnable { } } - public void setMockFlavour(String mockFlavour) { - this.mockFlavour = mockFlavour; - } - - public void setMockConfig(String mockConfig) { - this.mockConfig = mockConfig; - } - - public void setRepository(String repo) { - this.repository = repo; - } - - public void setRelease(String release) { - this.release = release; - } - - public void setLevel(String level) { - this.level = level; - } - public void setArch(String arch) { this.arch = arch; } @@ -194,14 +151,6 @@ public class BuildInMock implements Runnable { this.rpmPackage = rpmPackage; } - public void setMockVar(String mockVar) { - this.mockVar = mockVar; - } - - public void setMkdirs(Boolean mkdirs) { - this.mkdirs = mkdirs; - } - public void setBuildEnvironment(RpmBuildEnvironment buildEnvironment) { this.buildEnvironment = buildEnvironment; } @@ -209,5 +158,4 @@ public class BuildInMock implements Runnable { public void setExecutor(Executor executor) { this.executor = executor; } - -} +} \ No newline at end of file diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java new file mode 100644 index 000000000..cd893acd9 --- /dev/null +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java @@ -0,0 +1,16 @@ +package org.argeo.slc.rpmfactory.core; + +/** + * Gather RPMs from various sources (local builds or third party) into a + * consistent distributable set (typically to be used to generate an ISO). + */ +public class CreateRpmDistribution implements Runnable { + + + @Override + public void run() { + // TODO Auto-generated method stub + + } + +} diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmBuildEnvironment.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmBuildEnvironment.java index ebdf1de2b..61147f34b 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmBuildEnvironment.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmBuildEnvironment.java @@ -22,21 +22,229 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.jcr.Node; +import javax.jcr.Repository; +import javax.jcr.RepositoryException; +import javax.jcr.Session; + import org.apache.commons.io.FileUtils; +import org.argeo.jcr.JcrUtils; import org.argeo.slc.SlcException; +import org.argeo.slc.core.execution.tasks.SystemCall; +import org.argeo.slc.rpmfactory.RpmRepository; /** * Defines a build environment. This information is typically used by other * components performing the various actions related to RPM build. */ public class RpmBuildEnvironment { - static String defaultMacroFiles = "/usr/lib/rpm/macros:/usr/lib/rpm/ia32e-linux/macros:/usr/lib/rpm/redhat/macros:/etc/rpm/macros.*:/etc/rpm/macros:/etc/rpm/ia32e-linux/macros:~/.rpmmacros"; - private Map rpmmacros = new HashMap(); + private Repository rpmRepository; + private Repository distRepository; + private String id; + private String stagingBase = "/mnt/slc/repos/rpm"; + private String mockVar = "/var/lib/mock"; + private String mockEtc = "/etc/mock"; private List archs = new ArrayList(); - private String stagingBase = "/mnt/slc/repos/rpm"; + private String localUrlBase = "http://localhost:7070/"; + + private String yumConfigMainSection = "cachedir=/var/cache/yum\n" + + "debuglevel=1\n" + "reposdir=/dev/null\n" + + "logfile=/var/log/yum.log\n" + "retries=20\n" + "obsoletes=1\n" + + "gpgcheck=0\n" + "assumeyes=1\n" + "syslog_ident=mock\n" + + "syslog_device=\n"; + private List repositories = new ArrayList(); + + private String defaultMacroFiles = "/usr/lib/rpm/macros:" + + "/usr/lib/rpm/ia32e-linux/macros:" + + "/usr/lib/rpm/redhat/macros:" + "/etc/rpm/macros.*:" + + "/etc/rpm/macros:" + "/etc/rpm/ia32e-linux/macros:" + + "~/.rpmmacros"; + private Map rpmmacros = new HashMap(); + + // set by init + private String proxiedReposBase; + private String managedReposBase; + + private String stagingWorkspace; + private String testingWorkspace; + private String stableWorkspace; + + private File rpmFactoryBaseDir; + private File mockConfDir; + private File yumConfDir; + + public void init() { + proxiedReposBase = localUrlBase + "repo/rpm/"; + managedReposBase = localUrlBase + "data/public/rpm/"; + + stagingWorkspace = id + "-staging"; + testingWorkspace = id + "-testing"; + stableWorkspace = id; + + // rpmFactoryBaseDir = new File(System.getProperty("osgi.instance.area") + // .substring("file://".length()) + "/rpmfactory"); + rpmFactoryBaseDir.mkdirs(); + mockConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/mock"); + mockConfDir.mkdirs(); + yumConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/yum"); + yumConfDir.mkdirs(); + + initDistWorkspace(stagingWorkspace); + initRpmWorkspace(stagingWorkspace); + initRpmWorkspace(testingWorkspace); + initRpmWorkspace(stableWorkspace); + } + + protected void initRpmWorkspace(String workspace) { + Session session = null; + try { + session = JcrUtils.loginOrCreateWorkspace(rpmRepository, workspace); + JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read"); + + for (String arch : archs) { + Node archFolder = JcrUtils.mkfolders(session, "/" + arch); + if (!archFolder.hasNode("repodata")) { + SystemCall createrepo = new SystemCall(); + createrepo.arg("createrepo"); + createrepo.arg("-q"); + File archDir = new File(getStagingDir(), arch); + createrepo.arg(archDir.getAbsolutePath()); + } + } + } catch (RepositoryException e) { + throw new SlcException("Cannot initialize workspace " + workspace, + e); + } finally { + JcrUtils.logoutQuietly(session); + } + } + + protected void initDistWorkspace(String workspace) { + Session session = null; + try { + session = JcrUtils + .loginOrCreateWorkspace(distRepository, workspace); + JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read"); + } catch (RepositoryException e) { + throw new SlcException("Cannot initialize workspace " + workspace, + e); + } finally { + JcrUtils.logoutQuietly(session); + } + } + + public void destroy() { + + } + + public String generateMockConfigFile(String arch) { + StringBuffer buf = new StringBuffer(); + + buf.append("config_opts['root'] = '" + getIdWithArch(arch) + "'\n"); + buf.append("config_opts['target_arch'] = '" + arch + "'\n"); + buf.append("config_opts['legal_host_arches'] = ('" + arch + "',)\n"); + buf.append("config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'\n"); + // buf.append("config_opts['dist'] = 'el6'\n"); + + buf.append("config_opts['scm'] = False\n"); + buf.append("config_opts['scm_opts']['method'] = 'git'\n"); + buf.append("config_opts['scm_opts']['git_get'] = 'git clone -b elgis6 /home/mbaudier/dev/git/elgis.argeo.org/SCM_PKG SCM_PKG'\n"); + buf.append("config_opts['scm_opts']['spec'] = 'SCM_PKG.spec'\n"); + buf.append("config_opts['scm_opts']['ext_src_dir'] = '/home/mbaudier/dev/git/elgis.argeo.org/sources'\n"); + buf.append("config_opts['scm_opts']['git_timestamps'] = True\n"); + + buf.append("config_opts['yum.conf'] = \"\"\"\n"); + buf.append(generateYumConfigFile(arch)).append('\n'); + buf.append("\"\"\"\n"); + return buf.toString(); + } + + public String generateYumConfigFile(String arch) { + StringBuffer buf = new StringBuffer(); + buf.append("[main]\n"); + buf.append(yumConfigMainSection).append('\n'); + + for (RpmRepository repository : repositories) { + buf.append('[').append(repository.getId()).append("]\n"); + if (repository instanceof ThirdPartyRpmRepository) { + buf.append("#baseurl=").append(repository.getUrl()) + .append(arch).append('/').append("\n"); + buf.append("baseurl=").append(proxiedReposBase) + .append(repository.getId()).append('/').append(arch) + .append('/').append("\n"); + } + } + + // managed repos + buf.append('[').append(stagingWorkspace).append("]\n"); + buf.append("baseurl=").append(managedReposBase) + .append(stagingWorkspace).append('/').append(arch).append('/') + .append("\n"); + + buf.append('[').append(testingWorkspace).append("]\n"); + buf.append("baseurl=").append(managedReposBase) + .append(testingWorkspace).append('/').append(arch).append('/') + .append("\n"); + + buf.append('[').append(stableWorkspace).append("]\n"); + buf.append("baseurl=").append(managedReposBase).append(stableWorkspace) + .append('/').append(arch).append('/').append("\n"); + return buf.toString(); + } + + /** Creates a mock config file. */ + public File getMockConfigFile(String arch) { + File mockSiteDefaultsFile = new File(mockConfDir, "site-defaults.cfg"); + File mockLoggingFile = new File(mockConfDir, "logging.ini"); + File mockConfigFile = new File(mockConfDir, getIdWithArch(arch) + + ".cfg"); + try { + if (!mockSiteDefaultsFile.exists()) + mockSiteDefaultsFile.createNewFile(); + if (!mockLoggingFile.exists()) + FileUtils.copyFile(new File(mockEtc + "/logging.ini"), + mockLoggingFile); + + FileUtils.writeStringToFile(mockConfigFile, + generateMockConfigFile(arch)); + return mockConfigFile; + } catch (IOException e) { + throw new SlcException("Cannot write mock config file to " + + mockConfigFile, e); + } + } + + /** Creates a yum config file. */ + public File getYumConfigFile(String arch) { + File yumConfigFile = new File(yumConfDir, getIdWithArch(arch) + ".conf"); + try { + FileUtils.writeStringToFile(yumConfigFile, + generateYumConfigFile(arch)); + return yumConfigFile; + } catch (IOException e) { + throw new SlcException("Cannot write yum config file to " + + yumConfigFile, e); + } + } + + public File getResultDir(String arch) { + return new File(mockVar + "/" + getIdWithArch(arch) + "/result"); + } + + public File getStagingDir() { + return new File(stagingBase + "/" + stagingWorkspace); + } + + public String getMockConfig(String arch) { + return getIdWithArch(arch); + } + + public String getIdWithArch(String arch) { + return id + "-" + arch; + } /** Write (topdir)/rpmmacros and (topdir)/rpmrc */ public void writeRpmbuildConfigFiles(File topdir) { @@ -96,4 +304,37 @@ public class RpmBuildEnvironment { public void setStagingBase(String stagingBase) { this.stagingBase = stagingBase; } + + public void setId(String id) { + this.id = id; + } + + public void setMockVar(String mockVar) { + this.mockVar = mockVar; + } + + public void setRpmRepository(Repository rpmRepository) { + this.rpmRepository = rpmRepository; + } + + public void setDistRepository(Repository distRepository) { + this.distRepository = distRepository; + } + + public void setLocalUrlBase(String localUrlBase) { + this.localUrlBase = localUrlBase; + } + + public void setYumConfigMainSection(String yumConfigMainSection) { + this.yumConfigMainSection = yumConfigMainSection; + } + + public void setRepositories(List repositories) { + this.repositories = repositories; + } + + public void setRpmFactoryBaseDir(File rpmFactoryBaseDir) { + this.rpmFactoryBaseDir = rpmFactoryBaseDir; + } + } diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java new file mode 100644 index 000000000..0556f6a3a --- /dev/null +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java @@ -0,0 +1,6 @@ +package org.argeo.slc.rpmfactory.core; + +/** A consistent distributable set of RPM. */ +public class RpmDistribution { + +} diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmProxyServiceImpl.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmProxyServiceImpl.java index 14d399c27..d5b0bc78b 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmProxyServiceImpl.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmProxyServiceImpl.java @@ -29,7 +29,6 @@ import org.argeo.jcr.JcrUtils; import org.argeo.jcr.proxy.AbstractUrlProxy; import org.argeo.slc.SlcException; import org.argeo.slc.jcr.SlcNames; -import org.argeo.slc.jcr.SlcTypes; import org.argeo.slc.rpmfactory.RpmProxyService; import org.argeo.slc.rpmfactory.RpmRepository; @@ -69,14 +68,14 @@ public class RpmProxyServiceImpl extends AbstractUrlProxy implements String remoteUrl = baseUrl + relativePath; Node node = proxyUrl(session, remoteUrl, path); if (node != null) { - node.addMixin(SlcTypes.SLC_KNOWN_ORIGIN); - Node origin; - if (!node.hasNode(SLC_ORIGIN)) - origin = node.addNode(SLC_ORIGIN, SlcTypes.SLC_PROXIED); - else - origin = node.getNode(SLC_ORIGIN); - // origin.setProperty(SLC_PROXY, sourceRepo.getId()); - JcrUtils.urlToAddressProperties(origin, remoteUrl); + // node.addMixin(SlcTypes.SLC_KNOWN_ORIGIN); + // Node origin; + // if (!node.hasNode(SLC_ORIGIN)) + // origin = node.addNode(SLC_ORIGIN, SlcTypes.SLC_PROXIED); + // else + // origin = node.getNode(SLC_ORIGIN); + // // origin.setProperty(SLC_PROXY, sourceRepo.getId()); + // JcrUtils.urlToAddressProperties(origin, remoteUrl); if (log.isDebugEnabled()) log.debug("Imported " + remoteUrl + " to " + node); diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/StagingRpmRepository.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/StagingRpmRepository.java new file mode 100644 index 000000000..58977f463 --- /dev/null +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/StagingRpmRepository.java @@ -0,0 +1,6 @@ +package org.argeo.slc.rpmfactory.core; + +/** Local build repository, used only for builds. */ +public class StagingRpmRepository extends AbstractRpmRepository { + +} -- 2.39.2