From: Mathieu Baudier Date: Wed, 5 Jun 2013 19:45:13 +0000 (+0000) Subject: Create distribution working. X-Git-Tag: argeo-slc-2.1.7~338 X-Git-Url: http://git.argeo.org/?a=commitdiff_plain;ds=sidebyside;h=3752fba0edf87a770b5763f5a4aab43c4a0a4e4e;p=gpl%2Fargeo-slc.git Create distribution working. git-svn-id: https://svn.argeo.org/slc/trunk@6332 4cfe0d0a-d680-48aa-b62c-e0a02a3f76cc --- diff --git a/runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/tasks/SystemCall.java b/runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/tasks/SystemCall.java index 5d6d97658..ba21b9f25 100644 --- a/runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/tasks/SystemCall.java +++ b/runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/tasks/SystemCall.java @@ -253,10 +253,16 @@ public class SystemCall implements Runnable { public synchronized String function() { final StringBuffer buf = new StringBuffer(""); SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() { + private Long lineCount = 0l; + public void newLine(SystemCall systemCall, String line, Boolean isError) { - if (!isError) + if (!isError) { + if (lineCount != 0l) + buf.append('\n'); buf.append(line); + lineCount++; + } } }; addOutputListener(tempOutputListener); diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java index 32d28144a..70a5e705c 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/CreateRpmDistribution.java @@ -1,14 +1,130 @@ package org.argeo.slc.rpmfactory.core; +import java.io.File; +import java.io.InputStream; +import java.net.URL; +import java.util.StringTokenizer; + +import javax.jcr.Node; +import javax.jcr.Session; +import javax.jcr.nodetype.NodeType; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.argeo.jcr.JcrUtils; +import org.argeo.slc.SlcException; +import org.argeo.slc.core.execution.tasks.SystemCall; +import org.springframework.core.io.ByteArrayResource; + /** * Gather RPMs from various sources (local builds or third party) into a * consistent distributable set (typically to be used to generate an ISO). */ public class CreateRpmDistribution implements Runnable { - // private RpmDistribution rpmDistribution; + private final static Log log = LogFactory + .getLog(CreateRpmDistribution.class); + + private RpmFactory rpmFactory; + private RpmDistribution rpmDistribution; + + private String arch = "x86_64"; + + private String repoqueryExecutable = "/usr/bin/repoquery"; @Override public void run() { + Session session = null; + // Reader reader = null; + try { + Node baseFolder = rpmFactory.newDistribution(rpmDistribution + .getId()); + session = baseFolder.getSession(); + Node targetFolder = baseFolder.addNode(arch, NodeType.NT_FOLDER); + + SystemCall repoquery = new SystemCall(); + repoquery.arg(repoqueryExecutable); + + File yumConfigFile = rpmFactory.getYumConfigFile(arch); + repoquery.arg("-c", yumConfigFile.getAbsolutePath()); + repoquery.arg("--requires"); + repoquery.arg("--resolve"); + repoquery.arg("--location"); + + for (String rpmPackage : rpmDistribution.getPackages()) + repoquery.arg(rpmPackage); + + if (log.isDebugEnabled()) + log.debug("Command:\n" + repoquery.asCommand()); + + String output = repoquery.function(); + if (log.isDebugEnabled()) + log.debug(output + "\n"); + // reader = new StringReader(output); + StringTokenizer lines = new StringTokenizer(output, "\n"); + // List dependencies = IOUtils.readLines(reader); + dependencies: while (lines.hasMoreTokens()) { + String urlStr = lines.nextToken(); + InputStream in = null; + try { + URL url = new URL(urlStr); + String fileName = FilenameUtils.getName(url.getFile()); + String[] tokens = fileName.split("-"); + if (tokens.length < 3) + continue dependencies; + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < tokens.length - 2; i++) { + if (i != 0) + buf.append('-'); + buf.append(tokens[i]); + + } + String packageName = buf.toString(); + for (RpmPackageSet excluded : rpmDistribution + .getExcludedPackages()) { + if (excluded.contains(packageName)) { + if (log.isDebugEnabled()) + log.debug("Skipped " + packageName); + continue dependencies;// skip + } + } + in = url.openStream(); + JcrUtils.copyStreamAsFile(targetFolder, fileName, in); + targetFolder.getSession().save(); + if (log.isDebugEnabled()) + log.debug("Copied " + packageName); + } catch (Exception e) { + log.error("Cannot copy " + urlStr, e); + } finally { + IOUtils.closeQuietly(in); + } + } + + } catch (Exception e) { + throw new SlcException("Cannot generate distribution " + + rpmDistribution.getId(), e); + } finally { + JcrUtils.logoutQuietly(session); + // IOUtils.closeQuietly(reader); + } + } + + public void setRpmDistribution(RpmDistribution rpmDistribution) { + this.rpmDistribution = rpmDistribution; + } + + public void setRpmFactory(RpmFactory rpmFactory) { + this.rpmFactory = rpmFactory; } + + public void setArch(String arch) { + this.arch = arch; + } + + public void setRepoqueryExecutable(String yumdownloaderExecutable) { + this.repoqueryExecutable = yumdownloaderExecutable; + } + } diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java index 66d87346f..0df31c687 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmDistribution.java @@ -4,7 +4,9 @@ import java.util.List; /** A consistent distributable set of RPM. */ public class RpmDistribution { + private String id; private List packages; + private List excludedPackages; public List getPackages() { return packages; @@ -13,4 +15,21 @@ public class RpmDistribution { public void setPackages(List packages) { this.packages = packages; } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public List getExcludedPackages() { + return excludedPackages; + } + + public void setExcludedPackages(List excludedPackages) { + this.excludedPackages = excludedPackages; + } + } diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmFactory.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmFactory.java index ce1e93fce..b881cf46f 100644 --- a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmFactory.java +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmFactory.java @@ -17,7 +17,11 @@ package org.argeo.slc.rpmfactory.core; import java.io.File; import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; +import java.util.GregorianCalendar; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,6 +56,8 @@ public class RpmFactory { private String mockVar = "/var/lib/mock"; private String mockEtc = "/etc/mock"; + private DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmm"); + private String gitWorkspace = "git"; private String localUrlBase = "http://localhost:7070/"; @@ -144,6 +150,26 @@ public class RpmFactory { } } + /** Caller must logout the underlying session. */ + protected Node newDistribution(String distributionId) { + Session session = null; + try { + session = JcrUtils.loginOrCreateWorkspace(rpmRepository, + distributionId); + JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read"); + JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC, + "jcr:all"); + + Calendar now = new GregorianCalendar(); + String folderName = dateFormat.format(now.getTime()); + return JcrUtils.mkfolders(session, "/" + folderName); + } catch (Exception e) { + JcrUtils.logoutQuietly(session); + throw new SlcException("Cannot initialize distribution workspace " + + distributionId, e); + } + } + protected void initGitWorkspace() { Session session = null; try { diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmPackageSet.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmPackageSet.java new file mode 100644 index 000000000..16934132b --- /dev/null +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmPackageSet.java @@ -0,0 +1,6 @@ +package org.argeo.slc.rpmfactory.core; + +/** Set of RPM packages */ +public interface RpmPackageSet { + public Boolean contains(String rpmPackage); +} diff --git a/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/YumListParser.java b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/YumListParser.java new file mode 100644 index 000000000..c87b40e6c --- /dev/null +++ b/runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/YumListParser.java @@ -0,0 +1,97 @@ +package org.argeo.slc.rpmfactory.core; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.TreeSet; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.LineIterator; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.argeo.slc.SlcException; +import org.springframework.core.io.Resource; + +/** + * Reads the output of a 'yum list all' command and interpret the list of + * packages. + */ +public class YumListParser implements RpmPackageSet { + private final static Log log = LogFactory.getLog(YumListParser.class); + + private Set installed = new TreeSet(); + /** Not installed but available */ + private Set installable = new TreeSet(); + + private Resource yumListOutput; + + public void init() { + try { + if (yumListOutput != null) { + load(yumListOutput.getInputStream()); + if (log.isDebugEnabled()) + log.debug(installed.size() + " installed, " + + installable.size() + " installable, from " + + yumListOutput); + } + } catch (IOException e) { + throw new SlcException("Cannot initialize yum list parser", e); + } + } + + public Boolean contains(String packageName) { + if (installed.contains(packageName)) + return true; + else + return installable.contains(packageName); + } + + protected void load(InputStream in) { + try { + Boolean readingInstalled = false; + Boolean readingAvailable = false; + LineIterator it = IOUtils.lineIterator(in, "UTF-8"); + while (it.hasNext()) { + String line = it.nextLine(); + if (line.trim().equals("Installed Packages")) { + readingInstalled = true; + } else if (line.trim().equals("Available Packages")) { + readingAvailable = true; + readingInstalled = false; + } else if (readingAvailable) { + if (Character.isLetterOrDigit(line.charAt(0))) { + installable.add(extractRpmName(line)); + } + } else if (readingInstalled) { + if (Character.isLetterOrDigit(line.charAt(0))) { + installed.add(extractRpmName(line)); + } + } + } + } catch (IOException e) { + throw new SlcException("Cannot load yum list output", e); + } finally { + IOUtils.closeQuietly(in); + } + } + + protected String extractRpmName(String line) { + StringTokenizer st = new StringTokenizer(line, " \t"); + String packageName = st.nextToken(); + return packageName.split("\\.")[0]; + } + + public Set getInstalled() { + return installed; + } + + public Set getInstallable() { + return installable; + } + + public void setYumListOutput(Resource yumListOutput) { + this.yumListOutput = yumListOutput; + } + +}