cms/org.argeo.slc.factory/src/org/argeo/slc/rpmfactory/core/RpmFactoryImpl.java

   1 package org.argeo.slc.rpmfactory.core;
   2
   3 import java.io.File;
   4 import java.io.IOException;
   5 import java.text.DateFormat;
   6 import java.text.SimpleDateFormat;
   7 import java.util.ArrayList;
   8 import java.util.Calendar;
   9 import java.util.GregorianCalendar;
  10 import java.util.HashMap;
  11 import java.util.List;
  12 import java.util.Map;
  13
  14 import javax.jcr.Node;
  15 import javax.jcr.Repository;
  16 import javax.jcr.RepositoryException;
  17 import javax.jcr.Session;
  18
  19 import org.apache.commons.io.FileUtils;
  20 import org.apache.commons.logging.Log;
  21 import org.apache.commons.logging.LogFactory;
  22 import org.argeo.jcr.JcrUtils;
  23 import org.argeo.slc.SlcConstants;
  24 import org.argeo.slc.SlcException;
  25 import org.argeo.slc.repo.NodeIndexerVisitor;
  26 import org.argeo.slc.repo.RpmIndexer;
  27 import org.argeo.slc.rpmfactory.RpmFactory;
  28 import org.argeo.slc.rpmfactory.RpmRepository;
  29 import org.argeo.slc.runtime.tasks.SystemCall;
  30
  31 /**
  32  * Defines a build environment. This information is typically used by other
  33  * components performing the various actions related to RPM build.
  34  */
  35 public class RpmFactoryImpl implements RpmFactory {
  36         private Log log = LogFactory.getLog(RpmFactoryImpl.class);
  37
  38         private Repository rpmRepository;
  39         private Repository distRepository;
  40
  41         private String id;
  42         private List<RpmRepository> repositories = new ArrayList<RpmRepository>();
  43         private List<String> archs = new ArrayList<String>();
  44
  45         private String rpmBase = "/mnt/slc/repos/rpm";
  46         private String distBase = "/mnt/slc/repos/dist";
  47         private String mockVar = "/var/lib/mock";
  48         private String mockEtc = "/etc/mock";
  49
  50         private DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmm");
  51
  52         private String gitWorkspace = "git";
  53
  54         private String localUrlBase = "http://localhost:7070/";
  55         /** If not null or empty, this is a developer instance. */
  56         private String gitDevBaseUrl = null;
  57
  58         private Boolean withTestingRepository = false;
  59
  60         private String yumConfigMainSection = "cachedir=/var/cache/yum\n"
  61                         + "debuglevel=1\n" + "reposdir=/dev/null\n"
  62                         + "logfile=/var/log/yum.log\n" + "retries=20\n" + "obsoletes=1\n"
  63                         + "gpgcheck=0\n" + "assumeyes=1\n" + "syslog_ident=mock\n"
  64                         + "syslog_device=\n" + "http_caching=none\n";
  65
  66         private String defaultMacroFiles = "/usr/lib/rpm/macros:"
  67                         + "/usr/lib/rpm/ia32e-linux/macros:"
  68                         + "/usr/lib/rpm/redhat/macros:" + "/etc/rpm/macros.*:"
  69                         + "/etc/rpm/macros:" + "/etc/rpm/ia32e-linux/macros:"
  70                         + "~/.rpmmacros";
  71         private Map<String, String> rpmmacros = new HashMap<String, String>();
  72
  73         // set by init
  74         private String proxiedReposBase;
  75         private String managedReposBase;
  76
  77         private String stagingWorkspace;
  78         private String testingWorkspace;
  79         private String stableWorkspace;
  80
  81         private File rpmFactoryBaseDir;
  82         private File mockConfDir;
  83         private File yumConfDir;
  84
  85         public void init() {
  86                 // local URL bases
  87                 proxiedReposBase = localUrlBase + "repo/rpm/";
  88                 managedReposBase = localUrlBase + "data/public/rpm/";
  89
  90                 // local directories
  91                 rpmFactoryBaseDir.mkdirs();
  92                 mockConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/mock");
  93                 mockConfDir.mkdirs();
  94                 yumConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/yum");
  95                 yumConfDir.mkdirs();
  96
  97                 // managed repositories
  98                 stagingWorkspace = id + "-staging";
  99                 if (withTestingRepository)
 100                         testingWorkspace = id + "-testing";
 101                 stableWorkspace = id;
 102
 103                 initDistWorkspace(stableWorkspace);
 104                 initGitWorkspace();
 105                 initRpmWorkspace(stagingWorkspace);
 106                 if (withTestingRepository)
 107                         initRpmWorkspace(testingWorkspace);
 108                 initRpmWorkspace(stableWorkspace);
 109         }
 110
 111         protected void initRpmWorkspace(String workspace) {
 112                 Session session = null;
 113                 try {
 114                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository, workspace);
 115                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 116                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 117                                         "jcr:all");
 118
 119                         for (String arch : archs) {
 120                                 Node archFolder = JcrUtils.mkfolders(session, "/" + arch);
 121                                 session.save();
 122                                 File workspaceDir = getWorkspaceDir(workspace);
 123                                 try {
 124                                         if (!archFolder.hasNode("repodata")) {
 125                                                 // touch a file in order to make sure this is properly
 126                                                 // mounted.
 127                                                 File touch = new File(workspaceDir, ".touch");
 128                                                 touch.createNewFile();
 129                                                 touch.delete();
 130
 131                                                 SystemCall createrepo = new SystemCall();
 132                                                 createrepo.arg("createrepo");
 133                                                 createrepo.arg("-q");
 134                                                 File archDir = new File(workspaceDir, arch);
 135                                                 createrepo.arg(archDir.getAbsolutePath());
 136                                                 createrepo.run();
 137                                         }
 138                                 } catch (IOException e) {
 139                                         log.error(workspaceDir + " not properly mounted.", e);
 140                                 }
 141                         }
 142                 } catch (Exception e) {
 143                         throw new SlcException("Cannot initialize workspace " + workspace,
 144                                         e);
 145                 } finally {
 146                         JcrUtils.logoutQuietly(session);
 147                 }
 148         }
 149
 150         /** Caller must logout the underlying session. */
 151         public Node newDistribution(String distributionId) {
 152                 Session session = null;
 153                 try {
 154                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository,
 155                                         distributionId);
 156                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 157                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 158                                         "jcr:all");
 159
 160                         Calendar now = new GregorianCalendar();
 161                         String folderName = dateFormat.format(now.getTime());
 162                         return JcrUtils.mkfolders(session, "/" + folderName);
 163                 } catch (Exception e) {
 164                         JcrUtils.logoutQuietly(session);
 165                         throw new SlcException("Cannot initialize distribution workspace "
 166                                         + distributionId, e);
 167                 }
 168         }
 169
 170         protected void initGitWorkspace() {
 171                 Session session = null;
 172                 try {
 173                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository,
 174                                         gitWorkspace);
 175                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 176                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 177                                         "jcr:all");
 178                 } catch (Exception e) {
 179                         throw new SlcException("Cannot initialize workspace "
 180                                         + gitWorkspace, e);
 181                 } finally {
 182                         JcrUtils.logoutQuietly(session);
 183                 }
 184         }
 185
 186         protected void initDistWorkspace(String workspace) {
 187                 Session session = null;
 188                 try {
 189                         session = JcrUtils
 190                                         .loginOrCreateWorkspace(distRepository, workspace);
 191                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 192                 } catch (RepositoryException e) {
 193                         throw new SlcException("Cannot initialize workspace " + workspace,
 194                                         e);
 195                 } finally {
 196                         JcrUtils.logoutQuietly(session);
 197                 }
 198         }
 199
 200         public void destroy() {
 201
 202         }
 203
 204         public String generateMockConfigFile(String arch, String branch) {
 205                 StringBuffer buf = new StringBuffer();
 206
 207                 buf.append("config_opts['root'] = '" + getIdWithArch(arch) + "'\n");
 208                 buf.append("config_opts['target_arch'] = '" + arch + "'\n");
 209                 buf.append("config_opts['legal_host_arches'] = ('" + arch + "',)\n");
 210                 buf.append("config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'\n");
 211                 // buf.append("config_opts['dist'] = 'el6'\n");
 212                 buf.append("config_opts['plugin_conf']['yum_cache_enable'] = False\n");
 213
 214                 buf.append("config_opts['scm'] = False\n");
 215                 buf.append("config_opts['scm_opts']['method'] = 'git'\n");
 216                 buf.append("config_opts['scm_opts']['spec'] = 'SCM_PKG.spec'\n");
 217                 buf.append("config_opts['scm_opts']['ext_src_dir'] = '"
 218                                 + getSourcesDir().getAbsolutePath() + "'\n");
 219                 buf.append("config_opts['scm_opts']['git_timestamps'] = True\n");
 220
 221                 // development
 222                 if (gitDevBaseUrl != null && !gitDevBaseUrl.trim().equals(""))
 223                         buf.append("config_opts['scm_opts']['git_get'] = 'git clone "
 224                                         + (branch != null ? "-b " + branch : "") + " "
 225                                         + gitDevBaseUrl + "/SCM_PKG SCM_PKG'\n");
 226                 else
 227                         buf.append("config_opts['scm_opts']['git_get'] = 'git clone "
 228                                         + (branch != null ? "-b " + branch : "") + " "
 229                                         + getGitBaseUrl() + "/SCM_PKG.git SCM_PKG'\n");
 230
 231                 buf.append("\nconfig_opts['yum.conf'] = \"\"\"\n");
 232                 buf.append(generateYumConfigFile(arch)).append('\n');
 233                 buf.append("\"\"\"\n");
 234                 return buf.toString();
 235         }
 236
 237         public String generateYumConfigFile(String arch) {
 238                 StringBuffer buf = new StringBuffer();
 239                 buf.append("[main]\n");
 240                 buf.append(yumConfigMainSection).append('\n');
 241
 242                 for (RpmRepository repository : repositories) {
 243                         buf.append('[').append(repository.getId()).append("]\n");
 244                         buf.append("name=").append(repository.getId()).append('\n');
 245                         if (repository instanceof ThirdPartyRpmRepository) {
 246                                 buf.append("#baseurl=").append(repository.getUrl())
 247                                                 .append(arch).append('/').append("\n");
 248                                 buf.append("baseurl=").append(proxiedReposBase)
 249                                                 .append(repository.getId()).append('/').append(arch)
 250                                                 .append('/').append("\n");
 251                                 if (((ThirdPartyRpmRepository) repository).getYumConf() != null)
 252                                         buf.append(
 253                                                         ((ThirdPartyRpmRepository) repository).getYumConf()
 254                                                                         .trim()).append('\n');
 255                         }
 256                 }
 257
 258                 // managed repos
 259                 addManagedRepository(buf, stagingWorkspace, arch);
 260                 if (withTestingRepository)
 261                         addManagedRepository(buf, testingWorkspace, arch);
 262                 addManagedRepository(buf, stableWorkspace, arch);
 263                 return buf.toString();
 264         }
 265
 266         protected void addManagedRepository(StringBuffer buf, String workspace,
 267                         String arch) {
 268                 buf.append('[').append(workspace).append("]\n");
 269                 buf.append("baseurl=").append(managedReposBase).append(workspace)
 270                                 .append('/').append(arch).append('/').append("\n");
 271                 buf.append("gpgcheck=0").append("\n");
 272         }
 273
 274         /** Creates a mock config file. */
 275         public File getMockConfigFile(String arch, String branch) {
 276                 File mockSiteDefaultsFile = new File(mockConfDir, "site-defaults.cfg");
 277                 File mockLoggingFile = new File(mockConfDir, "logging.ini");
 278                 File mockConfigFile = new File(mockConfDir, getIdWithArch(arch)
 279                                 + ".cfg");
 280                 try {
 281                         if (!mockSiteDefaultsFile.exists())
 282                                 mockSiteDefaultsFile.createNewFile();
 283                         if (!mockLoggingFile.exists())
 284                                 FileUtils.copyFile(new File(mockEtc + "/logging.ini"),
 285                                                 mockLoggingFile);
 286
 287                         FileUtils.writeStringToFile(mockConfigFile,
 288                                         generateMockConfigFile(arch, branch));
 289                         return mockConfigFile;
 290                 } catch (IOException e) {
 291                         throw new SlcException("Cannot write mock config file to "
 292                                         + mockConfigFile, e);
 293                 }
 294         }
 295
 296         /** Creates a yum config file. */
 297         public File getYumRepoFile(String arch) {
 298                 File yumConfigFile = new File(yumConfDir, getIdWithArch(arch) + ".repo");
 299                 try {
 300                         FileUtils.writeStringToFile(yumConfigFile,
 301                                         generateYumConfigFile(arch));
 302                         return yumConfigFile;
 303                 } catch (IOException e) {
 304                         throw new SlcException("Cannot write yum config file to "
 305                                         + yumConfigFile, e);
 306                 }
 307         }
 308
 309         public File getResultDir(String arch) {
 310                 return new File(mockVar + "/" + getIdWithArch(arch) + "/result");
 311         }
 312
 313         public File getWorkspaceDir(String workspace) {
 314                 return new File(rpmBase + "/" + workspace);
 315         }
 316
 317         public File getSourcesDir() {
 318                 return new File(distBase + "/" + stableWorkspace);
 319         }
 320
 321         public String getMockConfig(String arch) {
 322                 return getIdWithArch(arch);
 323         }
 324
 325         public String getIdWithArch(String arch) {
 326                 return id + "-" + arch;
 327         }
 328
 329         public String getGitBaseUrl() {
 330                 return managedReposBase + gitWorkspace;
 331         }
 332
 333         public void indexWorkspace(String workspace) {
 334                 Session session = null;
 335                 try {
 336                         session = rpmRepository.login(workspace);
 337                         session.getRootNode().accept(
 338                                         new NodeIndexerVisitor(new RpmIndexer()));
 339                         if (log.isDebugEnabled())
 340                                 log.debug("Indexed workspace " + workspace);
 341                 } catch (RepositoryException e) {
 342                         throw new SlcException("Cannot index workspace " + workspace, e);
 343                 } finally {
 344                         JcrUtils.logoutQuietly(session);
 345                 }
 346         }
 347
 348         public Boolean isDeveloperInstance() {
 349                 return gitDevBaseUrl != null;
 350         }
 351
 352         /** Write (topdir)/rpmmacros and (topdir)/rpmrc */
 353         public void writeRpmbuildConfigFiles(File topdir) {
 354                 writeRpmbuildConfigFiles(topdir, new File(topdir, "rpmmacros"),
 355                                 new File(topdir, "rpmrc"));
 356         }
 357
 358         public void writeRpmbuildConfigFiles(File topdir, File rpmmacroFile,
 359                         File rpmrcFile) {
 360                 try {
 361                         List<String> macroLines = new ArrayList<String>();
 362                         macroLines.add("%_topdir " + topdir.getCanonicalPath());
 363                         for (String macroKey : rpmmacros.keySet()) {
 364                                 macroLines.add(macroKey + " " + rpmmacros.get(macroKey));
 365                         }
 366                         FileUtils.writeLines(rpmmacroFile, macroLines);
 367
 368                         List<String> rpmrcLines = new ArrayList<String>();
 369                         rpmrcLines.add("include: /usr/lib/rpm/rpmrc");
 370                         rpmrcLines.add("macrofiles: " + defaultMacroFiles + ":"
 371                                         + rpmmacroFile.getCanonicalPath());
 372                         FileUtils.writeLines(rpmrcFile, rpmrcLines);
 373                 } catch (IOException e) {
 374                         throw new SlcException("Cannot write rpmbuild config files", e);
 375                 }
 376
 377         }
 378
 379         public Map<String, String> getRpmmacros() {
 380                 return rpmmacros;
 381         }
 382
 383         public void setRpmmacros(Map<String, String> rpmmacros) {
 384                 this.rpmmacros = rpmmacros;
 385         }
 386
 387         public String getDefaultMacroFiles() {
 388                 return defaultMacroFiles;
 389         }
 390
 391         public void setDefaultMacroFiles(String defaultMacroFiles) {
 392                 this.defaultMacroFiles = defaultMacroFiles;
 393         }
 394
 395         public void setArchs(List<String> archs) {
 396                 this.archs = archs;
 397         }
 398
 399         public List<String> getArchs() {
 400                 return archs;
 401         }
 402
 403         public void setRpmBase(String stagingBase) {
 404                 this.rpmBase = stagingBase;
 405         }
 406
 407         public void setId(String id) {
 408                 this.id = id;
 409         }
 410
 411         public void setMockVar(String mockVar) {
 412                 this.mockVar = mockVar;
 413         }
 414
 415         public void setRpmRepository(Repository rpmRepository) {
 416                 this.rpmRepository = rpmRepository;
 417         }
 418
 419         public void setDistRepository(Repository distRepository) {
 420                 this.distRepository = distRepository;
 421         }
 422
 423         public void setLocalUrlBase(String localUrlBase) {
 424                 this.localUrlBase = localUrlBase;
 425         }
 426
 427         public void setYumConfigMainSection(String yumConfigMainSection) {
 428                 this.yumConfigMainSection = yumConfigMainSection;
 429         }
 430
 431         public void setRepositories(List<RpmRepository> repositories) {
 432                 this.repositories = repositories;
 433         }
 434
 435         public void setRpmFactoryBaseDir(File rpmFactoryBaseDir) {
 436                 this.rpmFactoryBaseDir = rpmFactoryBaseDir;
 437         }
 438
 439         public String getStagingWorkspace() {
 440                 return stagingWorkspace;
 441         }
 442
 443         public String getTestingWorkspace() {
 444                 return testingWorkspace;
 445         }
 446
 447         public String getStableWorkspace() {
 448                 return stableWorkspace;
 449         }
 450
 451         public void setWithTestingRepository(Boolean withTestingRepository) {
 452                 this.withTestingRepository = withTestingRepository;
 453         }
 454
 455         public void setGitDevBaseUrl(String gitBaseUrl) {
 456                 this.gitDevBaseUrl = gitBaseUrl;
 457         }
 458 }