runtime/org.argeo.slc.rpmfactory/src/main/java/org/argeo/slc/rpmfactory/core/RpmFactoryImpl.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.rpmfactory.core;
  17
  18 import java.io.File;
  19 import java.io.IOException;
  20 import java.text.DateFormat;
  21 import java.text.SimpleDateFormat;
  22 import java.util.ArrayList;
  23 import java.util.Calendar;
  24 import java.util.GregorianCalendar;
  25 import java.util.HashMap;
  26 import java.util.List;
  27 import java.util.Map;
  28
  29 import javax.jcr.Node;
  30 import javax.jcr.Repository;
  31 import javax.jcr.RepositoryException;
  32 import javax.jcr.Session;
  33
  34 import org.apache.commons.io.FileUtils;
  35 import org.apache.commons.logging.Log;
  36 import org.apache.commons.logging.LogFactory;
  37 import org.argeo.jcr.JcrUtils;
  38 import org.argeo.slc.SlcConstants;
  39 import org.argeo.slc.SlcException;
  40 import org.argeo.slc.core.execution.tasks.SystemCall;
  41 import org.argeo.slc.repo.NodeIndexerVisitor;
  42 import org.argeo.slc.rpmfactory.RpmFactory;
  43 import org.argeo.slc.rpmfactory.RpmRepository;
  44
  45 /**
  46  * Defines a build environment. This information is typically used by other
  47  * components performing the various actions related to RPM build.
  48  */
  49 public class RpmFactoryImpl implements RpmFactory {
  50         private Log log = LogFactory.getLog(RpmFactoryImpl.class);
  51
  52         private Repository rpmRepository;
  53         private Repository distRepository;
  54
  55         private String id;
  56         private List<RpmRepository> repositories = new ArrayList<RpmRepository>();
  57         private List<String> archs = new ArrayList<String>();
  58
  59         private String rpmBase = "/mnt/slc/repos/rpm";
  60         private String distBase = "/mnt/slc/repos/dist";
  61         private String mockVar = "/var/lib/mock";
  62         private String mockEtc = "/etc/mock";
  63
  64         private DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmm");
  65
  66         private String gitWorkspace = "git";
  67
  68         private String localUrlBase = "http://localhost:7070/";
  69         /** If not null or empty, this is a developer instance. */
  70         private String gitDevBaseUrl = null;
  71
  72         private Boolean withTestingRepository = false;
  73
  74         private String yumConfigMainSection = "cachedir=/var/cache/yum\n"
  75                         + "debuglevel=1\n" + "reposdir=/dev/null\n"
  76                         + "logfile=/var/log/yum.log\n" + "retries=20\n" + "obsoletes=1\n"
  77                         + "gpgcheck=0\n" + "assumeyes=1\n" + "syslog_ident=mock\n"
  78                         + "syslog_device=\n" + "http_caching=none\n";
  79
  80         private String defaultMacroFiles = "/usr/lib/rpm/macros:"
  81                         + "/usr/lib/rpm/ia32e-linux/macros:"
  82                         + "/usr/lib/rpm/redhat/macros:" + "/etc/rpm/macros.*:"
  83                         + "/etc/rpm/macros:" + "/etc/rpm/ia32e-linux/macros:"
  84                         + "~/.rpmmacros";
  85         private Map<String, String> rpmmacros = new HashMap<String, String>();
  86
  87         // set by init
  88         private String proxiedReposBase;
  89         private String managedReposBase;
  90
  91         private String stagingWorkspace;
  92         private String testingWorkspace;
  93         private String stableWorkspace;
  94
  95         private File rpmFactoryBaseDir;
  96         private File mockConfDir;
  97         private File yumConfDir;
  98
  99         public void init() {
 100                 // local URL bases
 101                 proxiedReposBase = localUrlBase + "repo/rpm/";
 102                 managedReposBase = localUrlBase + "data/public/rpm/";
 103
 104                 // local directories
 105                 rpmFactoryBaseDir.mkdirs();
 106                 mockConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/mock");
 107                 mockConfDir.mkdirs();
 108                 yumConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/yum");
 109                 yumConfDir.mkdirs();
 110
 111                 // managed repositories
 112                 stagingWorkspace = id + "-staging";
 113                 if (withTestingRepository)
 114                         testingWorkspace = id + "-testing";
 115                 stableWorkspace = id;
 116
 117                 initDistWorkspace(stableWorkspace);
 118                 initGitWorkspace();
 119                 initRpmWorkspace(stagingWorkspace);
 120                 if (withTestingRepository)
 121                         initRpmWorkspace(testingWorkspace);
 122                 initRpmWorkspace(stableWorkspace);
 123         }
 124
 125         protected void initRpmWorkspace(String workspace) {
 126                 Session session = null;
 127                 try {
 128                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository, workspace);
 129                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 130                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 131                                         "jcr:all");
 132
 133                         for (String arch : archs) {
 134                                 Node archFolder = JcrUtils.mkfolders(session, "/" + arch);
 135                                 session.save();
 136                                 if (!archFolder.hasNode("repodata")) {
 137                                         File workspaceDir = getWorkspaceDir(workspace);
 138                                         // touch a file in order to make sue this is properly
 139                                         // mounted.
 140                                         File touch = new File(workspaceDir, ".touch");
 141                                         touch.createNewFile();
 142                                         touch.delete();
 143
 144                                         SystemCall createrepo = new SystemCall();
 145                                         createrepo.arg("createrepo");
 146                                         createrepo.arg("-q");
 147                                         File archDir = new File(workspaceDir, arch);
 148                                         createrepo.arg(archDir.getAbsolutePath());
 149                                         createrepo.run();
 150                                 }
 151                         }
 152                 } catch (Exception e) {
 153                         throw new SlcException("Cannot initialize workspace " + workspace,
 154                                         e);
 155                 } finally {
 156                         JcrUtils.logoutQuietly(session);
 157                 }
 158         }
 159
 160         /** Caller must logout the underlying session. */
 161         public Node newDistribution(String distributionId) {
 162                 Session session = null;
 163                 try {
 164                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository,
 165                                         distributionId);
 166                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 167                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 168                                         "jcr:all");
 169
 170                         Calendar now = new GregorianCalendar();
 171                         String folderName = dateFormat.format(now.getTime());
 172                         return JcrUtils.mkfolders(session, "/" + folderName);
 173                 } catch (Exception e) {
 174                         JcrUtils.logoutQuietly(session);
 175                         throw new SlcException("Cannot initialize distribution workspace "
 176                                         + distributionId, e);
 177                 }
 178         }
 179
 180         protected void initGitWorkspace() {
 181                 Session session = null;
 182                 try {
 183                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository,
 184                                         gitWorkspace);
 185                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 186                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 187                                         "jcr:all");
 188                 } catch (Exception e) {
 189                         throw new SlcException("Cannot initialize workspace "
 190                                         + gitWorkspace, e);
 191                 } finally {
 192                         JcrUtils.logoutQuietly(session);
 193                 }
 194         }
 195
 196         protected void initDistWorkspace(String workspace) {
 197                 Session session = null;
 198                 try {
 199                         session = JcrUtils
 200                                         .loginOrCreateWorkspace(distRepository, workspace);
 201                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 202                 } catch (RepositoryException e) {
 203                         throw new SlcException("Cannot initialize workspace " + workspace,
 204                                         e);
 205                 } finally {
 206                         JcrUtils.logoutQuietly(session);
 207                 }
 208         }
 209
 210         public void destroy() {
 211
 212         }
 213
 214         public String generateMockConfigFile(String arch, String branch) {
 215                 StringBuffer buf = new StringBuffer();
 216
 217                 buf.append("config_opts['root'] = '" + getIdWithArch(arch) + "'\n");
 218                 buf.append("config_opts['target_arch'] = '" + arch + "'\n");
 219                 buf.append("config_opts['legal_host_arches'] = ('" + arch + "',)\n");
 220                 buf.append("config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'\n");
 221                 // buf.append("config_opts['dist'] = 'el6'\n");
 222                 buf.append("config_opts['plugin_conf']['yum_cache_enable'] = False\n");
 223
 224                 buf.append("config_opts['scm'] = False\n");
 225                 buf.append("config_opts['scm_opts']['method'] = 'git'\n");
 226                 buf.append("config_opts['scm_opts']['spec'] = 'SCM_PKG.spec'\n");
 227                 buf.append("config_opts['scm_opts']['ext_src_dir'] = '"
 228                                 + getSourcesDir().getAbsolutePath() + "'\n");
 229                 buf.append("config_opts['scm_opts']['git_timestamps'] = True\n");
 230
 231                 // development
 232                 if (gitDevBaseUrl != null && !gitDevBaseUrl.trim().equals(""))
 233                         buf.append("config_opts['scm_opts']['git_get'] = 'git clone "
 234                                         + (branch != null ? "-b " + branch : "") + " "
 235                                         + gitDevBaseUrl + "/SCM_PKG SCM_PKG'\n");
 236                 else
 237                         buf.append("config_opts['scm_opts']['git_get'] = 'git clone "
 238                                         + (branch != null ? "-b " + branch : "") + " "
 239                                         + getGitBaseUrl() + "/SCM_PKG.git SCM_PKG'\n");
 240
 241                 buf.append("\nconfig_opts['yum.conf'] = \"\"\"\n");
 242                 buf.append(generateYumConfigFile(arch)).append('\n');
 243                 buf.append("\"\"\"\n");
 244                 return buf.toString();
 245         }
 246
 247         public String generateYumConfigFile(String arch) {
 248                 StringBuffer buf = new StringBuffer();
 249                 buf.append("[main]\n");
 250                 buf.append(yumConfigMainSection).append('\n');
 251
 252                 for (RpmRepository repository : repositories) {
 253                         buf.append('[').append(repository.getId()).append("]\n");
 254                         buf.append("name=").append(repository.getId()).append('\n');
 255                         if (repository instanceof ThirdPartyRpmRepository) {
 256                                 buf.append("#baseurl=").append(repository.getUrl())
 257                                                 .append(arch).append('/').append("\n");
 258                                 buf.append("baseurl=").append(proxiedReposBase)
 259                                                 .append(repository.getId()).append('/').append(arch)
 260                                                 .append('/').append("\n");
 261                                 if (((ThirdPartyRpmRepository) repository).getYumConf() != null)
 262                                         buf.append(
 263                                                         ((ThirdPartyRpmRepository) repository).getYumConf()
 264                                                                         .trim()).append('\n');
 265                         }
 266                 }
 267
 268                 // managed repos
 269                 addManagedRepository(buf, stagingWorkspace, arch);
 270                 if (withTestingRepository)
 271                         addManagedRepository(buf, testingWorkspace, arch);
 272                 addManagedRepository(buf, stableWorkspace, arch);
 273                 return buf.toString();
 274         }
 275
 276         protected void addManagedRepository(StringBuffer buf, String workspace,
 277                         String arch) {
 278                 buf.append('[').append(workspace).append("]\n");
 279                 buf.append("baseurl=").append(managedReposBase).append(workspace)
 280                                 .append('/').append(arch).append('/').append("\n");
 281                 buf.append("gpgcheck=0").append("\n");
 282         }
 283
 284         /** Creates a mock config file. */
 285         public File getMockConfigFile(String arch, String branch) {
 286                 File mockSiteDefaultsFile = new File(mockConfDir, "site-defaults.cfg");
 287                 File mockLoggingFile = new File(mockConfDir, "logging.ini");
 288                 File mockConfigFile = new File(mockConfDir, getIdWithArch(arch)
 289                                 + ".cfg");
 290                 try {
 291                         if (!mockSiteDefaultsFile.exists())
 292                                 mockSiteDefaultsFile.createNewFile();
 293                         if (!mockLoggingFile.exists())
 294                                 FileUtils.copyFile(new File(mockEtc + "/logging.ini"),
 295                                                 mockLoggingFile);
 296
 297                         FileUtils.writeStringToFile(mockConfigFile,
 298                                         generateMockConfigFile(arch, branch));
 299                         return mockConfigFile;
 300                 } catch (IOException e) {
 301                         throw new SlcException("Cannot write mock config file to "
 302                                         + mockConfigFile, e);
 303                 }
 304         }
 305
 306         /** Creates a yum config file. */
 307         public File getYumRepoFile(String arch) {
 308                 File yumConfigFile = new File(yumConfDir, getIdWithArch(arch) + ".repo");
 309                 try {
 310                         FileUtils.writeStringToFile(yumConfigFile,
 311                                         generateYumConfigFile(arch));
 312                         return yumConfigFile;
 313                 } catch (IOException e) {
 314                         throw new SlcException("Cannot write yum config file to "
 315                                         + yumConfigFile, e);
 316                 }
 317         }
 318
 319         public File getResultDir(String arch) {
 320                 return new File(mockVar + "/" + getIdWithArch(arch) + "/result");
 321         }
 322
 323         public File getWorkspaceDir(String workspace) {
 324                 return new File(rpmBase + "/" + workspace);
 325         }
 326
 327         public File getSourcesDir() {
 328                 return new File(distBase + "/" + stableWorkspace);
 329         }
 330
 331         public String getMockConfig(String arch) {
 332                 return getIdWithArch(arch);
 333         }
 334
 335         public String getIdWithArch(String arch) {
 336                 return id + "-" + arch;
 337         }
 338
 339         public String getGitBaseUrl() {
 340                 return managedReposBase + gitWorkspace;
 341         }
 342
 343         public void indexWorkspace(String workspace) {
 344                 Session session = null;
 345                 try {
 346                         session = rpmRepository.login(workspace);
 347                         session.getRootNode().accept(
 348                                         new NodeIndexerVisitor(new RpmIndexer()));
 349                         if (log.isDebugEnabled())
 350                                 log.debug("Indexed workspace " + workspace);
 351                 } catch (RepositoryException e) {
 352                         throw new SlcException("Cannot index workspace " + workspace, e);
 353                 } finally {
 354                         JcrUtils.logoutQuietly(session);
 355                 }
 356         }
 357
 358         public Boolean isDeveloperInstance() {
 359                 return gitDevBaseUrl != null;
 360         }
 361
 362         /** Write (topdir)/rpmmacros and (topdir)/rpmrc */
 363         public void writeRpmbuildConfigFiles(File topdir) {
 364                 writeRpmbuildConfigFiles(topdir, new File(topdir, "rpmmacros"),
 365                                 new File(topdir, "rpmrc"));
 366         }
 367
 368         public void writeRpmbuildConfigFiles(File topdir, File rpmmacroFile,
 369                         File rpmrcFile) {
 370                 try {
 371                         List<String> macroLines = new ArrayList<String>();
 372                         macroLines.add("%_topdir " + topdir.getCanonicalPath());
 373                         for (String macroKey : rpmmacros.keySet()) {
 374                                 macroLines.add(macroKey + " " + rpmmacros.get(macroKey));
 375                         }
 376                         FileUtils.writeLines(rpmmacroFile, macroLines);
 377
 378                         List<String> rpmrcLines = new ArrayList<String>();
 379                         rpmrcLines.add("include: /usr/lib/rpm/rpmrc");
 380                         rpmrcLines.add("macrofiles: " + defaultMacroFiles + ":"
 381                                         + rpmmacroFile.getCanonicalPath());
 382                         FileUtils.writeLines(rpmrcFile, rpmrcLines);
 383                 } catch (IOException e) {
 384                         throw new SlcException("Cannot write rpmbuild config files", e);
 385                 }
 386
 387         }
 388
 389         public Map<String, String> getRpmmacros() {
 390                 return rpmmacros;
 391         }
 392
 393         public void setRpmmacros(Map<String, String> rpmmacros) {
 394                 this.rpmmacros = rpmmacros;
 395         }
 396
 397         public String getDefaultMacroFiles() {
 398                 return defaultMacroFiles;
 399         }
 400
 401         public void setDefaultMacroFiles(String defaultMacroFiles) {
 402                 this.defaultMacroFiles = defaultMacroFiles;
 403         }
 404
 405         public void setArchs(List<String> archs) {
 406                 this.archs = archs;
 407         }
 408
 409         public List<String> getArchs() {
 410                 return archs;
 411         }
 412
 413         public void setRpmBase(String stagingBase) {
 414                 this.rpmBase = stagingBase;
 415         }
 416
 417         public void setId(String id) {
 418                 this.id = id;
 419         }
 420
 421         public void setMockVar(String mockVar) {
 422                 this.mockVar = mockVar;
 423         }
 424
 425         public void setRpmRepository(Repository rpmRepository) {
 426                 this.rpmRepository = rpmRepository;
 427         }
 428
 429         public void setDistRepository(Repository distRepository) {
 430                 this.distRepository = distRepository;
 431         }
 432
 433         public void setLocalUrlBase(String localUrlBase) {
 434                 this.localUrlBase = localUrlBase;
 435         }
 436
 437         public void setYumConfigMainSection(String yumConfigMainSection) {
 438                 this.yumConfigMainSection = yumConfigMainSection;
 439         }
 440
 441         public void setRepositories(List<RpmRepository> repositories) {
 442                 this.repositories = repositories;
 443         }
 444
 445         public void setRpmFactoryBaseDir(File rpmFactoryBaseDir) {
 446                 this.rpmFactoryBaseDir = rpmFactoryBaseDir;
 447         }
 448
 449         public String getStagingWorkspace() {
 450                 return stagingWorkspace;
 451         }
 452
 453         public String getTestingWorkspace() {
 454                 return testingWorkspace;
 455         }
 456
 457         public String getStableWorkspace() {
 458                 return stableWorkspace;
 459         }
 460
 461         public void setWithTestingRepository(Boolean withTestingRepository) {
 462                 this.withTestingRepository = withTestingRepository;
 463         }
 464
 465         public void setGitDevBaseUrl(String gitBaseUrl) {
 466                 this.gitDevBaseUrl = gitBaseUrl;
 467         }
 468 }