org.argeo.slc.factory/src/org/argeo/slc/rpmfactory/core/RpmFactoryImpl.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.rpmfactory.core;
  17
  18 import java.io.File;
  19 import java.io.IOException;
  20 import java.text.DateFormat;
  21 import java.text.SimpleDateFormat;
  22 import java.util.ArrayList;
  23 import java.util.Calendar;
  24 import java.util.GregorianCalendar;
  25 import java.util.HashMap;
  26 import java.util.List;
  27 import java.util.Map;
  28
  29 import javax.jcr.Node;
  30 import javax.jcr.Repository;
  31 import javax.jcr.RepositoryException;
  32 import javax.jcr.Session;
  33
  34 import org.apache.commons.io.FileUtils;
  35 import org.apache.commons.logging.Log;
  36 import org.apache.commons.logging.LogFactory;
  37 import org.argeo.jcr.JcrUtils;
  38 import org.argeo.slc.SlcConstants;
  39 import org.argeo.slc.SlcException;
  40 import org.argeo.slc.core.execution.tasks.SystemCall;
  41 import org.argeo.slc.repo.NodeIndexerVisitor;
  42 import org.argeo.slc.repo.RpmIndexer;
  43 import org.argeo.slc.rpmfactory.RpmFactory;
  44 import org.argeo.slc.rpmfactory.RpmRepository;
  45
  46 /**
  47  * Defines a build environment. This information is typically used by other
  48  * components performing the various actions related to RPM build.
  49  */
  50 public class RpmFactoryImpl implements RpmFactory {
  51         private Log log = LogFactory.getLog(RpmFactoryImpl.class);
  52
  53         private Repository rpmRepository;
  54         private Repository distRepository;
  55
  56         private String id;
  57         private List<RpmRepository> repositories = new ArrayList<RpmRepository>();
  58         private List<String> archs = new ArrayList<String>();
  59
  60         private String rpmBase = "/mnt/slc/repos/rpm";
  61         private String distBase = "/mnt/slc/repos/dist";
  62         private String mockVar = "/var/lib/mock";
  63         private String mockEtc = "/etc/mock";
  64
  65         private DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmm");
  66
  67         private String gitWorkspace = "git";
  68
  69         private String localUrlBase = "http://localhost:7070/";
  70         /** If not null or empty, this is a developer instance. */
  71         private String gitDevBaseUrl = null;
  72
  73         private Boolean withTestingRepository = false;
  74
  75         private String yumConfigMainSection = "cachedir=/var/cache/yum\n"
  76                         + "debuglevel=1\n" + "reposdir=/dev/null\n"
  77                         + "logfile=/var/log/yum.log\n" + "retries=20\n" + "obsoletes=1\n"
  78                         + "gpgcheck=0\n" + "assumeyes=1\n" + "syslog_ident=mock\n"
  79                         + "syslog_device=\n" + "http_caching=none\n";
  80
  81         private String defaultMacroFiles = "/usr/lib/rpm/macros:"
  82                         + "/usr/lib/rpm/ia32e-linux/macros:"
  83                         + "/usr/lib/rpm/redhat/macros:" + "/etc/rpm/macros.*:"
  84                         + "/etc/rpm/macros:" + "/etc/rpm/ia32e-linux/macros:"
  85                         + "~/.rpmmacros";
  86         private Map<String, String> rpmmacros = new HashMap<String, String>();
  87
  88         // set by init
  89         private String proxiedReposBase;
  90         private String managedReposBase;
  91
  92         private String stagingWorkspace;
  93         private String testingWorkspace;
  94         private String stableWorkspace;
  95
  96         private File rpmFactoryBaseDir;
  97         private File mockConfDir;
  98         private File yumConfDir;
  99
 100         public void init() {
 101                 // local URL bases
 102                 proxiedReposBase = localUrlBase + "repo/rpm/";
 103                 managedReposBase = localUrlBase + "data/public/rpm/";
 104
 105                 // local directories
 106                 rpmFactoryBaseDir.mkdirs();
 107                 mockConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/mock");
 108                 mockConfDir.mkdirs();
 109                 yumConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/yum");
 110                 yumConfDir.mkdirs();
 111
 112                 // managed repositories
 113                 stagingWorkspace = id + "-staging";
 114                 if (withTestingRepository)
 115                         testingWorkspace = id + "-testing";
 116                 stableWorkspace = id;
 117
 118                 initDistWorkspace(stableWorkspace);
 119                 initGitWorkspace();
 120                 initRpmWorkspace(stagingWorkspace);
 121                 if (withTestingRepository)
 122                         initRpmWorkspace(testingWorkspace);
 123                 initRpmWorkspace(stableWorkspace);
 124         }
 125
 126         protected void initRpmWorkspace(String workspace) {
 127                 Session session = null;
 128                 try {
 129                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository, workspace);
 130                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 131                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 132                                         "jcr:all");
 133
 134                         for (String arch : archs) {
 135                                 Node archFolder = JcrUtils.mkfolders(session, "/" + arch);
 136                                 session.save();
 137                                 File workspaceDir = getWorkspaceDir(workspace);
 138                                 try {
 139                                         if (!archFolder.hasNode("repodata")) {
 140                                                 // touch a file in order to make sure this is properly
 141                                                 // mounted.
 142                                                 File touch = new File(workspaceDir, ".touch");
 143                                                 touch.createNewFile();
 144                                                 touch.delete();
 145
 146                                                 SystemCall createrepo = new SystemCall();
 147                                                 createrepo.arg("createrepo");
 148                                                 createrepo.arg("-q");
 149                                                 File archDir = new File(workspaceDir, arch);
 150                                                 createrepo.arg(archDir.getAbsolutePath());
 151                                                 createrepo.run();
 152                                         }
 153                                 } catch (IOException e) {
 154                                         log.error(workspaceDir + " not properly mounted.", e);
 155                                 }
 156                         }
 157                 } catch (Exception e) {
 158                         throw new SlcException("Cannot initialize workspace " + workspace,
 159                                         e);
 160                 } finally {
 161                         JcrUtils.logoutQuietly(session);
 162                 }
 163         }
 164
 165         /** Caller must logout the underlying session. */
 166         public Node newDistribution(String distributionId) {
 167                 Session session = null;
 168                 try {
 169                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository,
 170                                         distributionId);
 171                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 172                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 173                                         "jcr:all");
 174
 175                         Calendar now = new GregorianCalendar();
 176                         String folderName = dateFormat.format(now.getTime());
 177                         return JcrUtils.mkfolders(session, "/" + folderName);
 178                 } catch (Exception e) {
 179                         JcrUtils.logoutQuietly(session);
 180                         throw new SlcException("Cannot initialize distribution workspace "
 181                                         + distributionId, e);
 182                 }
 183         }
 184
 185         protected void initGitWorkspace() {
 186                 Session session = null;
 187                 try {
 188                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository,
 189                                         gitWorkspace);
 190                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 191                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 192                                         "jcr:all");
 193                 } catch (Exception e) {
 194                         throw new SlcException("Cannot initialize workspace "
 195                                         + gitWorkspace, e);
 196                 } finally {
 197                         JcrUtils.logoutQuietly(session);
 198                 }
 199         }
 200
 201         protected void initDistWorkspace(String workspace) {
 202                 Session session = null;
 203                 try {
 204                         session = JcrUtils
 205                                         .loginOrCreateWorkspace(distRepository, workspace);
 206                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 207                 } catch (RepositoryException e) {
 208                         throw new SlcException("Cannot initialize workspace " + workspace,
 209                                         e);
 210                 } finally {
 211                         JcrUtils.logoutQuietly(session);
 212                 }
 213         }
 214
 215         public void destroy() {
 216
 217         }
 218
 219         public String generateMockConfigFile(String arch, String branch) {
 220                 StringBuffer buf = new StringBuffer();
 221
 222                 buf.append("config_opts['root'] = '" + getIdWithArch(arch) + "'\n");
 223                 buf.append("config_opts['target_arch'] = '" + arch + "'\n");
 224                 buf.append("config_opts['legal_host_arches'] = ('" + arch + "',)\n");
 225                 buf.append("config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'\n");
 226                 // buf.append("config_opts['dist'] = 'el6'\n");
 227                 buf.append("config_opts['plugin_conf']['yum_cache_enable'] = False\n");
 228
 229                 buf.append("config_opts['scm'] = False\n");
 230                 buf.append("config_opts['scm_opts']['method'] = 'git'\n");
 231                 buf.append("config_opts['scm_opts']['spec'] = 'SCM_PKG.spec'\n");
 232                 buf.append("config_opts['scm_opts']['ext_src_dir'] = '"
 233                                 + getSourcesDir().getAbsolutePath() + "'\n");
 234                 buf.append("config_opts['scm_opts']['git_timestamps'] = True\n");
 235
 236                 // development
 237                 if (gitDevBaseUrl != null && !gitDevBaseUrl.trim().equals(""))
 238                         buf.append("config_opts['scm_opts']['git_get'] = 'git clone "
 239                                         + (branch != null ? "-b " + branch : "") + " "
 240                                         + gitDevBaseUrl + "/SCM_PKG SCM_PKG'\n");
 241                 else
 242                         buf.append("config_opts['scm_opts']['git_get'] = 'git clone "
 243                                         + (branch != null ? "-b " + branch : "") + " "
 244                                         + getGitBaseUrl() + "/SCM_PKG.git SCM_PKG'\n");
 245
 246                 buf.append("\nconfig_opts['yum.conf'] = \"\"\"\n");
 247                 buf.append(generateYumConfigFile(arch)).append('\n');
 248                 buf.append("\"\"\"\n");
 249                 return buf.toString();
 250         }
 251
 252         public String generateYumConfigFile(String arch) {
 253                 StringBuffer buf = new StringBuffer();
 254                 buf.append("[main]\n");
 255                 buf.append(yumConfigMainSection).append('\n');
 256
 257                 for (RpmRepository repository : repositories) {
 258                         buf.append('[').append(repository.getId()).append("]\n");
 259                         buf.append("name=").append(repository.getId()).append('\n');
 260                         if (repository instanceof ThirdPartyRpmRepository) {
 261                                 buf.append("#baseurl=").append(repository.getUrl())
 262                                                 .append(arch).append('/').append("\n");
 263                                 buf.append("baseurl=").append(proxiedReposBase)
 264                                                 .append(repository.getId()).append('/').append(arch)
 265                                                 .append('/').append("\n");
 266                                 if (((ThirdPartyRpmRepository) repository).getYumConf() != null)
 267                                         buf.append(
 268                                                         ((ThirdPartyRpmRepository) repository).getYumConf()
 269                                                                         .trim()).append('\n');
 270                         }
 271                 }
 272
 273                 // managed repos
 274                 addManagedRepository(buf, stagingWorkspace, arch);
 275                 if (withTestingRepository)
 276                         addManagedRepository(buf, testingWorkspace, arch);
 277                 addManagedRepository(buf, stableWorkspace, arch);
 278                 return buf.toString();
 279         }
 280
 281         protected void addManagedRepository(StringBuffer buf, String workspace,
 282                         String arch) {
 283                 buf.append('[').append(workspace).append("]\n");
 284                 buf.append("baseurl=").append(managedReposBase).append(workspace)
 285                                 .append('/').append(arch).append('/').append("\n");
 286                 buf.append("gpgcheck=0").append("\n");
 287         }
 288
 289         /** Creates a mock config file. */
 290         public File getMockConfigFile(String arch, String branch) {
 291                 File mockSiteDefaultsFile = new File(mockConfDir, "site-defaults.cfg");
 292                 File mockLoggingFile = new File(mockConfDir, "logging.ini");
 293                 File mockConfigFile = new File(mockConfDir, getIdWithArch(arch)
 294                                 + ".cfg");
 295                 try {
 296                         if (!mockSiteDefaultsFile.exists())
 297                                 mockSiteDefaultsFile.createNewFile();
 298                         if (!mockLoggingFile.exists())
 299                                 FileUtils.copyFile(new File(mockEtc + "/logging.ini"),
 300                                                 mockLoggingFile);
 301
 302                         FileUtils.writeStringToFile(mockConfigFile,
 303                                         generateMockConfigFile(arch, branch));
 304                         return mockConfigFile;
 305                 } catch (IOException e) {
 306                         throw new SlcException("Cannot write mock config file to "
 307                                         + mockConfigFile, e);
 308                 }
 309         }
 310
 311         /** Creates a yum config file. */
 312         public File getYumRepoFile(String arch) {
 313                 File yumConfigFile = new File(yumConfDir, getIdWithArch(arch) + ".repo");
 314                 try {
 315                         FileUtils.writeStringToFile(yumConfigFile,
 316                                         generateYumConfigFile(arch));
 317                         return yumConfigFile;
 318                 } catch (IOException e) {
 319                         throw new SlcException("Cannot write yum config file to "
 320                                         + yumConfigFile, e);
 321                 }
 322         }
 323
 324         public File getResultDir(String arch) {
 325                 return new File(mockVar + "/" + getIdWithArch(arch) + "/result");
 326         }
 327
 328         public File getWorkspaceDir(String workspace) {
 329                 return new File(rpmBase + "/" + workspace);
 330         }
 331
 332         public File getSourcesDir() {
 333                 return new File(distBase + "/" + stableWorkspace);
 334         }
 335
 336         public String getMockConfig(String arch) {
 337                 return getIdWithArch(arch);
 338         }
 339
 340         public String getIdWithArch(String arch) {
 341                 return id + "-" + arch;
 342         }
 343
 344         public String getGitBaseUrl() {
 345                 return managedReposBase + gitWorkspace;
 346         }
 347
 348         public void indexWorkspace(String workspace) {
 349                 Session session = null;
 350                 try {
 351                         session = rpmRepository.login(workspace);
 352                         session.getRootNode().accept(
 353                                         new NodeIndexerVisitor(new RpmIndexer()));
 354                         if (log.isDebugEnabled())
 355                                 log.debug("Indexed workspace " + workspace);
 356                 } catch (RepositoryException e) {
 357                         throw new SlcException("Cannot index workspace " + workspace, e);
 358                 } finally {
 359                         JcrUtils.logoutQuietly(session);
 360                 }
 361         }
 362
 363         public Boolean isDeveloperInstance() {
 364                 return gitDevBaseUrl != null;
 365         }
 366
 367         /** Write (topdir)/rpmmacros and (topdir)/rpmrc */
 368         public void writeRpmbuildConfigFiles(File topdir) {
 369                 writeRpmbuildConfigFiles(topdir, new File(topdir, "rpmmacros"),
 370                                 new File(topdir, "rpmrc"));
 371         }
 372
 373         public void writeRpmbuildConfigFiles(File topdir, File rpmmacroFile,
 374                         File rpmrcFile) {
 375                 try {
 376                         List<String> macroLines = new ArrayList<String>();
 377                         macroLines.add("%_topdir " + topdir.getCanonicalPath());
 378                         for (String macroKey : rpmmacros.keySet()) {
 379                                 macroLines.add(macroKey + " " + rpmmacros.get(macroKey));
 380                         }
 381                         FileUtils.writeLines(rpmmacroFile, macroLines);
 382
 383                         List<String> rpmrcLines = new ArrayList<String>();
 384                         rpmrcLines.add("include: /usr/lib/rpm/rpmrc");
 385                         rpmrcLines.add("macrofiles: " + defaultMacroFiles + ":"
 386                                         + rpmmacroFile.getCanonicalPath());
 387                         FileUtils.writeLines(rpmrcFile, rpmrcLines);
 388                 } catch (IOException e) {
 389                         throw new SlcException("Cannot write rpmbuild config files", e);
 390                 }
 391
 392         }
 393
 394         public Map<String, String> getRpmmacros() {
 395                 return rpmmacros;
 396         }
 397
 398         public void setRpmmacros(Map<String, String> rpmmacros) {
 399                 this.rpmmacros = rpmmacros;
 400         }
 401
 402         public String getDefaultMacroFiles() {
 403                 return defaultMacroFiles;
 404         }
 405
 406         public void setDefaultMacroFiles(String defaultMacroFiles) {
 407                 this.defaultMacroFiles = defaultMacroFiles;
 408         }
 409
 410         public void setArchs(List<String> archs) {
 411                 this.archs = archs;
 412         }
 413
 414         public List<String> getArchs() {
 415                 return archs;
 416         }
 417
 418         public void setRpmBase(String stagingBase) {
 419                 this.rpmBase = stagingBase;
 420         }
 421
 422         public void setId(String id) {
 423                 this.id = id;
 424         }
 425
 426         public void setMockVar(String mockVar) {
 427                 this.mockVar = mockVar;
 428         }
 429
 430         public void setRpmRepository(Repository rpmRepository) {
 431                 this.rpmRepository = rpmRepository;
 432         }
 433
 434         public void setDistRepository(Repository distRepository) {
 435                 this.distRepository = distRepository;
 436         }
 437
 438         public void setLocalUrlBase(String localUrlBase) {
 439                 this.localUrlBase = localUrlBase;
 440         }
 441
 442         public void setYumConfigMainSection(String yumConfigMainSection) {
 443                 this.yumConfigMainSection = yumConfigMainSection;
 444         }
 445
 446         public void setRepositories(List<RpmRepository> repositories) {
 447                 this.repositories = repositories;
 448         }
 449
 450         public void setRpmFactoryBaseDir(File rpmFactoryBaseDir) {
 451                 this.rpmFactoryBaseDir = rpmFactoryBaseDir;
 452         }
 453
 454         public String getStagingWorkspace() {
 455                 return stagingWorkspace;
 456         }
 457
 458         public String getTestingWorkspace() {
 459                 return testingWorkspace;
 460         }
 461
 462         public String getStableWorkspace() {
 463                 return stableWorkspace;
 464         }
 465
 466         public void setWithTestingRepository(Boolean withTestingRepository) {
 467                 this.withTestingRepository = withTestingRepository;
 468         }
 469
 470         public void setGitDevBaseUrl(String gitBaseUrl) {
 471                 this.gitDevBaseUrl = gitBaseUrl;
 472         }
 473 }