RpmFactory.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.rpmfactory.core;
  17
  18 import java.io.File;
  19 import java.io.IOException;
  20 import java.text.DateFormat;
  21 import java.text.SimpleDateFormat;
  22 import java.util.ArrayList;
  23 import java.util.Calendar;
  24 import java.util.GregorianCalendar;
  25 import java.util.HashMap;
  26 import java.util.List;
  27 import java.util.Map;
  28
  29 import javax.jcr.Node;
  30 import javax.jcr.Repository;
  31 import javax.jcr.RepositoryException;
  32 import javax.jcr.Session;
  33
  34 import org.apache.commons.io.FileUtils;
  35 import org.argeo.jcr.JcrUtils;
  36 import org.argeo.slc.SlcConstants;
  37 import org.argeo.slc.SlcException;
  38 import org.argeo.slc.core.execution.tasks.SystemCall;
  39 import org.argeo.slc.repo.NodeIndexerVisitor;
  40 import org.argeo.slc.rpmfactory.RpmRepository;
  41
  42 /**
  43  * Defines a build environment. This information is typically used by other
  44  * components performing the various actions related to RPM build.
  45  */
  46 public class RpmFactory {
  47         private Repository rpmRepository;
  48         private Repository distRepository;
  49
  50         private String id;
  51         private List<RpmRepository> repositories = new ArrayList<RpmRepository>();
  52         private List<String> archs = new ArrayList<String>();
  53
  54         private String rpmBase = "/mnt/slc/repos/rpm";
  55         private String distBase = "/mnt/slc/repos/dist";
  56         private String mockVar = "/var/lib/mock";
  57         private String mockEtc = "/etc/mock";
  58
  59         private DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmm");
  60
  61         private String gitWorkspace = "git";
  62
  63         private String localUrlBase = "http://localhost:7070/";
  64
  65         private Boolean withTestingRepository = false;
  66
  67         private String yumConfigMainSection = "cachedir=/var/cache/yum\n"
  68                         + "debuglevel=1\n" + "reposdir=/dev/null\n"
  69                         + "logfile=/var/log/yum.log\n" + "retries=20\n" + "obsoletes=1\n"
  70                         + "gpgcheck=0\n" + "assumeyes=1\n" + "syslog_ident=mock\n"
  71                         + "syslog_device=\n" + "http_caching=none\n";
  72
  73         private String defaultMacroFiles = "/usr/lib/rpm/macros:"
  74                         + "/usr/lib/rpm/ia32e-linux/macros:"
  75                         + "/usr/lib/rpm/redhat/macros:" + "/etc/rpm/macros.*:"
  76                         + "/etc/rpm/macros:" + "/etc/rpm/ia32e-linux/macros:"
  77                         + "~/.rpmmacros";
  78         private Map<String, String> rpmmacros = new HashMap<String, String>();
  79
  80         // set by init
  81         private String proxiedReposBase;
  82         private String managedReposBase;
  83
  84         private String stagingWorkspace;
  85         private String testingWorkspace;
  86         private String stableWorkspace;
  87
  88         private File rpmFactoryBaseDir;
  89         private File mockConfDir;
  90         private File yumConfDir;
  91
  92         public void init() {
  93                 // local URL bases
  94                 proxiedReposBase = localUrlBase + "repo/rpm/";
  95                 managedReposBase = localUrlBase + "data/public/rpm/";
  96
  97                 // local directories
  98                 rpmFactoryBaseDir.mkdirs();
  99                 mockConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/mock");
 100                 mockConfDir.mkdirs();
 101                 yumConfDir = new File(rpmFactoryBaseDir.getPath() + "/conf/yum");
 102                 yumConfDir.mkdirs();
 103
 104                 // managed repositories
 105                 stagingWorkspace = id + "-staging";
 106                 if (withTestingRepository)
 107                         testingWorkspace = id + "-testing";
 108                 stableWorkspace = id;
 109
 110                 initDistWorkspace(stableWorkspace);
 111                 initGitWorkspace();
 112                 initRpmWorkspace(stagingWorkspace);
 113                 if (withTestingRepository)
 114                         initRpmWorkspace(testingWorkspace);
 115                 initRpmWorkspace(stableWorkspace);
 116         }
 117
 118         protected void initRpmWorkspace(String workspace) {
 119                 Session session = null;
 120                 try {
 121                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository, workspace);
 122                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 123                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 124                                         "jcr:all");
 125
 126                         for (String arch : archs) {
 127                                 Node archFolder = JcrUtils.mkfolders(session, "/" + arch);
 128                                 session.save();
 129                                 if (!archFolder.hasNode("repodata")) {
 130                                         File workspaceDir = getWorkspaceDir(workspace);
 131                                         // touch a file in order to make sue this is properly
 132                                         // mounted.
 133                                         File touch = new File(workspaceDir, ".touch");
 134                                         touch.createNewFile();
 135                                         touch.delete();
 136
 137                                         SystemCall createrepo = new SystemCall();
 138                                         createrepo.arg("createrepo");
 139                                         createrepo.arg("-q");
 140                                         File archDir = new File(workspaceDir, arch);
 141                                         createrepo.arg(archDir.getAbsolutePath());
 142                                         createrepo.run();
 143                                 }
 144                         }
 145                 } catch (Exception e) {
 146                         throw new SlcException("Cannot initialize workspace " + workspace,
 147                                         e);
 148                 } finally {
 149                         JcrUtils.logoutQuietly(session);
 150                 }
 151         }
 152
 153         /** Caller must logout the underlying session. */
 154         protected Node newDistribution(String distributionId) {
 155                 Session session = null;
 156                 try {
 157                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository,
 158                                         distributionId);
 159                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 160                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 161                                         "jcr:all");
 162
 163                         Calendar now = new GregorianCalendar();
 164                         String folderName = dateFormat.format(now.getTime());
 165                         return JcrUtils.mkfolders(session, "/" + folderName);
 166                 } catch (Exception e) {
 167                         JcrUtils.logoutQuietly(session);
 168                         throw new SlcException("Cannot initialize distribution workspace "
 169                                         + distributionId, e);
 170                 }
 171         }
 172
 173         protected void initGitWorkspace() {
 174                 Session session = null;
 175                 try {
 176                         session = JcrUtils.loginOrCreateWorkspace(rpmRepository,
 177                                         gitWorkspace);
 178                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 179                         JcrUtils.addPrivilege(session, "/", SlcConstants.ROLE_SLC,
 180                                         "jcr:all");
 181                 } catch (Exception e) {
 182                         throw new SlcException("Cannot initialize workspace "
 183                                         + gitWorkspace, e);
 184                 } finally {
 185                         JcrUtils.logoutQuietly(session);
 186                 }
 187         }
 188
 189         protected void initDistWorkspace(String workspace) {
 190                 Session session = null;
 191                 try {
 192                         session = JcrUtils
 193                                         .loginOrCreateWorkspace(distRepository, workspace);
 194                         JcrUtils.addPrivilege(session, "/", "anonymous", "jcr:read");
 195                 } catch (RepositoryException e) {
 196                         throw new SlcException("Cannot initialize workspace " + workspace,
 197                                         e);
 198                 } finally {
 199                         JcrUtils.logoutQuietly(session);
 200                 }
 201         }
 202
 203         public void destroy() {
 204
 205         }
 206
 207         public String generateMockConfigFile(String arch, String branch) {
 208                 StringBuffer buf = new StringBuffer();
 209
 210                 buf.append("config_opts['root'] = '" + getIdWithArch(arch) + "'\n");
 211                 buf.append("config_opts['target_arch'] = '" + arch + "'\n");
 212                 buf.append("config_opts['legal_host_arches'] = ('" + arch + "',)\n");
 213                 buf.append("config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'\n");
 214                 // buf.append("config_opts['dist'] = 'el6'\n");
 215                 buf.append("config_opts['plugin_conf']['yum_cache_enable'] = False\n");
 216
 217                 buf.append("config_opts['scm'] = False\n");
 218                 buf.append("config_opts['scm_opts']['method'] = 'git'\n");
 219                 buf.append("config_opts['scm_opts']['spec'] = 'SCM_PKG.spec'\n");
 220                 buf.append("config_opts['scm_opts']['ext_src_dir'] = '"
 221                                 + getSourcesDir().getAbsolutePath() + "'\n");
 222                 buf.append("config_opts['scm_opts']['git_timestamps'] = True\n");
 223                 buf.append("config_opts['scm_opts']['git_get'] = 'git clone "
 224                                 + (branch != null ? "-b " + branch : "") + " "
 225                                 + getGitBaseUrl() + "/SCM_PKG.git SCM_PKG'\n");
 226
 227                 buf.append("\nconfig_opts['yum.conf'] = \"\"\"\n");
 228                 buf.append(generateYumConfigFile(arch)).append('\n');
 229                 buf.append("\"\"\"\n");
 230                 return buf.toString();
 231         }
 232
 233         public String generateYumConfigFile(String arch) {
 234                 StringBuffer buf = new StringBuffer();
 235                 buf.append("[main]\n");
 236                 buf.append(yumConfigMainSection).append('\n');
 237
 238                 for (RpmRepository repository : repositories) {
 239                         buf.append('[').append(repository.getId()).append("]\n");
 240                         if (repository instanceof ThirdPartyRpmRepository) {
 241                                 buf.append("#baseurl=").append(repository.getUrl())
 242                                                 .append(arch).append('/').append("\n");
 243                                 buf.append("baseurl=").append(proxiedReposBase)
 244                                                 .append(repository.getId()).append('/').append(arch)
 245                                                 .append('/').append("\n");
 246                                 if (((ThirdPartyRpmRepository) repository).getYumConf() != null)
 247                                         buf.append(
 248                                                         ((ThirdPartyRpmRepository) repository).getYumConf()
 249                                                                         .trim()).append('\n');
 250                         }
 251                 }
 252
 253                 // managed repos
 254                 addManagedRepository(buf, stagingWorkspace, arch);
 255                 if (withTestingRepository)
 256                         addManagedRepository(buf, testingWorkspace, arch);
 257                 addManagedRepository(buf, stableWorkspace, arch);
 258                 return buf.toString();
 259         }
 260
 261         protected void addManagedRepository(StringBuffer buf, String workspace,
 262                         String arch) {
 263                 buf.append('[').append(workspace).append("]\n");
 264                 buf.append("baseurl=").append(managedReposBase).append(workspace)
 265                                 .append('/').append(arch).append('/').append("\n");
 266                 buf.append("gpgcheck=0").append("\n");
 267         }
 268
 269         /** Creates a mock config file. */
 270         public File getMockConfigFile(String arch, String branch) {
 271                 File mockSiteDefaultsFile = new File(mockConfDir, "site-defaults.cfg");
 272                 File mockLoggingFile = new File(mockConfDir, "logging.ini");
 273                 File mockConfigFile = new File(mockConfDir, getIdWithArch(arch)
 274                                 + ".cfg");
 275                 try {
 276                         if (!mockSiteDefaultsFile.exists())
 277                                 mockSiteDefaultsFile.createNewFile();
 278                         if (!mockLoggingFile.exists())
 279                                 FileUtils.copyFile(new File(mockEtc + "/logging.ini"),
 280                                                 mockLoggingFile);
 281
 282                         FileUtils.writeStringToFile(mockConfigFile,
 283                                         generateMockConfigFile(arch, branch));
 284                         return mockConfigFile;
 285                 } catch (IOException e) {
 286                         throw new SlcException("Cannot write mock config file to "
 287                                         + mockConfigFile, e);
 288                 }
 289         }
 290
 291         /** Creates a yum config file. */
 292         public File getYumRepoFile(String arch) {
 293                 File yumConfigFile = new File(yumConfDir, getIdWithArch(arch) + ".repo");
 294                 try {
 295                         FileUtils.writeStringToFile(yumConfigFile,
 296                                         generateYumConfigFile(arch));
 297                         return yumConfigFile;
 298                 } catch (IOException e) {
 299                         throw new SlcException("Cannot write yum config file to "
 300                                         + yumConfigFile, e);
 301                 }
 302         }
 303
 304         public File getResultDir(String arch) {
 305                 return new File(mockVar + "/" + getIdWithArch(arch) + "/result");
 306         }
 307
 308         public File getWorkspaceDir(String workspace) {
 309                 return new File(rpmBase + "/" + workspace);
 310         }
 311
 312         public File getSourcesDir() {
 313                 return new File(distBase + "/" + stableWorkspace);
 314         }
 315
 316         public String getMockConfig(String arch) {
 317                 return getIdWithArch(arch);
 318         }
 319
 320         public String getIdWithArch(String arch) {
 321                 return id + "-" + arch;
 322         }
 323
 324         public String getGitBaseUrl() {
 325                 return managedReposBase + gitWorkspace;
 326         }
 327
 328         public void indexWorkspace(String workspace) {
 329                 Session session = null;
 330                 try {
 331                         session = rpmRepository.login(workspace);
 332                         session.getRootNode().accept(
 333                                         new NodeIndexerVisitor(new RpmIndexer()));
 334                 } catch (RepositoryException e) {
 335                         throw new SlcException("Cannot index workspace " + workspace, e);
 336                 } finally {
 337                         JcrUtils.logoutQuietly(session);
 338                 }
 339         }
 340
 341         /** Write (topdir)/rpmmacros and (topdir)/rpmrc */
 342         public void writeRpmbuildConfigFiles(File topdir) {
 343                 writeRpmbuildConfigFiles(topdir, new File(topdir, "rpmmacros"),
 344                                 new File(topdir, "rpmrc"));
 345         }
 346
 347         public void writeRpmbuildConfigFiles(File topdir, File rpmmacroFile,
 348                         File rpmrcFile) {
 349                 try {
 350                         List<String> macroLines = new ArrayList<String>();
 351                         macroLines.add("%_topdir " + topdir.getCanonicalPath());
 352                         for (String macroKey : rpmmacros.keySet()) {
 353                                 macroLines.add(macroKey + " " + rpmmacros.get(macroKey));
 354                         }
 355                         FileUtils.writeLines(rpmmacroFile, macroLines);
 356
 357                         List<String> rpmrcLines = new ArrayList<String>();
 358                         rpmrcLines.add("include: /usr/lib/rpm/rpmrc");
 359                         rpmrcLines.add("macrofiles: " + defaultMacroFiles + ":"
 360                                         + rpmmacroFile.getCanonicalPath());
 361                         FileUtils.writeLines(rpmrcFile, rpmrcLines);
 362                 } catch (IOException e) {
 363                         throw new SlcException("Cannot write rpmbuild config files", e);
 364                 }
 365
 366         }
 367
 368         public Map<String, String> getRpmmacros() {
 369                 return rpmmacros;
 370         }
 371
 372         public void setRpmmacros(Map<String, String> rpmmacros) {
 373                 this.rpmmacros = rpmmacros;
 374         }
 375
 376         public String getDefaultMacroFiles() {
 377                 return defaultMacroFiles;
 378         }
 379
 380         public void setDefaultMacroFiles(String defaultMacroFiles) {
 381                 this.defaultMacroFiles = defaultMacroFiles;
 382         }
 383
 384         public void setArchs(List<String> archs) {
 385                 this.archs = archs;
 386         }
 387
 388         public List<String> getArchs() {
 389                 return archs;
 390         }
 391
 392         public void setRpmBase(String stagingBase) {
 393                 this.rpmBase = stagingBase;
 394         }
 395
 396         public void setId(String id) {
 397                 this.id = id;
 398         }
 399
 400         public void setMockVar(String mockVar) {
 401                 this.mockVar = mockVar;
 402         }
 403
 404         public void setRpmRepository(Repository rpmRepository) {
 405                 this.rpmRepository = rpmRepository;
 406         }
 407
 408         public void setDistRepository(Repository distRepository) {
 409                 this.distRepository = distRepository;
 410         }
 411
 412         public void setLocalUrlBase(String localUrlBase) {
 413                 this.localUrlBase = localUrlBase;
 414         }
 415
 416         public void setYumConfigMainSection(String yumConfigMainSection) {
 417                 this.yumConfigMainSection = yumConfigMainSection;
 418         }
 419
 420         public void setRepositories(List<RpmRepository> repositories) {
 421                 this.repositories = repositories;
 422         }
 423
 424         public void setRpmFactoryBaseDir(File rpmFactoryBaseDir) {
 425                 this.rpmFactoryBaseDir = rpmFactoryBaseDir;
 426         }
 427
 428         public String getStagingWorkspace() {
 429                 return stagingWorkspace;
 430         }
 431
 432         public String getTestingWorkspace() {
 433                 return testingWorkspace;
 434         }
 435
 436         public String getStableWorkspace() {
 437                 return stableWorkspace;
 438         }
 439
 440         public void setWithTestingRepository(Boolean withTestingRepository) {
 441                 this.withTestingRepository = withTestingRepository;
 442         }
 443
 444 }