runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/tasks/SystemCall.java

   1 /*
   2  * Copyright (C) 2010 Mathieu Baudier <mbaudier@argeo.org>
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 package org.argeo.slc.core.execution.tasks;
  18
  19 import java.io.File;
  20 import java.io.FileOutputStream;
  21 import java.io.FileWriter;
  22 import java.io.IOException;
  23 import java.io.InputStream;
  24 import java.io.OutputStream;
  25 import java.io.PipedInputStream;
  26 import java.io.PipedOutputStream;
  27 import java.io.Writer;
  28 import java.util.ArrayList;
  29 import java.util.Collections;
  30 import java.util.HashMap;
  31 import java.util.List;
  32 import java.util.Map;
  33
  34 import org.apache.commons.exec.CommandLine;
  35 import org.apache.commons.exec.DefaultExecutor;
  36 import org.apache.commons.exec.ExecuteException;
  37 import org.apache.commons.exec.ExecuteResultHandler;
  38 import org.apache.commons.exec.ExecuteStreamHandler;
  39 import org.apache.commons.exec.ExecuteWatchdog;
  40 import org.apache.commons.exec.Executor;
  41 import org.apache.commons.exec.LogOutputStream;
  42 import org.apache.commons.exec.PumpStreamHandler;
  43 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
  44 import org.apache.commons.io.FileUtils;
  45 import org.apache.commons.io.IOUtils;
  46 import org.apache.commons.logging.Log;
  47 import org.apache.commons.logging.LogFactory;
  48 import org.argeo.slc.SlcException;
  49 import org.argeo.slc.UnsupportedException;
  50 import org.argeo.slc.core.execution.ExecutionResources;
  51 import org.argeo.slc.core.test.SimpleResultPart;
  52 import org.argeo.slc.test.TestResult;
  53 import org.argeo.slc.test.TestStatus;
  54 import org.springframework.core.io.Resource;
  55
  56 /** Execute an OS specific system call. */
  57 public class SystemCall implements Runnable {
  58         public final static String LOG_STDOUT = "System.out";
  59
  60         private final Log log = LogFactory.getLog(getClass());
  61
  62         private String execDir;
  63
  64         private String cmd = null;
  65         private List<Object> command = null;
  66
  67         private Executor executor = new DefaultExecutor();
  68         private Boolean synchronous = true;
  69
  70         private String stdErrLogLevel = "ERROR";
  71         private String stdOutLogLevel = "INFO";
  72
  73         private Resource stdOutFile = null;
  74         private Resource stdErrFile = null;
  75
  76         private Resource stdInFile = null;
  77         /**
  78          * If no {@link #stdInFile} provided, writing to this stream will write to
  79          * the stdin of the process.
  80          */
  81         private OutputStream stdInSink = null;
  82
  83         private Boolean redirectStdOut = false;
  84
  85         private List<SystemCallOutputListener> outputListeners = Collections
  86                         .synchronizedList(new ArrayList<SystemCallOutputListener>());
  87
  88         private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
  89         private Map<String, String> osCmds = new HashMap<String, String>();
  90         private Map<String, String> environmentVariables = new HashMap<String, String>();
  91
  92         private Boolean logCommand = false;
  93         private Boolean redirectStreams = true;
  94         private Boolean exceptionOnFailed = true;
  95         private Boolean mergeEnvironmentVariables = true;
  96
  97         private String osConsole = null;
  98         private String generateScript = null;
  99
 100         /** 24 hours */
 101         private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
 102
 103         private TestResult testResult;
 104
 105         private ExecutionResources executionResources;
 106
 107         /** Empty constructor */
 108         public SystemCall() {
 109
 110         }
 111
 112         /**
 113          * Constructor based on the provided command list.
 114          *
 115          * @param command
 116          *            the command list
 117          */
 118         public SystemCall(List<Object> command) {
 119                 this.command = command;
 120         }
 121
 122         /**
 123          * Constructor based on the provided command.
 124          *
 125          * @param cmd
 126          *            the command. If the provided string contains no space a
 127          *            command list is initialized with the argument as first
 128          *            component (useful for chained construction)
 129          */
 130         public SystemCall(String cmd) {
 131                 if (cmd.indexOf(' ') < 0) {
 132                         command = new ArrayList<Object>();
 133                         command.add(cmd);
 134                 } else {
 135                         this.cmd = cmd;
 136                 }
 137         }
 138
 139         /** Executes the system call. */
 140         public void run() {
 141                 // Manage streams
 142                 Writer stdOutWriter = null;
 143                 OutputStream stdOutputStream = null;
 144                 Writer stdErrWriter = null;
 145                 InputStream stdInStream = null;
 146                 if (stdOutFile != null)
 147                         if (redirectStdOut)
 148                                 stdOutputStream = createOutputStream(stdOutFile);
 149                         else
 150                                 stdOutWriter = createWriter(stdOutFile, true);
 151
 152                 if (stdErrFile != null) {
 153                         stdErrWriter = createWriter(stdErrFile, true);
 154                 } else {
 155                         if (stdOutFile != null && !redirectStdOut)
 156                                 stdErrWriter = createWriter(stdOutFile, true);
 157                 }
 158
 159                 try {
 160                         if (stdInFile != null)
 161                                 stdInStream = stdInFile.getInputStream();
 162                         else {
 163                                 stdInStream = new PipedInputStream();
 164                                 stdInSink = new PipedOutputStream(
 165                                                 (PipedInputStream) stdInStream);
 166                         }
 167                 } catch (IOException e2) {
 168                         throw new SlcException("Cannot open a stream for " + stdInFile, e2);
 169                 }
 170
 171                 if (log.isTraceEnabled()) {
 172                         log.debug("os.name=" + System.getProperty("os.name"));
 173                         log.debug("os.arch=" + System.getProperty("os.arch"));
 174                         log.debug("os.version=" + System.getProperty("os.version"));
 175                 }
 176
 177                 // Execution directory
 178                 File dir = new File(getExecDirToUse());
 179                 // if (!dir.exists())
 180                 // dir.mkdirs();
 181
 182                 // Watchdog to check for lost processes
 183                 Executor executorToUse;
 184                 if (executor != null)
 185                         executorToUse = executor;
 186                 else
 187                         executorToUse = new DefaultExecutor();
 188                 executorToUse.setWatchdog(new ExecuteWatchdog(watchdogTimeout));
 189
 190                 if (redirectStreams) {
 191                         // Redirect standard streams
 192                         executorToUse.setStreamHandler(createExecuteStreamHandler(
 193                                         stdOutWriter, stdOutputStream, stdErrWriter, stdInStream));
 194                 } else {
 195                         // Dummy stream handler (otherwise pump is used)
 196                         executorToUse.setStreamHandler(new DummyexecuteStreamHandler());
 197                 }
 198
 199                 executorToUse.setProcessDestroyer(new ShutdownHookProcessDestroyer());
 200                 executorToUse.setWorkingDirectory(dir);
 201
 202                 // Command line to use
 203                 final CommandLine commandLine = createCommandLine();
 204                 if (logCommand)
 205                         log.info("Execute command:\n" + commandLine
 206                                         + "\n in working directory: \n" + dir + "\n");
 207
 208                 // Env variables
 209                 Map<String, String> environmentVariablesToUse = null;
 210                 environmentVariablesToUse = new HashMap<String, String>();
 211                 if (mergeEnvironmentVariables)
 212                         environmentVariablesToUse.putAll(System.getenv());
 213                 if (environmentVariables.size() > 0)
 214                         environmentVariablesToUse.putAll(environmentVariables);
 215
 216                 // Execute
 217                 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
 218
 219                 //
 220                 // THE EXECUTION PROPER
 221                 //
 222                 try {
 223                         if (synchronous)
 224                                 try {
 225                                         int exitValue = executorToUse.execute(commandLine,
 226                                                         environmentVariablesToUse);
 227                                         executeResultHandler.onProcessComplete(exitValue);
 228                                 } catch (ExecuteException e1) {
 229                                         if (e1.getExitValue() == Executor.INVALID_EXITVALUE) {
 230                                                 Thread.currentThread().interrupt();
 231                                                 return;
 232                                         }
 233                                         executeResultHandler.onProcessFailed(e1);
 234                                 }
 235                         else
 236                                 executorToUse.execute(commandLine, environmentVariablesToUse,
 237                                                 executeResultHandler);
 238                 } catch (SlcException e) {
 239                         throw e;
 240                 } catch (Exception e) {
 241                         throw new SlcException("Could not execute command " + commandLine,
 242                                         e);
 243                 } finally {
 244                         IOUtils.closeQuietly(stdOutWriter);
 245                         IOUtils.closeQuietly(stdErrWriter);
 246                         IOUtils.closeQuietly(stdInStream);
 247                         IOUtils.closeQuietly(stdInSink);
 248                 }
 249
 250         }
 251
 252         public synchronized String function() {
 253                 final StringBuffer buf = new StringBuffer("");
 254                 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
 255                         public void newLine(SystemCall systemCall, String line,
 256                                         Boolean isError) {
 257                                 if (!isError)
 258                                         buf.append(line);
 259                         }
 260                 };
 261                 addOutputListener(tempOutputListener);
 262                 run();
 263                 removeOutputListener(tempOutputListener);
 264                 return buf.toString();
 265         }
 266
 267         public String asCommand() {
 268                 return createCommandLine().toString();
 269         }
 270
 271         @Override
 272         public String toString() {
 273                 return asCommand();
 274         }
 275
 276         /**
 277          * Build a command line based on the properties. Can be overridden by
 278          * specific command wrappers.
 279          */
 280         protected CommandLine createCommandLine() {
 281                 // Check if an OS specific command overrides
 282                 String osName = System.getProperty("os.name");
 283                 List<Object> commandToUse = null;
 284                 if (osCommands.containsKey(osName))
 285                         commandToUse = osCommands.get(osName);
 286                 else
 287                         commandToUse = command;
 288                 String cmdToUse = null;
 289                 if (osCmds.containsKey(osName))
 290                         cmdToUse = osCmds.get(osName);
 291                 else
 292                         cmdToUse = cmd;
 293
 294                 CommandLine commandLine = null;
 295
 296                 // Which command definition to use
 297                 if (commandToUse == null && cmdToUse == null)
 298                         throw new SlcException("Please specify a command.");
 299                 else if (commandToUse != null && cmdToUse != null)
 300                         throw new SlcException(
 301                                         "Specify the command either as a line or as a list.");
 302                 else if (cmdToUse != null) {
 303                         commandLine = CommandLine.parse(cmdToUse);
 304                 } else if (commandToUse != null) {
 305                         if (commandToUse.size() == 0)
 306                                 throw new SlcException("Command line is empty.");
 307
 308                         commandLine = new CommandLine(commandToUse.get(0).toString());
 309
 310                         for (int i = 1; i < commandToUse.size(); i++) {
 311                                 if (log.isTraceEnabled())
 312                                         log.debug(commandToUse.get(i));
 313                                 commandLine.addArgument(commandToUse.get(i).toString());
 314                         }
 315                 } else {
 316                         // all cases covered previously
 317                         throw new UnsupportedException();
 318                 }
 319
 320                 if (generateScript != null) {
 321                         File scriptFile = new File(getExecDirToUse() + File.separator
 322                                         + generateScript);
 323                         try {
 324                                 FileUtils.writeStringToFile(scriptFile,
 325                                                 (osConsole != null ? osConsole + " " : "")
 326                                                                 + commandLine.toString());
 327                         } catch (IOException e) {
 328                                 throw new SlcException("Could not generate script "
 329                                                 + scriptFile, e);
 330                         }
 331                         commandLine = new CommandLine(scriptFile);
 332                 } else {
 333                         if (osConsole != null)
 334                                 commandLine = CommandLine.parse(osConsole + " "
 335                                                 + commandLine.toString());
 336                 }
 337
 338                 return commandLine;
 339         }
 340
 341         /**
 342          * Creates a {@link PumpStreamHandler} which redirects streams to the custom
 343          * logging mechanism.
 344          */
 345         protected ExecuteStreamHandler createExecuteStreamHandler(
 346                         final Writer stdOutWriter, final OutputStream stdOutputStream,
 347                         final Writer stdErrWriter, final InputStream stdInStream) {
 348
 349                 // Log writers
 350
 351                 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(
 352                                 stdOutputStream != null ? stdOutputStream
 353                                                 : new LogOutputStream() {
 354                                                         protected void processLine(String line, int level) {
 355                                                                 if (line != null && !line.trim().equals(""))
 356                                                                         logStdOut(line);
 357                                                                 if (stdOutWriter != null)
 358                                                                         appendLineToFile(stdOutWriter, line);
 359                                                         }
 360                                                 }, new LogOutputStream() {
 361                                         protected void processLine(String line, int level) {
 362                                                 if (line != null && !line.trim().equals(""))
 363                                                         logStdErr(line);
 364                                                 if (stdErrWriter != null)
 365                                                         appendLineToFile(stdErrWriter, line);
 366                                         }
 367                                 }, stdInStream) {
 368
 369                         @Override
 370                         public void stop() {
 371                                 // prevents the method to block when joining stdin
 372                                 if (stdInSink != null)
 373                                         IOUtils.closeQuietly(stdInSink);
 374
 375                                 super.stop();
 376                         }
 377                 };
 378                 return pumpStreamHandler;
 379         }
 380
 381         /** Creates the default {@link ExecuteResultHandler}. */
 382         protected ExecuteResultHandler createExecuteResultHandler(
 383                         final CommandLine commandLine) {
 384                 return new ExecuteResultHandler() {
 385
 386                         public void onProcessComplete(int exitValue) {
 387                                 String msg = "System call '" + commandLine
 388                                                 + "' properly completed.";
 389                                 if (log.isTraceEnabled())
 390                                         log.trace(msg);
 391                                 if (testResult != null) {
 392                                         forwardPath(testResult);
 393                                         testResult.addResultPart(new SimpleResultPart(
 394                                                         TestStatus.PASSED, msg));
 395                                 }
 396                         }
 397
 398                         public void onProcessFailed(ExecuteException e) {
 399                                 String msg = "System call '" + commandLine + "' failed.";
 400                                 if (testResult != null) {
 401                                         forwardPath(testResult);
 402                                         testResult.addResultPart(new SimpleResultPart(
 403                                                         TestStatus.ERROR, msg, e));
 404                                 } else {
 405                                         if (exceptionOnFailed)
 406                                                 throw new SlcException(msg, e);
 407                                         else
 408                                                 log.error(msg, e);
 409                                 }
 410                         }
 411                 };
 412         }
 413
 414         protected void forwardPath(TestResult testResult) {
 415                 // TODO: allocate a TreeSPath
 416         }
 417
 418         /**
 419          * Shortcut method getting the execDir to use
 420          */
 421         protected String getExecDirToUse() {
 422                 try {
 423                         if (execDir != null) {
 424                                 return execDir;
 425                         }
 426                         return System.getProperty("user.dir");
 427                 } catch (Exception e) {
 428                         throw new SlcException("Cannot find exec dir", e);
 429                 }
 430         }
 431
 432         protected void logStdOut(String line) {
 433                 for (SystemCallOutputListener outputListener : outputListeners)
 434                         outputListener.newLine(this, line, false);
 435                 log(stdOutLogLevel, line);
 436         }
 437
 438         protected void logStdErr(String line) {
 439                 for (SystemCallOutputListener outputListener : outputListeners)
 440                         outputListener.newLine(this, line, true);
 441                 log(stdErrLogLevel, line);
 442         }
 443
 444         /** Log from the underlying streams. */
 445         protected void log(String logLevel, String line) {
 446                 if ("ERROR".equals(logLevel))
 447                         log.error(line);
 448                 else if ("WARN".equals(logLevel))
 449                         log.warn(line);
 450                 else if ("INFO".equals(logLevel))
 451                         log.info(line);
 452                 else if ("DEBUG".equals(logLevel))
 453                         log.debug(line);
 454                 else if ("TRACE".equals(logLevel))
 455                         log.trace(line);
 456                 else if (LOG_STDOUT.equals(logLevel))
 457                         System.out.println(line);
 458                 else if ("System.err".equals(logLevel))
 459                         System.err.println(line);
 460                 else
 461                         throw new SlcException("Unknown log level " + logLevel);
 462         }
 463
 464         /** Append line to a log file. */
 465         protected void appendLineToFile(Writer writer, String line) {
 466                 try {
 467                         writer.append(line).append('\n');
 468                 } catch (IOException e) {
 469                         log.error("Cannot write to log file", e);
 470                 }
 471         }
 472
 473         /** Creates the writer for the output/err files. */
 474         protected Writer createWriter(Resource target, Boolean append) {
 475                 FileWriter writer = null;
 476                 try {
 477
 478                         final File file;
 479                         if (executionResources != null)
 480                                 file = new File(executionResources.getAsOsPath(target, true));
 481                         else
 482                                 file = target.getFile();
 483                         writer = new FileWriter(file, append);
 484                 } catch (IOException e) {
 485                         log.error("Cannot get file for " + target, e);
 486                         IOUtils.closeQuietly(writer);
 487                 }
 488                 return writer;
 489         }
 490
 491         /** Creates an outputstream for the output/err files. */
 492         protected OutputStream createOutputStream(Resource target) {
 493                 FileOutputStream out = null;
 494                 try {
 495
 496                         final File file;
 497                         if (executionResources != null)
 498                                 file = new File(executionResources.getAsOsPath(target, true));
 499                         else
 500                                 file = target.getFile();
 501                         out = new FileOutputStream(file, false);
 502                 } catch (IOException e) {
 503                         log.error("Cannot get file for " + target, e);
 504                         IOUtils.closeQuietly(out);
 505                 }
 506                 return out;
 507         }
 508
 509         /** Append the argument (for chaining) */
 510         public SystemCall arg(String arg) {
 511                 if (command == null)
 512                         command = new ArrayList<Object>();
 513                 command.add(arg);
 514                 return this;
 515         }
 516
 517         /** Append the argument (for chaining) */
 518         public SystemCall arg(String arg, String value) {
 519                 if (command == null)
 520                         command = new ArrayList<Object>();
 521                 command.add(arg);
 522                 command.add(value);
 523                 return this;
 524         }
 525
 526         /** */
 527         public void setCmd(String command) {
 528                 this.cmd = command;
 529         }
 530
 531         public void setCommand(List<Object> command) {
 532                 this.command = command;
 533         }
 534
 535         public void setExecDir(String execdir) {
 536                 this.execDir = execdir;
 537         }
 538
 539         public void setStdErrLogLevel(String stdErrLogLevel) {
 540                 this.stdErrLogLevel = stdErrLogLevel;
 541         }
 542
 543         public void setStdOutLogLevel(String stdOutLogLevel) {
 544                 this.stdOutLogLevel = stdOutLogLevel;
 545         }
 546
 547         public void setSynchronous(Boolean synchronous) {
 548                 this.synchronous = synchronous;
 549         }
 550
 551         public void setOsCommands(Map<String, List<Object>> osCommands) {
 552                 this.osCommands = osCommands;
 553         }
 554
 555         public void setOsCmds(Map<String, String> osCmds) {
 556                 this.osCmds = osCmds;
 557         }
 558
 559         public void setEnvironmentVariables(Map<String, String> environmentVariables) {
 560                 this.environmentVariables = environmentVariables;
 561         }
 562
 563         public void setWatchdogTimeout(Long watchdogTimeout) {
 564                 this.watchdogTimeout = watchdogTimeout;
 565         }
 566
 567         public void setStdOutFile(Resource stdOutFile) {
 568                 this.stdOutFile = stdOutFile;
 569         }
 570
 571         public void setStdErrFile(Resource stdErrFile) {
 572                 this.stdErrFile = stdErrFile;
 573         }
 574
 575         public void setStdInFile(Resource stdInFile) {
 576                 this.stdInFile = stdInFile;
 577         }
 578
 579         public void setTestResult(TestResult testResult) {
 580                 this.testResult = testResult;
 581         }
 582
 583         public void setLogCommand(Boolean logCommand) {
 584                 this.logCommand = logCommand;
 585         }
 586
 587         public void setRedirectStreams(Boolean redirectStreams) {
 588                 this.redirectStreams = redirectStreams;
 589         }
 590
 591         public void setExceptionOnFailed(Boolean exceptionOnFailed) {
 592                 this.exceptionOnFailed = exceptionOnFailed;
 593         }
 594
 595         public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
 596                 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
 597         }
 598
 599         public void setOsConsole(String osConsole) {
 600                 this.osConsole = osConsole;
 601         }
 602
 603         public void setGenerateScript(String generateScript) {
 604                 this.generateScript = generateScript;
 605         }
 606
 607         public void setExecutionResources(ExecutionResources executionResources) {
 608                 this.executionResources = executionResources;
 609         }
 610
 611         public void setRedirectStdOut(Boolean redirectStdOut) {
 612                 this.redirectStdOut = redirectStdOut;
 613         }
 614
 615         public void addOutputListener(SystemCallOutputListener outputListener) {
 616                 outputListeners.add(outputListener);
 617         }
 618
 619         public void removeOutputListener(SystemCallOutputListener outputListener) {
 620                 outputListeners.remove(outputListener);
 621         }
 622
 623         public void setOutputListeners(
 624                         List<SystemCallOutputListener> outputListeners) {
 625                 this.outputListeners = outputListeners;
 626         }
 627
 628         public void setExecutor(Executor executor) {
 629                 this.executor = executor;
 630         }
 631
 632         private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
 633
 634                 public void setProcessErrorStream(InputStream is) throws IOException {
 635                 }
 636
 637                 public void setProcessInputStream(OutputStream os) throws IOException {
 638                 }
 639
 640                 public void setProcessOutputStream(InputStream is) throws IOException {
 641                 }
 642
 643                 public void start() throws IOException {
 644                 }
 645
 646                 public void stop() {
 647                 }
 648
 649         }
 650 }