runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/tasks/SystemCall.java

   1 /*
   2  * Copyright (C) 2007-2012 Argeo GmbH
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.argeo.slc.core.execution.tasks;
  17
  18 import java.io.File;
  19 import java.io.FileOutputStream;
  20 import java.io.FileWriter;
  21 import java.io.IOException;
  22 import java.io.InputStream;
  23 import java.io.OutputStream;
  24 import java.io.PipedInputStream;
  25 import java.io.PipedOutputStream;
  26 import java.io.Writer;
  27 import java.util.ArrayList;
  28 import java.util.Collections;
  29 import java.util.HashMap;
  30 import java.util.List;
  31 import java.util.Map;
  32
  33 import org.apache.commons.exec.CommandLine;
  34 import org.apache.commons.exec.DefaultExecutor;
  35 import org.apache.commons.exec.ExecuteException;
  36 import org.apache.commons.exec.ExecuteResultHandler;
  37 import org.apache.commons.exec.ExecuteStreamHandler;
  38 import org.apache.commons.exec.ExecuteWatchdog;
  39 import org.apache.commons.exec.Executor;
  40 import org.apache.commons.exec.LogOutputStream;
  41 import org.apache.commons.exec.PumpStreamHandler;
  42 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
  43 import org.apache.commons.io.FileUtils;
  44 import org.apache.commons.io.IOUtils;
  45 import org.apache.commons.logging.Log;
  46 import org.apache.commons.logging.LogFactory;
  47 import org.argeo.slc.SlcException;
  48 import org.argeo.slc.UnsupportedException;
  49 import org.argeo.slc.core.execution.ExecutionResources;
  50 import org.argeo.slc.core.test.SimpleResultPart;
  51 import org.argeo.slc.test.TestResult;
  52 import org.argeo.slc.test.TestStatus;
  53 import org.springframework.core.io.Resource;
  54
  55 /** Execute an OS specific system call. */
  56 public class SystemCall implements Runnable {
  57         public final static String LOG_STDOUT = "System.out";
  58
  59         private final Log log = LogFactory.getLog(getClass());
  60
  61         private String execDir;
  62
  63         private String cmd = null;
  64         private List<Object> command = null;
  65
  66         private Executor executor = new DefaultExecutor();
  67         private Boolean synchronous = true;
  68
  69         private String stdErrLogLevel = "ERROR";
  70         private String stdOutLogLevel = "INFO";
  71
  72         private Resource stdOutFile = null;
  73         private Resource stdErrFile = null;
  74
  75         private Resource stdInFile = null;
  76         /**
  77          * If no {@link #stdInFile} provided, writing to this stream will write to
  78          * the stdin of the process.
  79          */
  80         private OutputStream stdInSink = null;
  81
  82         private Boolean redirectStdOut = false;
  83
  84         private List<SystemCallOutputListener> outputListeners = Collections
  85                         .synchronizedList(new ArrayList<SystemCallOutputListener>());
  86
  87         private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
  88         private Map<String, String> osCmds = new HashMap<String, String>();
  89         private Map<String, String> environmentVariables = new HashMap<String, String>();
  90
  91         private Boolean logCommand = false;
  92         private Boolean redirectStreams = true;
  93         private Boolean exceptionOnFailed = true;
  94         private Boolean mergeEnvironmentVariables = true;
  95
  96         private String osConsole = null;
  97         private String generateScript = null;
  98
  99         /** 24 hours */
 100         private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
 101
 102         private TestResult testResult;
 103
 104         private ExecutionResources executionResources;
 105
 106         /** Empty constructor */
 107         public SystemCall() {
 108
 109         }
 110
 111         /**
 112          * Constructor based on the provided command list.
 113          *
 114          * @param command
 115          *            the command list
 116          */
 117         public SystemCall(List<Object> command) {
 118                 this.command = command;
 119         }
 120
 121         /**
 122          * Constructor based on the provided command.
 123          *
 124          * @param cmd
 125          *            the command. If the provided string contains no space a
 126          *            command list is initialized with the argument as first
 127          *            component (useful for chained construction)
 128          */
 129         public SystemCall(String cmd) {
 130                 if (cmd.indexOf(' ') < 0) {
 131                         command = new ArrayList<Object>();
 132                         command.add(cmd);
 133                 } else {
 134                         this.cmd = cmd;
 135                 }
 136         }
 137
 138         /** Executes the system call. */
 139         public void run() {
 140                 // Manage streams
 141                 Writer stdOutWriter = null;
 142                 OutputStream stdOutputStream = null;
 143                 Writer stdErrWriter = null;
 144                 InputStream stdInStream = null;
 145                 if (stdOutFile != null)
 146                         if (redirectStdOut)
 147                                 stdOutputStream = createOutputStream(stdOutFile);
 148                         else
 149                                 stdOutWriter = createWriter(stdOutFile, true);
 150
 151                 if (stdErrFile != null) {
 152                         stdErrWriter = createWriter(stdErrFile, true);
 153                 } else {
 154                         if (stdOutFile != null && !redirectStdOut)
 155                                 stdErrWriter = createWriter(stdOutFile, true);
 156                 }
 157
 158                 try {
 159                         if (stdInFile != null)
 160                                 stdInStream = stdInFile.getInputStream();
 161                         else {
 162                                 stdInStream = new PipedInputStream();
 163                                 stdInSink = new PipedOutputStream(
 164                                                 (PipedInputStream) stdInStream);
 165                         }
 166                 } catch (IOException e2) {
 167                         throw new SlcException("Cannot open a stream for " + stdInFile, e2);
 168                 }
 169
 170                 if (log.isTraceEnabled()) {
 171                         log.debug("os.name=" + System.getProperty("os.name"));
 172                         log.debug("os.arch=" + System.getProperty("os.arch"));
 173                         log.debug("os.version=" + System.getProperty("os.version"));
 174                 }
 175
 176                 // Execution directory
 177                 File dir = new File(getExecDirToUse());
 178                 // if (!dir.exists())
 179                 // dir.mkdirs();
 180
 181                 // Watchdog to check for lost processes
 182                 Executor executorToUse;
 183                 if (executor != null)
 184                         executorToUse = executor;
 185                 else
 186                         executorToUse = new DefaultExecutor();
 187                 executorToUse.setWatchdog(new ExecuteWatchdog(watchdogTimeout));
 188
 189                 if (redirectStreams) {
 190                         // Redirect standard streams
 191                         executorToUse.setStreamHandler(createExecuteStreamHandler(
 192                                         stdOutWriter, stdOutputStream, stdErrWriter, stdInStream));
 193                 } else {
 194                         // Dummy stream handler (otherwise pump is used)
 195                         executorToUse.setStreamHandler(new DummyexecuteStreamHandler());
 196                 }
 197
 198                 executorToUse.setProcessDestroyer(new ShutdownHookProcessDestroyer());
 199                 executorToUse.setWorkingDirectory(dir);
 200
 201                 // Command line to use
 202                 final CommandLine commandLine = createCommandLine();
 203                 if (logCommand)
 204                         log.info("Execute command:\n" + commandLine
 205                                         + "\n in working directory: \n" + dir + "\n");
 206
 207                 // Env variables
 208                 Map<String, String> environmentVariablesToUse = null;
 209                 environmentVariablesToUse = new HashMap<String, String>();
 210                 if (mergeEnvironmentVariables)
 211                         environmentVariablesToUse.putAll(System.getenv());
 212                 if (environmentVariables.size() > 0)
 213                         environmentVariablesToUse.putAll(environmentVariables);
 214
 215                 // Execute
 216                 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
 217
 218                 //
 219                 // THE EXECUTION PROPER
 220                 //
 221                 try {
 222                         if (synchronous)
 223                                 try {
 224                                         int exitValue = executorToUse.execute(commandLine,
 225                                                         environmentVariablesToUse);
 226                                         executeResultHandler.onProcessComplete(exitValue);
 227                                 } catch (ExecuteException e1) {
 228                                         if (e1.getExitValue() == Executor.INVALID_EXITVALUE) {
 229                                                 Thread.currentThread().interrupt();
 230                                                 return;
 231                                         }
 232                                         // Sleep 1s in order to make sure error logs are flushed
 233                                         Thread.sleep(1000);
 234                                         executeResultHandler.onProcessFailed(e1);
 235                                 }
 236                         else
 237                                 executorToUse.execute(commandLine, environmentVariablesToUse,
 238                                                 executeResultHandler);
 239                 } catch (SlcException e) {
 240                         throw e;
 241                 } catch (Exception e) {
 242                         throw new SlcException("Could not execute command " + commandLine,
 243                                         e);
 244                 } finally {
 245                         IOUtils.closeQuietly(stdOutWriter);
 246                         IOUtils.closeQuietly(stdErrWriter);
 247                         IOUtils.closeQuietly(stdInStream);
 248                         IOUtils.closeQuietly(stdInSink);
 249                 }
 250
 251         }
 252
 253         public synchronized String function() {
 254                 final StringBuffer buf = new StringBuffer("");
 255                 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
 256                         public void newLine(SystemCall systemCall, String line,
 257                                         Boolean isError) {
 258                                 if (!isError)
 259                                         buf.append(line);
 260                         }
 261                 };
 262                 addOutputListener(tempOutputListener);
 263                 run();
 264                 removeOutputListener(tempOutputListener);
 265                 return buf.toString();
 266         }
 267
 268         public String asCommand() {
 269                 return createCommandLine().toString();
 270         }
 271
 272         @Override
 273         public String toString() {
 274                 return asCommand();
 275         }
 276
 277         /**
 278          * Build a command line based on the properties. Can be overridden by
 279          * specific command wrappers.
 280          */
 281         protected CommandLine createCommandLine() {
 282                 // Check if an OS specific command overrides
 283                 String osName = System.getProperty("os.name");
 284                 List<Object> commandToUse = null;
 285                 if (osCommands.containsKey(osName))
 286                         commandToUse = osCommands.get(osName);
 287                 else
 288                         commandToUse = command;
 289                 String cmdToUse = null;
 290                 if (osCmds.containsKey(osName))
 291                         cmdToUse = osCmds.get(osName);
 292                 else
 293                         cmdToUse = cmd;
 294
 295                 CommandLine commandLine = null;
 296
 297                 // Which command definition to use
 298                 if (commandToUse == null && cmdToUse == null)
 299                         throw new SlcException("Please specify a command.");
 300                 else if (commandToUse != null && cmdToUse != null)
 301                         throw new SlcException(
 302                                         "Specify the command either as a line or as a list.");
 303                 else if (cmdToUse != null) {
 304                         commandLine = CommandLine.parse(cmdToUse);
 305                 } else if (commandToUse != null) {
 306                         if (commandToUse.size() == 0)
 307                                 throw new SlcException("Command line is empty.");
 308
 309                         commandLine = new CommandLine(commandToUse.get(0).toString());
 310
 311                         for (int i = 1; i < commandToUse.size(); i++) {
 312                                 if (log.isTraceEnabled())
 313                                         log.debug(commandToUse.get(i));
 314                                 commandLine.addArgument(commandToUse.get(i).toString());
 315                         }
 316                 } else {
 317                         // all cases covered previously
 318                         throw new UnsupportedException();
 319                 }
 320
 321                 if (generateScript != null) {
 322                         File scriptFile = new File(getExecDirToUse() + File.separator
 323                                         + generateScript);
 324                         try {
 325                                 FileUtils.writeStringToFile(scriptFile,
 326                                                 (osConsole != null ? osConsole + " " : "")
 327                                                                 + commandLine.toString());
 328                         } catch (IOException e) {
 329                                 throw new SlcException("Could not generate script "
 330                                                 + scriptFile, e);
 331                         }
 332                         commandLine = new CommandLine(scriptFile);
 333                 } else {
 334                         if (osConsole != null)
 335                                 commandLine = CommandLine.parse(osConsole + " "
 336                                                 + commandLine.toString());
 337                 }
 338
 339                 return commandLine;
 340         }
 341
 342         /**
 343          * Creates a {@link PumpStreamHandler} which redirects streams to the custom
 344          * logging mechanism.
 345          */
 346         protected ExecuteStreamHandler createExecuteStreamHandler(
 347                         final Writer stdOutWriter, final OutputStream stdOutputStream,
 348                         final Writer stdErrWriter, final InputStream stdInStream) {
 349
 350                 // Log writers
 351
 352                 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(
 353                                 stdOutputStream != null ? stdOutputStream
 354                                                 : new LogOutputStream() {
 355                                                         protected void processLine(String line, int level) {
 356                                                                 if (line != null && !line.trim().equals(""))
 357                                                                         logStdOut(line);
 358                                                                 if (stdOutWriter != null)
 359                                                                         appendLineToFile(stdOutWriter, line);
 360                                                         }
 361                                                 }, new LogOutputStream() {
 362                                         protected void processLine(String line, int level) {
 363                                                 if (line != null && !line.trim().equals(""))
 364                                                         logStdErr(line);
 365                                                 if (stdErrWriter != null)
 366                                                         appendLineToFile(stdErrWriter, line);
 367                                         }
 368                                 }, stdInStream) {
 369
 370                         @Override
 371                         public void stop() {
 372                                 // prevents the method to block when joining stdin
 373                                 if (stdInSink != null)
 374                                         IOUtils.closeQuietly(stdInSink);
 375
 376                                 super.stop();
 377                         }
 378                 };
 379                 return pumpStreamHandler;
 380         }
 381
 382         /** Creates the default {@link ExecuteResultHandler}. */
 383         protected ExecuteResultHandler createExecuteResultHandler(
 384                         final CommandLine commandLine) {
 385                 return new ExecuteResultHandler() {
 386
 387                         public void onProcessComplete(int exitValue) {
 388                                 String msg = "System call '" + commandLine
 389                                                 + "' properly completed.";
 390                                 if (log.isTraceEnabled())
 391                                         log.trace(msg);
 392                                 if (testResult != null) {
 393                                         forwardPath(testResult);
 394                                         testResult.addResultPart(new SimpleResultPart(
 395                                                         TestStatus.PASSED, msg));
 396                                 }
 397                         }
 398
 399                         public void onProcessFailed(ExecuteException e) {
 400
 401                                 String msg = "System call '" + commandLine + "' failed.";
 402                                 if (testResult != null) {
 403                                         forwardPath(testResult);
 404                                         testResult.addResultPart(new SimpleResultPart(
 405                                                         TestStatus.ERROR, msg, e));
 406                                 } else {
 407                                         if (exceptionOnFailed)
 408                                                 throw new SlcException(msg, e);
 409                                         else
 410                                                 log.error(msg, e);
 411                                 }
 412                         }
 413                 };
 414         }
 415
 416         protected void forwardPath(TestResult testResult) {
 417                 // TODO: allocate a TreeSPath
 418         }
 419
 420         /**
 421          * Shortcut method getting the execDir to use
 422          */
 423         protected String getExecDirToUse() {
 424                 try {
 425                         if (execDir != null) {
 426                                 return execDir;
 427                         }
 428                         return System.getProperty("user.dir");
 429                 } catch (Exception e) {
 430                         throw new SlcException("Cannot find exec dir", e);
 431                 }
 432         }
 433
 434         protected void logStdOut(String line) {
 435                 for (SystemCallOutputListener outputListener : outputListeners)
 436                         outputListener.newLine(this, line, false);
 437                 log(stdOutLogLevel, line);
 438         }
 439
 440         protected void logStdErr(String line) {
 441                 for (SystemCallOutputListener outputListener : outputListeners)
 442                         outputListener.newLine(this, line, true);
 443                 log(stdErrLogLevel, line);
 444         }
 445
 446         /** Log from the underlying streams. */
 447         protected void log(String logLevel, String line) {
 448                 if ("ERROR".equals(logLevel))
 449                         log.error(line);
 450                 else if ("WARN".equals(logLevel))
 451                         log.warn(line);
 452                 else if ("INFO".equals(logLevel))
 453                         log.info(line);
 454                 else if ("DEBUG".equals(logLevel))
 455                         log.debug(line);
 456                 else if ("TRACE".equals(logLevel))
 457                         log.trace(line);
 458                 else if (LOG_STDOUT.equals(logLevel))
 459                         System.out.println(line);
 460                 else if ("System.err".equals(logLevel))
 461                         System.err.println(line);
 462                 else
 463                         throw new SlcException("Unknown log level " + logLevel);
 464         }
 465
 466         /** Append line to a log file. */
 467         protected void appendLineToFile(Writer writer, String line) {
 468                 try {
 469                         writer.append(line).append('\n');
 470                 } catch (IOException e) {
 471                         log.error("Cannot write to log file", e);
 472                 }
 473         }
 474
 475         /** Creates the writer for the output/err files. */
 476         protected Writer createWriter(Resource target, Boolean append) {
 477                 FileWriter writer = null;
 478                 try {
 479
 480                         final File file;
 481                         if (executionResources != null)
 482                                 file = new File(executionResources.getAsOsPath(target, true));
 483                         else
 484                                 file = target.getFile();
 485                         writer = new FileWriter(file, append);
 486                 } catch (IOException e) {
 487                         log.error("Cannot get file for " + target, e);
 488                         IOUtils.closeQuietly(writer);
 489                 }
 490                 return writer;
 491         }
 492
 493         /** Creates an outputstream for the output/err files. */
 494         protected OutputStream createOutputStream(Resource target) {
 495                 FileOutputStream out = null;
 496                 try {
 497
 498                         final File file;
 499                         if (executionResources != null)
 500                                 file = new File(executionResources.getAsOsPath(target, true));
 501                         else
 502                                 file = target.getFile();
 503                         out = new FileOutputStream(file, false);
 504                 } catch (IOException e) {
 505                         log.error("Cannot get file for " + target, e);
 506                         IOUtils.closeQuietly(out);
 507                 }
 508                 return out;
 509         }
 510
 511         /** Append the argument (for chaining) */
 512         public SystemCall arg(String arg) {
 513                 if (command == null)
 514                         command = new ArrayList<Object>();
 515                 command.add(arg);
 516                 return this;
 517         }
 518
 519         /** Append the argument (for chaining) */
 520         public SystemCall arg(String arg, String value) {
 521                 if (command == null)
 522                         command = new ArrayList<Object>();
 523                 command.add(arg);
 524                 command.add(value);
 525                 return this;
 526         }
 527
 528         /** */
 529         public void setCmd(String command) {
 530                 this.cmd = command;
 531         }
 532
 533         public void setCommand(List<Object> command) {
 534                 this.command = command;
 535         }
 536
 537         public void setExecDir(String execdir) {
 538                 this.execDir = execdir;
 539         }
 540
 541         public void setStdErrLogLevel(String stdErrLogLevel) {
 542                 this.stdErrLogLevel = stdErrLogLevel;
 543         }
 544
 545         public void setStdOutLogLevel(String stdOutLogLevel) {
 546                 this.stdOutLogLevel = stdOutLogLevel;
 547         }
 548
 549         public void setSynchronous(Boolean synchronous) {
 550                 this.synchronous = synchronous;
 551         }
 552
 553         public void setOsCommands(Map<String, List<Object>> osCommands) {
 554                 this.osCommands = osCommands;
 555         }
 556
 557         public void setOsCmds(Map<String, String> osCmds) {
 558                 this.osCmds = osCmds;
 559         }
 560
 561         public void setEnvironmentVariables(Map<String, String> environmentVariables) {
 562                 this.environmentVariables = environmentVariables;
 563         }
 564
 565         public void setWatchdogTimeout(Long watchdogTimeout) {
 566                 this.watchdogTimeout = watchdogTimeout;
 567         }
 568
 569         public void setStdOutFile(Resource stdOutFile) {
 570                 this.stdOutFile = stdOutFile;
 571         }
 572
 573         public void setStdErrFile(Resource stdErrFile) {
 574                 this.stdErrFile = stdErrFile;
 575         }
 576
 577         public void setStdInFile(Resource stdInFile) {
 578                 this.stdInFile = stdInFile;
 579         }
 580
 581         public void setTestResult(TestResult testResult) {
 582                 this.testResult = testResult;
 583         }
 584
 585         public void setLogCommand(Boolean logCommand) {
 586                 this.logCommand = logCommand;
 587         }
 588
 589         public void setRedirectStreams(Boolean redirectStreams) {
 590                 this.redirectStreams = redirectStreams;
 591         }
 592
 593         public void setExceptionOnFailed(Boolean exceptionOnFailed) {
 594                 this.exceptionOnFailed = exceptionOnFailed;
 595         }
 596
 597         public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
 598                 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
 599         }
 600
 601         public void setOsConsole(String osConsole) {
 602                 this.osConsole = osConsole;
 603         }
 604
 605         public void setGenerateScript(String generateScript) {
 606                 this.generateScript = generateScript;
 607         }
 608
 609         public void setExecutionResources(ExecutionResources executionResources) {
 610                 this.executionResources = executionResources;
 611         }
 612
 613         public void setRedirectStdOut(Boolean redirectStdOut) {
 614                 this.redirectStdOut = redirectStdOut;
 615         }
 616
 617         public void addOutputListener(SystemCallOutputListener outputListener) {
 618                 outputListeners.add(outputListener);
 619         }
 620
 621         public void removeOutputListener(SystemCallOutputListener outputListener) {
 622                 outputListeners.remove(outputListener);
 623         }
 624
 625         public void setOutputListeners(
 626                         List<SystemCallOutputListener> outputListeners) {
 627                 this.outputListeners = outputListeners;
 628         }
 629
 630         public void setExecutor(Executor executor) {
 631                 this.executor = executor;
 632         }
 633
 634         private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
 635
 636                 public void setProcessErrorStream(InputStream is) throws IOException {
 637                 }
 638
 639                 public void setProcessInputStream(OutputStream os) throws IOException {
 640                 }
 641
 642                 public void setProcessOutputStream(InputStream is) throws IOException {
 643                 }
 644
 645                 public void start() throws IOException {
 646                 }
 647
 648                 public void stop() {
 649                 }
 650
 651         }
 652 }