runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/tasks/SystemCall.java

   1 package org.argeo.slc.core.execution.tasks;
   2
   3 import java.io.File;
   4 import java.io.FileOutputStream;
   5 import java.io.FileWriter;
   6 import java.io.IOException;
   7 import java.io.InputStream;
   8 import java.io.OutputStream;
   9 import java.io.Writer;
  10 import java.util.ArrayList;
  11 import java.util.Collections;
  12 import java.util.HashMap;
  13 import java.util.List;
  14 import java.util.Map;
  15
  16 import org.apache.commons.exec.CommandLine;
  17 import org.apache.commons.exec.DefaultExecutor;
  18 import org.apache.commons.exec.ExecuteException;
  19 import org.apache.commons.exec.ExecuteResultHandler;
  20 import org.apache.commons.exec.ExecuteStreamHandler;
  21 import org.apache.commons.exec.ExecuteWatchdog;
  22 import org.apache.commons.exec.Executor;
  23 import org.apache.commons.exec.LogOutputStream;
  24 import org.apache.commons.exec.PumpStreamHandler;
  25 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
  26 import org.apache.commons.io.FileUtils;
  27 import org.apache.commons.io.IOUtils;
  28 import org.apache.commons.logging.Log;
  29 import org.apache.commons.logging.LogFactory;
  30 import org.argeo.slc.SlcException;
  31 import org.argeo.slc.UnsupportedException;
  32 import org.argeo.slc.core.execution.ExecutionResources;
  33 import org.argeo.slc.core.structure.tree.TreeSRelatedHelper;
  34 import org.argeo.slc.core.test.SimpleResultPart;
  35 import org.argeo.slc.test.TestResult;
  36 import org.argeo.slc.test.TestStatus;
  37 import org.springframework.core.io.Resource;
  38
  39 /** Execute an OS specific system call. */
  40 public class SystemCall extends TreeSRelatedHelper implements Runnable {
  41         public final static String LOG_STDOUT = "System.out";
  42
  43         private final Log log = LogFactory.getLog(getClass());
  44
  45         private String execDir;
  46
  47         private String cmd = null;
  48         private List<Object> command = null;
  49
  50         private Boolean synchronous = true;
  51
  52         private String stdErrLogLevel = "ERROR";
  53         private String stdOutLogLevel = "INFO";
  54
  55         private Resource stdOutFile = null;
  56         private Resource stdErrFile = null;
  57         private Resource stdInFile = null;
  58         private Boolean redirectStdOut = false;
  59
  60         private List<SystemCallOutputListener> outputListeners = Collections
  61                         .synchronizedList(new ArrayList<SystemCallOutputListener>());
  62
  63         private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
  64         private Map<String, String> osCmds = new HashMap<String, String>();
  65         private Map<String, String> environmentVariables = new HashMap<String, String>();
  66
  67         private Boolean logCommand = false;
  68         private Boolean redirectStreams = true;
  69         private Boolean exceptionOnFailed = true;
  70         private Boolean mergeEnvironmentVariables = true;
  71
  72         private String osConsole = null;
  73         private String generateScript = null;
  74
  75         private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
  76
  77         private TestResult testResult;
  78
  79         private ExecutionResources executionResources;
  80
  81         /** Empty constructor */
  82         public SystemCall() {
  83
  84         }
  85
  86         /**
  87          * Constructor based on the provided command list.
  88          *
  89          * @param command
  90          *            the command list
  91          */
  92         public SystemCall(List<Object> command) {
  93                 this.command = command;
  94         }
  95
  96         /**
  97          * Constructor based on the provided command.
  98          *
  99          * @param cmd
 100          *            the command. If the provided string contains no space a
 101          *            command list is initialized with the argument as first
 102          *            component (useful for chained construction)
 103          */
 104         public SystemCall(String cmd) {
 105                 if (cmd.indexOf(' ') < 0) {
 106                         command = new ArrayList<Object>();
 107                         command.add(cmd);
 108                 } else {
 109                         this.cmd = cmd;
 110                 }
 111         }
 112
 113         /** Executes the system call. */
 114         public void run() {
 115                 // Manage streams
 116                 Writer stdOutWriter = null;
 117                 OutputStream stdOutputStream = null;
 118                 Writer stdErrWriter = null;
 119                 InputStream stdInStream = null;
 120                 if (stdOutFile != null)
 121                         if (redirectStdOut)
 122                                 stdOutputStream = createOutputStream(stdOutFile);
 123                         else
 124                                 stdOutWriter = createWriter(stdOutFile, true);
 125
 126                 if (stdErrFile != null) {
 127                         stdErrWriter = createWriter(stdErrFile, true);
 128                 } else {
 129                         if (stdOutFile != null && !redirectStdOut)
 130                                 stdErrWriter = createWriter(stdOutFile, true);
 131                 }
 132
 133                 if (stdInFile != null)
 134                         try {
 135                                 stdInStream = stdInFile.getInputStream();
 136                         } catch (IOException e2) {
 137                                 throw new SlcException("Cannot open a stream for " + stdInFile,
 138                                                 e2);
 139                         }
 140
 141                 if (log.isTraceEnabled()) {
 142                         log.debug("os.name=" + System.getProperty("os.name"));
 143                         log.debug("os.arch=" + System.getProperty("os.arch"));
 144                         log.debug("os.version=" + System.getProperty("os.version"));
 145                 }
 146
 147                 // Execution directory
 148                 File dir = new File(getExecDirToUse());
 149                 if (!dir.exists())
 150                         dir.mkdirs();
 151
 152                 // Watchdog to check for lost processes
 153                 Executor executor = new DefaultExecutor();
 154                 executor.setWatchdog(new ExecuteWatchdog(watchdogTimeout));
 155
 156                 if (redirectStreams) {
 157                         // Redirect standard streams
 158                         executor.setStreamHandler(createExecuteStreamHandler(stdOutWriter,
 159                                         stdOutputStream, stdErrWriter, stdInStream));
 160                 } else {
 161                         // Dummy stream handler (otherwise pump is used)
 162                         executor.setStreamHandler(new DummyexecuteStreamHandler());
 163                 }
 164
 165                 executor.setProcessDestroyer(new ShutdownHookProcessDestroyer());
 166                 executor.setWorkingDirectory(dir);
 167
 168                 // Command line to use
 169                 final CommandLine commandLine = createCommandLine();
 170                 if (logCommand)
 171                         log.info("Execute command:\n" + commandLine
 172                                         + "\n in working directory: \n" + dir + "\n");
 173
 174                 // Env variables
 175                 Map<String, String> environmentVariablesToUse = null;
 176                 if (environmentVariables.size() > 0) {
 177                         environmentVariablesToUse = new HashMap<String, String>();
 178                         if (mergeEnvironmentVariables)
 179                                 environmentVariablesToUse.putAll(System.getenv());
 180                         environmentVariablesToUse.putAll(environmentVariables);
 181                 }
 182
 183                 // Execute
 184                 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
 185
 186                 //
 187                 // THE EXECUTION PROPER
 188                 //
 189                 try {
 190                         if (synchronous)
 191                                 try {
 192                                         int exitValue = executor.execute(commandLine,
 193                                                         environmentVariablesToUse);
 194                                         executeResultHandler.onProcessComplete(exitValue);
 195                                 } catch (ExecuteException e1) {
 196                                         executeResultHandler.onProcessFailed(e1);
 197                                 }
 198                         else
 199                                 executor.execute(commandLine, environmentVariablesToUse,
 200                                                 executeResultHandler);
 201                 } catch (SlcException e) {
 202                         throw e;
 203                 } catch (Exception e) {
 204                         throw new SlcException("Could not execute command " + commandLine,
 205                                         e);
 206                 } finally {
 207                         IOUtils.closeQuietly(stdOutWriter);
 208                         IOUtils.closeQuietly(stdErrWriter);
 209                         IOUtils.closeQuietly(stdInStream);
 210                 }
 211
 212         }
 213
 214         public synchronized String function() {
 215                 final StringBuffer buf = new StringBuffer("");
 216                 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
 217                         public void newLine(SystemCall systemCall, String line,
 218                                         Boolean isError) {
 219                                 if (!isError)
 220                                         buf.append(line);
 221                         }
 222                 };
 223                 addOutputListener(tempOutputListener);
 224                 run();
 225                 removeOutputListener(tempOutputListener);
 226                 return buf.toString();
 227         }
 228
 229         public String asCommand() {
 230                 return createCommandLine().toString();
 231         }
 232
 233         @Override
 234         public String toString() {
 235                 return asCommand();
 236         }
 237
 238         /**
 239          * Build a command line based on the properties. Can be overridden by
 240          * specific command wrappers.
 241          */
 242         protected CommandLine createCommandLine() {
 243                 // Check if an OS specific command overrides
 244                 String osName = System.getProperty("os.name");
 245                 List<Object> commandToUse = null;
 246                 if (osCommands.containsKey(osName))
 247                         commandToUse = osCommands.get(osName);
 248                 else
 249                         commandToUse = command;
 250                 String cmdToUse = null;
 251                 if (osCmds.containsKey(osName))
 252                         cmdToUse = osCmds.get(osName);
 253                 else
 254                         cmdToUse = cmd;
 255
 256                 CommandLine commandLine = null;
 257
 258                 // Which command definition to use
 259                 if (commandToUse == null && cmdToUse == null)
 260                         throw new SlcException("Please specify a command.");
 261                 else if (commandToUse != null && cmdToUse != null)
 262                         throw new SlcException(
 263                                         "Specify the command either as a line or as a list.");
 264                 else if (cmdToUse != null) {
 265                         commandLine = CommandLine.parse(cmdToUse);
 266                 } else if (commandToUse != null) {
 267                         if (commandToUse.size() == 0)
 268                                 throw new SlcException("Command line is empty.");
 269
 270                         commandLine = new CommandLine(commandToUse.get(0).toString());
 271
 272                         for (int i = 1; i < commandToUse.size(); i++) {
 273                                 if (log.isTraceEnabled())
 274                                         log.debug(commandToUse.get(i));
 275                                 commandLine.addArgument(commandToUse.get(i).toString());
 276                         }
 277                 } else {
 278                         // all cases covered previously
 279                         throw new UnsupportedException();
 280                 }
 281
 282                 if (generateScript != null) {
 283                         File scriptFile = new File(getExecDirToUse() + File.separator
 284                                         + generateScript);
 285                         try {
 286                                 FileUtils.writeStringToFile(scriptFile,
 287                                                 (osConsole != null ? osConsole + " " : "")
 288                                                                 + commandLine.toString());
 289                         } catch (IOException e) {
 290                                 throw new SlcException("Could not generate script "
 291                                                 + scriptFile, e);
 292                         }
 293                         commandLine = new CommandLine(scriptFile);
 294                 } else {
 295                         if (osConsole != null)
 296                                 commandLine = CommandLine.parse(osConsole + " "
 297                                                 + commandLine.toString());
 298                 }
 299
 300                 return commandLine;
 301         }
 302
 303         /**
 304          * Creates a {@link PumpStreamHandler} which redirects streams to the custom
 305          * logging mechanism.
 306          */
 307         protected ExecuteStreamHandler createExecuteStreamHandler(
 308                         final Writer stdOutWriter, final OutputStream stdOutputStream,
 309                         final Writer stdErrWriter, final InputStream stdInStream) {
 310
 311                 // Log writers
 312
 313                 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(
 314                                 stdOutputStream != null ? stdOutputStream
 315                                                 : new LogOutputStream() {
 316                                                         protected void processLine(String line, int level) {
 317                                                                 if (line != null && !line.trim().equals(""))
 318                                                                         logStdOut(line);
 319                                                                 if (stdOutWriter != null)
 320                                                                         appendLineToFile(stdOutWriter, line);
 321                                                         }
 322                                                 }, new LogOutputStream() {
 323                                         protected void processLine(String line, int level) {
 324                                                 if (line != null && !line.trim().equals(""))
 325                                                         logStdErr(line);
 326                                                 if (stdErrWriter != null)
 327                                                         appendLineToFile(stdErrWriter, line);
 328                                         }
 329                                 }, stdInStream);
 330                 return pumpStreamHandler;
 331         }
 332
 333         /** Creates the default {@link ExecuteResultHandler}. */
 334         protected ExecuteResultHandler createExecuteResultHandler(
 335                         final CommandLine commandLine) {
 336                 return new ExecuteResultHandler() {
 337
 338                         public void onProcessComplete(int exitValue) {
 339                                 String msg = "System call '" + commandLine
 340                                                 + "' properly completed.";
 341                                 if (log.isTraceEnabled())
 342                                         log.trace(msg);
 343                                 if (testResult != null) {
 344                                         forwardPath(testResult, null);
 345                                         testResult.addResultPart(new SimpleResultPart(
 346                                                         TestStatus.PASSED, msg));
 347                                 }
 348                         }
 349
 350                         public void onProcessFailed(ExecuteException e) {
 351                                 String msg = "System call '" + commandLine + "' failed.";
 352                                 if (testResult != null) {
 353                                         forwardPath(testResult, null);
 354                                         testResult.addResultPart(new SimpleResultPart(
 355                                                         TestStatus.ERROR, msg, e));
 356                                 } else {
 357                                         if (exceptionOnFailed)
 358                                                 throw new SlcException(msg, e);
 359                                         else
 360                                                 log.error(msg, e);
 361                                 }
 362                         }
 363                 };
 364         }
 365
 366         /**
 367          * Shortcut method getting the execDir to use
 368          */
 369         protected String getExecDirToUse() {
 370                 try {
 371                         File dir = null;
 372                         if (execDir != null) {
 373                                 // Replace '/' by local file separator, for portability
 374                                 execDir.replace('/', File.separatorChar);
 375                                 dir = new File(execDir).getCanonicalFile();
 376                         }
 377
 378                         if (dir == null)
 379                                 return System.getProperty("user.dir");
 380                         else
 381                                 return dir.getPath();
 382                 } catch (Exception e) {
 383                         throw new SlcException("Cannot find exec dir", e);
 384                 }
 385         }
 386
 387         protected void logStdOut(String line) {
 388                 for (SystemCallOutputListener outputListener : outputListeners)
 389                         outputListener.newLine(this, line, false);
 390                 log(stdOutLogLevel, line);
 391         }
 392
 393         protected void logStdErr(String line) {
 394                 for (SystemCallOutputListener outputListener : outputListeners)
 395                         outputListener.newLine(this, line, true);
 396                 log(stdErrLogLevel, line);
 397         }
 398
 399         /** Log from the underlying streams. */
 400         protected void log(String logLevel, String line) {
 401                 if ("ERROR".equals(logLevel))
 402                         log.error(line);
 403                 else if ("WARN".equals(logLevel))
 404                         log.warn(line);
 405                 else if ("INFO".equals(logLevel))
 406                         log.info(line);
 407                 else if ("DEBUG".equals(logLevel))
 408                         log.debug(line);
 409                 else if ("TRACE".equals(logLevel))
 410                         log.trace(line);
 411                 else if (LOG_STDOUT.equals(logLevel))
 412                         System.out.println(line);
 413                 else if ("System.err".equals(logLevel))
 414                         System.err.println(line);
 415                 else
 416                         throw new SlcException("Unknown log level " + logLevel);
 417         }
 418
 419         /** Append line to a log file. */
 420         protected void appendLineToFile(Writer writer, String line) {
 421                 try {
 422                         writer.append(line).append('\n');
 423                 } catch (IOException e) {
 424                         log.error("Cannot write to log file", e);
 425                 }
 426         }
 427
 428         /** Creates the writer for the output/err files. */
 429         protected Writer createWriter(Resource target, Boolean append) {
 430                 FileWriter writer = null;
 431                 try {
 432
 433                         final File file;
 434                         if (executionResources != null)
 435                                 file = new File(executionResources.getAsOsPath(target, true));
 436                         else
 437                                 file = target.getFile();
 438                         writer = new FileWriter(file, append);
 439                 } catch (IOException e) {
 440                         log.error("Cannot get file for " + target, e);
 441                         IOUtils.closeQuietly(writer);
 442                 }
 443                 return writer;
 444         }
 445
 446         /** Creates an outputstream for the output/err files. */
 447         protected OutputStream createOutputStream(Resource target) {
 448                 FileOutputStream out = null;
 449                 try {
 450
 451                         final File file;
 452                         if (executionResources != null)
 453                                 file = new File(executionResources.getAsOsPath(target, true));
 454                         else
 455                                 file = target.getFile();
 456                         out = new FileOutputStream(file, false);
 457                 } catch (IOException e) {
 458                         log.error("Cannot get file for " + target, e);
 459                         IOUtils.closeQuietly(out);
 460                 }
 461                 return out;
 462         }
 463
 464         /** Append the argument (for chaining) */
 465         public SystemCall arg(String arg) {
 466                 command.add(arg);
 467                 return this;
 468         }
 469
 470         /** Append the argument (for chaining) */
 471         public SystemCall arg(String arg, String value) {
 472                 command.add(arg);
 473                 command.add(value);
 474                 return this;
 475         }
 476
 477         /** */
 478         public void setCmd(String command) {
 479                 this.cmd = command;
 480         }
 481
 482         public void setCommand(List<Object> command) {
 483                 this.command = command;
 484         }
 485
 486         public void setExecDir(String execdir) {
 487                 this.execDir = execdir;
 488         }
 489
 490         public void setStdErrLogLevel(String stdErrLogLevel) {
 491                 this.stdErrLogLevel = stdErrLogLevel;
 492         }
 493
 494         public void setStdOutLogLevel(String stdOutLogLevel) {
 495                 this.stdOutLogLevel = stdOutLogLevel;
 496         }
 497
 498         public void setSynchronous(Boolean synchronous) {
 499                 this.synchronous = synchronous;
 500         }
 501
 502         public void setOsCommands(Map<String, List<Object>> osCommands) {
 503                 this.osCommands = osCommands;
 504         }
 505
 506         public void setOsCmds(Map<String, String> osCmds) {
 507                 this.osCmds = osCmds;
 508         }
 509
 510         public void setEnvironmentVariables(Map<String, String> environmentVariables) {
 511                 this.environmentVariables = environmentVariables;
 512         }
 513
 514         public void setWatchdogTimeout(Long watchdogTimeout) {
 515                 this.watchdogTimeout = watchdogTimeout;
 516         }
 517
 518         public void setStdOutFile(Resource stdOutFile) {
 519                 this.stdOutFile = stdOutFile;
 520         }
 521
 522         public void setStdErrFile(Resource stdErrFile) {
 523                 this.stdErrFile = stdErrFile;
 524         }
 525
 526         public void setStdInFile(Resource stdInFile) {
 527                 this.stdInFile = stdInFile;
 528         }
 529
 530         public void setTestResult(TestResult testResult) {
 531                 this.testResult = testResult;
 532         }
 533
 534         public void setLogCommand(Boolean logCommand) {
 535                 this.logCommand = logCommand;
 536         }
 537
 538         public void setRedirectStreams(Boolean redirectStreams) {
 539                 this.redirectStreams = redirectStreams;
 540         }
 541
 542         public void setExceptionOnFailed(Boolean exceptionOnFailed) {
 543                 this.exceptionOnFailed = exceptionOnFailed;
 544         }
 545
 546         public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
 547                 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
 548         }
 549
 550         public void setOsConsole(String osConsole) {
 551                 this.osConsole = osConsole;
 552         }
 553
 554         public void setGenerateScript(String generateScript) {
 555                 this.generateScript = generateScript;
 556         }
 557
 558         public void setExecutionResources(ExecutionResources executionResources) {
 559                 this.executionResources = executionResources;
 560         }
 561
 562         public void setRedirectStdOut(Boolean redirectStdOut) {
 563                 this.redirectStdOut = redirectStdOut;
 564         }
 565
 566         public void addOutputListener(SystemCallOutputListener outputListener) {
 567                 outputListeners.add(outputListener);
 568         }
 569
 570         public void removeOutputListener(SystemCallOutputListener outputListener) {
 571                 outputListeners.remove(outputListener);
 572         }
 573
 574         public void setOutputListeners(
 575                         List<SystemCallOutputListener> outputListeners) {
 576                 this.outputListeners = outputListeners;
 577         }
 578
 579         private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
 580
 581                 public void setProcessErrorStream(InputStream is) throws IOException {
 582                 }
 583
 584                 public void setProcessInputStream(OutputStream os) throws IOException {
 585                 }
 586
 587                 public void setProcessOutputStream(InputStream is) throws IOException {
 588                 }
 589
 590                 public void start() throws IOException {
 591                 }
 592
 593                 public void stop() {
 594                 }
 595
 596         }
 597
 598 }