runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/tasks/SystemCall.java

   1 /*
   2  * Copyright (C) 2010 Mathieu Baudier <mbaudier@argeo.org>
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *         http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 package org.argeo.slc.core.execution.tasks;
  18
  19 import java.io.File;
  20 import java.io.FileOutputStream;
  21 import java.io.FileWriter;
  22 import java.io.IOException;
  23 import java.io.InputStream;
  24 import java.io.OutputStream;
  25 import java.io.Writer;
  26 import java.util.ArrayList;
  27 import java.util.Collections;
  28 import java.util.HashMap;
  29 import java.util.List;
  30 import java.util.Map;
  31
  32 import org.apache.commons.exec.CommandLine;
  33 import org.apache.commons.exec.DefaultExecutor;
  34 import org.apache.commons.exec.ExecuteException;
  35 import org.apache.commons.exec.ExecuteResultHandler;
  36 import org.apache.commons.exec.ExecuteStreamHandler;
  37 import org.apache.commons.exec.ExecuteWatchdog;
  38 import org.apache.commons.exec.Executor;
  39 import org.apache.commons.exec.LogOutputStream;
  40 import org.apache.commons.exec.PumpStreamHandler;
  41 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
  42 import org.apache.commons.io.FileUtils;
  43 import org.apache.commons.io.IOUtils;
  44 import org.apache.commons.logging.Log;
  45 import org.apache.commons.logging.LogFactory;
  46 import org.argeo.slc.SlcException;
  47 import org.argeo.slc.UnsupportedException;
  48 import org.argeo.slc.core.execution.ExecutionResources;
  49 import org.argeo.slc.core.test.SimpleResultPart;
  50 import org.argeo.slc.test.TestResult;
  51 import org.argeo.slc.test.TestStatus;
  52 import org.springframework.core.io.Resource;
  53
  54 /** Execute an OS specific system call. */
  55 public class SystemCall implements Runnable {
  56         public final static String LOG_STDOUT = "System.out";
  57
  58         private final Log log = LogFactory.getLog(getClass());
  59
  60         private String execDir;
  61
  62         private String cmd = null;
  63         private List<Object> command = null;
  64
  65         private Boolean synchronous = true;
  66
  67         private String stdErrLogLevel = "ERROR";
  68         private String stdOutLogLevel = "INFO";
  69
  70         private Resource stdOutFile = null;
  71         private Resource stdErrFile = null;
  72         private Resource stdInFile = null;
  73         private Boolean redirectStdOut = false;
  74
  75         private List<SystemCallOutputListener> outputListeners = Collections
  76                         .synchronizedList(new ArrayList<SystemCallOutputListener>());
  77
  78         private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
  79         private Map<String, String> osCmds = new HashMap<String, String>();
  80         private Map<String, String> environmentVariables = new HashMap<String, String>();
  81
  82         private Boolean logCommand = false;
  83         private Boolean redirectStreams = true;
  84         private Boolean exceptionOnFailed = true;
  85         private Boolean mergeEnvironmentVariables = true;
  86
  87         private String osConsole = null;
  88         private String generateScript = null;
  89
  90         private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
  91
  92         private TestResult testResult;
  93
  94         private ExecutionResources executionResources;
  95
  96         /** Empty constructor */
  97         public SystemCall() {
  98
  99         }
 100
 101         /**
 102          * Constructor based on the provided command list.
 103          *
 104          * @param command
 105          *            the command list
 106          */
 107         public SystemCall(List<Object> command) {
 108                 this.command = command;
 109         }
 110
 111         /**
 112          * Constructor based on the provided command.
 113          *
 114          * @param cmd
 115          *            the command. If the provided string contains no space a
 116          *            command list is initialized with the argument as first
 117          *            component (useful for chained construction)
 118          */
 119         public SystemCall(String cmd) {
 120                 if (cmd.indexOf(' ') < 0) {
 121                         command = new ArrayList<Object>();
 122                         command.add(cmd);
 123                 } else {
 124                         this.cmd = cmd;
 125                 }
 126         }
 127
 128         /** Executes the system call. */
 129         public void run() {
 130                 // Manage streams
 131                 Writer stdOutWriter = null;
 132                 OutputStream stdOutputStream = null;
 133                 Writer stdErrWriter = null;
 134                 InputStream stdInStream = null;
 135                 if (stdOutFile != null)
 136                         if (redirectStdOut)
 137                                 stdOutputStream = createOutputStream(stdOutFile);
 138                         else
 139                                 stdOutWriter = createWriter(stdOutFile, true);
 140
 141                 if (stdErrFile != null) {
 142                         stdErrWriter = createWriter(stdErrFile, true);
 143                 } else {
 144                         if (stdOutFile != null && !redirectStdOut)
 145                                 stdErrWriter = createWriter(stdOutFile, true);
 146                 }
 147
 148                 if (stdInFile != null)
 149                         try {
 150                                 stdInStream = stdInFile.getInputStream();
 151                         } catch (IOException e2) {
 152                                 throw new SlcException("Cannot open a stream for " + stdInFile,
 153                                                 e2);
 154                         }
 155
 156                 if (log.isTraceEnabled()) {
 157                         log.debug("os.name=" + System.getProperty("os.name"));
 158                         log.debug("os.arch=" + System.getProperty("os.arch"));
 159                         log.debug("os.version=" + System.getProperty("os.version"));
 160                 }
 161
 162                 // Execution directory
 163                 File dir = new File(getExecDirToUse());
 164                 if (!dir.exists())
 165                         dir.mkdirs();
 166
 167                 // Watchdog to check for lost processes
 168                 Executor executor = new DefaultExecutor();
 169                 executor.setWatchdog(new ExecuteWatchdog(watchdogTimeout));
 170
 171                 if (redirectStreams) {
 172                         // Redirect standard streams
 173                         executor.setStreamHandler(createExecuteStreamHandler(stdOutWriter,
 174                                         stdOutputStream, stdErrWriter, stdInStream));
 175                 } else {
 176                         // Dummy stream handler (otherwise pump is used)
 177                         executor.setStreamHandler(new DummyexecuteStreamHandler());
 178                 }
 179
 180                 executor.setProcessDestroyer(new ShutdownHookProcessDestroyer());
 181                 executor.setWorkingDirectory(dir);
 182
 183                 // Command line to use
 184                 final CommandLine commandLine = createCommandLine();
 185                 if (logCommand)
 186                         log.info("Execute command:\n" + commandLine
 187                                         + "\n in working directory: \n" + dir + "\n");
 188
 189                 // Env variables
 190                 Map<String, String> environmentVariablesToUse = null;
 191                 if (environmentVariables.size() > 0) {
 192                         environmentVariablesToUse = new HashMap<String, String>();
 193                         if (mergeEnvironmentVariables)
 194                                 environmentVariablesToUse.putAll(System.getenv());
 195                         environmentVariablesToUse.putAll(environmentVariables);
 196                 }
 197
 198                 // Execute
 199                 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
 200
 201                 //
 202                 // THE EXECUTION PROPER
 203                 //
 204                 try {
 205                         if (synchronous)
 206                                 try {
 207                                         int exitValue = executor.execute(commandLine,
 208                                                         environmentVariablesToUse);
 209                                         executeResultHandler.onProcessComplete(exitValue);
 210                                 } catch (ExecuteException e1) {
 211                                         if (e1.getExitValue() == Executor.INVALID_EXITVALUE) {
 212                                                 Thread.currentThread().interrupt();
 213                                                 return;
 214                                         }
 215                                         executeResultHandler.onProcessFailed(e1);
 216                                 }
 217                         else
 218                                 executor.execute(commandLine, environmentVariablesToUse,
 219                                                 executeResultHandler);
 220                 } catch (SlcException e) {
 221                         throw e;
 222                 } catch (Exception e) {
 223                         throw new SlcException("Could not execute command " + commandLine,
 224                                         e);
 225                 } finally {
 226                         IOUtils.closeQuietly(stdOutWriter);
 227                         IOUtils.closeQuietly(stdErrWriter);
 228                         IOUtils.closeQuietly(stdInStream);
 229                 }
 230
 231         }
 232
 233         public synchronized String function() {
 234                 final StringBuffer buf = new StringBuffer("");
 235                 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
 236                         public void newLine(SystemCall systemCall, String line,
 237                                         Boolean isError) {
 238                                 if (!isError)
 239                                         buf.append(line);
 240                         }
 241                 };
 242                 addOutputListener(tempOutputListener);
 243                 run();
 244                 removeOutputListener(tempOutputListener);
 245                 return buf.toString();
 246         }
 247
 248         public String asCommand() {
 249                 return createCommandLine().toString();
 250         }
 251
 252         @Override
 253         public String toString() {
 254                 return asCommand();
 255         }
 256
 257         /**
 258          * Build a command line based on the properties. Can be overridden by
 259          * specific command wrappers.
 260          */
 261         protected CommandLine createCommandLine() {
 262                 // Check if an OS specific command overrides
 263                 String osName = System.getProperty("os.name");
 264                 List<Object> commandToUse = null;
 265                 if (osCommands.containsKey(osName))
 266                         commandToUse = osCommands.get(osName);
 267                 else
 268                         commandToUse = command;
 269                 String cmdToUse = null;
 270                 if (osCmds.containsKey(osName))
 271                         cmdToUse = osCmds.get(osName);
 272                 else
 273                         cmdToUse = cmd;
 274
 275                 CommandLine commandLine = null;
 276
 277                 // Which command definition to use
 278                 if (commandToUse == null && cmdToUse == null)
 279                         throw new SlcException("Please specify a command.");
 280                 else if (commandToUse != null && cmdToUse != null)
 281                         throw new SlcException(
 282                                         "Specify the command either as a line or as a list.");
 283                 else if (cmdToUse != null) {
 284                         commandLine = CommandLine.parse(cmdToUse);
 285                 } else if (commandToUse != null) {
 286                         if (commandToUse.size() == 0)
 287                                 throw new SlcException("Command line is empty.");
 288
 289                         commandLine = new CommandLine(commandToUse.get(0).toString());
 290
 291                         for (int i = 1; i < commandToUse.size(); i++) {
 292                                 if (log.isTraceEnabled())
 293                                         log.debug(commandToUse.get(i));
 294                                 commandLine.addArgument(commandToUse.get(i).toString());
 295                         }
 296                 } else {
 297                         // all cases covered previously
 298                         throw new UnsupportedException();
 299                 }
 300
 301                 if (generateScript != null) {
 302                         File scriptFile = new File(getExecDirToUse() + File.separator
 303                                         + generateScript);
 304                         try {
 305                                 FileUtils.writeStringToFile(scriptFile,
 306                                                 (osConsole != null ? osConsole + " " : "")
 307                                                                 + commandLine.toString());
 308                         } catch (IOException e) {
 309                                 throw new SlcException("Could not generate script "
 310                                                 + scriptFile, e);
 311                         }
 312                         commandLine = new CommandLine(scriptFile);
 313                 } else {
 314                         if (osConsole != null)
 315                                 commandLine = CommandLine.parse(osConsole + " "
 316                                                 + commandLine.toString());
 317                 }
 318
 319                 return commandLine;
 320         }
 321
 322         /**
 323          * Creates a {@link PumpStreamHandler} which redirects streams to the custom
 324          * logging mechanism.
 325          */
 326         protected ExecuteStreamHandler createExecuteStreamHandler(
 327                         final Writer stdOutWriter, final OutputStream stdOutputStream,
 328                         final Writer stdErrWriter, final InputStream stdInStream) {
 329
 330                 // Log writers
 331
 332                 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(
 333                                 stdOutputStream != null ? stdOutputStream
 334                                                 : new LogOutputStream() {
 335                                                         protected void processLine(String line, int level) {
 336                                                                 if (line != null && !line.trim().equals(""))
 337                                                                         logStdOut(line);
 338                                                                 if (stdOutWriter != null)
 339                                                                         appendLineToFile(stdOutWriter, line);
 340                                                         }
 341                                                 }, new LogOutputStream() {
 342                                         protected void processLine(String line, int level) {
 343                                                 if (line != null && !line.trim().equals(""))
 344                                                         logStdErr(line);
 345                                                 if (stdErrWriter != null)
 346                                                         appendLineToFile(stdErrWriter, line);
 347                                         }
 348                                 }, stdInStream);
 349                 return pumpStreamHandler;
 350         }
 351
 352         /** Creates the default {@link ExecuteResultHandler}. */
 353         protected ExecuteResultHandler createExecuteResultHandler(
 354                         final CommandLine commandLine) {
 355                 return new ExecuteResultHandler() {
 356
 357                         public void onProcessComplete(int exitValue) {
 358                                 String msg = "System call '" + commandLine
 359                                                 + "' properly completed.";
 360                                 if (log.isTraceEnabled())
 361                                         log.trace(msg);
 362                                 if (testResult != null) {
 363                                         forwardPath(testResult);
 364                                         testResult.addResultPart(new SimpleResultPart(
 365                                                         TestStatus.PASSED, msg));
 366                                 }
 367                         }
 368
 369                         public void onProcessFailed(ExecuteException e) {
 370                                 String msg = "System call '" + commandLine + "' failed.";
 371                                 if (testResult != null) {
 372                                         forwardPath(testResult);
 373                                         testResult.addResultPart(new SimpleResultPart(
 374                                                         TestStatus.ERROR, msg, e));
 375                                 } else {
 376                                         if (exceptionOnFailed)
 377                                                 throw new SlcException(msg, e);
 378                                         else
 379                                                 log.error(msg, e);
 380                                 }
 381                         }
 382                 };
 383         }
 384
 385         protected void forwardPath(TestResult testResult) {
 386                 // TODO: allocate a TreeSPath
 387         }
 388
 389         /**
 390          * Shortcut method getting the execDir to use
 391          */
 392         protected String getExecDirToUse() {
 393                 try {
 394                         File dir = null;
 395                         if (execDir != null) {
 396                                 // Replace '/' by local file separator, for portability
 397                                 execDir.replace('/', File.separatorChar);
 398                                 dir = new File(execDir).getCanonicalFile();
 399                         }
 400
 401                         if (dir == null)
 402                                 return System.getProperty("user.dir");
 403                         else
 404                                 return dir.getPath();
 405                 } catch (Exception e) {
 406                         throw new SlcException("Cannot find exec dir", e);
 407                 }
 408         }
 409
 410         protected void logStdOut(String line) {
 411                 for (SystemCallOutputListener outputListener : outputListeners)
 412                         outputListener.newLine(this, line, false);
 413                 log(stdOutLogLevel, line);
 414         }
 415
 416         protected void logStdErr(String line) {
 417                 for (SystemCallOutputListener outputListener : outputListeners)
 418                         outputListener.newLine(this, line, true);
 419                 log(stdErrLogLevel, line);
 420         }
 421
 422         /** Log from the underlying streams. */
 423         protected void log(String logLevel, String line) {
 424                 if ("ERROR".equals(logLevel))
 425                         log.error(line);
 426                 else if ("WARN".equals(logLevel))
 427                         log.warn(line);
 428                 else if ("INFO".equals(logLevel))
 429                         log.info(line);
 430                 else if ("DEBUG".equals(logLevel))
 431                         log.debug(line);
 432                 else if ("TRACE".equals(logLevel))
 433                         log.trace(line);
 434                 else if (LOG_STDOUT.equals(logLevel))
 435                         System.out.println(line);
 436                 else if ("System.err".equals(logLevel))
 437                         System.err.println(line);
 438                 else
 439                         throw new SlcException("Unknown log level " + logLevel);
 440         }
 441
 442         /** Append line to a log file. */
 443         protected void appendLineToFile(Writer writer, String line) {
 444                 try {
 445                         writer.append(line).append('\n');
 446                 } catch (IOException e) {
 447                         log.error("Cannot write to log file", e);
 448                 }
 449         }
 450
 451         /** Creates the writer for the output/err files. */
 452         protected Writer createWriter(Resource target, Boolean append) {
 453                 FileWriter writer = null;
 454                 try {
 455
 456                         final File file;
 457                         if (executionResources != null)
 458                                 file = new File(executionResources.getAsOsPath(target, true));
 459                         else
 460                                 file = target.getFile();
 461                         writer = new FileWriter(file, append);
 462                 } catch (IOException e) {
 463                         log.error("Cannot get file for " + target, e);
 464                         IOUtils.closeQuietly(writer);
 465                 }
 466                 return writer;
 467         }
 468
 469         /** Creates an outputstream for the output/err files. */
 470         protected OutputStream createOutputStream(Resource target) {
 471                 FileOutputStream out = null;
 472                 try {
 473
 474                         final File file;
 475                         if (executionResources != null)
 476                                 file = new File(executionResources.getAsOsPath(target, true));
 477                         else
 478                                 file = target.getFile();
 479                         out = new FileOutputStream(file, false);
 480                 } catch (IOException e) {
 481                         log.error("Cannot get file for " + target, e);
 482                         IOUtils.closeQuietly(out);
 483                 }
 484                 return out;
 485         }
 486
 487         /** Append the argument (for chaining) */
 488         public SystemCall arg(String arg) {
 489                 if (command == null)
 490                         command = new ArrayList<Object>();
 491                 command.add(arg);
 492                 return this;
 493         }
 494
 495         /** Append the argument (for chaining) */
 496         public SystemCall arg(String arg, String value) {
 497                 if (command == null)
 498                         command = new ArrayList<Object>();
 499                 command.add(arg);
 500                 command.add(value);
 501                 return this;
 502         }
 503
 504         /** */
 505         public void setCmd(String command) {
 506                 this.cmd = command;
 507         }
 508
 509         public void setCommand(List<Object> command) {
 510                 this.command = command;
 511         }
 512
 513         public void setExecDir(String execdir) {
 514                 this.execDir = execdir;
 515         }
 516
 517         public void setStdErrLogLevel(String stdErrLogLevel) {
 518                 this.stdErrLogLevel = stdErrLogLevel;
 519         }
 520
 521         public void setStdOutLogLevel(String stdOutLogLevel) {
 522                 this.stdOutLogLevel = stdOutLogLevel;
 523         }
 524
 525         public void setSynchronous(Boolean synchronous) {
 526                 this.synchronous = synchronous;
 527         }
 528
 529         public void setOsCommands(Map<String, List<Object>> osCommands) {
 530                 this.osCommands = osCommands;
 531         }
 532
 533         public void setOsCmds(Map<String, String> osCmds) {
 534                 this.osCmds = osCmds;
 535         }
 536
 537         public void setEnvironmentVariables(Map<String, String> environmentVariables) {
 538                 this.environmentVariables = environmentVariables;
 539         }
 540
 541         public void setWatchdogTimeout(Long watchdogTimeout) {
 542                 this.watchdogTimeout = watchdogTimeout;
 543         }
 544
 545         public void setStdOutFile(Resource stdOutFile) {
 546                 this.stdOutFile = stdOutFile;
 547         }
 548
 549         public void setStdErrFile(Resource stdErrFile) {
 550                 this.stdErrFile = stdErrFile;
 551         }
 552
 553         public void setStdInFile(Resource stdInFile) {
 554                 this.stdInFile = stdInFile;
 555         }
 556
 557         public void setTestResult(TestResult testResult) {
 558                 this.testResult = testResult;
 559         }
 560
 561         public void setLogCommand(Boolean logCommand) {
 562                 this.logCommand = logCommand;
 563         }
 564
 565         public void setRedirectStreams(Boolean redirectStreams) {
 566                 this.redirectStreams = redirectStreams;
 567         }
 568
 569         public void setExceptionOnFailed(Boolean exceptionOnFailed) {
 570                 this.exceptionOnFailed = exceptionOnFailed;
 571         }
 572
 573         public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
 574                 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
 575         }
 576
 577         public void setOsConsole(String osConsole) {
 578                 this.osConsole = osConsole;
 579         }
 580
 581         public void setGenerateScript(String generateScript) {
 582                 this.generateScript = generateScript;
 583         }
 584
 585         public void setExecutionResources(ExecutionResources executionResources) {
 586                 this.executionResources = executionResources;
 587         }
 588
 589         public void setRedirectStdOut(Boolean redirectStdOut) {
 590                 this.redirectStdOut = redirectStdOut;
 591         }
 592
 593         public void addOutputListener(SystemCallOutputListener outputListener) {
 594                 outputListeners.add(outputListener);
 595         }
 596
 597         public void removeOutputListener(SystemCallOutputListener outputListener) {
 598                 outputListeners.remove(outputListener);
 599         }
 600
 601         public void setOutputListeners(
 602                         List<SystemCallOutputListener> outputListeners) {
 603                 this.outputListeners = outputListeners;
 604         }
 605
 606         private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
 607
 608                 public void setProcessErrorStream(InputStream is) throws IOException {
 609                 }
 610
 611                 public void setProcessInputStream(OutputStream os) throws IOException {
 612                 }
 613
 614                 public void setProcessOutputStream(InputStream is) throws IOException {
 615                 }
 616
 617                 public void start() throws IOException {
 618                 }
 619
 620                 public void stop() {
 621                 }
 622
 623         }
 624
 625 }