1 package org
.argeo
.slc
.runtime
.tasks
;
3 import static java
.lang
.System
.Logger
.Level
.DEBUG
;
4 import static java
.lang
.System
.Logger
.Level
.ERROR
;
5 import static java
.lang
.System
.Logger
.Level
.INFO
;
6 import static java
.lang
.System
.Logger
.Level
.TRACE
;
7 import static java
.lang
.System
.Logger
.Level
.WARNING
;
10 import java
.io
.FileOutputStream
;
11 import java
.io
.FileWriter
;
12 import java
.io
.IOException
;
13 import java
.io
.InputStream
;
14 import java
.io
.OutputStream
;
15 import java
.io
.PipedInputStream
;
16 import java
.io
.PipedOutputStream
;
17 import java
.io
.Writer
;
18 import java
.lang
.System
.Logger
;
19 import java
.nio
.file
.Files
;
20 import java
.nio
.file
.Path
;
21 import java
.util
.ArrayList
;
22 import java
.util
.Collections
;
23 import java
.util
.HashMap
;
24 import java
.util
.List
;
26 import java
.util
.UUID
;
28 import javax
.security
.auth
.callback
.CallbackHandler
;
30 import org
.apache
.commons
.exec
.CommandLine
;
31 import org
.apache
.commons
.exec
.DefaultExecutor
;
32 import org
.apache
.commons
.exec
.ExecuteException
;
33 import org
.apache
.commons
.exec
.ExecuteResultHandler
;
34 import org
.apache
.commons
.exec
.ExecuteStreamHandler
;
35 import org
.apache
.commons
.exec
.ExecuteWatchdog
;
36 import org
.apache
.commons
.exec
.Executor
;
37 import org
.apache
.commons
.exec
.LogOutputStream
;
38 import org
.apache
.commons
.exec
.PumpStreamHandler
;
39 import org
.apache
.commons
.exec
.ShutdownHookProcessDestroyer
;
40 import org
.apache
.commons
.io
.FileUtils
;
41 import org
.apache
.commons
.io
.IOUtils
;
42 import org
.argeo
.slc
.SlcException
;
43 import org
.argeo
.slc
.UnsupportedException
;
44 import org
.argeo
.slc
.execution
.ExecutionResources
;
45 import org
.argeo
.slc
.runtime
.test
.SimpleResultPart
;
46 import org
.argeo
.slc
.test
.TestResult
;
47 import org
.argeo
.slc
.test
.TestStatus
;
49 /** Execute an OS specific system call. */
50 public class SystemCall
implements Runnable
{
51 public final static String LOG_STDOUT
= "System.out";
53 private final Logger logger
= System
.getLogger(getClass().getName());
55 private String execDir
;
57 private String cmd
= null;
58 private List
<Object
> command
= null;
60 private Executor executor
= new DefaultExecutor();
61 private Boolean synchronous
= true;
63 private String stdErrLogLevel
= "ERROR";
64 private String stdOutLogLevel
= "INFO";
66 private Path stdOutFile
= null;
67 private Path stdErrFile
= null;
69 private Path stdInFile
= null;
71 * If no {@link #stdInFile} provided, writing to this stream will write to the
72 * stdin of the process.
74 private OutputStream stdInSink
= null;
76 private Boolean redirectStdOut
= false;
78 private List
<SystemCallOutputListener
> outputListeners
= Collections
79 .synchronizedList(new ArrayList
<SystemCallOutputListener
>());
81 private Map
<String
, List
<Object
>> osCommands
= new HashMap
<String
, List
<Object
>>();
82 private Map
<String
, String
> osCmds
= new HashMap
<String
, String
>();
83 private Map
<String
, String
> environmentVariables
= new HashMap
<String
, String
>();
85 private Boolean logCommand
= false;
86 private Boolean redirectStreams
= true;
87 private Boolean exceptionOnFailed
= true;
88 private Boolean mergeEnvironmentVariables
= true;
90 // private Authentication authentication;
92 private String osConsole
= null;
93 private String generateScript
= null;
96 private Long watchdogTimeout
= 24 * 60 * 60 * 1000l;
98 private TestResult testResult
;
100 private ExecutionResources executionResources
;
102 /** Sudo the command, as root if empty or as user if not. */
103 private String sudo
= null;
104 // TODO make it more secure and robust, test only once
105 private final String sudoPrompt
= UUID
.randomUUID().toString();
106 private String askPassProgram
= "/usr/libexec/openssh/ssh-askpass";
107 @SuppressWarnings("unused")
108 private boolean firstLine
= true;
109 @SuppressWarnings("unused")
110 private CallbackHandler callbackHandler
;
111 /** Chroot to the this path (must not be empty) */
112 private String chroot
= null;
115 /** Current watchdog, null if process is completed */
116 ExecuteWatchdog currentWatchdog
= null;
118 /** Empty constructor */
119 public SystemCall() {
124 * Constructor based on the provided command list.
126 * @param command the command list
128 public SystemCall(List
<Object
> command
) {
129 this.command
= command
;
133 * Constructor based on the provided command.
135 * @param cmd the command. If the provided string contains no space a command
136 * list is initialized with the argument as first component (useful
137 * for chained construction)
139 public SystemCall(String cmd
) {
140 if (cmd
.indexOf(' ') < 0) {
141 command
= new ArrayList
<Object
>();
148 /** Executes the system call. */
150 // authentication = SecurityContextHolder.getContext().getAuthentication();
153 Writer stdOutWriter
= null;
154 OutputStream stdOutputStream
= null;
155 Writer stdErrWriter
= null;
156 InputStream stdInStream
= null;
157 if (stdOutFile
!= null)
159 stdOutputStream
= createOutputStream(stdOutFile
);
161 stdOutWriter
= createWriter(stdOutFile
, true);
163 if (stdErrFile
!= null) {
164 stdErrWriter
= createWriter(stdErrFile
, true);
166 if (stdOutFile
!= null && !redirectStdOut
)
167 stdErrWriter
= createWriter(stdOutFile
, true);
171 if (stdInFile
!= null)
172 stdInStream
= Files
.newInputStream(stdInFile
);
174 stdInStream
= new PipedInputStream();
175 stdInSink
= new PipedOutputStream((PipedInputStream
) stdInStream
);
177 } catch (IOException e2
) {
178 throw new SlcException("Cannot open a stream for " + stdInFile
, e2
);
181 logger
.log(TRACE
, () -> "os.name=" + System
.getProperty("os.name"));
182 logger
.log(TRACE
, () -> "os.arch=" + System
.getProperty("os.arch"));
183 logger
.log(TRACE
, () -> "os.version=" + System
.getProperty("os.version"));
185 // Execution directory
186 File dir
= new File(getExecDirToUse());
187 // if (!dir.exists())
190 // Watchdog to check for lost processes
191 Executor executorToUse
;
192 if (executor
!= null)
193 executorToUse
= executor
;
195 executorToUse
= new DefaultExecutor();
196 executorToUse
.setWatchdog(createWatchdog());
198 if (redirectStreams
) {
199 // Redirect standard streams
200 executorToUse
.setStreamHandler(
201 createExecuteStreamHandler(stdOutWriter
, stdOutputStream
, stdErrWriter
, stdInStream
));
203 // Dummy stream handler (otherwise pump is used)
204 executorToUse
.setStreamHandler(new DummyexecuteStreamHandler());
207 executorToUse
.setProcessDestroyer(new ShutdownHookProcessDestroyer());
208 executorToUse
.setWorkingDirectory(dir
);
210 // Command line to use
211 final CommandLine commandLine
= createCommandLine();
213 logger
.log(INFO
, "Execute command:\n" + commandLine
+ "\n in working directory: \n" + dir
+ "\n");
216 Map
<String
, String
> environmentVariablesToUse
= null;
217 environmentVariablesToUse
= new HashMap
<String
, String
>();
218 if (mergeEnvironmentVariables
)
219 environmentVariablesToUse
.putAll(System
.getenv());
220 if (environmentVariables
.size() > 0)
221 environmentVariablesToUse
.putAll(environmentVariables
);
224 ExecuteResultHandler executeResultHandler
= createExecuteResultHandler(commandLine
);
227 // THE EXECUTION PROPER
232 int exitValue
= executorToUse
.execute(commandLine
, environmentVariablesToUse
);
233 executeResultHandler
.onProcessComplete(exitValue
);
234 } catch (ExecuteException e1
) {
235 if (e1
.getExitValue() == Executor
.INVALID_EXITVALUE
) {
236 Thread
.currentThread().interrupt();
239 // Sleep 1s in order to make sure error logs are flushed
241 executeResultHandler
.onProcessFailed(e1
);
244 executorToUse
.execute(commandLine
, environmentVariablesToUse
, executeResultHandler
);
245 } catch (SlcException e
) {
247 } catch (Exception e
) {
248 throw new SlcException("Could not execute command " + commandLine
, e
);
250 IOUtils
.closeQuietly(stdOutWriter
);
251 IOUtils
.closeQuietly(stdErrWriter
);
252 IOUtils
.closeQuietly(stdInStream
);
253 IOUtils
.closeQuietly(stdInSink
);
258 public synchronized String
function() {
259 final StringBuffer buf
= new StringBuffer("");
260 SystemCallOutputListener tempOutputListener
= new SystemCallOutputListener() {
261 private Long lineCount
= 0l;
263 public void newLine(SystemCall systemCall
, String line
, Boolean isError
) {
272 addOutputListener(tempOutputListener
);
274 removeOutputListener(tempOutputListener
);
275 return buf
.toString();
278 public String
asCommand() {
279 return createCommandLine().toString();
283 public String
toString() {
288 * Build a command line based on the properties. Can be overridden by specific
291 protected CommandLine
createCommandLine() {
292 // Check if an OS specific command overrides
293 String osName
= System
.getProperty("os.name");
294 List
<Object
> commandToUse
= null;
295 if (osCommands
.containsKey(osName
))
296 commandToUse
= osCommands
.get(osName
);
298 commandToUse
= command
;
299 String cmdToUse
= null;
300 if (osCmds
.containsKey(osName
))
301 cmdToUse
= osCmds
.get(osName
);
305 CommandLine commandLine
= null;
307 // Which command definition to use
308 if (commandToUse
== null && cmdToUse
== null)
309 throw new SlcException("Please specify a command.");
310 else if (commandToUse
!= null && cmdToUse
!= null)
311 throw new SlcException("Specify the command either as a line or as a list.");
312 else if (cmdToUse
!= null) {
313 if (chroot
!= null && !chroot
.trim().equals(""))
314 cmdToUse
= "chroot \"" + chroot
+ "\" " + cmdToUse
;
316 environmentVariables
.put("SUDO_ASKPASS", askPassProgram
);
317 if (!sudo
.trim().equals(""))
318 cmdToUse
= "sudo -p " + sudoPrompt
+ " -u " + sudo
+ " " + cmdToUse
;
320 cmdToUse
= "sudo -p " + sudoPrompt
+ " " + cmdToUse
;
323 // GENERATE COMMAND LINE
324 commandLine
= CommandLine
.parse(cmdToUse
);
325 } else if (commandToUse
!= null) {
326 if (commandToUse
.size() == 0)
327 throw new SlcException("Command line is empty.");
329 if (chroot
!= null && sudo
!= null) {
330 commandToUse
.add(0, "chroot");
331 commandToUse
.add(1, chroot
);
335 environmentVariables
.put("SUDO_ASKPASS", askPassProgram
);
336 commandToUse
.add(0, "sudo");
337 commandToUse
.add(1, "-p");
338 commandToUse
.add(2, sudoPrompt
);
339 if (!sudo
.trim().equals("")) {
340 commandToUse
.add(3, "-u");
341 commandToUse
.add(4, sudo
);
345 // GENERATE COMMAND LINE
346 commandLine
= new CommandLine(commandToUse
.get(0).toString());
348 for (int i
= 1; i
< commandToUse
.size(); i
++) {
349 if (logger
.isLoggable(TRACE
))
350 logger
.log(TRACE
, commandToUse
.get(i
));
351 commandLine
.addArgument(commandToUse
.get(i
).toString());
354 // all cases covered previously
355 throw new UnsupportedException();
358 if (generateScript
!= null) {
359 File scriptFile
= new File(getExecDirToUse() + File
.separator
+ generateScript
);
361 FileUtils
.writeStringToFile(scriptFile
,
362 (osConsole
!= null ? osConsole
+ " " : "") + commandLine
.toString());
363 } catch (IOException e
) {
364 throw new SlcException("Could not generate script " + scriptFile
, e
);
366 commandLine
= new CommandLine(scriptFile
);
368 if (osConsole
!= null)
369 commandLine
= CommandLine
.parse(osConsole
+ " " + commandLine
.toString());
376 * Creates a {@link PumpStreamHandler} which redirects streams to the custom
379 protected ExecuteStreamHandler
createExecuteStreamHandler(final Writer stdOutWriter
,
380 final OutputStream stdOutputStream
, final Writer stdErrWriter
, final InputStream stdInStream
) {
383 OutputStream stdout
= stdOutputStream
!= null ? stdOutputStream
: new LogOutputStream() {
384 protected void processLine(String line
, int level
) {
386 // if (sudo != null && callbackHandler != null
387 // && line.startsWith(sudoPrompt)) {
389 // PasswordCallback pc = new PasswordCallback(
390 // "sudo password", false);
391 // Callback[] cbs = { pc };
392 // callbackHandler.handle(cbs);
393 // char[] pwd = pc.getPassword();
394 // char[] arr = Arrays.copyOf(pwd,
396 // arr[arr.length - 1] = '\n';
397 // IOUtils.write(arr, stdInSink);
398 // stdInSink.flush();
399 // } catch (Exception e) {
400 // throw new SlcException(
401 // "Cannot retrieve sudo password", e);
404 // firstLine = false;
407 if (line
!= null && !line
.trim().equals(""))
410 if (stdOutWriter
!= null)
411 appendLineToFile(stdOutWriter
, line
);
415 OutputStream stderr
= new LogOutputStream() {
416 protected void processLine(String line
, int level
) {
417 if (line
!= null && !line
.trim().equals(""))
419 if (stdErrWriter
!= null)
420 appendLineToFile(stdErrWriter
, line
);
424 PumpStreamHandler pumpStreamHandler
= new PumpStreamHandler(stdout
, stderr
, stdInStream
) {
427 public void stop() throws IOException
{
428 // prevents the method to block when joining stdin
429 if (stdInSink
!= null)
430 IOUtils
.closeQuietly(stdInSink
);
435 return pumpStreamHandler
;
438 /** Creates the default {@link ExecuteResultHandler}. */
439 protected ExecuteResultHandler
createExecuteResultHandler(final CommandLine commandLine
) {
440 return new ExecuteResultHandler() {
442 public void onProcessComplete(int exitValue
) {
443 String msg
= "System call '" + commandLine
+ "' properly completed.";
444 logger
.log(TRACE
, () -> msg
);
445 if (testResult
!= null) {
446 forwardPath(testResult
);
447 testResult
.addResultPart(new SimpleResultPart(TestStatus
.PASSED
, msg
));
452 public void onProcessFailed(ExecuteException e
) {
454 String msg
= "System call '" + commandLine
+ "' failed.";
455 if (testResult
!= null) {
456 forwardPath(testResult
);
457 testResult
.addResultPart(new SimpleResultPart(TestStatus
.ERROR
, msg
, e
));
459 if (exceptionOnFailed
)
460 throw new SlcException(msg
, e
);
462 logger
.log(ERROR
, msg
, e
);
470 protected void forwardPath(TestResult testResult
) {
471 // TODO: allocate a TreeSPath
475 * Shortcut method getting the execDir to use
477 protected String
getExecDirToUse() {
479 if (execDir
!= null) {
482 return System
.getProperty("user.dir");
483 } catch (Exception e
) {
484 throw new SlcException("Cannot find exec dir", e
);
488 protected void logStdOut(String line
) {
489 for (SystemCallOutputListener outputListener
: outputListeners
)
490 outputListener
.newLine(this, line
, false);
491 log(stdOutLogLevel
, line
);
494 protected void logStdErr(String line
) {
495 for (SystemCallOutputListener outputListener
: outputListeners
)
496 outputListener
.newLine(this, line
, true);
497 log(stdErrLogLevel
, line
);
500 /** Log from the underlying streams. */
501 protected void log(String logLevel
, String line
) {
503 // if (SecurityContextHolder.getContext().getAuthentication() == null) {
504 // SecurityContextHolder.getContext()
505 // .setAuthentication(authentication);
508 if ("ERROR".equals(logLevel
))
509 logger
.log(ERROR
, line
);
510 else if ("WARN".equals(logLevel
))
511 logger
.log(WARNING
, line
);
512 else if ("WARNING".equals(logLevel
))
513 logger
.log(WARNING
, line
);
514 else if ("INFO".equals(logLevel
))
515 logger
.log(INFO
, line
);
516 else if ("DEBUG".equals(logLevel
))
517 logger
.log(DEBUG
, line
);
518 else if ("TRACE".equals(logLevel
))
519 logger
.log(TRACE
, line
);
520 else if (LOG_STDOUT
.equals(logLevel
))
521 System
.out
.println(line
);
522 else if ("System.err".equals(logLevel
))
523 System
.err
.println(line
);
525 throw new SlcException("Unknown log level " + logLevel
);
528 /** Append line to a log file. */
529 protected void appendLineToFile(Writer writer
, String line
) {
531 writer
.append(line
).append('\n');
532 } catch (IOException e
) {
533 logger
.log(ERROR
, "Cannot write to log file", e
);
537 /** Creates the writer for the output/err files. */
538 protected Writer
createWriter(Path target
, Boolean append
) {
539 FileWriter writer
= null;
543 if (executionResources
!= null)
544 file
= new File(executionResources
.getAsOsPath(target
, true));
546 file
= target
.toFile();
547 writer
= new FileWriter(file
, append
);
548 } catch (IOException e
) {
549 logger
.log(ERROR
, "Cannot get file for " + target
, e
);
550 IOUtils
.closeQuietly(writer
);
555 /** Creates an outputstream for the output/err files. */
556 protected OutputStream
createOutputStream(Path target
) {
557 FileOutputStream out
= null;
561 if (executionResources
!= null)
562 file
= new File(executionResources
.getAsOsPath(target
, true));
564 file
= target
.toFile();
565 out
= new FileOutputStream(file
, false);
566 } catch (IOException e
) {
567 logger
.log(ERROR
, "Cannot get file for " + target
, e
);
568 IOUtils
.closeQuietly(out
);
573 /** Append the argument (for chaining) */
574 public SystemCall
arg(String arg
) {
576 command
= new ArrayList
<Object
>();
581 /** Append the argument (for chaining) */
582 public SystemCall
arg(String arg
, String value
) {
584 command
= new ArrayList
<Object
>();
591 public synchronized Boolean
isRunning() {
592 return currentWatchdog
!= null;
595 private synchronized ExecuteWatchdog
createWatchdog() {
596 // if (currentWatchdog != null)
597 // throw new SlcException("A process is already being monitored");
598 currentWatchdog
= new ExecuteWatchdog(watchdogTimeout
);
599 return currentWatchdog
;
602 private synchronized void releaseWatchdog() {
603 currentWatchdog
= null;
606 public synchronized void kill() {
607 if (currentWatchdog
!= null)
608 currentWatchdog
.destroyProcess();
612 public void setCmd(String command
) {
616 public void setCommand(List
<Object
> command
) {
617 this.command
= command
;
620 public void setExecDir(String execdir
) {
621 this.execDir
= execdir
;
624 public void setStdErrLogLevel(String stdErrLogLevel
) {
625 this.stdErrLogLevel
= stdErrLogLevel
;
628 public void setStdOutLogLevel(String stdOutLogLevel
) {
629 this.stdOutLogLevel
= stdOutLogLevel
;
632 public void setSynchronous(Boolean synchronous
) {
633 this.synchronous
= synchronous
;
636 public void setOsCommands(Map
<String
, List
<Object
>> osCommands
) {
637 this.osCommands
= osCommands
;
640 public void setOsCmds(Map
<String
, String
> osCmds
) {
641 this.osCmds
= osCmds
;
644 public void setEnvironmentVariables(Map
<String
, String
> environmentVariables
) {
645 this.environmentVariables
= environmentVariables
;
648 public Map
<String
, String
> getEnvironmentVariables() {
649 return environmentVariables
;
652 public void setWatchdogTimeout(Long watchdogTimeout
) {
653 this.watchdogTimeout
= watchdogTimeout
;
656 public void setStdOutFile(Path stdOutFile
) {
657 this.stdOutFile
= stdOutFile
;
660 public void setStdErrFile(Path stdErrFile
) {
661 this.stdErrFile
= stdErrFile
;
664 public void setStdInFile(Path stdInFile
) {
665 this.stdInFile
= stdInFile
;
668 public void setTestResult(TestResult testResult
) {
669 this.testResult
= testResult
;
672 public void setLogCommand(Boolean logCommand
) {
673 this.logCommand
= logCommand
;
676 public void setRedirectStreams(Boolean redirectStreams
) {
677 this.redirectStreams
= redirectStreams
;
680 public void setExceptionOnFailed(Boolean exceptionOnFailed
) {
681 this.exceptionOnFailed
= exceptionOnFailed
;
684 public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables
) {
685 this.mergeEnvironmentVariables
= mergeEnvironmentVariables
;
688 public void setOsConsole(String osConsole
) {
689 this.osConsole
= osConsole
;
692 public void setGenerateScript(String generateScript
) {
693 this.generateScript
= generateScript
;
696 public void setExecutionResources(ExecutionResources executionResources
) {
697 this.executionResources
= executionResources
;
700 public void setRedirectStdOut(Boolean redirectStdOut
) {
701 this.redirectStdOut
= redirectStdOut
;
704 public void addOutputListener(SystemCallOutputListener outputListener
) {
705 outputListeners
.add(outputListener
);
708 public void removeOutputListener(SystemCallOutputListener outputListener
) {
709 outputListeners
.remove(outputListener
);
712 public void setOutputListeners(List
<SystemCallOutputListener
> outputListeners
) {
713 this.outputListeners
= outputListeners
;
716 public void setExecutor(Executor executor
) {
717 this.executor
= executor
;
720 public void setSudo(String sudo
) {
724 public void setCallbackHandler(CallbackHandler callbackHandler
) {
725 this.callbackHandler
= callbackHandler
;
728 public void setChroot(String chroot
) {
729 this.chroot
= chroot
;
732 private class DummyexecuteStreamHandler
implements ExecuteStreamHandler
{
734 public void setProcessErrorStream(InputStream is
) throws IOException
{
737 public void setProcessInputStream(OutputStream os
) throws IOException
{
740 public void setProcessOutputStream(InputStream is
) throws IOException
{
743 public void start() throws IOException
{