]> git.argeo.org Git - gpl/argeo-slc.git/blob - org.argeo.slc.runtime/src/org/argeo/slc/runtime/tasks/SystemCall.java
Clarify overall project structure.
[gpl/argeo-slc.git] / org.argeo.slc.runtime / src / org / argeo / slc / runtime / tasks / SystemCall.java
1 package org.argeo.slc.runtime.tasks;
2
3 import java.io.File;
4 import java.io.FileOutputStream;
5 import java.io.FileWriter;
6 import java.io.IOException;
7 import java.io.InputStream;
8 import java.io.OutputStream;
9 import java.io.PipedInputStream;
10 import java.io.PipedOutputStream;
11 import java.io.Writer;
12 import java.nio.file.Files;
13 import java.nio.file.Path;
14 import java.util.ArrayList;
15 import java.util.Collections;
16 import java.util.HashMap;
17 import java.util.List;
18 import java.util.Map;
19 import java.util.UUID;
20
21 import javax.security.auth.callback.CallbackHandler;
22
23 import org.apache.commons.exec.CommandLine;
24 import org.apache.commons.exec.DefaultExecutor;
25 import org.apache.commons.exec.ExecuteException;
26 import org.apache.commons.exec.ExecuteResultHandler;
27 import org.apache.commons.exec.ExecuteStreamHandler;
28 import org.apache.commons.exec.ExecuteWatchdog;
29 import org.apache.commons.exec.Executor;
30 import org.apache.commons.exec.LogOutputStream;
31 import org.apache.commons.exec.PumpStreamHandler;
32 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
33 import org.apache.commons.io.FileUtils;
34 import org.apache.commons.io.IOUtils;
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.argeo.slc.SlcException;
38 import org.argeo.slc.UnsupportedException;
39 import org.argeo.slc.execution.ExecutionResources;
40 import org.argeo.slc.runtime.test.SimpleResultPart;
41 import org.argeo.slc.test.TestResult;
42 import org.argeo.slc.test.TestStatus;
43
44 /** Execute an OS specific system call. */
45 public class SystemCall implements Runnable {
46 public final static String LOG_STDOUT = "System.out";
47
48 private final Log log = LogFactory.getLog(getClass());
49
50 private String execDir;
51
52 private String cmd = null;
53 private List<Object> command = null;
54
55 private Executor executor = new DefaultExecutor();
56 private Boolean synchronous = true;
57
58 private String stdErrLogLevel = "ERROR";
59 private String stdOutLogLevel = "INFO";
60
61 private Path stdOutFile = null;
62 private Path stdErrFile = null;
63
64 private Path stdInFile = null;
65 /**
66 * If no {@link #stdInFile} provided, writing to this stream will write to the
67 * stdin of the process.
68 */
69 private OutputStream stdInSink = null;
70
71 private Boolean redirectStdOut = false;
72
73 private List<SystemCallOutputListener> outputListeners = Collections
74 .synchronizedList(new ArrayList<SystemCallOutputListener>());
75
76 private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
77 private Map<String, String> osCmds = new HashMap<String, String>();
78 private Map<String, String> environmentVariables = new HashMap<String, String>();
79
80 private Boolean logCommand = false;
81 private Boolean redirectStreams = true;
82 private Boolean exceptionOnFailed = true;
83 private Boolean mergeEnvironmentVariables = true;
84
85 // private Authentication authentication;
86
87 private String osConsole = null;
88 private String generateScript = null;
89
90 /** 24 hours */
91 private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
92
93 private TestResult testResult;
94
95 private ExecutionResources executionResources;
96
97 /** Sudo the command, as root if empty or as user if not. */
98 private String sudo = null;
99 // TODO make it more secure and robust, test only once
100 private final String sudoPrompt = UUID.randomUUID().toString();
101 private String askPassProgram = "/usr/libexec/openssh/ssh-askpass";
102 @SuppressWarnings("unused")
103 private boolean firstLine = true;
104 @SuppressWarnings("unused")
105 private CallbackHandler callbackHandler;
106 /** Chroot to the this path (must not be empty) */
107 private String chroot = null;
108
109 // Current
110 /** Current watchdog, null if process is completed */
111 ExecuteWatchdog currentWatchdog = null;
112
113 /** Empty constructor */
114 public SystemCall() {
115
116 }
117
118 /**
119 * Constructor based on the provided command list.
120 *
121 * @param command the command list
122 */
123 public SystemCall(List<Object> command) {
124 this.command = command;
125 }
126
127 /**
128 * Constructor based on the provided command.
129 *
130 * @param cmd the command. If the provided string contains no space a command
131 * list is initialized with the argument as first component (useful
132 * for chained construction)
133 */
134 public SystemCall(String cmd) {
135 if (cmd.indexOf(' ') < 0) {
136 command = new ArrayList<Object>();
137 command.add(cmd);
138 } else {
139 this.cmd = cmd;
140 }
141 }
142
143 /** Executes the system call. */
144 public void run() {
145 // authentication = SecurityContextHolder.getContext().getAuthentication();
146
147 // Manage streams
148 Writer stdOutWriter = null;
149 OutputStream stdOutputStream = null;
150 Writer stdErrWriter = null;
151 InputStream stdInStream = null;
152 if (stdOutFile != null)
153 if (redirectStdOut)
154 stdOutputStream = createOutputStream(stdOutFile);
155 else
156 stdOutWriter = createWriter(stdOutFile, true);
157
158 if (stdErrFile != null) {
159 stdErrWriter = createWriter(stdErrFile, true);
160 } else {
161 if (stdOutFile != null && !redirectStdOut)
162 stdErrWriter = createWriter(stdOutFile, true);
163 }
164
165 try {
166 if (stdInFile != null)
167 stdInStream = Files.newInputStream(stdInFile);
168 else {
169 stdInStream = new PipedInputStream();
170 stdInSink = new PipedOutputStream((PipedInputStream) stdInStream);
171 }
172 } catch (IOException e2) {
173 throw new SlcException("Cannot open a stream for " + stdInFile, e2);
174 }
175
176 if (log.isTraceEnabled()) {
177 log.debug("os.name=" + System.getProperty("os.name"));
178 log.debug("os.arch=" + System.getProperty("os.arch"));
179 log.debug("os.version=" + System.getProperty("os.version"));
180 }
181
182 // Execution directory
183 File dir = new File(getExecDirToUse());
184 // if (!dir.exists())
185 // dir.mkdirs();
186
187 // Watchdog to check for lost processes
188 Executor executorToUse;
189 if (executor != null)
190 executorToUse = executor;
191 else
192 executorToUse = new DefaultExecutor();
193 executorToUse.setWatchdog(createWatchdog());
194
195 if (redirectStreams) {
196 // Redirect standard streams
197 executorToUse.setStreamHandler(
198 createExecuteStreamHandler(stdOutWriter, stdOutputStream, stdErrWriter, stdInStream));
199 } else {
200 // Dummy stream handler (otherwise pump is used)
201 executorToUse.setStreamHandler(new DummyexecuteStreamHandler());
202 }
203
204 executorToUse.setProcessDestroyer(new ShutdownHookProcessDestroyer());
205 executorToUse.setWorkingDirectory(dir);
206
207 // Command line to use
208 final CommandLine commandLine = createCommandLine();
209 if (logCommand)
210 log.info("Execute command:\n" + commandLine + "\n in working directory: \n" + dir + "\n");
211
212 // Env variables
213 Map<String, String> environmentVariablesToUse = null;
214 environmentVariablesToUse = new HashMap<String, String>();
215 if (mergeEnvironmentVariables)
216 environmentVariablesToUse.putAll(System.getenv());
217 if (environmentVariables.size() > 0)
218 environmentVariablesToUse.putAll(environmentVariables);
219
220 // Execute
221 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
222
223 //
224 // THE EXECUTION PROPER
225 //
226 try {
227 if (synchronous)
228 try {
229 int exitValue = executorToUse.execute(commandLine, environmentVariablesToUse);
230 executeResultHandler.onProcessComplete(exitValue);
231 } catch (ExecuteException e1) {
232 if (e1.getExitValue() == Executor.INVALID_EXITVALUE) {
233 Thread.currentThread().interrupt();
234 return;
235 }
236 // Sleep 1s in order to make sure error logs are flushed
237 Thread.sleep(1000);
238 executeResultHandler.onProcessFailed(e1);
239 }
240 else
241 executorToUse.execute(commandLine, environmentVariablesToUse, executeResultHandler);
242 } catch (SlcException e) {
243 throw e;
244 } catch (Exception e) {
245 throw new SlcException("Could not execute command " + commandLine, e);
246 } finally {
247 IOUtils.closeQuietly(stdOutWriter);
248 IOUtils.closeQuietly(stdErrWriter);
249 IOUtils.closeQuietly(stdInStream);
250 IOUtils.closeQuietly(stdInSink);
251 }
252
253 }
254
255 public synchronized String function() {
256 final StringBuffer buf = new StringBuffer("");
257 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
258 private Long lineCount = 0l;
259
260 public void newLine(SystemCall systemCall, String line, Boolean isError) {
261 if (!isError) {
262 if (lineCount != 0l)
263 buf.append('\n');
264 buf.append(line);
265 lineCount++;
266 }
267 }
268 };
269 addOutputListener(tempOutputListener);
270 run();
271 removeOutputListener(tempOutputListener);
272 return buf.toString();
273 }
274
275 public String asCommand() {
276 return createCommandLine().toString();
277 }
278
279 @Override
280 public String toString() {
281 return asCommand();
282 }
283
284 /**
285 * Build a command line based on the properties. Can be overridden by specific
286 * command wrappers.
287 */
288 protected CommandLine createCommandLine() {
289 // Check if an OS specific command overrides
290 String osName = System.getProperty("os.name");
291 List<Object> commandToUse = null;
292 if (osCommands.containsKey(osName))
293 commandToUse = osCommands.get(osName);
294 else
295 commandToUse = command;
296 String cmdToUse = null;
297 if (osCmds.containsKey(osName))
298 cmdToUse = osCmds.get(osName);
299 else
300 cmdToUse = cmd;
301
302 CommandLine commandLine = null;
303
304 // Which command definition to use
305 if (commandToUse == null && cmdToUse == null)
306 throw new SlcException("Please specify a command.");
307 else if (commandToUse != null && cmdToUse != null)
308 throw new SlcException("Specify the command either as a line or as a list.");
309 else if (cmdToUse != null) {
310 if (chroot != null && !chroot.trim().equals(""))
311 cmdToUse = "chroot \"" + chroot + "\" " + cmdToUse;
312 if (sudo != null) {
313 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
314 if (!sudo.trim().equals(""))
315 cmdToUse = "sudo -p " + sudoPrompt + " -u " + sudo + " " + cmdToUse;
316 else
317 cmdToUse = "sudo -p " + sudoPrompt + " " + cmdToUse;
318 }
319
320 // GENERATE COMMAND LINE
321 commandLine = CommandLine.parse(cmdToUse);
322 } else if (commandToUse != null) {
323 if (commandToUse.size() == 0)
324 throw new SlcException("Command line is empty.");
325
326 if (chroot != null && sudo != null) {
327 commandToUse.add(0, "chroot");
328 commandToUse.add(1, chroot);
329 }
330
331 if (sudo != null) {
332 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
333 commandToUse.add(0, "sudo");
334 commandToUse.add(1, "-p");
335 commandToUse.add(2, sudoPrompt);
336 if (!sudo.trim().equals("")) {
337 commandToUse.add(3, "-u");
338 commandToUse.add(4, sudo);
339 }
340 }
341
342 // GENERATE COMMAND LINE
343 commandLine = new CommandLine(commandToUse.get(0).toString());
344
345 for (int i = 1; i < commandToUse.size(); i++) {
346 if (log.isTraceEnabled())
347 log.debug(commandToUse.get(i));
348 commandLine.addArgument(commandToUse.get(i).toString());
349 }
350 } else {
351 // all cases covered previously
352 throw new UnsupportedException();
353 }
354
355 if (generateScript != null) {
356 File scriptFile = new File(getExecDirToUse() + File.separator + generateScript);
357 try {
358 FileUtils.writeStringToFile(scriptFile,
359 (osConsole != null ? osConsole + " " : "") + commandLine.toString());
360 } catch (IOException e) {
361 throw new SlcException("Could not generate script " + scriptFile, e);
362 }
363 commandLine = new CommandLine(scriptFile);
364 } else {
365 if (osConsole != null)
366 commandLine = CommandLine.parse(osConsole + " " + commandLine.toString());
367 }
368
369 return commandLine;
370 }
371
372 /**
373 * Creates a {@link PumpStreamHandler} which redirects streams to the custom
374 * logging mechanism.
375 */
376 protected ExecuteStreamHandler createExecuteStreamHandler(final Writer stdOutWriter,
377 final OutputStream stdOutputStream, final Writer stdErrWriter, final InputStream stdInStream) {
378
379 // Log writers
380 OutputStream stdout = stdOutputStream != null ? stdOutputStream : new LogOutputStream() {
381 protected void processLine(String line, int level) {
382 // if (firstLine) {
383 // if (sudo != null && callbackHandler != null
384 // && line.startsWith(sudoPrompt)) {
385 // try {
386 // PasswordCallback pc = new PasswordCallback(
387 // "sudo password", false);
388 // Callback[] cbs = { pc };
389 // callbackHandler.handle(cbs);
390 // char[] pwd = pc.getPassword();
391 // char[] arr = Arrays.copyOf(pwd,
392 // pwd.length + 1);
393 // arr[arr.length - 1] = '\n';
394 // IOUtils.write(arr, stdInSink);
395 // stdInSink.flush();
396 // } catch (Exception e) {
397 // throw new SlcException(
398 // "Cannot retrieve sudo password", e);
399 // }
400 // }
401 // firstLine = false;
402 // }
403
404 if (line != null && !line.trim().equals(""))
405 logStdOut(line);
406
407 if (stdOutWriter != null)
408 appendLineToFile(stdOutWriter, line);
409 }
410 };
411
412 OutputStream stderr = new LogOutputStream() {
413 protected void processLine(String line, int level) {
414 if (line != null && !line.trim().equals(""))
415 logStdErr(line);
416 if (stdErrWriter != null)
417 appendLineToFile(stdErrWriter, line);
418 }
419 };
420
421 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(stdout, stderr, stdInStream) {
422
423 @Override
424 public void stop() throws IOException {
425 // prevents the method to block when joining stdin
426 if (stdInSink != null)
427 IOUtils.closeQuietly(stdInSink);
428
429 super.stop();
430 }
431 };
432 return pumpStreamHandler;
433 }
434
435 /** Creates the default {@link ExecuteResultHandler}. */
436 protected ExecuteResultHandler createExecuteResultHandler(final CommandLine commandLine) {
437 return new ExecuteResultHandler() {
438
439 public void onProcessComplete(int exitValue) {
440 String msg = "System call '" + commandLine + "' properly completed.";
441 if (log.isTraceEnabled())
442 log.trace(msg);
443 if (testResult != null) {
444 forwardPath(testResult);
445 testResult.addResultPart(new SimpleResultPart(TestStatus.PASSED, msg));
446 }
447 releaseWatchdog();
448 }
449
450 public void onProcessFailed(ExecuteException e) {
451
452 String msg = "System call '" + commandLine + "' failed.";
453 if (testResult != null) {
454 forwardPath(testResult);
455 testResult.addResultPart(new SimpleResultPart(TestStatus.ERROR, msg, e));
456 } else {
457 if (exceptionOnFailed)
458 throw new SlcException(msg, e);
459 else
460 log.error(msg, e);
461 }
462 releaseWatchdog();
463 }
464 };
465 }
466
467 @Deprecated
468 protected void forwardPath(TestResult testResult) {
469 // TODO: allocate a TreeSPath
470 }
471
472 /**
473 * Shortcut method getting the execDir to use
474 */
475 protected String getExecDirToUse() {
476 try {
477 if (execDir != null) {
478 return execDir;
479 }
480 return System.getProperty("user.dir");
481 } catch (Exception e) {
482 throw new SlcException("Cannot find exec dir", e);
483 }
484 }
485
486 protected void logStdOut(String line) {
487 for (SystemCallOutputListener outputListener : outputListeners)
488 outputListener.newLine(this, line, false);
489 log(stdOutLogLevel, line);
490 }
491
492 protected void logStdErr(String line) {
493 for (SystemCallOutputListener outputListener : outputListeners)
494 outputListener.newLine(this, line, true);
495 log(stdErrLogLevel, line);
496 }
497
498 /** Log from the underlying streams. */
499 protected void log(String logLevel, String line) {
500 // TODO optimize
501 // if (SecurityContextHolder.getContext().getAuthentication() == null) {
502 // SecurityContextHolder.getContext()
503 // .setAuthentication(authentication);
504 // }
505
506 if ("ERROR".equals(logLevel))
507 log.error(line);
508 else if ("WARN".equals(logLevel))
509 log.warn(line);
510 else if ("INFO".equals(logLevel))
511 log.info(line);
512 else if ("DEBUG".equals(logLevel))
513 log.debug(line);
514 else if ("TRACE".equals(logLevel))
515 log.trace(line);
516 else if (LOG_STDOUT.equals(logLevel))
517 System.out.println(line);
518 else if ("System.err".equals(logLevel))
519 System.err.println(line);
520 else
521 throw new SlcException("Unknown log level " + logLevel);
522 }
523
524 /** Append line to a log file. */
525 protected void appendLineToFile(Writer writer, String line) {
526 try {
527 writer.append(line).append('\n');
528 } catch (IOException e) {
529 log.error("Cannot write to log file", e);
530 }
531 }
532
533 /** Creates the writer for the output/err files. */
534 protected Writer createWriter(Path target, Boolean append) {
535 FileWriter writer = null;
536 try {
537
538 final File file;
539 if (executionResources != null)
540 file = new File(executionResources.getAsOsPath(target, true));
541 else
542 file = target.toFile();
543 writer = new FileWriter(file, append);
544 } catch (IOException e) {
545 log.error("Cannot get file for " + target, e);
546 IOUtils.closeQuietly(writer);
547 }
548 return writer;
549 }
550
551 /** Creates an outputstream for the output/err files. */
552 protected OutputStream createOutputStream(Path target) {
553 FileOutputStream out = null;
554 try {
555
556 final File file;
557 if (executionResources != null)
558 file = new File(executionResources.getAsOsPath(target, true));
559 else
560 file = target.toFile();
561 out = new FileOutputStream(file, false);
562 } catch (IOException e) {
563 log.error("Cannot get file for " + target, e);
564 IOUtils.closeQuietly(out);
565 }
566 return out;
567 }
568
569 /** Append the argument (for chaining) */
570 public SystemCall arg(String arg) {
571 if (command == null)
572 command = new ArrayList<Object>();
573 command.add(arg);
574 return this;
575 }
576
577 /** Append the argument (for chaining) */
578 public SystemCall arg(String arg, String value) {
579 if (command == null)
580 command = new ArrayList<Object>();
581 command.add(arg);
582 command.add(value);
583 return this;
584 }
585
586 // CONTROL
587 public synchronized Boolean isRunning() {
588 return currentWatchdog != null;
589 }
590
591 private synchronized ExecuteWatchdog createWatchdog() {
592 // if (currentWatchdog != null)
593 // throw new SlcException("A process is already being monitored");
594 currentWatchdog = new ExecuteWatchdog(watchdogTimeout);
595 return currentWatchdog;
596 }
597
598 private synchronized void releaseWatchdog() {
599 currentWatchdog = null;
600 }
601
602 public synchronized void kill() {
603 if (currentWatchdog != null)
604 currentWatchdog.destroyProcess();
605 }
606
607 /** */
608 public void setCmd(String command) {
609 this.cmd = command;
610 }
611
612 public void setCommand(List<Object> command) {
613 this.command = command;
614 }
615
616 public void setExecDir(String execdir) {
617 this.execDir = execdir;
618 }
619
620 public void setStdErrLogLevel(String stdErrLogLevel) {
621 this.stdErrLogLevel = stdErrLogLevel;
622 }
623
624 public void setStdOutLogLevel(String stdOutLogLevel) {
625 this.stdOutLogLevel = stdOutLogLevel;
626 }
627
628 public void setSynchronous(Boolean synchronous) {
629 this.synchronous = synchronous;
630 }
631
632 public void setOsCommands(Map<String, List<Object>> osCommands) {
633 this.osCommands = osCommands;
634 }
635
636 public void setOsCmds(Map<String, String> osCmds) {
637 this.osCmds = osCmds;
638 }
639
640 public void setEnvironmentVariables(Map<String, String> environmentVariables) {
641 this.environmentVariables = environmentVariables;
642 }
643
644 public Map<String, String> getEnvironmentVariables() {
645 return environmentVariables;
646 }
647
648 public void setWatchdogTimeout(Long watchdogTimeout) {
649 this.watchdogTimeout = watchdogTimeout;
650 }
651
652 public void setStdOutFile(Path stdOutFile) {
653 this.stdOutFile = stdOutFile;
654 }
655
656 public void setStdErrFile(Path stdErrFile) {
657 this.stdErrFile = stdErrFile;
658 }
659
660 public void setStdInFile(Path stdInFile) {
661 this.stdInFile = stdInFile;
662 }
663
664 public void setTestResult(TestResult testResult) {
665 this.testResult = testResult;
666 }
667
668 public void setLogCommand(Boolean logCommand) {
669 this.logCommand = logCommand;
670 }
671
672 public void setRedirectStreams(Boolean redirectStreams) {
673 this.redirectStreams = redirectStreams;
674 }
675
676 public void setExceptionOnFailed(Boolean exceptionOnFailed) {
677 this.exceptionOnFailed = exceptionOnFailed;
678 }
679
680 public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
681 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
682 }
683
684 public void setOsConsole(String osConsole) {
685 this.osConsole = osConsole;
686 }
687
688 public void setGenerateScript(String generateScript) {
689 this.generateScript = generateScript;
690 }
691
692 public void setExecutionResources(ExecutionResources executionResources) {
693 this.executionResources = executionResources;
694 }
695
696 public void setRedirectStdOut(Boolean redirectStdOut) {
697 this.redirectStdOut = redirectStdOut;
698 }
699
700 public void addOutputListener(SystemCallOutputListener outputListener) {
701 outputListeners.add(outputListener);
702 }
703
704 public void removeOutputListener(SystemCallOutputListener outputListener) {
705 outputListeners.remove(outputListener);
706 }
707
708 public void setOutputListeners(List<SystemCallOutputListener> outputListeners) {
709 this.outputListeners = outputListeners;
710 }
711
712 public void setExecutor(Executor executor) {
713 this.executor = executor;
714 }
715
716 public void setSudo(String sudo) {
717 this.sudo = sudo;
718 }
719
720 public void setCallbackHandler(CallbackHandler callbackHandler) {
721 this.callbackHandler = callbackHandler;
722 }
723
724 public void setChroot(String chroot) {
725 this.chroot = chroot;
726 }
727
728 private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
729
730 public void setProcessErrorStream(InputStream is) throws IOException {
731 }
732
733 public void setProcessInputStream(OutputStream os) throws IOException {
734 }
735
736 public void setProcessOutputStream(InputStream is) throws IOException {
737 }
738
739 public void start() throws IOException {
740 }
741
742 public void stop() {
743 }
744
745 }
746 }