]> git.argeo.org Git - gpl/argeo-slc.git/blob - SystemCall.java
633f8f950b22a16ad1f3168ac9a425ca597425ee
[gpl/argeo-slc.git] / SystemCall.java
1 package org.argeo.slc.core.execution.tasks;
2
3 import java.io.File;
4 import java.io.FileOutputStream;
5 import java.io.FileWriter;
6 import java.io.IOException;
7 import java.io.InputStream;
8 import java.io.OutputStream;
9 import java.io.PipedInputStream;
10 import java.io.PipedOutputStream;
11 import java.io.Writer;
12 import java.util.ArrayList;
13 import java.util.Collections;
14 import java.util.HashMap;
15 import java.util.List;
16 import java.util.Map;
17 import java.util.UUID;
18
19 import javax.security.auth.callback.CallbackHandler;
20
21 import org.apache.commons.exec.CommandLine;
22 import org.apache.commons.exec.DefaultExecutor;
23 import org.apache.commons.exec.ExecuteException;
24 import org.apache.commons.exec.ExecuteResultHandler;
25 import org.apache.commons.exec.ExecuteStreamHandler;
26 import org.apache.commons.exec.ExecuteWatchdog;
27 import org.apache.commons.exec.Executor;
28 import org.apache.commons.exec.LogOutputStream;
29 import org.apache.commons.exec.PumpStreamHandler;
30 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
31 import org.apache.commons.io.FileUtils;
32 import org.apache.commons.io.IOUtils;
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.argeo.slc.SlcException;
36 import org.argeo.slc.UnsupportedException;
37 import org.argeo.slc.core.execution.ExecutionResources;
38 import org.argeo.slc.runtime.test.SimpleResultPart;
39 import org.argeo.slc.test.TestResult;
40 import org.argeo.slc.test.TestStatus;
41 import org.springframework.core.io.Resource;
42
43 /** Execute an OS specific system call. */
44 public class SystemCall implements Runnable {
45 public final static String LOG_STDOUT = "System.out";
46
47 private final Log log = LogFactory.getLog(getClass());
48
49 private String execDir;
50
51 private String cmd = null;
52 private List<Object> command = null;
53
54 private Executor executor = new DefaultExecutor();
55 private Boolean synchronous = true;
56
57 private String stdErrLogLevel = "ERROR";
58 private String stdOutLogLevel = "INFO";
59
60 private Resource stdOutFile = null;
61 private Resource stdErrFile = null;
62
63 private Resource stdInFile = null;
64 /**
65 * If no {@link #stdInFile} provided, writing to this stream will write to
66 * the stdin of the process.
67 */
68 private OutputStream stdInSink = null;
69
70 private Boolean redirectStdOut = false;
71
72 private List<SystemCallOutputListener> outputListeners = Collections
73 .synchronizedList(new ArrayList<SystemCallOutputListener>());
74
75 private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
76 private Map<String, String> osCmds = new HashMap<String, String>();
77 private Map<String, String> environmentVariables = new HashMap<String, String>();
78
79 private Boolean logCommand = false;
80 private Boolean redirectStreams = true;
81 private Boolean exceptionOnFailed = true;
82 private Boolean mergeEnvironmentVariables = true;
83
84 // private Authentication authentication;
85
86 private String osConsole = null;
87 private String generateScript = null;
88
89 /** 24 hours */
90 private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
91
92 private TestResult testResult;
93
94 private ExecutionResources executionResources;
95
96 /** Sudo the command, as root if empty or as user if not. */
97 private String sudo = null;
98 // TODO make it more secure and robust, test only once
99 private final String sudoPrompt = UUID.randomUUID().toString();
100 private String askPassProgram = "/usr/libexec/openssh/ssh-askpass";
101 @SuppressWarnings("unused")
102 private boolean firstLine = true;
103 @SuppressWarnings("unused")
104 private CallbackHandler callbackHandler;
105 /** Chroot to the this path (must not be empty) */
106 private String chroot = null;
107
108 // Current
109 /** Current watchdog, null if process is completed */
110 ExecuteWatchdog currentWatchdog = null;
111
112 /** Empty constructor */
113 public SystemCall() {
114
115 }
116
117 /**
118 * Constructor based on the provided command list.
119 *
120 * @param command
121 * the command list
122 */
123 public SystemCall(List<Object> command) {
124 this.command = command;
125 }
126
127 /**
128 * Constructor based on the provided command.
129 *
130 * @param cmd
131 * the command. If the provided string contains no space a
132 * command list is initialized with the argument as first
133 * component (useful for chained construction)
134 */
135 public SystemCall(String cmd) {
136 if (cmd.indexOf(' ') < 0) {
137 command = new ArrayList<Object>();
138 command.add(cmd);
139 } else {
140 this.cmd = cmd;
141 }
142 }
143
144 /** Executes the system call. */
145 public void run() {
146 // authentication = SecurityContextHolder.getContext().getAuthentication();
147
148 // Manage streams
149 Writer stdOutWriter = null;
150 OutputStream stdOutputStream = null;
151 Writer stdErrWriter = null;
152 InputStream stdInStream = null;
153 if (stdOutFile != null)
154 if (redirectStdOut)
155 stdOutputStream = createOutputStream(stdOutFile);
156 else
157 stdOutWriter = createWriter(stdOutFile, true);
158
159 if (stdErrFile != null) {
160 stdErrWriter = createWriter(stdErrFile, true);
161 } else {
162 if (stdOutFile != null && !redirectStdOut)
163 stdErrWriter = createWriter(stdOutFile, true);
164 }
165
166 try {
167 if (stdInFile != null)
168 stdInStream = stdInFile.getInputStream();
169 else {
170 stdInStream = new PipedInputStream();
171 stdInSink = new PipedOutputStream(
172 (PipedInputStream) stdInStream);
173 }
174 } catch (IOException e2) {
175 throw new SlcException("Cannot open a stream for " + stdInFile, e2);
176 }
177
178 if (log.isTraceEnabled()) {
179 log.debug("os.name=" + System.getProperty("os.name"));
180 log.debug("os.arch=" + System.getProperty("os.arch"));
181 log.debug("os.version=" + System.getProperty("os.version"));
182 }
183
184 // Execution directory
185 File dir = new File(getExecDirToUse());
186 // if (!dir.exists())
187 // dir.mkdirs();
188
189 // Watchdog to check for lost processes
190 Executor executorToUse;
191 if (executor != null)
192 executorToUse = executor;
193 else
194 executorToUse = new DefaultExecutor();
195 executorToUse.setWatchdog(createWatchdog());
196
197 if (redirectStreams) {
198 // Redirect standard streams
199 executorToUse.setStreamHandler(createExecuteStreamHandler(
200 stdOutWriter, stdOutputStream, stdErrWriter, stdInStream));
201 } else {
202 // Dummy stream handler (otherwise pump is used)
203 executorToUse.setStreamHandler(new DummyexecuteStreamHandler());
204 }
205
206 executorToUse.setProcessDestroyer(new ShutdownHookProcessDestroyer());
207 executorToUse.setWorkingDirectory(dir);
208
209 // Command line to use
210 final CommandLine commandLine = createCommandLine();
211 if (logCommand)
212 log.info("Execute command:\n" + commandLine
213 + "\n in working directory: \n" + dir + "\n");
214
215 // Env variables
216 Map<String, String> environmentVariablesToUse = null;
217 environmentVariablesToUse = new HashMap<String, String>();
218 if (mergeEnvironmentVariables)
219 environmentVariablesToUse.putAll(System.getenv());
220 if (environmentVariables.size() > 0)
221 environmentVariablesToUse.putAll(environmentVariables);
222
223 // Execute
224 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
225
226 //
227 // THE EXECUTION PROPER
228 //
229 try {
230 if (synchronous)
231 try {
232 int exitValue = executorToUse.execute(commandLine,
233 environmentVariablesToUse);
234 executeResultHandler.onProcessComplete(exitValue);
235 } catch (ExecuteException e1) {
236 if (e1.getExitValue() == Executor.INVALID_EXITVALUE) {
237 Thread.currentThread().interrupt();
238 return;
239 }
240 // Sleep 1s in order to make sure error logs are flushed
241 Thread.sleep(1000);
242 executeResultHandler.onProcessFailed(e1);
243 }
244 else
245 executorToUse.execute(commandLine, environmentVariablesToUse,
246 executeResultHandler);
247 } catch (SlcException e) {
248 throw e;
249 } catch (Exception e) {
250 throw new SlcException("Could not execute command " + commandLine,
251 e);
252 } finally {
253 IOUtils.closeQuietly(stdOutWriter);
254 IOUtils.closeQuietly(stdErrWriter);
255 IOUtils.closeQuietly(stdInStream);
256 IOUtils.closeQuietly(stdInSink);
257 }
258
259 }
260
261 public synchronized String function() {
262 final StringBuffer buf = new StringBuffer("");
263 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
264 private Long lineCount = 0l;
265
266 public void newLine(SystemCall systemCall, String line,
267 Boolean isError) {
268 if (!isError) {
269 if (lineCount != 0l)
270 buf.append('\n');
271 buf.append(line);
272 lineCount++;
273 }
274 }
275 };
276 addOutputListener(tempOutputListener);
277 run();
278 removeOutputListener(tempOutputListener);
279 return buf.toString();
280 }
281
282 public String asCommand() {
283 return createCommandLine().toString();
284 }
285
286 @Override
287 public String toString() {
288 return asCommand();
289 }
290
291 /**
292 * Build a command line based on the properties. Can be overridden by
293 * specific command wrappers.
294 */
295 protected CommandLine createCommandLine() {
296 // Check if an OS specific command overrides
297 String osName = System.getProperty("os.name");
298 List<Object> commandToUse = null;
299 if (osCommands.containsKey(osName))
300 commandToUse = osCommands.get(osName);
301 else
302 commandToUse = command;
303 String cmdToUse = null;
304 if (osCmds.containsKey(osName))
305 cmdToUse = osCmds.get(osName);
306 else
307 cmdToUse = cmd;
308
309 CommandLine commandLine = null;
310
311 // Which command definition to use
312 if (commandToUse == null && cmdToUse == null)
313 throw new SlcException("Please specify a command.");
314 else if (commandToUse != null && cmdToUse != null)
315 throw new SlcException(
316 "Specify the command either as a line or as a list.");
317 else if (cmdToUse != null) {
318 if (chroot != null && !chroot.trim().equals(""))
319 cmdToUse = "chroot \"" + chroot + "\" " + cmdToUse;
320 if (sudo != null) {
321 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
322 if (!sudo.trim().equals(""))
323 cmdToUse = "sudo -p " + sudoPrompt + " -u " + sudo + " "
324 + cmdToUse;
325 else
326 cmdToUse = "sudo -p " + sudoPrompt + " " + cmdToUse;
327 }
328
329 // GENERATE COMMAND LINE
330 commandLine = CommandLine.parse(cmdToUse);
331 } else if (commandToUse != null) {
332 if (commandToUse.size() == 0)
333 throw new SlcException("Command line is empty.");
334
335 if (chroot != null && sudo != null) {
336 commandToUse.add(0, "chroot");
337 commandToUse.add(1, chroot);
338 }
339
340 if (sudo != null) {
341 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
342 commandToUse.add(0, "sudo");
343 commandToUse.add(1, "-p");
344 commandToUse.add(2, sudoPrompt);
345 if (!sudo.trim().equals("")) {
346 commandToUse.add(3, "-u");
347 commandToUse.add(4, sudo);
348 }
349 }
350
351 // GENERATE COMMAND LINE
352 commandLine = new CommandLine(commandToUse.get(0).toString());
353
354 for (int i = 1; i < commandToUse.size(); i++) {
355 if (log.isTraceEnabled())
356 log.debug(commandToUse.get(i));
357 commandLine.addArgument(commandToUse.get(i).toString());
358 }
359 } else {
360 // all cases covered previously
361 throw new UnsupportedException();
362 }
363
364 if (generateScript != null) {
365 File scriptFile = new File(getExecDirToUse() + File.separator
366 + generateScript);
367 try {
368 FileUtils.writeStringToFile(scriptFile,
369 (osConsole != null ? osConsole + " " : "")
370 + commandLine.toString());
371 } catch (IOException e) {
372 throw new SlcException("Could not generate script "
373 + scriptFile, e);
374 }
375 commandLine = new CommandLine(scriptFile);
376 } else {
377 if (osConsole != null)
378 commandLine = CommandLine.parse(osConsole + " "
379 + commandLine.toString());
380 }
381
382 return commandLine;
383 }
384
385 /**
386 * Creates a {@link PumpStreamHandler} which redirects streams to the custom
387 * logging mechanism.
388 */
389 protected ExecuteStreamHandler createExecuteStreamHandler(
390 final Writer stdOutWriter, final OutputStream stdOutputStream,
391 final Writer stdErrWriter, final InputStream stdInStream) {
392
393 // Log writers
394 OutputStream stdout = stdOutputStream != null ? stdOutputStream
395 : new LogOutputStream() {
396 protected void processLine(String line, int level) {
397 // if (firstLine) {
398 // if (sudo != null && callbackHandler != null
399 // && line.startsWith(sudoPrompt)) {
400 // try {
401 // PasswordCallback pc = new PasswordCallback(
402 // "sudo password", false);
403 // Callback[] cbs = { pc };
404 // callbackHandler.handle(cbs);
405 // char[] pwd = pc.getPassword();
406 // char[] arr = Arrays.copyOf(pwd,
407 // pwd.length + 1);
408 // arr[arr.length - 1] = '\n';
409 // IOUtils.write(arr, stdInSink);
410 // stdInSink.flush();
411 // } catch (Exception e) {
412 // throw new SlcException(
413 // "Cannot retrieve sudo password", e);
414 // }
415 // }
416 // firstLine = false;
417 // }
418
419 if (line != null && !line.trim().equals(""))
420 logStdOut(line);
421
422 if (stdOutWriter != null)
423 appendLineToFile(stdOutWriter, line);
424 }
425 };
426
427 OutputStream stderr = new LogOutputStream() {
428 protected void processLine(String line, int level) {
429 if (line != null && !line.trim().equals(""))
430 logStdErr(line);
431 if (stdErrWriter != null)
432 appendLineToFile(stdErrWriter, line);
433 }
434 };
435
436 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(stdout,
437 stderr, stdInStream) {
438
439 @Override
440 public void stop() throws IOException {
441 // prevents the method to block when joining stdin
442 if (stdInSink != null)
443 IOUtils.closeQuietly(stdInSink);
444
445 super.stop();
446 }
447 };
448 return pumpStreamHandler;
449 }
450
451 /** Creates the default {@link ExecuteResultHandler}. */
452 protected ExecuteResultHandler createExecuteResultHandler(
453 final CommandLine commandLine) {
454 return new ExecuteResultHandler() {
455
456 public void onProcessComplete(int exitValue) {
457 String msg = "System call '" + commandLine
458 + "' properly completed.";
459 if (log.isTraceEnabled())
460 log.trace(msg);
461 if (testResult != null) {
462 forwardPath(testResult);
463 testResult.addResultPart(new SimpleResultPart(
464 TestStatus.PASSED, msg));
465 }
466 releaseWatchdog();
467 }
468
469 public void onProcessFailed(ExecuteException e) {
470
471 String msg = "System call '" + commandLine + "' failed.";
472 if (testResult != null) {
473 forwardPath(testResult);
474 testResult.addResultPart(new SimpleResultPart(
475 TestStatus.ERROR, msg, e));
476 } else {
477 if (exceptionOnFailed)
478 throw new SlcException(msg, e);
479 else
480 log.error(msg, e);
481 }
482 releaseWatchdog();
483 }
484 };
485 }
486
487 @Deprecated
488 protected void forwardPath(TestResult testResult) {
489 // TODO: allocate a TreeSPath
490 }
491
492 /**
493 * Shortcut method getting the execDir to use
494 */
495 protected String getExecDirToUse() {
496 try {
497 if (execDir != null) {
498 return execDir;
499 }
500 return System.getProperty("user.dir");
501 } catch (Exception e) {
502 throw new SlcException("Cannot find exec dir", e);
503 }
504 }
505
506 protected void logStdOut(String line) {
507 for (SystemCallOutputListener outputListener : outputListeners)
508 outputListener.newLine(this, line, false);
509 log(stdOutLogLevel, line);
510 }
511
512 protected void logStdErr(String line) {
513 for (SystemCallOutputListener outputListener : outputListeners)
514 outputListener.newLine(this, line, true);
515 log(stdErrLogLevel, line);
516 }
517
518 /** Log from the underlying streams. */
519 protected void log(String logLevel, String line) {
520 // TODO optimize
521 // if (SecurityContextHolder.getContext().getAuthentication() == null) {
522 // SecurityContextHolder.getContext()
523 // .setAuthentication(authentication);
524 // }
525
526 if ("ERROR".equals(logLevel))
527 log.error(line);
528 else if ("WARN".equals(logLevel))
529 log.warn(line);
530 else if ("INFO".equals(logLevel))
531 log.info(line);
532 else if ("DEBUG".equals(logLevel))
533 log.debug(line);
534 else if ("TRACE".equals(logLevel))
535 log.trace(line);
536 else if (LOG_STDOUT.equals(logLevel))
537 System.out.println(line);
538 else if ("System.err".equals(logLevel))
539 System.err.println(line);
540 else
541 throw new SlcException("Unknown log level " + logLevel);
542 }
543
544 /** Append line to a log file. */
545 protected void appendLineToFile(Writer writer, String line) {
546 try {
547 writer.append(line).append('\n');
548 } catch (IOException e) {
549 log.error("Cannot write to log file", e);
550 }
551 }
552
553 /** Creates the writer for the output/err files. */
554 protected Writer createWriter(Resource target, Boolean append) {
555 FileWriter writer = null;
556 try {
557
558 final File file;
559 if (executionResources != null)
560 file = new File(executionResources.getAsOsPath(target, true));
561 else
562 file = target.getFile();
563 writer = new FileWriter(file, append);
564 } catch (IOException e) {
565 log.error("Cannot get file for " + target, e);
566 IOUtils.closeQuietly(writer);
567 }
568 return writer;
569 }
570
571 /** Creates an outputstream for the output/err files. */
572 protected OutputStream createOutputStream(Resource target) {
573 FileOutputStream out = null;
574 try {
575
576 final File file;
577 if (executionResources != null)
578 file = new File(executionResources.getAsOsPath(target, true));
579 else
580 file = target.getFile();
581 out = new FileOutputStream(file, false);
582 } catch (IOException e) {
583 log.error("Cannot get file for " + target, e);
584 IOUtils.closeQuietly(out);
585 }
586 return out;
587 }
588
589 /** Append the argument (for chaining) */
590 public SystemCall arg(String arg) {
591 if (command == null)
592 command = new ArrayList<Object>();
593 command.add(arg);
594 return this;
595 }
596
597 /** Append the argument (for chaining) */
598 public SystemCall arg(String arg, String value) {
599 if (command == null)
600 command = new ArrayList<Object>();
601 command.add(arg);
602 command.add(value);
603 return this;
604 }
605
606 // CONTROL
607 public synchronized Boolean isRunning() {
608 return currentWatchdog != null;
609 }
610
611 private synchronized ExecuteWatchdog createWatchdog() {
612 // if (currentWatchdog != null)
613 // throw new SlcException("A process is already being monitored");
614 currentWatchdog = new ExecuteWatchdog(watchdogTimeout);
615 return currentWatchdog;
616 }
617
618 private synchronized void releaseWatchdog() {
619 currentWatchdog = null;
620 }
621
622 public synchronized void kill() {
623 if (currentWatchdog != null)
624 currentWatchdog.destroyProcess();
625 }
626
627 /** */
628 public void setCmd(String command) {
629 this.cmd = command;
630 }
631
632 public void setCommand(List<Object> command) {
633 this.command = command;
634 }
635
636 public void setExecDir(String execdir) {
637 this.execDir = execdir;
638 }
639
640 public void setStdErrLogLevel(String stdErrLogLevel) {
641 this.stdErrLogLevel = stdErrLogLevel;
642 }
643
644 public void setStdOutLogLevel(String stdOutLogLevel) {
645 this.stdOutLogLevel = stdOutLogLevel;
646 }
647
648 public void setSynchronous(Boolean synchronous) {
649 this.synchronous = synchronous;
650 }
651
652 public void setOsCommands(Map<String, List<Object>> osCommands) {
653 this.osCommands = osCommands;
654 }
655
656 public void setOsCmds(Map<String, String> osCmds) {
657 this.osCmds = osCmds;
658 }
659
660 public void setEnvironmentVariables(Map<String, String> environmentVariables) {
661 this.environmentVariables = environmentVariables;
662 }
663
664 public Map<String, String> getEnvironmentVariables() {
665 return environmentVariables;
666 }
667
668 public void setWatchdogTimeout(Long watchdogTimeout) {
669 this.watchdogTimeout = watchdogTimeout;
670 }
671
672 public void setStdOutFile(Resource stdOutFile) {
673 this.stdOutFile = stdOutFile;
674 }
675
676 public void setStdErrFile(Resource stdErrFile) {
677 this.stdErrFile = stdErrFile;
678 }
679
680 public void setStdInFile(Resource stdInFile) {
681 this.stdInFile = stdInFile;
682 }
683
684 public void setTestResult(TestResult testResult) {
685 this.testResult = testResult;
686 }
687
688 public void setLogCommand(Boolean logCommand) {
689 this.logCommand = logCommand;
690 }
691
692 public void setRedirectStreams(Boolean redirectStreams) {
693 this.redirectStreams = redirectStreams;
694 }
695
696 public void setExceptionOnFailed(Boolean exceptionOnFailed) {
697 this.exceptionOnFailed = exceptionOnFailed;
698 }
699
700 public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
701 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
702 }
703
704 public void setOsConsole(String osConsole) {
705 this.osConsole = osConsole;
706 }
707
708 public void setGenerateScript(String generateScript) {
709 this.generateScript = generateScript;
710 }
711
712 public void setExecutionResources(ExecutionResources executionResources) {
713 this.executionResources = executionResources;
714 }
715
716 public void setRedirectStdOut(Boolean redirectStdOut) {
717 this.redirectStdOut = redirectStdOut;
718 }
719
720 public void addOutputListener(SystemCallOutputListener outputListener) {
721 outputListeners.add(outputListener);
722 }
723
724 public void removeOutputListener(SystemCallOutputListener outputListener) {
725 outputListeners.remove(outputListener);
726 }
727
728 public void setOutputListeners(
729 List<SystemCallOutputListener> outputListeners) {
730 this.outputListeners = outputListeners;
731 }
732
733 public void setExecutor(Executor executor) {
734 this.executor = executor;
735 }
736
737 public void setSudo(String sudo) {
738 this.sudo = sudo;
739 }
740
741 public void setCallbackHandler(CallbackHandler callbackHandler) {
742 this.callbackHandler = callbackHandler;
743 }
744
745 public void setChroot(String chroot) {
746 this.chroot = chroot;
747 }
748
749 private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
750
751 public void setProcessErrorStream(InputStream is) throws IOException {
752 }
753
754 public void setProcessInputStream(OutputStream os) throws IOException {
755 }
756
757 public void setProcessOutputStream(InputStream is) throws IOException {
758 }
759
760 public void start() throws IOException {
761 }
762
763 public void stop() {
764 }
765
766 }
767 }