]> git.argeo.org Git - gpl/argeo-slc.git/blob - legacy/org.argeo.slc.spring/src/org/argeo/slc/core/execution/tasks/SystemCall.java
Massive Argeo APIs refactoring
[gpl/argeo-slc.git] / legacy / org.argeo.slc.spring / src / org / argeo / slc / core / execution / tasks / SystemCall.java
1 package org.argeo.slc.core.execution.tasks;
2
3 import java.io.File;
4 import java.io.FileOutputStream;
5 import java.io.FileWriter;
6 import java.io.IOException;
7 import java.io.InputStream;
8 import java.io.OutputStream;
9 import java.io.PipedInputStream;
10 import java.io.PipedOutputStream;
11 import java.io.Writer;
12 import java.util.ArrayList;
13 import java.util.Collections;
14 import java.util.HashMap;
15 import java.util.List;
16 import java.util.Map;
17 import java.util.UUID;
18
19 import javax.security.auth.callback.CallbackHandler;
20
21 import org.apache.commons.exec.CommandLine;
22 import org.apache.commons.exec.DefaultExecutor;
23 import org.apache.commons.exec.ExecuteException;
24 import org.apache.commons.exec.ExecuteResultHandler;
25 import org.apache.commons.exec.ExecuteStreamHandler;
26 import org.apache.commons.exec.ExecuteWatchdog;
27 import org.apache.commons.exec.Executor;
28 import org.apache.commons.exec.LogOutputStream;
29 import org.apache.commons.exec.PumpStreamHandler;
30 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
31 import org.apache.commons.io.FileUtils;
32 import org.apache.commons.io.IOUtils;
33 import org.argeo.api.cms.CmsLog;
34 import org.argeo.slc.SlcException;
35 import org.argeo.slc.UnsupportedException;
36 import org.argeo.slc.core.execution.ExecutionResources;
37 import org.argeo.slc.runtime.test.SimpleResultPart;
38 import org.argeo.slc.test.TestResult;
39 import org.argeo.slc.test.TestStatus;
40 import org.springframework.core.io.Resource;
41
42 /** Execute an OS specific system call. */
43 public class SystemCall implements Runnable {
44 public final static String LOG_STDOUT = "System.out";
45
46 private final CmsLog log = CmsLog.getLog(getClass());
47
48 private String execDir;
49
50 private String cmd = null;
51 private List<Object> command = null;
52
53 private Executor executor = new DefaultExecutor();
54 private Boolean synchronous = true;
55
56 private String stdErrLogLevel = "ERROR";
57 private String stdOutLogLevel = "INFO";
58
59 private Resource stdOutFile = null;
60 private Resource stdErrFile = null;
61
62 private Resource stdInFile = null;
63 /**
64 * If no {@link #stdInFile} provided, writing to this stream will write to
65 * the stdin of the process.
66 */
67 private OutputStream stdInSink = null;
68
69 private Boolean redirectStdOut = false;
70
71 private List<SystemCallOutputListener> outputListeners = Collections
72 .synchronizedList(new ArrayList<SystemCallOutputListener>());
73
74 private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
75 private Map<String, String> osCmds = new HashMap<String, String>();
76 private Map<String, String> environmentVariables = new HashMap<String, String>();
77
78 private Boolean logCommand = false;
79 private Boolean redirectStreams = true;
80 private Boolean exceptionOnFailed = true;
81 private Boolean mergeEnvironmentVariables = true;
82
83 // private Authentication authentication;
84
85 private String osConsole = null;
86 private String generateScript = null;
87
88 /** 24 hours */
89 private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
90
91 private TestResult testResult;
92
93 private ExecutionResources executionResources;
94
95 /** Sudo the command, as root if empty or as user if not. */
96 private String sudo = null;
97 // TODO make it more secure and robust, test only once
98 private final String sudoPrompt = UUID.randomUUID().toString();
99 private String askPassProgram = "/usr/libexec/openssh/ssh-askpass";
100 @SuppressWarnings("unused")
101 private boolean firstLine = true;
102 @SuppressWarnings("unused")
103 private CallbackHandler callbackHandler;
104 /** Chroot to the this path (must not be empty) */
105 private String chroot = null;
106
107 // Current
108 /** Current watchdog, null if process is completed */
109 ExecuteWatchdog currentWatchdog = null;
110
111 /** Empty constructor */
112 public SystemCall() {
113
114 }
115
116 /**
117 * Constructor based on the provided command list.
118 *
119 * @param command
120 * the command list
121 */
122 public SystemCall(List<Object> command) {
123 this.command = command;
124 }
125
126 /**
127 * Constructor based on the provided command.
128 *
129 * @param cmd
130 * the command. If the provided string contains no space a
131 * command list is initialized with the argument as first
132 * component (useful for chained construction)
133 */
134 public SystemCall(String cmd) {
135 if (cmd.indexOf(' ') < 0) {
136 command = new ArrayList<Object>();
137 command.add(cmd);
138 } else {
139 this.cmd = cmd;
140 }
141 }
142
143 /** Executes the system call. */
144 public void run() {
145 // authentication = SecurityContextHolder.getContext().getAuthentication();
146
147 // Manage streams
148 Writer stdOutWriter = null;
149 OutputStream stdOutputStream = null;
150 Writer stdErrWriter = null;
151 InputStream stdInStream = null;
152 if (stdOutFile != null)
153 if (redirectStdOut)
154 stdOutputStream = createOutputStream(stdOutFile);
155 else
156 stdOutWriter = createWriter(stdOutFile, true);
157
158 if (stdErrFile != null) {
159 stdErrWriter = createWriter(stdErrFile, true);
160 } else {
161 if (stdOutFile != null && !redirectStdOut)
162 stdErrWriter = createWriter(stdOutFile, true);
163 }
164
165 try {
166 if (stdInFile != null)
167 stdInStream = stdInFile.getInputStream();
168 else {
169 stdInStream = new PipedInputStream();
170 stdInSink = new PipedOutputStream(
171 (PipedInputStream) stdInStream);
172 }
173 } catch (IOException e2) {
174 throw new SlcException("Cannot open a stream for " + stdInFile, e2);
175 }
176
177 if (log.isTraceEnabled()) {
178 log.debug("os.name=" + System.getProperty("os.name"));
179 log.debug("os.arch=" + System.getProperty("os.arch"));
180 log.debug("os.version=" + System.getProperty("os.version"));
181 }
182
183 // Execution directory
184 File dir = new File(getExecDirToUse());
185 // if (!dir.exists())
186 // dir.mkdirs();
187
188 // Watchdog to check for lost processes
189 Executor executorToUse;
190 if (executor != null)
191 executorToUse = executor;
192 else
193 executorToUse = new DefaultExecutor();
194 executorToUse.setWatchdog(createWatchdog());
195
196 if (redirectStreams) {
197 // Redirect standard streams
198 executorToUse.setStreamHandler(createExecuteStreamHandler(
199 stdOutWriter, stdOutputStream, stdErrWriter, stdInStream));
200 } else {
201 // Dummy stream handler (otherwise pump is used)
202 executorToUse.setStreamHandler(new DummyexecuteStreamHandler());
203 }
204
205 executorToUse.setProcessDestroyer(new ShutdownHookProcessDestroyer());
206 executorToUse.setWorkingDirectory(dir);
207
208 // Command line to use
209 final CommandLine commandLine = createCommandLine();
210 if (logCommand)
211 log.info("Execute command:\n" + commandLine
212 + "\n in working directory: \n" + dir + "\n");
213
214 // Env variables
215 Map<String, String> environmentVariablesToUse = null;
216 environmentVariablesToUse = new HashMap<String, String>();
217 if (mergeEnvironmentVariables)
218 environmentVariablesToUse.putAll(System.getenv());
219 if (environmentVariables.size() > 0)
220 environmentVariablesToUse.putAll(environmentVariables);
221
222 // Execute
223 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
224
225 //
226 // THE EXECUTION PROPER
227 //
228 try {
229 if (synchronous)
230 try {
231 int exitValue = executorToUse.execute(commandLine,
232 environmentVariablesToUse);
233 executeResultHandler.onProcessComplete(exitValue);
234 } catch (ExecuteException e1) {
235 if (e1.getExitValue() == Executor.INVALID_EXITVALUE) {
236 Thread.currentThread().interrupt();
237 return;
238 }
239 // Sleep 1s in order to make sure error logs are flushed
240 Thread.sleep(1000);
241 executeResultHandler.onProcessFailed(e1);
242 }
243 else
244 executorToUse.execute(commandLine, environmentVariablesToUse,
245 executeResultHandler);
246 } catch (SlcException e) {
247 throw e;
248 } catch (Exception e) {
249 throw new SlcException("Could not execute command " + commandLine,
250 e);
251 } finally {
252 IOUtils.closeQuietly(stdOutWriter);
253 IOUtils.closeQuietly(stdErrWriter);
254 IOUtils.closeQuietly(stdInStream);
255 IOUtils.closeQuietly(stdInSink);
256 }
257
258 }
259
260 public synchronized String function() {
261 final StringBuffer buf = new StringBuffer("");
262 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
263 private Long lineCount = 0l;
264
265 public void newLine(SystemCall systemCall, String line,
266 Boolean isError) {
267 if (!isError) {
268 if (lineCount != 0l)
269 buf.append('\n');
270 buf.append(line);
271 lineCount++;
272 }
273 }
274 };
275 addOutputListener(tempOutputListener);
276 run();
277 removeOutputListener(tempOutputListener);
278 return buf.toString();
279 }
280
281 public String asCommand() {
282 return createCommandLine().toString();
283 }
284
285 @Override
286 public String toString() {
287 return asCommand();
288 }
289
290 /**
291 * Build a command line based on the properties. Can be overridden by
292 * specific command wrappers.
293 */
294 protected CommandLine createCommandLine() {
295 // Check if an OS specific command overrides
296 String osName = System.getProperty("os.name");
297 List<Object> commandToUse = null;
298 if (osCommands.containsKey(osName))
299 commandToUse = osCommands.get(osName);
300 else
301 commandToUse = command;
302 String cmdToUse = null;
303 if (osCmds.containsKey(osName))
304 cmdToUse = osCmds.get(osName);
305 else
306 cmdToUse = cmd;
307
308 CommandLine commandLine = null;
309
310 // Which command definition to use
311 if (commandToUse == null && cmdToUse == null)
312 throw new SlcException("Please specify a command.");
313 else if (commandToUse != null && cmdToUse != null)
314 throw new SlcException(
315 "Specify the command either as a line or as a list.");
316 else if (cmdToUse != null) {
317 if (chroot != null && !chroot.trim().equals(""))
318 cmdToUse = "chroot \"" + chroot + "\" " + cmdToUse;
319 if (sudo != null) {
320 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
321 if (!sudo.trim().equals(""))
322 cmdToUse = "sudo -p " + sudoPrompt + " -u " + sudo + " "
323 + cmdToUse;
324 else
325 cmdToUse = "sudo -p " + sudoPrompt + " " + cmdToUse;
326 }
327
328 // GENERATE COMMAND LINE
329 commandLine = CommandLine.parse(cmdToUse);
330 } else if (commandToUse != null) {
331 if (commandToUse.size() == 0)
332 throw new SlcException("Command line is empty.");
333
334 if (chroot != null && sudo != null) {
335 commandToUse.add(0, "chroot");
336 commandToUse.add(1, chroot);
337 }
338
339 if (sudo != null) {
340 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
341 commandToUse.add(0, "sudo");
342 commandToUse.add(1, "-p");
343 commandToUse.add(2, sudoPrompt);
344 if (!sudo.trim().equals("")) {
345 commandToUse.add(3, "-u");
346 commandToUse.add(4, sudo);
347 }
348 }
349
350 // GENERATE COMMAND LINE
351 commandLine = new CommandLine(commandToUse.get(0).toString());
352
353 for (int i = 1; i < commandToUse.size(); i++) {
354 if (log.isTraceEnabled())
355 log.debug(commandToUse.get(i));
356 commandLine.addArgument(commandToUse.get(i).toString());
357 }
358 } else {
359 // all cases covered previously
360 throw new UnsupportedException();
361 }
362
363 if (generateScript != null) {
364 File scriptFile = new File(getExecDirToUse() + File.separator
365 + generateScript);
366 try {
367 FileUtils.writeStringToFile(scriptFile,
368 (osConsole != null ? osConsole + " " : "")
369 + commandLine.toString());
370 } catch (IOException e) {
371 throw new SlcException("Could not generate script "
372 + scriptFile, e);
373 }
374 commandLine = new CommandLine(scriptFile);
375 } else {
376 if (osConsole != null)
377 commandLine = CommandLine.parse(osConsole + " "
378 + commandLine.toString());
379 }
380
381 return commandLine;
382 }
383
384 /**
385 * Creates a {@link PumpStreamHandler} which redirects streams to the custom
386 * logging mechanism.
387 */
388 protected ExecuteStreamHandler createExecuteStreamHandler(
389 final Writer stdOutWriter, final OutputStream stdOutputStream,
390 final Writer stdErrWriter, final InputStream stdInStream) {
391
392 // Log writers
393 OutputStream stdout = stdOutputStream != null ? stdOutputStream
394 : new LogOutputStream() {
395 protected void processLine(String line, int level) {
396 // if (firstLine) {
397 // if (sudo != null && callbackHandler != null
398 // && line.startsWith(sudoPrompt)) {
399 // try {
400 // PasswordCallback pc = new PasswordCallback(
401 // "sudo password", false);
402 // Callback[] cbs = { pc };
403 // callbackHandler.handle(cbs);
404 // char[] pwd = pc.getPassword();
405 // char[] arr = Arrays.copyOf(pwd,
406 // pwd.length + 1);
407 // arr[arr.length - 1] = '\n';
408 // IOUtils.write(arr, stdInSink);
409 // stdInSink.flush();
410 // } catch (Exception e) {
411 // throw new SlcException(
412 // "Cannot retrieve sudo password", e);
413 // }
414 // }
415 // firstLine = false;
416 // }
417
418 if (line != null && !line.trim().equals(""))
419 logStdOut(line);
420
421 if (stdOutWriter != null)
422 appendLineToFile(stdOutWriter, line);
423 }
424 };
425
426 OutputStream stderr = new LogOutputStream() {
427 protected void processLine(String line, int level) {
428 if (line != null && !line.trim().equals(""))
429 logStdErr(line);
430 if (stdErrWriter != null)
431 appendLineToFile(stdErrWriter, line);
432 }
433 };
434
435 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(stdout,
436 stderr, stdInStream) {
437
438 @Override
439 public void stop() throws IOException {
440 // prevents the method to block when joining stdin
441 if (stdInSink != null)
442 IOUtils.closeQuietly(stdInSink);
443
444 super.stop();
445 }
446 };
447 return pumpStreamHandler;
448 }
449
450 /** Creates the default {@link ExecuteResultHandler}. */
451 protected ExecuteResultHandler createExecuteResultHandler(
452 final CommandLine commandLine) {
453 return new ExecuteResultHandler() {
454
455 public void onProcessComplete(int exitValue) {
456 String msg = "System call '" + commandLine
457 + "' properly completed.";
458 if (log.isTraceEnabled())
459 log.trace(msg);
460 if (testResult != null) {
461 forwardPath(testResult);
462 testResult.addResultPart(new SimpleResultPart(
463 TestStatus.PASSED, msg));
464 }
465 releaseWatchdog();
466 }
467
468 public void onProcessFailed(ExecuteException e) {
469
470 String msg = "System call '" + commandLine + "' failed.";
471 if (testResult != null) {
472 forwardPath(testResult);
473 testResult.addResultPart(new SimpleResultPart(
474 TestStatus.ERROR, msg, e));
475 } else {
476 if (exceptionOnFailed)
477 throw new SlcException(msg, e);
478 else
479 log.error(msg, e);
480 }
481 releaseWatchdog();
482 }
483 };
484 }
485
486 @Deprecated
487 protected void forwardPath(TestResult testResult) {
488 // TODO: allocate a TreeSPath
489 }
490
491 /**
492 * Shortcut method getting the execDir to use
493 */
494 protected String getExecDirToUse() {
495 try {
496 if (execDir != null) {
497 return execDir;
498 }
499 return System.getProperty("user.dir");
500 } catch (Exception e) {
501 throw new SlcException("Cannot find exec dir", e);
502 }
503 }
504
505 protected void logStdOut(String line) {
506 for (SystemCallOutputListener outputListener : outputListeners)
507 outputListener.newLine(this, line, false);
508 log(stdOutLogLevel, line);
509 }
510
511 protected void logStdErr(String line) {
512 for (SystemCallOutputListener outputListener : outputListeners)
513 outputListener.newLine(this, line, true);
514 log(stdErrLogLevel, line);
515 }
516
517 /** Log from the underlying streams. */
518 protected void log(String logLevel, String line) {
519 // TODO optimize
520 // if (SecurityContextHolder.getContext().getAuthentication() == null) {
521 // SecurityContextHolder.getContext()
522 // .setAuthentication(authentication);
523 // }
524
525 if ("ERROR".equals(logLevel))
526 log.error(line);
527 else if ("WARN".equals(logLevel))
528 log.warn(line);
529 else if ("INFO".equals(logLevel))
530 log.info(line);
531 else if ("DEBUG".equals(logLevel))
532 log.debug(line);
533 else if ("TRACE".equals(logLevel))
534 log.trace(line);
535 else if (LOG_STDOUT.equals(logLevel))
536 System.out.println(line);
537 else if ("System.err".equals(logLevel))
538 System.err.println(line);
539 else
540 throw new SlcException("Unknown log level " + logLevel);
541 }
542
543 /** Append line to a log file. */
544 protected void appendLineToFile(Writer writer, String line) {
545 try {
546 writer.append(line).append('\n');
547 } catch (IOException e) {
548 log.error("Cannot write to log file", e);
549 }
550 }
551
552 /** Creates the writer for the output/err files. */
553 protected Writer createWriter(Resource target, Boolean append) {
554 FileWriter writer = null;
555 try {
556
557 final File file;
558 if (executionResources != null)
559 file = new File(executionResources.getAsOsPath(target, true));
560 else
561 file = target.getFile();
562 writer = new FileWriter(file, append);
563 } catch (IOException e) {
564 log.error("Cannot get file for " + target, e);
565 IOUtils.closeQuietly(writer);
566 }
567 return writer;
568 }
569
570 /** Creates an outputstream for the output/err files. */
571 protected OutputStream createOutputStream(Resource target) {
572 FileOutputStream out = null;
573 try {
574
575 final File file;
576 if (executionResources != null)
577 file = new File(executionResources.getAsOsPath(target, true));
578 else
579 file = target.getFile();
580 out = new FileOutputStream(file, false);
581 } catch (IOException e) {
582 log.error("Cannot get file for " + target, e);
583 IOUtils.closeQuietly(out);
584 }
585 return out;
586 }
587
588 /** Append the argument (for chaining) */
589 public SystemCall arg(String arg) {
590 if (command == null)
591 command = new ArrayList<Object>();
592 command.add(arg);
593 return this;
594 }
595
596 /** Append the argument (for chaining) */
597 public SystemCall arg(String arg, String value) {
598 if (command == null)
599 command = new ArrayList<Object>();
600 command.add(arg);
601 command.add(value);
602 return this;
603 }
604
605 // CONTROL
606 public synchronized Boolean isRunning() {
607 return currentWatchdog != null;
608 }
609
610 private synchronized ExecuteWatchdog createWatchdog() {
611 // if (currentWatchdog != null)
612 // throw new SlcException("A process is already being monitored");
613 currentWatchdog = new ExecuteWatchdog(watchdogTimeout);
614 return currentWatchdog;
615 }
616
617 private synchronized void releaseWatchdog() {
618 currentWatchdog = null;
619 }
620
621 public synchronized void kill() {
622 if (currentWatchdog != null)
623 currentWatchdog.destroyProcess();
624 }
625
626 /** */
627 public void setCmd(String command) {
628 this.cmd = command;
629 }
630
631 public void setCommand(List<Object> command) {
632 this.command = command;
633 }
634
635 public void setExecDir(String execdir) {
636 this.execDir = execdir;
637 }
638
639 public void setStdErrLogLevel(String stdErrLogLevel) {
640 this.stdErrLogLevel = stdErrLogLevel;
641 }
642
643 public void setStdOutLogLevel(String stdOutLogLevel) {
644 this.stdOutLogLevel = stdOutLogLevel;
645 }
646
647 public void setSynchronous(Boolean synchronous) {
648 this.synchronous = synchronous;
649 }
650
651 public void setOsCommands(Map<String, List<Object>> osCommands) {
652 this.osCommands = osCommands;
653 }
654
655 public void setOsCmds(Map<String, String> osCmds) {
656 this.osCmds = osCmds;
657 }
658
659 public void setEnvironmentVariables(Map<String, String> environmentVariables) {
660 this.environmentVariables = environmentVariables;
661 }
662
663 public Map<String, String> getEnvironmentVariables() {
664 return environmentVariables;
665 }
666
667 public void setWatchdogTimeout(Long watchdogTimeout) {
668 this.watchdogTimeout = watchdogTimeout;
669 }
670
671 public void setStdOutFile(Resource stdOutFile) {
672 this.stdOutFile = stdOutFile;
673 }
674
675 public void setStdErrFile(Resource stdErrFile) {
676 this.stdErrFile = stdErrFile;
677 }
678
679 public void setStdInFile(Resource stdInFile) {
680 this.stdInFile = stdInFile;
681 }
682
683 public void setTestResult(TestResult testResult) {
684 this.testResult = testResult;
685 }
686
687 public void setLogCommand(Boolean logCommand) {
688 this.logCommand = logCommand;
689 }
690
691 public void setRedirectStreams(Boolean redirectStreams) {
692 this.redirectStreams = redirectStreams;
693 }
694
695 public void setExceptionOnFailed(Boolean exceptionOnFailed) {
696 this.exceptionOnFailed = exceptionOnFailed;
697 }
698
699 public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
700 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
701 }
702
703 public void setOsConsole(String osConsole) {
704 this.osConsole = osConsole;
705 }
706
707 public void setGenerateScript(String generateScript) {
708 this.generateScript = generateScript;
709 }
710
711 public void setExecutionResources(ExecutionResources executionResources) {
712 this.executionResources = executionResources;
713 }
714
715 public void setRedirectStdOut(Boolean redirectStdOut) {
716 this.redirectStdOut = redirectStdOut;
717 }
718
719 public void addOutputListener(SystemCallOutputListener outputListener) {
720 outputListeners.add(outputListener);
721 }
722
723 public void removeOutputListener(SystemCallOutputListener outputListener) {
724 outputListeners.remove(outputListener);
725 }
726
727 public void setOutputListeners(
728 List<SystemCallOutputListener> outputListeners) {
729 this.outputListeners = outputListeners;
730 }
731
732 public void setExecutor(Executor executor) {
733 this.executor = executor;
734 }
735
736 public void setSudo(String sudo) {
737 this.sudo = sudo;
738 }
739
740 public void setCallbackHandler(CallbackHandler callbackHandler) {
741 this.callbackHandler = callbackHandler;
742 }
743
744 public void setChroot(String chroot) {
745 this.chroot = chroot;
746 }
747
748 private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
749
750 public void setProcessErrorStream(InputStream is) throws IOException {
751 }
752
753 public void setProcessInputStream(OutputStream os) throws IOException {
754 }
755
756 public void setProcessOutputStream(InputStream is) throws IOException {
757 }
758
759 public void start() throws IOException {
760 }
761
762 public void stop() {
763 }
764
765 }
766 }