]> git.argeo.org Git - gpl/argeo-slc.git/blob - org.argeo.slc.runtime/src/org/argeo/slc/runtime/tasks/SystemCall.java
Move backup to SLC runtime.
[gpl/argeo-slc.git] / org.argeo.slc.runtime / src / org / argeo / slc / runtime / tasks / SystemCall.java
1 package org.argeo.slc.runtime.tasks;
2
3 import static java.lang.System.Logger.Level.DEBUG;
4 import static java.lang.System.Logger.Level.ERROR;
5 import static java.lang.System.Logger.Level.INFO;
6 import static java.lang.System.Logger.Level.TRACE;
7 import static java.lang.System.Logger.Level.WARNING;
8
9 import java.io.File;
10 import java.io.FileOutputStream;
11 import java.io.FileWriter;
12 import java.io.IOException;
13 import java.io.InputStream;
14 import java.io.OutputStream;
15 import java.io.PipedInputStream;
16 import java.io.PipedOutputStream;
17 import java.io.Writer;
18 import java.lang.System.Logger;
19 import java.nio.file.Files;
20 import java.nio.file.Path;
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.HashMap;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.UUID;
27
28 import javax.security.auth.callback.CallbackHandler;
29
30 import org.apache.commons.exec.CommandLine;
31 import org.apache.commons.exec.DefaultExecutor;
32 import org.apache.commons.exec.ExecuteException;
33 import org.apache.commons.exec.ExecuteResultHandler;
34 import org.apache.commons.exec.ExecuteStreamHandler;
35 import org.apache.commons.exec.ExecuteWatchdog;
36 import org.apache.commons.exec.Executor;
37 import org.apache.commons.exec.LogOutputStream;
38 import org.apache.commons.exec.PumpStreamHandler;
39 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
40 import org.apache.commons.io.FileUtils;
41 import org.apache.commons.io.IOUtils;
42 import org.argeo.slc.SlcException;
43 import org.argeo.slc.UnsupportedException;
44 import org.argeo.slc.execution.ExecutionResources;
45 import org.argeo.slc.runtime.test.SimpleResultPart;
46 import org.argeo.slc.test.TestResult;
47 import org.argeo.slc.test.TestStatus;
48
49 /** Execute an OS specific system call. */
50 public class SystemCall implements Runnable {
51 public final static String LOG_STDOUT = "System.out";
52
53 private final Logger logger = System.getLogger(getClass().getName());
54
55 private String execDir;
56
57 private String cmd = null;
58 private List<Object> command = null;
59
60 private Executor executor = new DefaultExecutor();
61 private Boolean synchronous = true;
62
63 private String stdErrLogLevel = "ERROR";
64 private String stdOutLogLevel = "INFO";
65
66 private Path stdOutFile = null;
67 private Path stdErrFile = null;
68
69 private Path stdInFile = null;
70 /**
71 * If no {@link #stdInFile} provided, writing to this stream will write to the
72 * stdin of the process.
73 */
74 private OutputStream stdInSink = null;
75
76 private Boolean redirectStdOut = false;
77
78 private List<SystemCallOutputListener> outputListeners = Collections
79 .synchronizedList(new ArrayList<SystemCallOutputListener>());
80
81 private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
82 private Map<String, String> osCmds = new HashMap<String, String>();
83 private Map<String, String> environmentVariables = new HashMap<String, String>();
84
85 private Boolean logCommand = false;
86 private Boolean redirectStreams = true;
87 private Boolean exceptionOnFailed = true;
88 private Boolean mergeEnvironmentVariables = true;
89
90 // private Authentication authentication;
91
92 private String osConsole = null;
93 private String generateScript = null;
94
95 /** 24 hours */
96 private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
97
98 private TestResult testResult;
99
100 private ExecutionResources executionResources;
101
102 /** Sudo the command, as root if empty or as user if not. */
103 private String sudo = null;
104 // TODO make it more secure and robust, test only once
105 private final String sudoPrompt = UUID.randomUUID().toString();
106 private String askPassProgram = "/usr/libexec/openssh/ssh-askpass";
107 @SuppressWarnings("unused")
108 private boolean firstLine = true;
109 @SuppressWarnings("unused")
110 private CallbackHandler callbackHandler;
111 /** Chroot to the this path (must not be empty) */
112 private String chroot = null;
113
114 // Current
115 /** Current watchdog, null if process is completed */
116 ExecuteWatchdog currentWatchdog = null;
117
118 /** Empty constructor */
119 public SystemCall() {
120
121 }
122
123 /**
124 * Constructor based on the provided command list.
125 *
126 * @param command the command list
127 */
128 public SystemCall(List<Object> command) {
129 this.command = command;
130 }
131
132 /**
133 * Constructor based on the provided command.
134 *
135 * @param cmd the command. If the provided string contains no space a command
136 * list is initialized with the argument as first component (useful
137 * for chained construction)
138 */
139 public SystemCall(String cmd) {
140 if (cmd.indexOf(' ') < 0) {
141 command = new ArrayList<Object>();
142 command.add(cmd);
143 } else {
144 this.cmd = cmd;
145 }
146 }
147
148 /** Executes the system call. */
149 public void run() {
150 // authentication = SecurityContextHolder.getContext().getAuthentication();
151
152 // Manage streams
153 Writer stdOutWriter = null;
154 OutputStream stdOutputStream = null;
155 Writer stdErrWriter = null;
156 InputStream stdInStream = null;
157 if (stdOutFile != null)
158 if (redirectStdOut)
159 stdOutputStream = createOutputStream(stdOutFile);
160 else
161 stdOutWriter = createWriter(stdOutFile, true);
162
163 if (stdErrFile != null) {
164 stdErrWriter = createWriter(stdErrFile, true);
165 } else {
166 if (stdOutFile != null && !redirectStdOut)
167 stdErrWriter = createWriter(stdOutFile, true);
168 }
169
170 try {
171 if (stdInFile != null)
172 stdInStream = Files.newInputStream(stdInFile);
173 else {
174 stdInStream = new PipedInputStream();
175 stdInSink = new PipedOutputStream((PipedInputStream) stdInStream);
176 }
177 } catch (IOException e2) {
178 throw new SlcException("Cannot open a stream for " + stdInFile, e2);
179 }
180
181 logger.log(TRACE, () -> "os.name=" + System.getProperty("os.name"));
182 logger.log(TRACE, () -> "os.arch=" + System.getProperty("os.arch"));
183 logger.log(TRACE, () -> "os.version=" + System.getProperty("os.version"));
184
185 // Execution directory
186 File dir = new File(getExecDirToUse());
187 // if (!dir.exists())
188 // dir.mkdirs();
189
190 // Watchdog to check for lost processes
191 Executor executorToUse;
192 if (executor != null)
193 executorToUse = executor;
194 else
195 executorToUse = new DefaultExecutor();
196 executorToUse.setWatchdog(createWatchdog());
197
198 if (redirectStreams) {
199 // Redirect standard streams
200 executorToUse.setStreamHandler(
201 createExecuteStreamHandler(stdOutWriter, stdOutputStream, stdErrWriter, stdInStream));
202 } else {
203 // Dummy stream handler (otherwise pump is used)
204 executorToUse.setStreamHandler(new DummyexecuteStreamHandler());
205 }
206
207 executorToUse.setProcessDestroyer(new ShutdownHookProcessDestroyer());
208 executorToUse.setWorkingDirectory(dir);
209
210 // Command line to use
211 final CommandLine commandLine = createCommandLine();
212 if (logCommand)
213 logger.log(INFO, "Execute command:\n" + commandLine + "\n in working directory: \n" + dir + "\n");
214
215 // Env variables
216 Map<String, String> environmentVariablesToUse = null;
217 environmentVariablesToUse = new HashMap<String, String>();
218 if (mergeEnvironmentVariables)
219 environmentVariablesToUse.putAll(System.getenv());
220 if (environmentVariables.size() > 0)
221 environmentVariablesToUse.putAll(environmentVariables);
222
223 // Execute
224 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
225
226 //
227 // THE EXECUTION PROPER
228 //
229 try {
230 if (synchronous)
231 try {
232 int exitValue = executorToUse.execute(commandLine, environmentVariablesToUse);
233 executeResultHandler.onProcessComplete(exitValue);
234 } catch (ExecuteException e1) {
235 if (e1.getExitValue() == Executor.INVALID_EXITVALUE) {
236 Thread.currentThread().interrupt();
237 return;
238 }
239 // Sleep 1s in order to make sure error logs are flushed
240 Thread.sleep(1000);
241 executeResultHandler.onProcessFailed(e1);
242 }
243 else
244 executorToUse.execute(commandLine, environmentVariablesToUse, executeResultHandler);
245 } catch (SlcException e) {
246 throw e;
247 } catch (Exception e) {
248 throw new SlcException("Could not execute command " + commandLine, e);
249 } finally {
250 IOUtils.closeQuietly(stdOutWriter);
251 IOUtils.closeQuietly(stdErrWriter);
252 IOUtils.closeQuietly(stdInStream);
253 IOUtils.closeQuietly(stdInSink);
254 }
255
256 }
257
258 public synchronized String function() {
259 final StringBuffer buf = new StringBuffer("");
260 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
261 private Long lineCount = 0l;
262
263 public void newLine(SystemCall systemCall, String line, Boolean isError) {
264 if (!isError) {
265 if (lineCount != 0l)
266 buf.append('\n');
267 buf.append(line);
268 lineCount++;
269 }
270 }
271 };
272 addOutputListener(tempOutputListener);
273 run();
274 removeOutputListener(tempOutputListener);
275 return buf.toString();
276 }
277
278 public String asCommand() {
279 return createCommandLine().toString();
280 }
281
282 @Override
283 public String toString() {
284 return asCommand();
285 }
286
287 /**
288 * Build a command line based on the properties. Can be overridden by specific
289 * command wrappers.
290 */
291 protected CommandLine createCommandLine() {
292 // Check if an OS specific command overrides
293 String osName = System.getProperty("os.name");
294 List<Object> commandToUse = null;
295 if (osCommands.containsKey(osName))
296 commandToUse = osCommands.get(osName);
297 else
298 commandToUse = command;
299 String cmdToUse = null;
300 if (osCmds.containsKey(osName))
301 cmdToUse = osCmds.get(osName);
302 else
303 cmdToUse = cmd;
304
305 CommandLine commandLine = null;
306
307 // Which command definition to use
308 if (commandToUse == null && cmdToUse == null)
309 throw new SlcException("Please specify a command.");
310 else if (commandToUse != null && cmdToUse != null)
311 throw new SlcException("Specify the command either as a line or as a list.");
312 else if (cmdToUse != null) {
313 if (chroot != null && !chroot.trim().equals(""))
314 cmdToUse = "chroot \"" + chroot + "\" " + cmdToUse;
315 if (sudo != null) {
316 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
317 if (!sudo.trim().equals(""))
318 cmdToUse = "sudo -p " + sudoPrompt + " -u " + sudo + " " + cmdToUse;
319 else
320 cmdToUse = "sudo -p " + sudoPrompt + " " + cmdToUse;
321 }
322
323 // GENERATE COMMAND LINE
324 commandLine = CommandLine.parse(cmdToUse);
325 } else if (commandToUse != null) {
326 if (commandToUse.size() == 0)
327 throw new SlcException("Command line is empty.");
328
329 if (chroot != null && sudo != null) {
330 commandToUse.add(0, "chroot");
331 commandToUse.add(1, chroot);
332 }
333
334 if (sudo != null) {
335 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
336 commandToUse.add(0, "sudo");
337 commandToUse.add(1, "-p");
338 commandToUse.add(2, sudoPrompt);
339 if (!sudo.trim().equals("")) {
340 commandToUse.add(3, "-u");
341 commandToUse.add(4, sudo);
342 }
343 }
344
345 // GENERATE COMMAND LINE
346 commandLine = new CommandLine(commandToUse.get(0).toString());
347
348 for (int i = 1; i < commandToUse.size(); i++) {
349 if (logger.isLoggable(TRACE))
350 logger.log(TRACE, commandToUse.get(i));
351 commandLine.addArgument(commandToUse.get(i).toString());
352 }
353 } else {
354 // all cases covered previously
355 throw new UnsupportedException();
356 }
357
358 if (generateScript != null) {
359 File scriptFile = new File(getExecDirToUse() + File.separator + generateScript);
360 try {
361 FileUtils.writeStringToFile(scriptFile,
362 (osConsole != null ? osConsole + " " : "") + commandLine.toString());
363 } catch (IOException e) {
364 throw new SlcException("Could not generate script " + scriptFile, e);
365 }
366 commandLine = new CommandLine(scriptFile);
367 } else {
368 if (osConsole != null)
369 commandLine = CommandLine.parse(osConsole + " " + commandLine.toString());
370 }
371
372 return commandLine;
373 }
374
375 /**
376 * Creates a {@link PumpStreamHandler} which redirects streams to the custom
377 * logging mechanism.
378 */
379 protected ExecuteStreamHandler createExecuteStreamHandler(final Writer stdOutWriter,
380 final OutputStream stdOutputStream, final Writer stdErrWriter, final InputStream stdInStream) {
381
382 // Log writers
383 OutputStream stdout = stdOutputStream != null ? stdOutputStream : new LogOutputStream() {
384 protected void processLine(String line, int level) {
385 // if (firstLine) {
386 // if (sudo != null && callbackHandler != null
387 // && line.startsWith(sudoPrompt)) {
388 // try {
389 // PasswordCallback pc = new PasswordCallback(
390 // "sudo password", false);
391 // Callback[] cbs = { pc };
392 // callbackHandler.handle(cbs);
393 // char[] pwd = pc.getPassword();
394 // char[] arr = Arrays.copyOf(pwd,
395 // pwd.length + 1);
396 // arr[arr.length - 1] = '\n';
397 // IOUtils.write(arr, stdInSink);
398 // stdInSink.flush();
399 // } catch (Exception e) {
400 // throw new SlcException(
401 // "Cannot retrieve sudo password", e);
402 // }
403 // }
404 // firstLine = false;
405 // }
406
407 if (line != null && !line.trim().equals(""))
408 logStdOut(line);
409
410 if (stdOutWriter != null)
411 appendLineToFile(stdOutWriter, line);
412 }
413 };
414
415 OutputStream stderr = new LogOutputStream() {
416 protected void processLine(String line, int level) {
417 if (line != null && !line.trim().equals(""))
418 logStdErr(line);
419 if (stdErrWriter != null)
420 appendLineToFile(stdErrWriter, line);
421 }
422 };
423
424 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(stdout, stderr, stdInStream) {
425
426 @Override
427 public void stop() throws IOException {
428 // prevents the method to block when joining stdin
429 if (stdInSink != null)
430 IOUtils.closeQuietly(stdInSink);
431
432 super.stop();
433 }
434 };
435 return pumpStreamHandler;
436 }
437
438 /** Creates the default {@link ExecuteResultHandler}. */
439 protected ExecuteResultHandler createExecuteResultHandler(final CommandLine commandLine) {
440 return new ExecuteResultHandler() {
441
442 public void onProcessComplete(int exitValue) {
443 String msg = "System call '" + commandLine + "' properly completed.";
444 logger.log(TRACE, () -> msg);
445 if (testResult != null) {
446 forwardPath(testResult);
447 testResult.addResultPart(new SimpleResultPart(TestStatus.PASSED, msg));
448 }
449 releaseWatchdog();
450 }
451
452 public void onProcessFailed(ExecuteException e) {
453
454 String msg = "System call '" + commandLine + "' failed.";
455 if (testResult != null) {
456 forwardPath(testResult);
457 testResult.addResultPart(new SimpleResultPart(TestStatus.ERROR, msg, e));
458 } else {
459 if (exceptionOnFailed)
460 throw new SlcException(msg, e);
461 else
462 logger.log(ERROR, msg, e);
463 }
464 releaseWatchdog();
465 }
466 };
467 }
468
469 @Deprecated
470 protected void forwardPath(TestResult testResult) {
471 // TODO: allocate a TreeSPath
472 }
473
474 /**
475 * Shortcut method getting the execDir to use
476 */
477 protected String getExecDirToUse() {
478 try {
479 if (execDir != null) {
480 return execDir;
481 }
482 return System.getProperty("user.dir");
483 } catch (Exception e) {
484 throw new SlcException("Cannot find exec dir", e);
485 }
486 }
487
488 protected void logStdOut(String line) {
489 for (SystemCallOutputListener outputListener : outputListeners)
490 outputListener.newLine(this, line, false);
491 log(stdOutLogLevel, line);
492 }
493
494 protected void logStdErr(String line) {
495 for (SystemCallOutputListener outputListener : outputListeners)
496 outputListener.newLine(this, line, true);
497 log(stdErrLogLevel, line);
498 }
499
500 /** Log from the underlying streams. */
501 protected void log(String logLevel, String line) {
502 // TODO optimize
503 // if (SecurityContextHolder.getContext().getAuthentication() == null) {
504 // SecurityContextHolder.getContext()
505 // .setAuthentication(authentication);
506 // }
507
508 if ("ERROR".equals(logLevel))
509 logger.log(ERROR, line);
510 else if ("WARN".equals(logLevel))
511 logger.log(WARNING, line);
512 else if ("WARNING".equals(logLevel))
513 logger.log(WARNING, line);
514 else if ("INFO".equals(logLevel))
515 logger.log(INFO, line);
516 else if ("DEBUG".equals(logLevel))
517 logger.log(DEBUG, line);
518 else if ("TRACE".equals(logLevel))
519 logger.log(TRACE, line);
520 else if (LOG_STDOUT.equals(logLevel))
521 System.out.println(line);
522 else if ("System.err".equals(logLevel))
523 System.err.println(line);
524 else
525 throw new SlcException("Unknown log level " + logLevel);
526 }
527
528 /** Append line to a log file. */
529 protected void appendLineToFile(Writer writer, String line) {
530 try {
531 writer.append(line).append('\n');
532 } catch (IOException e) {
533 logger.log(ERROR, "Cannot write to log file", e);
534 }
535 }
536
537 /** Creates the writer for the output/err files. */
538 protected Writer createWriter(Path target, Boolean append) {
539 FileWriter writer = null;
540 try {
541
542 final File file;
543 if (executionResources != null)
544 file = new File(executionResources.getAsOsPath(target, true));
545 else
546 file = target.toFile();
547 writer = new FileWriter(file, append);
548 } catch (IOException e) {
549 logger.log(ERROR, "Cannot get file for " + target, e);
550 IOUtils.closeQuietly(writer);
551 }
552 return writer;
553 }
554
555 /** Creates an outputstream for the output/err files. */
556 protected OutputStream createOutputStream(Path target) {
557 FileOutputStream out = null;
558 try {
559
560 final File file;
561 if (executionResources != null)
562 file = new File(executionResources.getAsOsPath(target, true));
563 else
564 file = target.toFile();
565 out = new FileOutputStream(file, false);
566 } catch (IOException e) {
567 logger.log(ERROR, "Cannot get file for " + target, e);
568 IOUtils.closeQuietly(out);
569 }
570 return out;
571 }
572
573 /** Append the argument (for chaining) */
574 public SystemCall arg(String arg) {
575 if (command == null)
576 command = new ArrayList<Object>();
577 command.add(arg);
578 return this;
579 }
580
581 /** Append the argument (for chaining) */
582 public SystemCall arg(String arg, String value) {
583 if (command == null)
584 command = new ArrayList<Object>();
585 command.add(arg);
586 command.add(value);
587 return this;
588 }
589
590 // CONTROL
591 public synchronized Boolean isRunning() {
592 return currentWatchdog != null;
593 }
594
595 private synchronized ExecuteWatchdog createWatchdog() {
596 // if (currentWatchdog != null)
597 // throw new SlcException("A process is already being monitored");
598 currentWatchdog = new ExecuteWatchdog(watchdogTimeout);
599 return currentWatchdog;
600 }
601
602 private synchronized void releaseWatchdog() {
603 currentWatchdog = null;
604 }
605
606 public synchronized void kill() {
607 if (currentWatchdog != null)
608 currentWatchdog.destroyProcess();
609 }
610
611 /** */
612 public void setCmd(String command) {
613 this.cmd = command;
614 }
615
616 public void setCommand(List<Object> command) {
617 this.command = command;
618 }
619
620 public void setExecDir(String execdir) {
621 this.execDir = execdir;
622 }
623
624 public void setStdErrLogLevel(String stdErrLogLevel) {
625 this.stdErrLogLevel = stdErrLogLevel;
626 }
627
628 public void setStdOutLogLevel(String stdOutLogLevel) {
629 this.stdOutLogLevel = stdOutLogLevel;
630 }
631
632 public void setSynchronous(Boolean synchronous) {
633 this.synchronous = synchronous;
634 }
635
636 public void setOsCommands(Map<String, List<Object>> osCommands) {
637 this.osCommands = osCommands;
638 }
639
640 public void setOsCmds(Map<String, String> osCmds) {
641 this.osCmds = osCmds;
642 }
643
644 public void setEnvironmentVariables(Map<String, String> environmentVariables) {
645 this.environmentVariables = environmentVariables;
646 }
647
648 public Map<String, String> getEnvironmentVariables() {
649 return environmentVariables;
650 }
651
652 public void setWatchdogTimeout(Long watchdogTimeout) {
653 this.watchdogTimeout = watchdogTimeout;
654 }
655
656 public void setStdOutFile(Path stdOutFile) {
657 this.stdOutFile = stdOutFile;
658 }
659
660 public void setStdErrFile(Path stdErrFile) {
661 this.stdErrFile = stdErrFile;
662 }
663
664 public void setStdInFile(Path stdInFile) {
665 this.stdInFile = stdInFile;
666 }
667
668 public void setTestResult(TestResult testResult) {
669 this.testResult = testResult;
670 }
671
672 public void setLogCommand(Boolean logCommand) {
673 this.logCommand = logCommand;
674 }
675
676 public void setRedirectStreams(Boolean redirectStreams) {
677 this.redirectStreams = redirectStreams;
678 }
679
680 public void setExceptionOnFailed(Boolean exceptionOnFailed) {
681 this.exceptionOnFailed = exceptionOnFailed;
682 }
683
684 public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
685 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
686 }
687
688 public void setOsConsole(String osConsole) {
689 this.osConsole = osConsole;
690 }
691
692 public void setGenerateScript(String generateScript) {
693 this.generateScript = generateScript;
694 }
695
696 public void setExecutionResources(ExecutionResources executionResources) {
697 this.executionResources = executionResources;
698 }
699
700 public void setRedirectStdOut(Boolean redirectStdOut) {
701 this.redirectStdOut = redirectStdOut;
702 }
703
704 public void addOutputListener(SystemCallOutputListener outputListener) {
705 outputListeners.add(outputListener);
706 }
707
708 public void removeOutputListener(SystemCallOutputListener outputListener) {
709 outputListeners.remove(outputListener);
710 }
711
712 public void setOutputListeners(List<SystemCallOutputListener> outputListeners) {
713 this.outputListeners = outputListeners;
714 }
715
716 public void setExecutor(Executor executor) {
717 this.executor = executor;
718 }
719
720 public void setSudo(String sudo) {
721 this.sudo = sudo;
722 }
723
724 public void setCallbackHandler(CallbackHandler callbackHandler) {
725 this.callbackHandler = callbackHandler;
726 }
727
728 public void setChroot(String chroot) {
729 this.chroot = chroot;
730 }
731
732 private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
733
734 public void setProcessErrorStream(InputStream is) throws IOException {
735 }
736
737 public void setProcessInputStream(OutputStream os) throws IOException {
738 }
739
740 public void setProcessOutputStream(InputStream is) throws IOException {
741 }
742
743 public void start() throws IOException {
744 }
745
746 public void stop() {
747 }
748
749 }
750 }