]> git.argeo.org Git - gpl/argeo-slc.git/blob - org.argeo.slc.runtime/src/org/argeo/slc/runtime/tasks/SystemCall.java
Continue to remove dependencies with Spring.
[gpl/argeo-slc.git] / org.argeo.slc.runtime / src / org / argeo / slc / runtime / tasks / SystemCall.java
1 /*
2 * Copyright (C) 2007-2012 Argeo GmbH
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package org.argeo.slc.runtime.tasks;
17
18 import java.io.File;
19 import java.io.FileOutputStream;
20 import java.io.FileWriter;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.io.PipedInputStream;
25 import java.io.PipedOutputStream;
26 import java.io.Writer;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.HashMap;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.UUID;
35
36 import javax.security.auth.callback.CallbackHandler;
37
38 import org.apache.commons.exec.CommandLine;
39 import org.apache.commons.exec.DefaultExecutor;
40 import org.apache.commons.exec.ExecuteException;
41 import org.apache.commons.exec.ExecuteResultHandler;
42 import org.apache.commons.exec.ExecuteStreamHandler;
43 import org.apache.commons.exec.ExecuteWatchdog;
44 import org.apache.commons.exec.Executor;
45 import org.apache.commons.exec.LogOutputStream;
46 import org.apache.commons.exec.PumpStreamHandler;
47 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
48 import org.apache.commons.io.FileUtils;
49 import org.apache.commons.io.IOUtils;
50 import org.apache.commons.logging.Log;
51 import org.apache.commons.logging.LogFactory;
52 import org.argeo.slc.SlcException;
53 import org.argeo.slc.UnsupportedException;
54 import org.argeo.slc.execution.ExecutionResources;
55 import org.argeo.slc.runtime.test.SimpleResultPart;
56 import org.argeo.slc.test.TestResult;
57 import org.argeo.slc.test.TestStatus;
58
59 /** Execute an OS specific system call. */
60 public class SystemCall implements Runnable {
61 public final static String LOG_STDOUT = "System.out";
62
63 private final Log log = LogFactory.getLog(getClass());
64
65 private String execDir;
66
67 private String cmd = null;
68 private List<Object> command = null;
69
70 private Executor executor = new DefaultExecutor();
71 private Boolean synchronous = true;
72
73 private String stdErrLogLevel = "ERROR";
74 private String stdOutLogLevel = "INFO";
75
76 private Path stdOutFile = null;
77 private Path stdErrFile = null;
78
79 private Path stdInFile = null;
80 /**
81 * If no {@link #stdInFile} provided, writing to this stream will write to the
82 * stdin of the process.
83 */
84 private OutputStream stdInSink = null;
85
86 private Boolean redirectStdOut = false;
87
88 private List<SystemCallOutputListener> outputListeners = Collections
89 .synchronizedList(new ArrayList<SystemCallOutputListener>());
90
91 private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
92 private Map<String, String> osCmds = new HashMap<String, String>();
93 private Map<String, String> environmentVariables = new HashMap<String, String>();
94
95 private Boolean logCommand = false;
96 private Boolean redirectStreams = true;
97 private Boolean exceptionOnFailed = true;
98 private Boolean mergeEnvironmentVariables = true;
99
100 // private Authentication authentication;
101
102 private String osConsole = null;
103 private String generateScript = null;
104
105 /** 24 hours */
106 private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
107
108 private TestResult testResult;
109
110 private ExecutionResources executionResources;
111
112 /** Sudo the command, as root if empty or as user if not. */
113 private String sudo = null;
114 // TODO make it more secure and robust, test only once
115 private final String sudoPrompt = UUID.randomUUID().toString();
116 private String askPassProgram = "/usr/libexec/openssh/ssh-askpass";
117 @SuppressWarnings("unused")
118 private boolean firstLine = true;
119 @SuppressWarnings("unused")
120 private CallbackHandler callbackHandler;
121 /** Chroot to the this path (must not be empty) */
122 private String chroot = null;
123
124 // Current
125 /** Current watchdog, null if process is completed */
126 ExecuteWatchdog currentWatchdog = null;
127
128 /** Empty constructor */
129 public SystemCall() {
130
131 }
132
133 /**
134 * Constructor based on the provided command list.
135 *
136 * @param command the command list
137 */
138 public SystemCall(List<Object> command) {
139 this.command = command;
140 }
141
142 /**
143 * Constructor based on the provided command.
144 *
145 * @param cmd the command. If the provided string contains no space a command
146 * list is initialized with the argument as first component (useful
147 * for chained construction)
148 */
149 public SystemCall(String cmd) {
150 if (cmd.indexOf(' ') < 0) {
151 command = new ArrayList<Object>();
152 command.add(cmd);
153 } else {
154 this.cmd = cmd;
155 }
156 }
157
158 /** Executes the system call. */
159 public void run() {
160 // authentication = SecurityContextHolder.getContext().getAuthentication();
161
162 // Manage streams
163 Writer stdOutWriter = null;
164 OutputStream stdOutputStream = null;
165 Writer stdErrWriter = null;
166 InputStream stdInStream = null;
167 if (stdOutFile != null)
168 if (redirectStdOut)
169 stdOutputStream = createOutputStream(stdOutFile);
170 else
171 stdOutWriter = createWriter(stdOutFile, true);
172
173 if (stdErrFile != null) {
174 stdErrWriter = createWriter(stdErrFile, true);
175 } else {
176 if (stdOutFile != null && !redirectStdOut)
177 stdErrWriter = createWriter(stdOutFile, true);
178 }
179
180 try {
181 if (stdInFile != null)
182 stdInStream = Files.newInputStream(stdInFile);
183 else {
184 stdInStream = new PipedInputStream();
185 stdInSink = new PipedOutputStream((PipedInputStream) stdInStream);
186 }
187 } catch (IOException e2) {
188 throw new SlcException("Cannot open a stream for " + stdInFile, e2);
189 }
190
191 if (log.isTraceEnabled()) {
192 log.debug("os.name=" + System.getProperty("os.name"));
193 log.debug("os.arch=" + System.getProperty("os.arch"));
194 log.debug("os.version=" + System.getProperty("os.version"));
195 }
196
197 // Execution directory
198 File dir = new File(getExecDirToUse());
199 // if (!dir.exists())
200 // dir.mkdirs();
201
202 // Watchdog to check for lost processes
203 Executor executorToUse;
204 if (executor != null)
205 executorToUse = executor;
206 else
207 executorToUse = new DefaultExecutor();
208 executorToUse.setWatchdog(createWatchdog());
209
210 if (redirectStreams) {
211 // Redirect standard streams
212 executorToUse.setStreamHandler(
213 createExecuteStreamHandler(stdOutWriter, stdOutputStream, stdErrWriter, stdInStream));
214 } else {
215 // Dummy stream handler (otherwise pump is used)
216 executorToUse.setStreamHandler(new DummyexecuteStreamHandler());
217 }
218
219 executorToUse.setProcessDestroyer(new ShutdownHookProcessDestroyer());
220 executorToUse.setWorkingDirectory(dir);
221
222 // Command line to use
223 final CommandLine commandLine = createCommandLine();
224 if (logCommand)
225 log.info("Execute command:\n" + commandLine + "\n in working directory: \n" + dir + "\n");
226
227 // Env variables
228 Map<String, String> environmentVariablesToUse = null;
229 environmentVariablesToUse = new HashMap<String, String>();
230 if (mergeEnvironmentVariables)
231 environmentVariablesToUse.putAll(System.getenv());
232 if (environmentVariables.size() > 0)
233 environmentVariablesToUse.putAll(environmentVariables);
234
235 // Execute
236 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
237
238 //
239 // THE EXECUTION PROPER
240 //
241 try {
242 if (synchronous)
243 try {
244 int exitValue = executorToUse.execute(commandLine, environmentVariablesToUse);
245 executeResultHandler.onProcessComplete(exitValue);
246 } catch (ExecuteException e1) {
247 if (e1.getExitValue() == Executor.INVALID_EXITVALUE) {
248 Thread.currentThread().interrupt();
249 return;
250 }
251 // Sleep 1s in order to make sure error logs are flushed
252 Thread.sleep(1000);
253 executeResultHandler.onProcessFailed(e1);
254 }
255 else
256 executorToUse.execute(commandLine, environmentVariablesToUse, executeResultHandler);
257 } catch (SlcException e) {
258 throw e;
259 } catch (Exception e) {
260 throw new SlcException("Could not execute command " + commandLine, e);
261 } finally {
262 IOUtils.closeQuietly(stdOutWriter);
263 IOUtils.closeQuietly(stdErrWriter);
264 IOUtils.closeQuietly(stdInStream);
265 IOUtils.closeQuietly(stdInSink);
266 }
267
268 }
269
270 public synchronized String function() {
271 final StringBuffer buf = new StringBuffer("");
272 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
273 private Long lineCount = 0l;
274
275 public void newLine(SystemCall systemCall, String line, Boolean isError) {
276 if (!isError) {
277 if (lineCount != 0l)
278 buf.append('\n');
279 buf.append(line);
280 lineCount++;
281 }
282 }
283 };
284 addOutputListener(tempOutputListener);
285 run();
286 removeOutputListener(tempOutputListener);
287 return buf.toString();
288 }
289
290 public String asCommand() {
291 return createCommandLine().toString();
292 }
293
294 @Override
295 public String toString() {
296 return asCommand();
297 }
298
299 /**
300 * Build a command line based on the properties. Can be overridden by specific
301 * command wrappers.
302 */
303 protected CommandLine createCommandLine() {
304 // Check if an OS specific command overrides
305 String osName = System.getProperty("os.name");
306 List<Object> commandToUse = null;
307 if (osCommands.containsKey(osName))
308 commandToUse = osCommands.get(osName);
309 else
310 commandToUse = command;
311 String cmdToUse = null;
312 if (osCmds.containsKey(osName))
313 cmdToUse = osCmds.get(osName);
314 else
315 cmdToUse = cmd;
316
317 CommandLine commandLine = null;
318
319 // Which command definition to use
320 if (commandToUse == null && cmdToUse == null)
321 throw new SlcException("Please specify a command.");
322 else if (commandToUse != null && cmdToUse != null)
323 throw new SlcException("Specify the command either as a line or as a list.");
324 else if (cmdToUse != null) {
325 if (chroot != null && !chroot.trim().equals(""))
326 cmdToUse = "chroot \"" + chroot + "\" " + cmdToUse;
327 if (sudo != null) {
328 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
329 if (!sudo.trim().equals(""))
330 cmdToUse = "sudo -p " + sudoPrompt + " -u " + sudo + " " + cmdToUse;
331 else
332 cmdToUse = "sudo -p " + sudoPrompt + " " + cmdToUse;
333 }
334
335 // GENERATE COMMAND LINE
336 commandLine = CommandLine.parse(cmdToUse);
337 } else if (commandToUse != null) {
338 if (commandToUse.size() == 0)
339 throw new SlcException("Command line is empty.");
340
341 if (chroot != null && sudo != null) {
342 commandToUse.add(0, "chroot");
343 commandToUse.add(1, chroot);
344 }
345
346 if (sudo != null) {
347 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
348 commandToUse.add(0, "sudo");
349 commandToUse.add(1, "-p");
350 commandToUse.add(2, sudoPrompt);
351 if (!sudo.trim().equals("")) {
352 commandToUse.add(3, "-u");
353 commandToUse.add(4, sudo);
354 }
355 }
356
357 // GENERATE COMMAND LINE
358 commandLine = new CommandLine(commandToUse.get(0).toString());
359
360 for (int i = 1; i < commandToUse.size(); i++) {
361 if (log.isTraceEnabled())
362 log.debug(commandToUse.get(i));
363 commandLine.addArgument(commandToUse.get(i).toString());
364 }
365 } else {
366 // all cases covered previously
367 throw new UnsupportedException();
368 }
369
370 if (generateScript != null) {
371 File scriptFile = new File(getExecDirToUse() + File.separator + generateScript);
372 try {
373 FileUtils.writeStringToFile(scriptFile,
374 (osConsole != null ? osConsole + " " : "") + commandLine.toString());
375 } catch (IOException e) {
376 throw new SlcException("Could not generate script " + scriptFile, e);
377 }
378 commandLine = new CommandLine(scriptFile);
379 } else {
380 if (osConsole != null)
381 commandLine = CommandLine.parse(osConsole + " " + commandLine.toString());
382 }
383
384 return commandLine;
385 }
386
387 /**
388 * Creates a {@link PumpStreamHandler} which redirects streams to the custom
389 * logging mechanism.
390 */
391 protected ExecuteStreamHandler createExecuteStreamHandler(final Writer stdOutWriter,
392 final OutputStream stdOutputStream, final Writer stdErrWriter, final InputStream stdInStream) {
393
394 // Log writers
395 OutputStream stdout = stdOutputStream != null ? stdOutputStream : new LogOutputStream() {
396 protected void processLine(String line, int level) {
397 // if (firstLine) {
398 // if (sudo != null && callbackHandler != null
399 // && line.startsWith(sudoPrompt)) {
400 // try {
401 // PasswordCallback pc = new PasswordCallback(
402 // "sudo password", false);
403 // Callback[] cbs = { pc };
404 // callbackHandler.handle(cbs);
405 // char[] pwd = pc.getPassword();
406 // char[] arr = Arrays.copyOf(pwd,
407 // pwd.length + 1);
408 // arr[arr.length - 1] = '\n';
409 // IOUtils.write(arr, stdInSink);
410 // stdInSink.flush();
411 // } catch (Exception e) {
412 // throw new SlcException(
413 // "Cannot retrieve sudo password", e);
414 // }
415 // }
416 // firstLine = false;
417 // }
418
419 if (line != null && !line.trim().equals(""))
420 logStdOut(line);
421
422 if (stdOutWriter != null)
423 appendLineToFile(stdOutWriter, line);
424 }
425 };
426
427 OutputStream stderr = new LogOutputStream() {
428 protected void processLine(String line, int level) {
429 if (line != null && !line.trim().equals(""))
430 logStdErr(line);
431 if (stdErrWriter != null)
432 appendLineToFile(stdErrWriter, line);
433 }
434 };
435
436 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(stdout, stderr, stdInStream) {
437
438 @Override
439 public void stop() throws IOException {
440 // prevents the method to block when joining stdin
441 if (stdInSink != null)
442 IOUtils.closeQuietly(stdInSink);
443
444 super.stop();
445 }
446 };
447 return pumpStreamHandler;
448 }
449
450 /** Creates the default {@link ExecuteResultHandler}. */
451 protected ExecuteResultHandler createExecuteResultHandler(final CommandLine commandLine) {
452 return new ExecuteResultHandler() {
453
454 public void onProcessComplete(int exitValue) {
455 String msg = "System call '" + commandLine + "' properly completed.";
456 if (log.isTraceEnabled())
457 log.trace(msg);
458 if (testResult != null) {
459 forwardPath(testResult);
460 testResult.addResultPart(new SimpleResultPart(TestStatus.PASSED, msg));
461 }
462 releaseWatchdog();
463 }
464
465 public void onProcessFailed(ExecuteException e) {
466
467 String msg = "System call '" + commandLine + "' failed.";
468 if (testResult != null) {
469 forwardPath(testResult);
470 testResult.addResultPart(new SimpleResultPart(TestStatus.ERROR, msg, e));
471 } else {
472 if (exceptionOnFailed)
473 throw new SlcException(msg, e);
474 else
475 log.error(msg, e);
476 }
477 releaseWatchdog();
478 }
479 };
480 }
481
482 @Deprecated
483 protected void forwardPath(TestResult testResult) {
484 // TODO: allocate a TreeSPath
485 }
486
487 /**
488 * Shortcut method getting the execDir to use
489 */
490 protected String getExecDirToUse() {
491 try {
492 if (execDir != null) {
493 return execDir;
494 }
495 return System.getProperty("user.dir");
496 } catch (Exception e) {
497 throw new SlcException("Cannot find exec dir", e);
498 }
499 }
500
501 protected void logStdOut(String line) {
502 for (SystemCallOutputListener outputListener : outputListeners)
503 outputListener.newLine(this, line, false);
504 log(stdOutLogLevel, line);
505 }
506
507 protected void logStdErr(String line) {
508 for (SystemCallOutputListener outputListener : outputListeners)
509 outputListener.newLine(this, line, true);
510 log(stdErrLogLevel, line);
511 }
512
513 /** Log from the underlying streams. */
514 protected void log(String logLevel, String line) {
515 // TODO optimize
516 // if (SecurityContextHolder.getContext().getAuthentication() == null) {
517 // SecurityContextHolder.getContext()
518 // .setAuthentication(authentication);
519 // }
520
521 if ("ERROR".equals(logLevel))
522 log.error(line);
523 else if ("WARN".equals(logLevel))
524 log.warn(line);
525 else if ("INFO".equals(logLevel))
526 log.info(line);
527 else if ("DEBUG".equals(logLevel))
528 log.debug(line);
529 else if ("TRACE".equals(logLevel))
530 log.trace(line);
531 else if (LOG_STDOUT.equals(logLevel))
532 System.out.println(line);
533 else if ("System.err".equals(logLevel))
534 System.err.println(line);
535 else
536 throw new SlcException("Unknown log level " + logLevel);
537 }
538
539 /** Append line to a log file. */
540 protected void appendLineToFile(Writer writer, String line) {
541 try {
542 writer.append(line).append('\n');
543 } catch (IOException e) {
544 log.error("Cannot write to log file", e);
545 }
546 }
547
548 /** Creates the writer for the output/err files. */
549 protected Writer createWriter(Path target, Boolean append) {
550 FileWriter writer = null;
551 try {
552
553 final File file;
554 if (executionResources != null)
555 file = new File(executionResources.getAsOsPath(target, true));
556 else
557 file = target.toFile();
558 writer = new FileWriter(file, append);
559 } catch (IOException e) {
560 log.error("Cannot get file for " + target, e);
561 IOUtils.closeQuietly(writer);
562 }
563 return writer;
564 }
565
566 /** Creates an outputstream for the output/err files. */
567 protected OutputStream createOutputStream(Path target) {
568 FileOutputStream out = null;
569 try {
570
571 final File file;
572 if (executionResources != null)
573 file = new File(executionResources.getAsOsPath(target, true));
574 else
575 file = target.toFile();
576 out = new FileOutputStream(file, false);
577 } catch (IOException e) {
578 log.error("Cannot get file for " + target, e);
579 IOUtils.closeQuietly(out);
580 }
581 return out;
582 }
583
584 /** Append the argument (for chaining) */
585 public SystemCall arg(String arg) {
586 if (command == null)
587 command = new ArrayList<Object>();
588 command.add(arg);
589 return this;
590 }
591
592 /** Append the argument (for chaining) */
593 public SystemCall arg(String arg, String value) {
594 if (command == null)
595 command = new ArrayList<Object>();
596 command.add(arg);
597 command.add(value);
598 return this;
599 }
600
601 // CONTROL
602 public synchronized Boolean isRunning() {
603 return currentWatchdog != null;
604 }
605
606 private synchronized ExecuteWatchdog createWatchdog() {
607 // if (currentWatchdog != null)
608 // throw new SlcException("A process is already being monitored");
609 currentWatchdog = new ExecuteWatchdog(watchdogTimeout);
610 return currentWatchdog;
611 }
612
613 private synchronized void releaseWatchdog() {
614 currentWatchdog = null;
615 }
616
617 public synchronized void kill() {
618 if (currentWatchdog != null)
619 currentWatchdog.destroyProcess();
620 }
621
622 /** */
623 public void setCmd(String command) {
624 this.cmd = command;
625 }
626
627 public void setCommand(List<Object> command) {
628 this.command = command;
629 }
630
631 public void setExecDir(String execdir) {
632 this.execDir = execdir;
633 }
634
635 public void setStdErrLogLevel(String stdErrLogLevel) {
636 this.stdErrLogLevel = stdErrLogLevel;
637 }
638
639 public void setStdOutLogLevel(String stdOutLogLevel) {
640 this.stdOutLogLevel = stdOutLogLevel;
641 }
642
643 public void setSynchronous(Boolean synchronous) {
644 this.synchronous = synchronous;
645 }
646
647 public void setOsCommands(Map<String, List<Object>> osCommands) {
648 this.osCommands = osCommands;
649 }
650
651 public void setOsCmds(Map<String, String> osCmds) {
652 this.osCmds = osCmds;
653 }
654
655 public void setEnvironmentVariables(Map<String, String> environmentVariables) {
656 this.environmentVariables = environmentVariables;
657 }
658
659 public Map<String, String> getEnvironmentVariables() {
660 return environmentVariables;
661 }
662
663 public void setWatchdogTimeout(Long watchdogTimeout) {
664 this.watchdogTimeout = watchdogTimeout;
665 }
666
667 public void setStdOutFile(Path stdOutFile) {
668 this.stdOutFile = stdOutFile;
669 }
670
671 public void setStdErrFile(Path stdErrFile) {
672 this.stdErrFile = stdErrFile;
673 }
674
675 public void setStdInFile(Path stdInFile) {
676 this.stdInFile = stdInFile;
677 }
678
679 public void setTestResult(TestResult testResult) {
680 this.testResult = testResult;
681 }
682
683 public void setLogCommand(Boolean logCommand) {
684 this.logCommand = logCommand;
685 }
686
687 public void setRedirectStreams(Boolean redirectStreams) {
688 this.redirectStreams = redirectStreams;
689 }
690
691 public void setExceptionOnFailed(Boolean exceptionOnFailed) {
692 this.exceptionOnFailed = exceptionOnFailed;
693 }
694
695 public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
696 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
697 }
698
699 public void setOsConsole(String osConsole) {
700 this.osConsole = osConsole;
701 }
702
703 public void setGenerateScript(String generateScript) {
704 this.generateScript = generateScript;
705 }
706
707 public void setExecutionResources(ExecutionResources executionResources) {
708 this.executionResources = executionResources;
709 }
710
711 public void setRedirectStdOut(Boolean redirectStdOut) {
712 this.redirectStdOut = redirectStdOut;
713 }
714
715 public void addOutputListener(SystemCallOutputListener outputListener) {
716 outputListeners.add(outputListener);
717 }
718
719 public void removeOutputListener(SystemCallOutputListener outputListener) {
720 outputListeners.remove(outputListener);
721 }
722
723 public void setOutputListeners(List<SystemCallOutputListener> outputListeners) {
724 this.outputListeners = outputListeners;
725 }
726
727 public void setExecutor(Executor executor) {
728 this.executor = executor;
729 }
730
731 public void setSudo(String sudo) {
732 this.sudo = sudo;
733 }
734
735 public void setCallbackHandler(CallbackHandler callbackHandler) {
736 this.callbackHandler = callbackHandler;
737 }
738
739 public void setChroot(String chroot) {
740 this.chroot = chroot;
741 }
742
743 private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
744
745 public void setProcessErrorStream(InputStream is) throws IOException {
746 }
747
748 public void setProcessInputStream(OutputStream os) throws IOException {
749 }
750
751 public void setProcessOutputStream(InputStream is) throws IOException {
752 }
753
754 public void start() throws IOException {
755 }
756
757 public void stop() {
758 }
759
760 }
761 }