]> git.argeo.org Git - gpl/argeo-slc.git/blob - runtime/org.argeo.slc.core/src/main/java/org/argeo/slc/core/execution/tasks/SystemCall.java
Improve RunInOSGi command
[gpl/argeo-slc.git] / runtime / org.argeo.slc.core / src / main / java / org / argeo / slc / core / execution / tasks / SystemCall.java
1 /*
2 * Copyright (C) 2007-2012 Argeo GmbH
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package org.argeo.slc.core.execution.tasks;
17
18 import java.io.File;
19 import java.io.FileOutputStream;
20 import java.io.FileWriter;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.io.PipedInputStream;
25 import java.io.PipedOutputStream;
26 import java.io.Writer;
27 import java.util.ArrayList;
28 import java.util.Collections;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.UUID;
33
34 import javax.security.auth.callback.CallbackHandler;
35
36 import org.apache.commons.exec.CommandLine;
37 import org.apache.commons.exec.DefaultExecutor;
38 import org.apache.commons.exec.ExecuteException;
39 import org.apache.commons.exec.ExecuteResultHandler;
40 import org.apache.commons.exec.ExecuteStreamHandler;
41 import org.apache.commons.exec.ExecuteWatchdog;
42 import org.apache.commons.exec.Executor;
43 import org.apache.commons.exec.LogOutputStream;
44 import org.apache.commons.exec.PumpStreamHandler;
45 import org.apache.commons.exec.ShutdownHookProcessDestroyer;
46 import org.apache.commons.io.FileUtils;
47 import org.apache.commons.io.IOUtils;
48 import org.apache.commons.logging.Log;
49 import org.apache.commons.logging.LogFactory;
50 import org.argeo.slc.SlcException;
51 import org.argeo.slc.UnsupportedException;
52 import org.argeo.slc.core.execution.ExecutionResources;
53 import org.argeo.slc.core.test.SimpleResultPart;
54 import org.argeo.slc.test.TestResult;
55 import org.argeo.slc.test.TestStatus;
56 import org.springframework.core.io.Resource;
57 import org.springframework.security.Authentication;
58 import org.springframework.security.context.SecurityContextHolder;
59
60 /** Execute an OS specific system call. */
61 public class SystemCall implements Runnable {
62 public final static String LOG_STDOUT = "System.out";
63
64 private final Log log = LogFactory.getLog(getClass());
65
66 private String execDir;
67
68 private String cmd = null;
69 private List<Object> command = null;
70
71 private Executor executor = new DefaultExecutor();
72 private Boolean synchronous = true;
73
74 private String stdErrLogLevel = "ERROR";
75 private String stdOutLogLevel = "INFO";
76
77 private Resource stdOutFile = null;
78 private Resource stdErrFile = null;
79
80 private Resource stdInFile = null;
81 /**
82 * If no {@link #stdInFile} provided, writing to this stream will write to
83 * the stdin of the process.
84 */
85 private OutputStream stdInSink = null;
86
87 private Boolean redirectStdOut = false;
88
89 private List<SystemCallOutputListener> outputListeners = Collections
90 .synchronizedList(new ArrayList<SystemCallOutputListener>());
91
92 private Map<String, List<Object>> osCommands = new HashMap<String, List<Object>>();
93 private Map<String, String> osCmds = new HashMap<String, String>();
94 private Map<String, String> environmentVariables = new HashMap<String, String>();
95
96 private Boolean logCommand = false;
97 private Boolean redirectStreams = true;
98 private Boolean exceptionOnFailed = true;
99 private Boolean mergeEnvironmentVariables = true;
100
101 private Authentication authentication;
102
103 private String osConsole = null;
104 private String generateScript = null;
105
106 /** 24 hours */
107 private Long watchdogTimeout = 24 * 60 * 60 * 1000l;
108
109 private TestResult testResult;
110
111 private ExecutionResources executionResources;
112
113 /** Sudo the command, as root if empty or as user if not. */
114 private String sudo = null;
115 // TODO make it more secure and robust, test only once
116 private final String sudoPrompt = UUID.randomUUID().toString();
117 private String askPassProgram = "/usr/libexec/openssh/ssh-askpass";
118 private boolean firstLine = true;
119 private CallbackHandler callbackHandler;
120 /** Chroot to the this path (must not be empty) */
121 private String chroot = null;
122
123 /** Empty constructor */
124 public SystemCall() {
125
126 }
127
128 /**
129 * Constructor based on the provided command list.
130 *
131 * @param command
132 * the command list
133 */
134 public SystemCall(List<Object> command) {
135 this.command = command;
136 }
137
138 /**
139 * Constructor based on the provided command.
140 *
141 * @param cmd
142 * the command. If the provided string contains no space a
143 * command list is initialized with the argument as first
144 * component (useful for chained construction)
145 */
146 public SystemCall(String cmd) {
147 if (cmd.indexOf(' ') < 0) {
148 command = new ArrayList<Object>();
149 command.add(cmd);
150 } else {
151 this.cmd = cmd;
152 }
153 }
154
155 /** Executes the system call. */
156 public void run() {
157 authentication = SecurityContextHolder.getContext().getAuthentication();
158
159 // Manage streams
160 Writer stdOutWriter = null;
161 OutputStream stdOutputStream = null;
162 Writer stdErrWriter = null;
163 InputStream stdInStream = null;
164 if (stdOutFile != null)
165 if (redirectStdOut)
166 stdOutputStream = createOutputStream(stdOutFile);
167 else
168 stdOutWriter = createWriter(stdOutFile, true);
169
170 if (stdErrFile != null) {
171 stdErrWriter = createWriter(stdErrFile, true);
172 } else {
173 if (stdOutFile != null && !redirectStdOut)
174 stdErrWriter = createWriter(stdOutFile, true);
175 }
176
177 try {
178 if (stdInFile != null)
179 stdInStream = stdInFile.getInputStream();
180 else {
181 stdInStream = new PipedInputStream();
182 stdInSink = new PipedOutputStream(
183 (PipedInputStream) stdInStream);
184 }
185 } catch (IOException e2) {
186 throw new SlcException("Cannot open a stream for " + stdInFile, e2);
187 }
188
189 if (log.isTraceEnabled()) {
190 log.debug("os.name=" + System.getProperty("os.name"));
191 log.debug("os.arch=" + System.getProperty("os.arch"));
192 log.debug("os.version=" + System.getProperty("os.version"));
193 }
194
195 // Execution directory
196 File dir = new File(getExecDirToUse());
197 // if (!dir.exists())
198 // dir.mkdirs();
199
200 // Watchdog to check for lost processes
201 Executor executorToUse;
202 if (executor != null)
203 executorToUse = executor;
204 else
205 executorToUse = new DefaultExecutor();
206 executorToUse.setWatchdog(new ExecuteWatchdog(watchdogTimeout));
207
208 if (redirectStreams) {
209 // Redirect standard streams
210 executorToUse.setStreamHandler(createExecuteStreamHandler(
211 stdOutWriter, stdOutputStream, stdErrWriter, stdInStream));
212 } else {
213 // Dummy stream handler (otherwise pump is used)
214 executorToUse.setStreamHandler(new DummyexecuteStreamHandler());
215 }
216
217 executorToUse.setProcessDestroyer(new ShutdownHookProcessDestroyer());
218 executorToUse.setWorkingDirectory(dir);
219
220 // Command line to use
221 final CommandLine commandLine = createCommandLine();
222 if (logCommand)
223 log.info("Execute command:\n" + commandLine
224 + "\n in working directory: \n" + dir + "\n");
225
226 // Env variables
227 Map<String, String> environmentVariablesToUse = null;
228 environmentVariablesToUse = new HashMap<String, String>();
229 if (mergeEnvironmentVariables)
230 environmentVariablesToUse.putAll(System.getenv());
231 if (environmentVariables.size() > 0)
232 environmentVariablesToUse.putAll(environmentVariables);
233
234 // Execute
235 ExecuteResultHandler executeResultHandler = createExecuteResultHandler(commandLine);
236
237 //
238 // THE EXECUTION PROPER
239 //
240 try {
241 if (synchronous)
242 try {
243 int exitValue = executorToUse.execute(commandLine,
244 environmentVariablesToUse);
245 executeResultHandler.onProcessComplete(exitValue);
246 } catch (ExecuteException e1) {
247 if (e1.getExitValue() == Executor.INVALID_EXITVALUE) {
248 Thread.currentThread().interrupt();
249 return;
250 }
251 // Sleep 1s in order to make sure error logs are flushed
252 Thread.sleep(1000);
253 executeResultHandler.onProcessFailed(e1);
254 }
255 else
256 executorToUse.execute(commandLine, environmentVariablesToUse,
257 executeResultHandler);
258 } catch (SlcException e) {
259 throw e;
260 } catch (Exception e) {
261 throw new SlcException("Could not execute command " + commandLine,
262 e);
263 } finally {
264 IOUtils.closeQuietly(stdOutWriter);
265 IOUtils.closeQuietly(stdErrWriter);
266 IOUtils.closeQuietly(stdInStream);
267 IOUtils.closeQuietly(stdInSink);
268 }
269
270 }
271
272 public synchronized String function() {
273 final StringBuffer buf = new StringBuffer("");
274 SystemCallOutputListener tempOutputListener = new SystemCallOutputListener() {
275 private Long lineCount = 0l;
276
277 public void newLine(SystemCall systemCall, String line,
278 Boolean isError) {
279 if (!isError) {
280 if (lineCount != 0l)
281 buf.append('\n');
282 buf.append(line);
283 lineCount++;
284 }
285 }
286 };
287 addOutputListener(tempOutputListener);
288 run();
289 removeOutputListener(tempOutputListener);
290 return buf.toString();
291 }
292
293 public String asCommand() {
294 return createCommandLine().toString();
295 }
296
297 @Override
298 public String toString() {
299 return asCommand();
300 }
301
302 /**
303 * Build a command line based on the properties. Can be overridden by
304 * specific command wrappers.
305 */
306 protected CommandLine createCommandLine() {
307 // Check if an OS specific command overrides
308 String osName = System.getProperty("os.name");
309 List<Object> commandToUse = null;
310 if (osCommands.containsKey(osName))
311 commandToUse = osCommands.get(osName);
312 else
313 commandToUse = command;
314 String cmdToUse = null;
315 if (osCmds.containsKey(osName))
316 cmdToUse = osCmds.get(osName);
317 else
318 cmdToUse = cmd;
319
320 CommandLine commandLine = null;
321
322 // Which command definition to use
323 if (commandToUse == null && cmdToUse == null)
324 throw new SlcException("Please specify a command.");
325 else if (commandToUse != null && cmdToUse != null)
326 throw new SlcException(
327 "Specify the command either as a line or as a list.");
328 else if (cmdToUse != null) {
329 if (chroot != null && !chroot.trim().equals(""))
330 cmdToUse = "chroot \"" + chroot + "\" " + cmdToUse;
331 if (sudo != null) {
332 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
333 if (!sudo.trim().equals(""))
334 cmdToUse = "sudo -p " + sudoPrompt + " -u " + sudo + " "
335 + cmdToUse;
336 else
337 cmdToUse = "sudo -p " + sudoPrompt + " " + cmdToUse;
338 }
339
340 // GENERATE COMMAND LINE
341 commandLine = CommandLine.parse(cmdToUse);
342 } else if (commandToUse != null) {
343 if (commandToUse.size() == 0)
344 throw new SlcException("Command line is empty.");
345
346 if (chroot != null && sudo != null) {
347 commandToUse.add(0, "chroot");
348 commandToUse.add(1, chroot);
349 }
350
351 if (sudo != null) {
352 environmentVariables.put("SUDO_ASKPASS", askPassProgram);
353 commandToUse.add(0, "sudo");
354 commandToUse.add(1, "-p");
355 commandToUse.add(2, sudoPrompt);
356 if (!sudo.trim().equals("")) {
357 commandToUse.add(3, "-u");
358 commandToUse.add(4, sudo);
359 }
360 }
361
362 // GENERATE COMMAND LINE
363 commandLine = new CommandLine(commandToUse.get(0).toString());
364
365 for (int i = 1; i < commandToUse.size(); i++) {
366 if (log.isTraceEnabled())
367 log.debug(commandToUse.get(i));
368 commandLine.addArgument(commandToUse.get(i).toString());
369 }
370 } else {
371 // all cases covered previously
372 throw new UnsupportedException();
373 }
374
375 if (generateScript != null) {
376 File scriptFile = new File(getExecDirToUse() + File.separator
377 + generateScript);
378 try {
379 FileUtils.writeStringToFile(scriptFile,
380 (osConsole != null ? osConsole + " " : "")
381 + commandLine.toString());
382 } catch (IOException e) {
383 throw new SlcException("Could not generate script "
384 + scriptFile, e);
385 }
386 commandLine = new CommandLine(scriptFile);
387 } else {
388 if (osConsole != null)
389 commandLine = CommandLine.parse(osConsole + " "
390 + commandLine.toString());
391 }
392
393 return commandLine;
394 }
395
396 /**
397 * Creates a {@link PumpStreamHandler} which redirects streams to the custom
398 * logging mechanism.
399 */
400 protected ExecuteStreamHandler createExecuteStreamHandler(
401 final Writer stdOutWriter, final OutputStream stdOutputStream,
402 final Writer stdErrWriter, final InputStream stdInStream) {
403
404 // Log writers
405 OutputStream stdout = stdOutputStream != null ? stdOutputStream
406 : new LogOutputStream() {
407 protected void processLine(String line, int level) {
408 // if (firstLine) {
409 // if (sudo != null && callbackHandler != null
410 // && line.startsWith(sudoPrompt)) {
411 // try {
412 // PasswordCallback pc = new PasswordCallback(
413 // "sudo password", false);
414 // Callback[] cbs = { pc };
415 // callbackHandler.handle(cbs);
416 // char[] pwd = pc.getPassword();
417 // char[] arr = Arrays.copyOf(pwd,
418 // pwd.length + 1);
419 // arr[arr.length - 1] = '\n';
420 // IOUtils.write(arr, stdInSink);
421 // stdInSink.flush();
422 // } catch (Exception e) {
423 // throw new SlcException(
424 // "Cannot retrieve sudo password", e);
425 // }
426 // }
427 // firstLine = false;
428 // }
429
430 if (line != null && !line.trim().equals(""))
431 logStdOut(line);
432
433 if (stdOutWriter != null)
434 appendLineToFile(stdOutWriter, line);
435 }
436 };
437
438 OutputStream stderr = new LogOutputStream() {
439 protected void processLine(String line, int level) {
440 if (line != null && !line.trim().equals(""))
441 logStdErr(line);
442 if (stdErrWriter != null)
443 appendLineToFile(stdErrWriter, line);
444 }
445 };
446
447 PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(stdout,
448 stderr, stdInStream) {
449
450 @Override
451 public void stop() {
452 // prevents the method to block when joining stdin
453 if (stdInSink != null)
454 IOUtils.closeQuietly(stdInSink);
455
456 super.stop();
457 }
458 };
459 return pumpStreamHandler;
460 }
461
462 /** Creates the default {@link ExecuteResultHandler}. */
463 protected ExecuteResultHandler createExecuteResultHandler(
464 final CommandLine commandLine) {
465 return new ExecuteResultHandler() {
466
467 public void onProcessComplete(int exitValue) {
468 String msg = "System call '" + commandLine
469 + "' properly completed.";
470 if (log.isTraceEnabled())
471 log.trace(msg);
472 if (testResult != null) {
473 forwardPath(testResult);
474 testResult.addResultPart(new SimpleResultPart(
475 TestStatus.PASSED, msg));
476 }
477 }
478
479 public void onProcessFailed(ExecuteException e) {
480
481 String msg = "System call '" + commandLine + "' failed.";
482 if (testResult != null) {
483 forwardPath(testResult);
484 testResult.addResultPart(new SimpleResultPart(
485 TestStatus.ERROR, msg, e));
486 } else {
487 if (exceptionOnFailed)
488 throw new SlcException(msg, e);
489 else
490 log.error(msg, e);
491 }
492 }
493 };
494 }
495
496 @Deprecated
497 protected void forwardPath(TestResult testResult) {
498 // TODO: allocate a TreeSPath
499 }
500
501 /**
502 * Shortcut method getting the execDir to use
503 */
504 protected String getExecDirToUse() {
505 try {
506 if (execDir != null) {
507 return execDir;
508 }
509 return System.getProperty("user.dir");
510 } catch (Exception e) {
511 throw new SlcException("Cannot find exec dir", e);
512 }
513 }
514
515 protected void logStdOut(String line) {
516 for (SystemCallOutputListener outputListener : outputListeners)
517 outputListener.newLine(this, line, false);
518 log(stdOutLogLevel, line);
519 }
520
521 protected void logStdErr(String line) {
522 for (SystemCallOutputListener outputListener : outputListeners)
523 outputListener.newLine(this, line, true);
524 log(stdErrLogLevel, line);
525 }
526
527 /** Log from the underlying streams. */
528 protected void log(String logLevel, String line) {
529 // TODO optimize
530 if (SecurityContextHolder.getContext().getAuthentication() == null) {
531 SecurityContextHolder.getContext()
532 .setAuthentication(authentication);
533 }
534
535 if ("ERROR".equals(logLevel))
536 log.error(line);
537 else if ("WARN".equals(logLevel))
538 log.warn(line);
539 else if ("INFO".equals(logLevel))
540 log.info(line);
541 else if ("DEBUG".equals(logLevel))
542 log.debug(line);
543 else if ("TRACE".equals(logLevel))
544 log.trace(line);
545 else if (LOG_STDOUT.equals(logLevel))
546 System.out.println(line);
547 else if ("System.err".equals(logLevel))
548 System.err.println(line);
549 else
550 throw new SlcException("Unknown log level " + logLevel);
551 }
552
553 /** Append line to a log file. */
554 protected void appendLineToFile(Writer writer, String line) {
555 try {
556 writer.append(line).append('\n');
557 } catch (IOException e) {
558 log.error("Cannot write to log file", e);
559 }
560 }
561
562 /** Creates the writer for the output/err files. */
563 protected Writer createWriter(Resource target, Boolean append) {
564 FileWriter writer = null;
565 try {
566
567 final File file;
568 if (executionResources != null)
569 file = new File(executionResources.getAsOsPath(target, true));
570 else
571 file = target.getFile();
572 writer = new FileWriter(file, append);
573 } catch (IOException e) {
574 log.error("Cannot get file for " + target, e);
575 IOUtils.closeQuietly(writer);
576 }
577 return writer;
578 }
579
580 /** Creates an outputstream for the output/err files. */
581 protected OutputStream createOutputStream(Resource target) {
582 FileOutputStream out = null;
583 try {
584
585 final File file;
586 if (executionResources != null)
587 file = new File(executionResources.getAsOsPath(target, true));
588 else
589 file = target.getFile();
590 out = new FileOutputStream(file, false);
591 } catch (IOException e) {
592 log.error("Cannot get file for " + target, e);
593 IOUtils.closeQuietly(out);
594 }
595 return out;
596 }
597
598 /** Append the argument (for chaining) */
599 public SystemCall arg(String arg) {
600 if (command == null)
601 command = new ArrayList<Object>();
602 command.add(arg);
603 return this;
604 }
605
606 /** Append the argument (for chaining) */
607 public SystemCall arg(String arg, String value) {
608 if (command == null)
609 command = new ArrayList<Object>();
610 command.add(arg);
611 command.add(value);
612 return this;
613 }
614
615 /** */
616 public void setCmd(String command) {
617 this.cmd = command;
618 }
619
620 public void setCommand(List<Object> command) {
621 this.command = command;
622 }
623
624 public void setExecDir(String execdir) {
625 this.execDir = execdir;
626 }
627
628 public void setStdErrLogLevel(String stdErrLogLevel) {
629 this.stdErrLogLevel = stdErrLogLevel;
630 }
631
632 public void setStdOutLogLevel(String stdOutLogLevel) {
633 this.stdOutLogLevel = stdOutLogLevel;
634 }
635
636 public void setSynchronous(Boolean synchronous) {
637 this.synchronous = synchronous;
638 }
639
640 public void setOsCommands(Map<String, List<Object>> osCommands) {
641 this.osCommands = osCommands;
642 }
643
644 public void setOsCmds(Map<String, String> osCmds) {
645 this.osCmds = osCmds;
646 }
647
648 public void setEnvironmentVariables(Map<String, String> environmentVariables) {
649 this.environmentVariables = environmentVariables;
650 }
651
652 public Map<String, String> getEnvironmentVariables() {
653 return environmentVariables;
654 }
655
656 public void setWatchdogTimeout(Long watchdogTimeout) {
657 this.watchdogTimeout = watchdogTimeout;
658 }
659
660 public void setStdOutFile(Resource stdOutFile) {
661 this.stdOutFile = stdOutFile;
662 }
663
664 public void setStdErrFile(Resource stdErrFile) {
665 this.stdErrFile = stdErrFile;
666 }
667
668 public void setStdInFile(Resource stdInFile) {
669 this.stdInFile = stdInFile;
670 }
671
672 public void setTestResult(TestResult testResult) {
673 this.testResult = testResult;
674 }
675
676 public void setLogCommand(Boolean logCommand) {
677 this.logCommand = logCommand;
678 }
679
680 public void setRedirectStreams(Boolean redirectStreams) {
681 this.redirectStreams = redirectStreams;
682 }
683
684 public void setExceptionOnFailed(Boolean exceptionOnFailed) {
685 this.exceptionOnFailed = exceptionOnFailed;
686 }
687
688 public void setMergeEnvironmentVariables(Boolean mergeEnvironmentVariables) {
689 this.mergeEnvironmentVariables = mergeEnvironmentVariables;
690 }
691
692 public void setOsConsole(String osConsole) {
693 this.osConsole = osConsole;
694 }
695
696 public void setGenerateScript(String generateScript) {
697 this.generateScript = generateScript;
698 }
699
700 public void setExecutionResources(ExecutionResources executionResources) {
701 this.executionResources = executionResources;
702 }
703
704 public void setRedirectStdOut(Boolean redirectStdOut) {
705 this.redirectStdOut = redirectStdOut;
706 }
707
708 public void addOutputListener(SystemCallOutputListener outputListener) {
709 outputListeners.add(outputListener);
710 }
711
712 public void removeOutputListener(SystemCallOutputListener outputListener) {
713 outputListeners.remove(outputListener);
714 }
715
716 public void setOutputListeners(
717 List<SystemCallOutputListener> outputListeners) {
718 this.outputListeners = outputListeners;
719 }
720
721 public void setExecutor(Executor executor) {
722 this.executor = executor;
723 }
724
725 public void setSudo(String sudo) {
726 this.sudo = sudo;
727 }
728
729 public void setCallbackHandler(CallbackHandler callbackHandler) {
730 this.callbackHandler = callbackHandler;
731 }
732
733 public void setChroot(String chroot) {
734 this.chroot = chroot;
735 }
736
737 private class DummyexecuteStreamHandler implements ExecuteStreamHandler {
738
739 public void setProcessErrorStream(InputStream is) throws IOException {
740 }
741
742 public void setProcessInputStream(OutputStream os) throws IOException {
743 }
744
745 public void setProcessOutputStream(InputStream is) throws IOException {
746 }
747
748 public void start() throws IOException {
749 }
750
751 public void stop() {
752 }
753
754 }
755 }