From: Mathieu Baudier Date: Wed, 13 Mar 2019 15:20:01 +0000 (+0100) Subject: Add SFTP support to sync X-Git-Tag: argeo-commons-2.1.77~26 X-Git-Url: http://git.argeo.org/?p=lgpl%2Fargeo-commons.git;a=commitdiff_plain;h=5c6333d04de4985c349197852414faa0f4ee33ee Add SFTP support to sync --- diff --git a/org.argeo.core/src/org/argeo/ssh/AbstractSsh.java b/org.argeo.core/src/org/argeo/ssh/AbstractSsh.java new file mode 100644 index 000000000..9c4ec567a --- /dev/null +++ b/org.argeo.core/src/org/argeo/ssh/AbstractSsh.java @@ -0,0 +1,125 @@ +package org.argeo.ssh; + +import java.io.IOException; +import java.net.URI; +import java.security.GeneralSecurityException; +import java.security.KeyPair; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sshd.client.SshClient; +import org.apache.sshd.client.config.keys.ClientIdentityLoader; +import org.apache.sshd.client.future.ConnectFuture; +import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.client.subsystem.sftp.SftpFileSystemProvider; +import org.apache.sshd.common.config.keys.FilePasswordProvider; + +abstract class AbstractSsh { + private final static Log log = LogFactory.getLog(AbstractSsh.class); + + private static SshClient sshClient; + private static SftpFileSystemProvider sftpFileSystemProvider; + + private boolean passwordSet = false; + private ClientSession session; + + synchronized SshClient getSshClient() { + if (sshClient == null) { + long begin = System.currentTimeMillis(); + sshClient = SshClient.setUpDefaultClient(); + sshClient.start(); + long duration = System.currentTimeMillis() - begin; + if (log.isDebugEnabled()) + log.debug("SSH client started in " + duration + " ms"); + Runtime.getRuntime().addShutdownHook(new Thread(() -> sshClient.stop(), "Stop SSH client")); + } + return sshClient; + } + + synchronized SftpFileSystemProvider getSftpFileSystemProvider() { + if(sftpFileSystemProvider==null) { + sftpFileSystemProvider = new SftpFileSystemProvider(sshClient); + } + return sftpFileSystemProvider; + } + + void authenticate() { + try { + session.auth().verify(1000l); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + void addPassword(String password) { + session.addPasswordIdentity(password); + } + + void loadKey(String password) { + loadKey(password, System.getProperty("user.home") + "/.ssh/id_rsa"); + } + + void loadKey(String password, String keyPath) { + try { + KeyPair keyPair = ClientIdentityLoader.DEFAULT.loadClientIdentity(keyPath, + FilePasswordProvider.of(password)); + session.addPublicKeyIdentity(keyPair); + } catch (IOException | GeneralSecurityException e) { + throw new IllegalStateException(e); + } + } + + void openSession(URI uri) { + openSession(uri.getUserInfo(), uri.getHost(), uri.getPort() > 0 ? uri.getPort() : null); + } + + void openSession(String login, String host, Integer port) { + if (session != null) + throw new IllegalStateException("Session is already open"); + + if (host == null) + host = "localhost"; + if (port == null) + port = 22; + if (login == null) + login = System.getProperty("user.name"); + String password = null; + int sepIndex = login.indexOf(':'); + if (sepIndex > 0) + if (sepIndex + 1 < login.length()) { + password = login.substring(sepIndex + 1); + login = login.substring(0, sepIndex); + } else { + throw new IllegalArgumentException("Illegal authority: " + login); + } + try { + ConnectFuture connectFuture = getSshClient().connect(login, host, port); + connectFuture.await(); + ClientSession session = connectFuture.getSession(); + if (password != null) { + session.addPasswordIdentity(password); + passwordSet = true; + } + this.session = session; + } catch (IOException e) { + throw new IllegalStateException("Cannot connect to " + host + ":" + port); + } + } + + void closeSession() { + if (session != null) + throw new IllegalStateException("No session is open"); + try { + session.close(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + session = null; + } + } + + ClientSession getSession() { + return session; + } + +} diff --git a/org.argeo.core/src/org/argeo/ssh/Sftp.java b/org.argeo.core/src/org/argeo/ssh/Sftp.java new file mode 100644 index 000000000..3ad3e9855 --- /dev/null +++ b/org.argeo.core/src/org/argeo/ssh/Sftp.java @@ -0,0 +1,37 @@ +package org.argeo.ssh; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.Path; + +import org.apache.sshd.client.subsystem.sftp.SftpFileSystem; + +public class Sftp extends AbstractSsh { + private URI uri; + + private SftpFileSystem fileSystem; + + public Sftp(URI uri) { + this.uri = uri; + openSession(uri); + } + + public FileSystem getFileSystem() { + if (fileSystem == null) { + try { + authenticate(); + fileSystem = getSftpFileSystemProvider().newFileSystem(getSession()); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + return fileSystem; + } + + public Path getBasePath() { + String p = uri.getPath() != null ? uri.getPath() : "/"; + return getFileSystem().getPath(p); + } + +} diff --git a/org.argeo.core/src/org/argeo/ssh/Ssh.java b/org.argeo.core/src/org/argeo/ssh/Ssh.java new file mode 100644 index 000000000..584294cc0 --- /dev/null +++ b/org.argeo.core/src/org/argeo/ssh/Ssh.java @@ -0,0 +1,98 @@ +package org.argeo.ssh; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.sshd.client.channel.ClientChannel; +import org.apache.sshd.client.channel.ClientChannelEvent; +import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.common.util.io.NoCloseInputStream; +import org.apache.sshd.common.util.io.NoCloseOutputStream; + +public class Ssh extends AbstractSsh { + private final URI uri; + + public Ssh(URI uri) { + this.uri = uri; + openSession(uri); + } + + static void openShell(ClientSession session) { + try (ClientChannel channel = session.createChannel(ClientChannel.CHANNEL_SHELL)) { + channel.setIn(new NoCloseInputStream(System.in)); + channel.setOut(new NoCloseOutputStream(System.out)); + channel.setErr(new NoCloseOutputStream(System.err)); + channel.open(); + + Set events = new HashSet<>(); + events.add(ClientChannelEvent.CLOSED); + channel.waitFor(events, 0); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } finally { + session.close(false); + } + } + + public static void main(String[] args) { + Options options = getOptions(); + CommandLineParser parser = new DefaultParser(); + try { + CommandLine line = parser.parse(options, args); + List remaining = line.getArgList(); + if (remaining.size() == 0) { + System.err.println("There must be at least one argument"); + printHelp(options); + System.exit(1); + } + URI uri = new URI("ssh://" + remaining.get(0)); + List command = new ArrayList<>(); + if (remaining.size() > 1) { + for (int i = 1; i < remaining.size(); i++) { + command.add(remaining.get(i)); + } + } + + // auth + Ssh ssh = new Ssh(uri); + ssh.authenticate(); + + if (command.size() == 0) {// shell + openShell(ssh.getSession()); + } else {// execute command + + } + ssh.closeSession(); + } catch (Exception exp) { + exp.printStackTrace(); + printHelp(options); + System.exit(1); + } finally { + + } + } + + public static Options getOptions() { + Options options = new Options(); +// options.addOption("p", true, "port"); + options.addOption(Option.builder("p").hasArg().argName("port").desc("port of the SSH server").build()); + + return options; + } + + public static void printHelp(Options options) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("ssh [username@]hostname", options, true); + } +} diff --git a/org.argeo.core/src/org/argeo/sync/cli/Sync.java b/org.argeo.core/src/org/argeo/sync/cli/Sync.java index 6bb098b19..d21b5a255 100644 --- a/org.argeo.core/src/org/argeo/sync/cli/Sync.java +++ b/org.argeo.core/src/org/argeo/sync/cli/Sync.java @@ -1,11 +1,13 @@ package org.argeo.sync.cli; import java.net.URI; +import java.nio.file.Paths; import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.argeo.sync.fs.PathSync; @@ -15,21 +17,36 @@ public class Sync { public static void main(String[] args) { Options options = new Options(); options.addOption("r", "recursive", false, "recurse into directories"); - options.addOption(Option.builder().longOpt("progress").hasArg(false).desc("").build()); + options.addOption(Option.builder().longOpt("progress").hasArg(false).desc("show progress").build()); CommandLineParser parser = new DefaultParser(); try { CommandLine line = parser.parse(options, args); List remaining = line.getArgList(); - + if (remaining.size() == 0) { + System.err.println("There must be at least one argument"); + printHelp(options); + System.exit(1); + } URI sourceUri = new URI(remaining.get(0)); - URI targetUri = new URI(remaining.get(1)); + URI targetUri; + if (remaining.size() == 1) { + targetUri = Paths.get(System.getProperty("user.dir")).toUri(); + } else { + targetUri = new URI(remaining.get(1)); + } PathSync pathSync = new PathSync(sourceUri, targetUri); pathSync.run(); } catch (Exception exp) { exp.printStackTrace(); - + printHelp(options); + System.exit(1); } } + public static void printHelp(Options options) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("sync SRC [DEST]", options, true); + } + } diff --git a/org.argeo.core/src/org/argeo/sync/fs/PathSync.java b/org.argeo.core/src/org/argeo/sync/fs/PathSync.java index 1dc30db65..99d5de81f 100644 --- a/org.argeo.core/src/org/argeo/sync/fs/PathSync.java +++ b/org.argeo.core/src/org/argeo/sync/fs/PathSync.java @@ -11,6 +11,7 @@ import java.time.ZonedDateTime; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.argeo.jackrabbit.fs.DavexFsProvider; +import org.argeo.ssh.Sftp; import org.argeo.sync.SyncException; import org.argeo.util.LangUtils; @@ -27,12 +28,8 @@ public class PathSync implements Runnable { @Override public void run() { try { - FileSystemProvider sourceFsProvider = createFsProvider(sourceUri); - FileSystemProvider targetFsProvider = createFsProvider(targetUri); - Path sourceBasePath = sourceUri.getScheme() != null ? sourceFsProvider.getPath(sourceUri) - : Paths.get(sourceUri.getPath()); - Path targetBasePath = targetUri.getScheme() != null ? targetFsProvider.getPath(targetUri) - : Paths.get(targetUri.getPath()); + Path sourceBasePath = createPath(sourceUri); + Path targetBasePath = createPath(targetUri); SyncFileVisitor syncFileVisitor = new SyncFileVisitor(sourceBasePath, targetBasePath); ZonedDateTime begin = ZonedDateTime.now(); Files.walkFileTree(sourceBasePath, syncFileVisitor); @@ -43,15 +40,22 @@ public class PathSync implements Runnable { } } - private static FileSystemProvider createFsProvider(URI uri) { - FileSystemProvider fsProvider; - if (uri.getScheme() == null || uri.getScheme().equals("file")) - fsProvider = FileSystems.getDefault().provider(); - else if (uri.getScheme().equals("davex")) - fsProvider = new DavexFsProvider(); - else + private static Path createPath(URI uri) { + Path path; + if (uri.getScheme() == null) { + path = Paths.get(uri.getPath()); + } else if (uri.getScheme().equals("file")) { + FileSystemProvider fsProvider = FileSystems.getDefault().provider(); + path = fsProvider.getPath(uri); + } else if (uri.getScheme().equals("davex")) { + FileSystemProvider fsProvider = new DavexFsProvider(); + path = fsProvider.getPath(uri); + } else if (uri.getScheme().equals("sftp")) { + Sftp sftp = new Sftp(uri); + path = sftp.getBasePath(); + } else throw new SyncException("URI scheme not supported for " + uri); - return fsProvider; + return path; } static enum Arg { diff --git a/org.argeo.core/src/org/argeo/sync/fs/SshSync.java b/org.argeo.core/src/org/argeo/sync/fs/SshSync.java index 43af45036..773f32d34 100644 --- a/org.argeo.core/src/org/argeo/sync/fs/SshSync.java +++ b/org.argeo.core/src/org/argeo/sync/fs/SshSync.java @@ -143,21 +143,4 @@ public class SshSync { } - static void openShell(ClientSession session) { - try (ClientChannel channel = session.createChannel(ClientChannel.CHANNEL_SHELL)) { - channel.setIn(new NoCloseInputStream(System.in)); - channel.setOut(new NoCloseOutputStream(System.out)); - channel.setErr(new NoCloseOutputStream(System.err)); - channel.open(); - - Set events = new HashSet<>(); - events.add(ClientChannelEvent.CLOSED); - channel.waitFor(events, 0); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } finally { - session.close(false); - } - } }