diff options
author | Jason R. Coombs <jaraco@jaraco.com> | 2012-12-02 06:48:32 -0500 |
---|---|---|
committer | Jason R. Coombs <jaraco@jaraco.com> | 2012-12-02 06:48:32 -0500 |
commit | 7bde7840dd4ffeae3c794eca216244975c856f94 (patch) | |
tree | 4768627ba0e5e74234d8bba2f4674715b87d1772 | |
parent | 45aa88b530cff0ad09f5da83efd4697ba7986563 (diff) | |
parent | 0ae0e9800c7bfb3f8ea40fa0d33ebf6dff49f759 (diff) |
Merge with master
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | .travis.yml | 2 | ||||
-rw-r--r-- | Makefile | 2 | ||||
-rw-r--r-- | NEWS | 34 | ||||
-rw-r--r-- | paramiko/__init__.py | 7 | ||||
-rw-r--r-- | paramiko/client.py | 45 | ||||
-rw-r--r-- | paramiko/config.py | 58 | ||||
-rw-r--r-- | paramiko/file.py | 4 | ||||
-rw-r--r-- | paramiko/packet.py | 4 | ||||
-rw-r--r-- | paramiko/proxy.py | 91 | ||||
-rw-r--r-- | paramiko/sftp_client.py | 128 | ||||
-rw-r--r-- | paramiko/ssh_exception.py | 17 | ||||
-rw-r--r-- | paramiko/transport.py | 5 | ||||
-rw-r--r-- | setup.py | 2 | ||||
-rw-r--r-- | tests/test_util.py | 48 |
15 files changed, 357 insertions, 91 deletions
@@ -3,3 +3,4 @@ build/ dist/ paramiko.egg-info/ test.log +docs/ diff --git a/.travis.yml b/.travis.yml index 312a1846..6896b897 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,6 @@ install: script: python test.py notifications: irc: - channels: "irc.freenode.org#fabric" + channels: "irc.freenode.org#paramiko" on_success: change on_failure: change @@ -1,7 +1,7 @@ release: docs python setup.py sdist register upload -docs: +docs: paramiko/* epydoc --no-private -o docs/ paramiko clean: @@ -12,12 +12,36 @@ Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/. Releases ======== -v1.9.0 (DD MM YYYY) -------------------- +v1.10.0 (DD MM YYYY) +-------------------- +* #71: Add `SFTPClient.putfo` and `.getfo` methods to allow direct + uploading/downloading of file-like objects. Thanks to Eric Buehl for the + patch. +* #113: Add `timeout` parameter to `SSHClient.exec_command` for easier setting + of the command's internal channel object's timeout. Thanks to Cernov Vladimir + for the patch. +* #94: Remove duplication of SSH port constant. Thanks to Olle Lundberg for the + catch. +* #80: Expose the internal "is closed" property of the file transfer class + `BufferedFile` as `.closed`, better conforming to Python's file interface. + Thanks to `@smunaut` and James Hiscock for catch & patch. + +v1.9.0 (6th Nov 2012) +--------------------- -v1.8.1 (DD MM YYYY) -------------------- +* #97 (with a little #93): Improve config parsing of `ProxyCommand` directives + and provide a wrapper class to allow subprocess-driven proxy commands to be + used as `sock=` arguments for `SSHClient.connect`. +* #77: Allow `SSHClient.connect()` to take an explicit `sock` parameter + overriding creation of an internal, implicit socket object. +* Thanks in no particular order to Erwin Bolwidt, Oskari Saarenmaa, Steven + Noonan, Vladimir Lazarenko, Lincoln de Sousa, Valentino Volonghi, Olle + Lundberg, and Github user `@acrish` for the various and sundry patches + leading to the above changes. + +v1.8.1 (6th Nov 2012) +--------------------- * #90: Ensure that callbacks handed to `SFTPClient.get()` always fire at least once, even for zero-length files downloaded. Thanks to Github user `@enB` for @@ -32,6 +56,8 @@ v1.8.1 (DD MM YYYY) v1.8.0 (3rd Oct 2012) --------------------- +* #17 ('ssh' 28): Fix spurious `NoneType has no attribute 'error'` and similar + exceptions that crop up on interpreter exit. * 'ssh' 32: Raise a more useful error explaining which `known_hosts` key line was problematic, when encountering `binascii` issues decoding known host keys. Thanks to `@thomasvs` for catch & patch. diff --git a/paramiko/__init__.py b/paramiko/__init__.py index 8c158538..7d7dcbf4 100644 --- a/paramiko/__init__.py +++ b/paramiko/__init__.py @@ -55,7 +55,7 @@ if sys.version_info < (2, 2): __author__ = "Jeff Forcier <jeff@bitprophet.org>" -__version__ = "1.8.0" +__version__ = "1.10.0" __license__ = "GNU Lesser General Public License (LGPL)" @@ -65,7 +65,7 @@ from auth_handler import AuthHandler from channel import Channel, ChannelFile from ssh_exception import SSHException, PasswordRequiredException, \ BadAuthenticationType, ChannelException, BadHostKeyException, \ - AuthenticationException + AuthenticationException, ProxyCommandFailure from server import ServerInterface, SubsystemHandler, InteractiveQuery from rsakey import RSAKey from dsskey import DSSKey @@ -83,6 +83,7 @@ from agent import Agent, AgentKey from pkey import PKey from hostkeys import HostKeys from config import SSHConfig +from proxy import ProxyCommand # fix module names for epydoc for c in locals().values(): @@ -119,6 +120,8 @@ __all__ = [ 'Transport', 'BadAuthenticationType', 'ChannelException', 'BadHostKeyException', + 'ProxyCommand', + 'ProxyCommandFailure', 'SFTP', 'SFTPFile', 'SFTPHandle', diff --git a/paramiko/client.py b/paramiko/client.py index 0f408977..a777b45b 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -28,6 +28,7 @@ import warnings from paramiko.agent import Agent from paramiko.common import * +from paramiko.config import SSH_PORT from paramiko.dsskey import DSSKey from paramiko.hostkeys import HostKeys from paramiko.resource import ResourceManager @@ -37,8 +38,6 @@ from paramiko.transport import Transport from paramiko.util import retry_on_signal -SSH_PORT = 22 - class MissingHostKeyPolicy (object): """ Interface for defining the policy that L{SSHClient} should use when the @@ -229,7 +228,7 @@ class SSHClient (object): def connect(self, hostname, port=SSH_PORT, username=None, password=None, pkey=None, key_filename=None, timeout=None, allow_agent=True, look_for_keys=True, - compress=False): + compress=False, sock=None): """ Connect to an SSH server and authenticate to it. The server's host key is checked against the system host keys (see L{load_system_host_keys}) @@ -272,6 +271,9 @@ class SSHClient (object): @type look_for_keys: bool @param compress: set to True to turn on compression @type compress: bool + @param sock: an open socket or socket-like object (such as a + L{Channel}) to use for communication to the target host + @type sock: socket @raise BadHostKeyException: if the server's host key could not be verified @@ -280,21 +282,23 @@ class SSHClient (object): establishing an SSH session @raise socket.error: if a socket error occurred while connecting """ - for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM): - if socktype == socket.SOCK_STREAM: - af = family - addr = sockaddr - break - else: - # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :( - af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM) - sock = socket.socket(af, socket.SOCK_STREAM) - if timeout is not None: - try: - sock.settimeout(timeout) - except: - pass - retry_on_signal(lambda: sock.connect(addr)) + if not sock: + for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM): + if socktype == socket.SOCK_STREAM: + af = family + addr = sockaddr + break + else: + # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :( + af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + sock = socket.socket(af, socket.SOCK_STREAM) + if timeout is not None: + try: + sock.settimeout(timeout) + except: + pass + retry_on_signal(lambda: sock.connect(addr)) + t = self._transport = Transport(sock) t.use_compression(compress=compress) if self._log_channel is not None: @@ -345,7 +349,7 @@ class SSHClient (object): self._agent.close() self._agent = None - def exec_command(self, command, bufsize=-1): + def exec_command(self, command, bufsize=-1, timeout=None): """ Execute a command on the SSH server. A new L{Channel} is opened and the requested command is executed. The command's input and output @@ -356,12 +360,15 @@ class SSHClient (object): @type command: str @param bufsize: interpreted the same way as by the built-in C{file()} function in python @type bufsize: int + @param timeout: set command's channel timeout. See L{Channel.settimeout}.settimeout + @type timeout: int @return: the stdin, stdout, and stderr of the executing command @rtype: tuple(L{ChannelFile}, L{ChannelFile}, L{ChannelFile}) @raise SSHException: if the server fails to execute the command """ chan = self._transport.open_session() + chan.settimeout(timeout) chan.exec_command(command) stdin = chan.makefile('wb', bufsize) stdout = chan.makefile('rb', bufsize) diff --git a/paramiko/config.py b/paramiko/config.py index 458d5dd0..d1ce9490 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -22,9 +22,12 @@ L{SSHConfig}. import fnmatch import os +import re import socket -SSH_PORT=22 +SSH_PORT = 22 +proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) + class SSHConfig (object): """ @@ -56,8 +59,13 @@ class SSHConfig (object): if (line == '') or (line[0] == '#'): continue if '=' in line: - key, value = line.split('=', 1) - key = key.strip().lower() + # Ensure ProxyCommand gets properly split + if line.lower().strip().startswith('proxycommand'): + match = proxy_re.match(line) + key, value = match.group(1).lower(), match.group(2) + else: + key, value = line.split('=', 1) + key = key.strip().lower() else: # find first whitespace, and split there i = 0 @@ -149,26 +157,30 @@ class SSHConfig (object): host = socket.gethostname().split('.')[0] fqdn = socket.getfqdn() homedir = os.path.expanduser('~') - replacements = {'controlpath' : - [ - ('%h', config['hostname']), - ('%l', fqdn), - ('%L', host), - ('%n', hostname), - ('%p', port), - ('%r', remoteuser), - ('%u', user) - ], - 'identityfile' : - [ - ('~', homedir), - ('%d', homedir), - ('%h', config['hostname']), - ('%l', fqdn), - ('%u', user), - ('%r', remoteuser) - ] - } + replacements = { + 'controlpath': [ + ('%h', config['hostname']), + ('%l', fqdn), + ('%L', host), + ('%n', hostname), + ('%p', port), + ('%r', remoteuser), + ('%u', user) + ], + 'identityfile': [ + ('~', homedir), + ('%d', homedir), + ('%h', config['hostname']), + ('%l', fqdn), + ('%u', user), + ('%r', remoteuser) + ], + 'proxycommand': [ + ('%h', config['hostname']), + ('%p', port), + ('%r', remoteuser), + ], + } for k in config: if k in replacements: for find, replace in replacements[k]: diff --git a/paramiko/file.py b/paramiko/file.py index d4aec8e3..7e2904e1 100644 --- a/paramiko/file.py +++ b/paramiko/file.py @@ -354,6 +354,10 @@ class BufferedFile (object): """ return self + @property + def closed(self): + return self._closed + ### overrides... diff --git a/paramiko/packet.py b/paramiko/packet.py index 97820619..5d918e2a 100644 --- a/paramiko/packet.py +++ b/paramiko/packet.py @@ -29,7 +29,7 @@ import time from paramiko.common import * from paramiko import util -from paramiko.ssh_exception import SSHException +from paramiko.ssh_exception import SSHException, ProxyCommandFailure from paramiko.message import Message @@ -254,6 +254,8 @@ class Packetizer (object): retry_write = True else: n = -1 + except ProxyCommandFailure: + raise # so it doesn't get swallowed by the below catchall except Exception: # could be: (32, 'Broken pipe') n = -1 diff --git a/paramiko/proxy.py b/paramiko/proxy.py new file mode 100644 index 00000000..218b76e2 --- /dev/null +++ b/paramiko/proxy.py @@ -0,0 +1,91 @@ +# Copyright (C) 2012 Yipit, Inc <coders@yipit.com> +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{ProxyCommand}. +""" + +import os +from shlex import split as shlsplit +import signal +from subprocess import Popen, PIPE + +from paramiko.ssh_exception import ProxyCommandFailure + + +class ProxyCommand(object): + """ + Wraps a subprocess running ProxyCommand-driven programs. + + This class implements a the socket-like interface needed by the + L{Transport} and L{Packetizer} classes. Using this class instead of a + regular socket makes it possible to talk with a Popen'd command that will + proxy traffic between the client and a server hosted in another machine. + """ + def __init__(self, command_line): + """ + Create a new CommandProxy instance. The instance created by this + class can be passed as an argument to the L{Transport} class. + + @param command_line: the command that should be executed and + used as the proxy. + @type command_line: str + """ + self.cmd = shlsplit(command_line) + self.process = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + + def send(self, content): + """ + Write the content received from the SSH client to the standard + input of the forked command. + + @param content: string to be sent to the forked command + @type content: str + """ + try: + self.process.stdin.write(content) + except IOError, e: + # There was a problem with the child process. It probably + # died and we can't proceed. The best option here is to + # raise an exception informing the user that the informed + # ProxyCommand is not working. + raise BadProxyCommand(' '.join(self.cmd), e.strerror) + return len(content) + + def recv(self, size): + """ + Read from the standard output of the forked program. + + @param size: how many chars should be read + @type size: int + + @return: the length of the read content + @rtype: int + """ + try: + return os.read(self.process.stdout.fileno(), size) + except IOError, e: + raise BadProxyCommand(' '.join(self.cmd), e.strerror) + + def close(self): + os.kill(self.process.pid, signal.SIGTERM) + + def settimeout(self, timeout): + # Timeouts are meaningless for this implementation, but are part of the + # spec, so must be present. + pass diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 3eaefc9c..8cb8ceaf 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -533,6 +533,56 @@ class SFTPClient (BaseSFTP): """ return self._cwd + def putfo(self, fl, remotepath, file_size=0, callback=None, confirm=True): + """ + Copy the contents of an open file object (C{fl}) to the SFTP server as + C{remotepath}. Any exception raised by operations will be passed through. + + The SFTP operations use pipelining for speed. + + @param fl: opened file or file-like object to copy + @type localpath: object + @param remotepath: the destination path on the SFTP server + @type remotepath: str + @param file_size: optional size parameter passed to callback. If none is + specified, size defaults to 0 + @type file_size: int + @param callback: optional callback function that accepts the bytes + transferred so far and the total bytes to be transferred + (since 1.7.4) + @type callback: function(int, int) + @param confirm: whether to do a stat() on the file afterwards to + confirm the file size (since 1.7.7) + @type confirm: bool + + @return: an object containing attributes about the given file + (since 1.7.4) + @rtype: SFTPAttributes + + @since: 1.4 + """ + fr = self.file(remotepath, 'wb') + fr.set_pipelined(True) + size = 0 + try: + while True: + data = fl.read(32768) + fr.write(data) + size += len(data) + if callback is not None: + callback(size, file_size) + if len(data) == 0: + break + finally: + fr.close() + if confirm and file_size: + s = self.stat(remotepath) + if s.st_size != size: + raise IOError('size mismatch in put! %d != %d' % (s.st_size, size)) + else: + s = SFTPAttributes() + return s + def put(self, localpath, remotepath, callback=None, confirm=True): """ Copy a local file (C{localpath}) to the SFTP server as C{remotepath}. @@ -562,29 +612,46 @@ class SFTPClient (BaseSFTP): file_size = os.stat(localpath).st_size fl = file(localpath, 'rb') try: - fr = self.file(remotepath, 'wb') - fr.set_pipelined(True) - size = 0 - try: - while True: - data = fl.read(32768) - if len(data) == 0: - break - fr.write(data) - size += len(data) - if callback is not None: - callback(size, file_size) - finally: - fr.close() + return self.putfo(fl, remotepath, os.stat(localpath).st_size, callback, confirm) finally: fl.close() - if confirm: - s = self.stat(remotepath) - if s.st_size != size: - raise IOError('size mismatch in put! %d != %d' % (s.st_size, size)) - else: - s = SFTPAttributes() - return s + + def getfo(self, remotepath, fl, callback=None): + """ + Copy a remote file (C{remotepath}) from the SFTP server and write to + an open file or file-like object, C{fl}. Any exception raised by + operations will be passed through. This method is primarily provided + as a convenience. + + @param remotepath: opened file or file-like object to copy to + @type remotepath: object + @param fl: the destination path on the local host or open file + object + @type localpath: str + @param callback: optional callback function that accepts the bytes + transferred so far and the total bytes to be transferred + (since 1.7.4) + @type callback: function(int, int) + @return: the number of bytes written to the opened file object + + @since: 1.4 + """ + fr = self.file(remotepath, 'rb') + file_size = self.stat(remotepath).st_size + fr.prefetch() + try: + size = 0 + while True: + data = fr.read(32768) + fl.write(data) + size += len(data) + if callback is not None: + callback(size, file_size) + if len(data) == 0: + break + finally: + fr.close() + return size def get(self, remotepath, localpath, callback=None): """ @@ -603,25 +670,12 @@ class SFTPClient (BaseSFTP): @since: 1.4 """ - fr = self.file(remotepath, 'rb') file_size = self.stat(remotepath).st_size - fr.prefetch() + fl = file(localpath, 'wb') try: - fl = file(localpath, 'wb') - try: - size = 0 - while True: - data = fr.read(32768) - fl.write(data) - size += len(data) - if callback is not None: - callback(size, file_size) - if len(data) == 0: - break - finally: - fl.close() + size = self.getfo(remotepath, fl, callback) finally: - fr.close() + fl.close() s = os.stat(localpath) if s.st_size != size: raise IOError('size mismatch in get! %d != %d' % (s.st_size, size)) diff --git a/paramiko/ssh_exception.py b/paramiko/ssh_exception.py index 68924d0f..f2406dcf 100644 --- a/paramiko/ssh_exception.py +++ b/paramiko/ssh_exception.py @@ -113,3 +113,20 @@ class BadHostKeyException (SSHException): self.key = got_key self.expected_key = expected_key + +class ProxyCommandFailure (SSHException): + """ + The "ProxyCommand" found in the .ssh/config file returned an error. + + @ivar command: The command line that is generating this exception. + @type command: str + @ivar error: The error captured from the proxy command output. + @type error: str + """ + def __init__(self, command, error): + SSHException.__init__(self, + '"ProxyCommand (%s)" returned non-zero exit status: %s' % ( + command, error + ) + ) + self.error = error diff --git a/paramiko/transport.py b/paramiko/transport.py index 04680a9f..c8010312 100644 --- a/paramiko/transport.py +++ b/paramiko/transport.py @@ -44,7 +44,8 @@ from paramiko.primes import ModulusPack from paramiko.rsakey import RSAKey from paramiko.server import ServerInterface from paramiko.sftp_client import SFTPClient -from paramiko.ssh_exception import SSHException, BadAuthenticationType, ChannelException +from paramiko.ssh_exception import (SSHException, BadAuthenticationType, + ChannelException, ProxyCommandFailure) from paramiko.util import retry_on_signal from Crypto import Random @@ -1674,6 +1675,8 @@ class Transport (threading.Thread): timeout = 2 try: buf = self.packetizer.readline(timeout) + except ProxyCommandFailure: + raise except Exception, x: raise SSHException('Error reading SSH protocol banner' + str(x)) if buf[:4] == 'SSH-': @@ -52,7 +52,7 @@ if sys.platform == 'darwin': setup(name = "paramiko", - version = "1.8.0", + version = "1.10.0", description = "SSH2 protocol library", author = "Jeff Forcier", author_email = "jeff@bitprophet.org", diff --git a/tests/test_util.py b/tests/test_util.py index 458709b2..093a2157 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -27,6 +27,7 @@ import os import unittest from Crypto.Hash import SHA import paramiko.util +from paramiko.util import lookup_ssh_host_config as host_config from util import ParamikoTest @@ -151,7 +152,7 @@ class UtilTest(ParamikoTest): x = rng.read(32) self.assertEquals(len(x), 32) - def test_7_host_config_expose_issue_33(self): + def test_7_host_config_expose_ssh_issue_33(self): test_config_file = """ Host www13.* Port 22 @@ -194,3 +195,48 @@ Host * raise AssertionError('foo') self.assertRaises(AssertionError, lambda: paramiko.util.retry_on_signal(raises_other_exception)) + + def test_9_proxycommand_config_equals_parsing(self): + """ + ProxyCommand should not split on equals signs within the value. + """ + conf = """ +Host space-delimited + ProxyCommand foo bar=biz baz + +Host equals-delimited + ProxyCommand=foo bar=biz baz +""" + f = cStringIO.StringIO(conf) + config = paramiko.util.parse_ssh_config(f) + for host in ('space-delimited', 'equals-delimited'): + self.assertEquals( + host_config(host, config)['proxycommand'], + 'foo bar=biz baz' + ) + + def test_10_proxycommand_interpolation(self): + """ + ProxyCommand should perform interpolation on the value + """ + config = paramiko.util.parse_ssh_config(cStringIO.StringIO(""" +Host * + Port 25 + ProxyCommand host %h port %p + +Host specific + Port 37 + ProxyCommand host %h port %p lol + +Host portonly + Port 155 +""")) + for host, val in ( + ('foo.com', "host foo.com port 25"), + ('specific', "host specific port 37 lol"), + ('portonly', "host portonly port 155"), + ): + self.assertEquals( + host_config(host, config)['proxycommand'], + val + ) |