diff options
| -rw-r--r-- | paramiko/_version.py | 2 | ||||
| -rw-r--r-- | paramiko/_winapi.py | 25 | ||||
| -rw-r--r-- | paramiko/buffered_pipe.py | 19 | ||||
| -rw-r--r-- | paramiko/channel.py | 8 | ||||
| -rw-r--r-- | paramiko/client.py | 141 | ||||
| -rw-r--r-- | paramiko/kex_gex.py | 13 | ||||
| -rw-r--r-- | paramiko/kex_group1.py | 1 | ||||
| -rw-r--r-- | paramiko/kex_group14.py | 2 | ||||
| -rw-r--r-- | paramiko/packet.py | 7 | ||||
| -rw-r--r-- | paramiko/proxy.py | 26 | ||||
| -rw-r--r-- | paramiko/sftp_client.py | 8 | ||||
| -rw-r--r-- | paramiko/sftp_file.py | 25 | ||||
| -rw-r--r-- | paramiko/ssh_exception.py | 46 | ||||
| -rw-r--r-- | paramiko/transport.py | 241 | ||||
| -rw-r--r-- | setup.cfg | 3 | ||||
| -rw-r--r-- | setup_helper.py | 62 | ||||
| -rw-r--r-- | sites/shared_conf.py | 1 | ||||
| -rw-r--r-- | sites/www/changelog.rst | 84 | ||||
| -rw-r--r-- | sites/www/faq.rst | 10 | ||||
| -rwxr-xr-x | test.py | 4 | ||||
| -rw-r--r-- | tests/loop.py | 2 | ||||
| -rw-r--r-- | tests/test_client.py | 2 | ||||
| -rw-r--r-- | tests/test_kex.py | 120 | ||||
| -rwxr-xr-x | tests/test_sftp.py | 3 | ||||
| -rw-r--r-- | tests/test_sftp_big.py | 18 | ||||
| -rw-r--r-- | tests/test_ssh_exception.py | 31 | ||||
| -rw-r--r-- | tests/test_transport.py | 27 |
27 files changed, 798 insertions, 133 deletions
diff --git a/paramiko/_version.py b/paramiko/_version.py index c573fd3d..65353bda 100644 --- a/paramiko/_version.py +++ b/paramiko/_version.py @@ -1,2 +1,2 @@ -__version_info__ = (1, 15, 5) +__version_info__ = (1, 16, 3) __version__ = '.'.join(map(str, __version_info__)) diff --git a/paramiko/_winapi.py b/paramiko/_winapi.py index 9a8bdedd..94bf6017 100644 --- a/paramiko/_winapi.py +++ b/paramiko/_winapi.py @@ -1,6 +1,6 @@ """ Windows API functions implemented as ctypes functions and classes as found -in jaraco.windows (3.3). +in jaraco.windows (3.4.1). If you encounter issues with this module, please consider reporting the issues in jaraco.windows and asking the author to port the fixes back here. @@ -118,6 +118,18 @@ CreateFileMapping.restype = ctypes.wintypes.HANDLE MapViewOfFile = ctypes.windll.kernel32.MapViewOfFile MapViewOfFile.restype = ctypes.wintypes.HANDLE +UnmapViewOfFile = ctypes.windll.kernel32.UnmapViewOfFile +UnmapViewOfFile.argtypes = ctypes.wintypes.HANDLE, + +RtlMoveMemory = ctypes.windll.kernel32.RtlMoveMemory +RtlMoveMemory.argtypes = ( + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_size_t, +) + +ctypes.windll.kernel32.LocalFree.argtypes = ctypes.wintypes.HLOCAL, + ##################### # jaraco.windows.mmap @@ -158,7 +170,7 @@ class MemoryMap(object): if self.pos + n >= self.length: # A little safety. raise ValueError("Refusing to write %d bytes" % n) dest = self.view + self.pos - length = ctypes.wintypes.SIZE(n) + length = ctypes.c_size_t(n) ctypes.windll.kernel32.RtlMoveMemory(dest, msg, length) self.pos += n @@ -168,7 +180,7 @@ class MemoryMap(object): """ out = ctypes.create_string_buffer(n) source = self.view + self.pos - length = ctypes.wintypes.SIZE(n) + length = ctypes.c_size_t(n) ctypes.windll.kernel32.RtlMoveMemory(out, source, length) self.pos += n return out.raw @@ -307,6 +319,13 @@ class SECURITY_ATTRIBUTES(ctypes.Structure): self._descriptor = value self.lpSecurityDescriptor = ctypes.addressof(value) + +ctypes.windll.advapi32.SetSecurityDescriptorOwner.argtypes = ( + ctypes.POINTER(SECURITY_DESCRIPTOR), + ctypes.c_void_p, + ctypes.wintypes.BOOL, +) + ######################### # jaraco.windows.security diff --git a/paramiko/buffered_pipe.py b/paramiko/buffered_pipe.py index d5fe164e..605f51e9 100644 --- a/paramiko/buffered_pipe.py +++ b/paramiko/buffered_pipe.py @@ -70,11 +70,20 @@ class BufferedPipe (object): :param threading.Event event: the event to set/clear """ - self._event = event - if len(self._buffer) > 0: - event.set() - else: - event.clear() + self._lock.acquire() + try: + self._event = event + # Make sure the event starts in `set` state if we appear to already + # be closed; otherwise, if we start in `clear` state & are closed, + # nothing will ever call `.feed` and the event (& OS pipe, if we're + # wrapping one - see `Channel.fileno`) will permanently stay in + # `clear`, causing deadlock if e.g. `select`ed upon. + if self._closed or len(self._buffer) > 0: + event.set() + else: + event.clear() + finally: + self._lock.release() def feed(self, data): """ diff --git a/paramiko/channel.py b/paramiko/channel.py index f4540bcd..3a05bdc4 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -887,6 +887,12 @@ class Channel (ClosingContextManager): """ self.shutdown(1) + @property + def _closed(self): + # Concession to Python 3's socket API, which has a private ._closed + # attribute instead of a semipublic .closed attribute. + return self.closed + ### calls from Transport def _set_transport(self, transport): @@ -992,7 +998,7 @@ class Channel (ClosingContextManager): else: ok = server.check_channel_env_request(self, name, value) elif key == 'exec': - cmd = m.get_text() + cmd = m.get_string() if server is None: ok = False else: diff --git a/paramiko/client.py b/paramiko/client.py index 4e806bb8..ebf21b08 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -25,6 +25,7 @@ import getpass import os import socket import warnings +from errno import ECONNREFUSED, EHOSTUNREACH from paramiko.agent import Agent from paramiko.common import DEBUG @@ -35,7 +36,9 @@ from paramiko.hostkeys import HostKeys from paramiko.py3compat import string_types from paramiko.resource import ResourceManager from paramiko.rsakey import RSAKey -from paramiko.ssh_exception import SSHException, BadHostKeyException +from paramiko.ssh_exception import ( + SSHException, BadHostKeyException, NoValidConnectionsError +) from paramiko.transport import Transport from paramiko.util import retry_on_signal, ClosingContextManager @@ -185,10 +188,46 @@ class SSHClient (ClosingContextManager): """ self._policy = policy - def connect(self, hostname, port=SSH_PORT, username=None, password=None, pkey=None, - key_filename=None, timeout=None, allow_agent=True, look_for_keys=True, - compress=False, sock=None, gss_auth=False, gss_kex=False, - gss_deleg_creds=True, gss_host=None, banner_timeout=None): + def _families_and_addresses(self, hostname, port): + """ + Yield pairs of address families and addresses to try for connecting. + + :param str hostname: the server to connect to + :param int port: the server port to connect to + :returns: Yields an iterable of ``(family, address)`` tuples + """ + guess = True + addrinfos = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + for (family, socktype, proto, canonname, sockaddr) in addrinfos: + if socktype == socket.SOCK_STREAM: + yield family, sockaddr + guess = False + + # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :( + # We only do this if we did not get a single result marked as socktype == SOCK_STREAM. + if guess: + for family, _, _, _, sockaddr in addrinfos: + yield family, sockaddr + + def connect( + self, + hostname, + port=SSH_PORT, + username=None, + password=None, + pkey=None, + key_filename=None, + timeout=None, + allow_agent=True, + look_for_keys=True, + compress=False, + sock=None, + gss_auth=False, + gss_kex=False, + gss_deleg_creds=True, + gss_host=None, + banner_timeout=None + ): """ Connect to an SSH server and authenticate to it. The server's host key is checked against the system host keys (see `load_system_host_keys`) @@ -219,8 +258,10 @@ class SSHClient (ClosingContextManager): :param str key_filename: the filename, or list of filenames, of optional private key(s) to try for authentication - :param float timeout: an optional timeout (in seconds) for the TCP connect - :param bool allow_agent: set to False to disable connecting to the SSH agent + :param float timeout: + an optional timeout (in seconds) for the TCP connect + :param bool allow_agent: + set to False to disable connecting to the SSH agent :param bool look_for_keys: set to False to disable searching for discoverable private key files in ``~/.ssh/`` @@ -228,10 +269,13 @@ class SSHClient (ClosingContextManager): :param socket sock: an open socket or socket-like object (such as a `.Channel`) to use for communication to the target host - :param bool gss_auth: ``True`` if you want to use GSS-API authentication - :param bool gss_kex: Perform GSS-API Key Exchange and user authentication + :param bool gss_auth: + ``True`` if you want to use GSS-API authentication + :param bool gss_kex: + Perform GSS-API Key Exchange and user authentication :param bool gss_deleg_creds: Delegate GSS-API client credentials or not - :param str gss_host: The targets name in the kerberos database. default: hostname + :param str gss_host: + The targets name in the kerberos database. default: hostname :param float banner_timeout: an optional timeout (in seconds) to wait for the SSH banner to be presented. @@ -247,21 +291,37 @@ class SSHClient (ClosingContextManager): ``gss_deleg_creds`` and ``gss_host`` arguments. """ if not sock: - for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM): - if socktype == socket.SOCK_STREAM: - af = family - addr = sockaddr - break - else: - # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :( - af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM) - sock = socket.socket(af, socket.SOCK_STREAM) - if timeout is not None: + errors = {} + # Try multiple possible address families (e.g. IPv4 vs IPv6) + to_try = list(self._families_and_addresses(hostname, port)) + for af, addr in to_try: try: - sock.settimeout(timeout) - except: - pass - retry_on_signal(lambda: sock.connect(addr)) + sock = socket.socket(af, socket.SOCK_STREAM) + if timeout is not None: + try: + sock.settimeout(timeout) + except: + pass + retry_on_signal(lambda: sock.connect(addr)) + # Break out of the loop on success + break + except socket.error as e: + # Raise anything that isn't a straight up connection error + # (such as a resolution error) + if e.errno not in (ECONNREFUSED, EHOSTUNREACH): + raise + # Capture anything else so we know how the run looks once + # iteration is complete. Retain info about which attempt + # this was. + errors[addr] = e + + # Make sure we explode usefully if no address family attempts + # succeeded. We've no way of knowing which error is the "right" + # one, so we construct a hybrid exception containing all the real + # ones, of a subclass that client code should still be watching for + # (socket.error) + if len(errors) == len(to_try): + raise NoValidConnectionsError(errors) t = self._transport = Transport(sock, gss_kex=gss_kex, gss_deleg_creds=gss_deleg_creds) t.use_compression(compress=compress) @@ -322,6 +382,12 @@ class SSHClient (ClosingContextManager): def close(self): """ Close this SSHClient and its underlying `.Transport`. + + .. warning:: + Failure to do this may, in some situations, cause your Python + interpreter to hang at shutdown (often due to race conditions). + It's good practice to `close` your client objects anytime you're + done using them, instead of relying on garbage collection. """ if self._transport is None: return @@ -417,7 +483,8 @@ class SSHClient (ClosingContextManager): """ saved_exception = None two_factor = False - allowed_types = [] + allowed_types = set() + two_factor_types = set(['keyboard-interactive','password']) # If GSS-API support and GSS-PI Key Exchange was performed, we attempt # authentication with gssapi-keyex. @@ -443,8 +510,8 @@ class SSHClient (ClosingContextManager): if pkey is not None: try: self._log(DEBUG, 'Trying SSH key %s' % hexlify(pkey.get_fingerprint())) - allowed_types = self._transport.auth_publickey(username, pkey) - two_factor = (allowed_types == ['password']) + allowed_types = set(self._transport.auth_publickey(username, pkey)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return except SSHException as e: @@ -456,8 +523,8 @@ class SSHClient (ClosingContextManager): try: key = pkey_class.from_private_key_file(key_filename, password) self._log(DEBUG, 'Trying key %s from %s' % (hexlify(key.get_fingerprint()), key_filename)) - self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) + allowed_types = set(self._transport.auth_publickey(username, key)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return break @@ -471,9 +538,9 @@ class SSHClient (ClosingContextManager): for key in self._agent.get_keys(): try: self._log(DEBUG, 'Trying SSH agent key %s' % hexlify(key.get_fingerprint())) - # for 2-factor auth a successfully auth'd key will result in ['password'] - allowed_types = self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) + # for 2-factor auth a successfully auth'd key password will return an allowed 2fac auth method + allowed_types = set(self._transport.auth_publickey(username, key)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return break @@ -510,8 +577,8 @@ class SSHClient (ClosingContextManager): key = pkey_class.from_private_key_file(filename, password) self._log(DEBUG, 'Trying discovered key %s in %s' % (hexlify(key.get_fingerprint()), filename)) # for 2-factor auth a successfully auth'd key will result in ['password'] - allowed_types = self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) + allowed_types = set(self._transport.auth_publickey(username, key)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return break @@ -525,7 +592,11 @@ class SSHClient (ClosingContextManager): except SSHException as e: saved_exception = e elif two_factor: - raise SSHException('Two-factor authentication requires a password') + try: + self._transport.auth_interactive_dumb(username) + return + except SSHException as e: + saved_exception = e # if we got an auth-failed exception earlier, re-raise it if saved_exception is not None: diff --git a/paramiko/kex_gex.py b/paramiko/kex_gex.py index cb548f33..c980b690 100644 --- a/paramiko/kex_gex.py +++ b/paramiko/kex_gex.py @@ -23,7 +23,7 @@ client side, and a **lot** more on the server side. """ import os -from hashlib import sha1 +from hashlib import sha1, sha256 from paramiko import util from paramiko.common import DEBUG @@ -44,6 +44,7 @@ class KexGex (object): min_bits = 1024 max_bits = 8192 preferred_bits = 2048 + hash_algo = sha1 def __init__(self, transport): self.transport = transport @@ -87,7 +88,7 @@ class KexGex (object): return self._parse_kexdh_gex_reply(m) elif ptype == _MSG_KEXDH_GEX_REQUEST_OLD: return self._parse_kexdh_gex_request_old(m) - raise SSHException('KexGex asked to handle packet type %d' % ptype) + raise SSHException('KexGex %s asked to handle packet type %d' % self.name, ptype) ### internals... @@ -204,7 +205,7 @@ class KexGex (object): hm.add_mpint(self.e) hm.add_mpint(self.f) hm.add_mpint(K) - H = sha1(hm.asbytes()).digest() + H = self.hash_algo(hm.asbytes()).digest() self.transport._set_K_H(K, H) # sign it sig = self.transport.get_server_key().sign_ssh_data(H) @@ -239,6 +240,10 @@ class KexGex (object): hm.add_mpint(self.e) hm.add_mpint(self.f) hm.add_mpint(K) - self.transport._set_K_H(K, sha1(hm.asbytes()).digest()) + self.transport._set_K_H(K, self.hash_algo(hm.asbytes()).digest()) self.transport._verify_key(host_key, sig) self.transport._activate_outbound() + +class KexGexSHA256(KexGex): + name = 'diffie-hellman-group-exchange-sha256' + hash_algo = sha256 diff --git a/paramiko/kex_group1.py b/paramiko/kex_group1.py index a88f00d2..9eee066c 100644 --- a/paramiko/kex_group1.py +++ b/paramiko/kex_group1.py @@ -45,6 +45,7 @@ class KexGroup1(object): G = 2 name = 'diffie-hellman-group1-sha1' + hash_algo = sha1 def __init__(self, transport): self.transport = transport diff --git a/paramiko/kex_group14.py b/paramiko/kex_group14.py index a914aeaf..9f7dd216 100644 --- a/paramiko/kex_group14.py +++ b/paramiko/kex_group14.py @@ -22,6 +22,7 @@ Standard SSH key exchange ("kex" if you wanna sound cool). Diffie-Hellman of """ from paramiko.kex_group1 import KexGroup1 +from hashlib import sha1 class KexGroup14(KexGroup1): @@ -31,3 +32,4 @@ class KexGroup14(KexGroup1): G = 2 name = 'diffie-hellman-group14-sha1' + hash_algo = sha1 diff --git a/paramiko/packet.py b/paramiko/packet.py index 8aaab75a..5dd4c49f 100644 --- a/paramiko/packet.py +++ b/paramiko/packet.py @@ -103,6 +103,10 @@ class Packetizer (object): self.__handshake_complete = False self.__timer_expired = False + @property + def closed(self): + return self.__closed + def set_log(self, log): """ Set the Python log object to use for logging. @@ -390,7 +394,8 @@ class Packetizer (object): if self.__dump_packets: self._log(DEBUG, util.format_binary(header, 'IN: ')) packet_size = struct.unpack('>I', header[:4])[0] - # leftover contains decrypted bytes from the first block (after the length field) + # leftover contains decrypted bytes from the first block (after the + # length field) leftover = header[4:] if (packet_size - len(leftover)) % self.__block_size_in != 0: raise SSHException('Invalid packet blocking') diff --git a/paramiko/proxy.py b/paramiko/proxy.py index ca602c4c..d3ae436f 100644 --- a/paramiko/proxy.py +++ b/paramiko/proxy.py @@ -38,7 +38,7 @@ class ProxyCommand(ClosingContextManager): `.Transport` and `.Packetizer` classes. Using this class instead of a regular socket makes it possible to talk with a Popen'd command that will proxy traffic between the client and a server hosted in another machine. - + Instances of this class may be used as context managers. """ def __init__(self, command_line): @@ -50,9 +50,9 @@ class ProxyCommand(ClosingContextManager): the command that should be executed and used as the proxy. """ self.cmd = shlsplit(command_line) - self.process = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + self.process = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, + bufsize=0) self.timeout = None - self.buffer = [] def send(self, content): """ @@ -77,11 +77,12 @@ class ProxyCommand(ClosingContextManager): :param int size: how many chars should be read - :return: the length of the read content, as an `int` + :return: the string of bytes read, which may be shorter than requested """ try: + buffer = b'' start = time.time() - while len(self.buffer) < size: + while len(buffer) < size: select_timeout = None if self.timeout is not None: elapsed = (time.time() - start) @@ -92,16 +93,13 @@ class ProxyCommand(ClosingContextManager): r, w, x = select( [self.process.stdout], [], [], select_timeout) if r and r[0] == self.process.stdout: - b = os.read( - self.process.stdout.fileno(), size - len(self.buffer)) - # Store in class-level buffer for persistence across - # timeouts; this makes us act more like a real socket - # (where timeouts don't actually drop data.) - self.buffer.extend(b) - result = ''.join(self.buffer) - self.buffer = [] - return result + buffer += os.read( + self.process.stdout.fileno(), size - len(buffer)) + return buffer except socket.timeout: + if buffer: + # Don't raise socket.timeout, return partial result instead + return buffer raise # socket.timeout is a subclass of IOError except IOError as e: raise ProxyCommandFailure(' '.join(self.cmd), e.strerror) diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 25abe590..0df94389 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -589,6 +589,7 @@ class SFTPClient(BaseSFTP, ClosingContextManager): .. versionadded:: 1.4 """ + # TODO: make class initialize with self._cwd set to self.normalize('.') return self._cwd and u(self._cwd) def _transfer_with_callback(self, reader, writer, file_size, callback): @@ -691,13 +692,15 @@ class SFTPClient(BaseSFTP, ClosingContextManager): .. versionadded:: 1.10 """ + file_size = self.stat(remotepath).st_size with self.open(remotepath, 'rb') as fr: - file_size = self.stat(remotepath).st_size - fr.prefetch() + fr.prefetch(file_size) return self._transfer_with_callback( reader=fr, writer=fl, file_size=file_size, callback=callback ) + return size + def get(self, remotepath, localpath, callback=None): """ Copy a remote file (``remotepath``) from the SFTP server to the local @@ -714,7 +717,6 @@ class SFTPClient(BaseSFTP, ClosingContextManager): .. versionchanged:: 1.7.4 Added the ``callback`` param """ - file_size = self.stat(remotepath).st_size with open(localpath, 'wb') as fl: size = self.getfo(remotepath, fl, callback) s = os.stat(localpath) diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index b3c1d648..8d147342 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -378,8 +378,8 @@ class SFTPFile (BufferedFile): .. versionadded:: 1.5 """ self.pipelined = pipelined - - def prefetch(self): + + def prefetch(self, file_size=None): """ Pre-fetch the remaining contents of this file in anticipation of future `.read` calls. If reading the entire file, pre-fetching can @@ -391,14 +391,29 @@ class SFTPFile (BufferedFile): data may be read in a random order (using `.seek`); chunks of the buffer that haven't been read will continue to be buffered. + :param int file_size: + When this is ``None`` (the default), this method calls `stat` to + determine the remote file size. In some situations, doing so can + cause exceptions or hangs (see `#562 + <https://github.com/paramiko/paramiko/pull/562>`_); as a + workaround, one may call `stat` explicitly and pass its value in + via this parameter. + .. versionadded:: 1.5.1 + .. versionchanged:: 1.16.0 + The ``file_size`` parameter was added (with no default value). + .. versionchanged:: 1.16.1 + The ``file_size`` parameter was made optional for backwards + compatibility. """ - size = self.stat().st_size + if file_size is None: + file_size = self.stat().st_size; + # queue up async reads for the rest of the file chunks = [] n = self._realpos - while n < size: - chunk = min(self.MAX_REQUEST_SIZE, size - n) + while n < file_size: + chunk = min(self.MAX_REQUEST_SIZE, file_size - n) chunks.append((n, chunk)) n += chunk if len(chunks) > 0: diff --git a/paramiko/ssh_exception.py b/paramiko/ssh_exception.py index e120a45e..ed36a952 100644 --- a/paramiko/ssh_exception.py +++ b/paramiko/ssh_exception.py @@ -16,6 +16,8 @@ # along with Paramiko; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. +import socket + class SSHException (Exception): """ @@ -133,3 +135,47 @@ class ProxyCommandFailure (SSHException): self.error = error # for unpickling self.args = (command, error, ) + + +class NoValidConnectionsError(socket.error): + """ + Multiple connection attempts were made and no families succeeded. + + This exception class wraps multiple "real" underlying connection errors, + all of which represent failed connection attempts. Because these errors are + not guaranteed to all be of the same error type (i.e. different errno, + `socket.error` subclass, message, etc) we expose a single unified error + message and a ``None`` errno so that instances of this class match most + normal handling of `socket.error` objects. + + To see the wrapped exception objects, access the ``errors`` attribute. + ``errors`` is a dict whose keys are address tuples (e.g. ``('127.0.0.1', + 22)``) and whose values are the exception encountered trying to connect to + that address. + + It is implied/assumed that all the errors given to a single instance of + this class are from connecting to the same hostname + port (and thus that + the differences are in the resolution of the hostname - e.g. IPv4 vs v6). + + .. versionadded:: 1.16 + """ + def __init__(self, errors): + """ + :param dict errors: + The errors dict to store, as described by class docstring. + """ + addrs = sorted(errors.keys()) + body = ', '.join([x[0] for x in addrs[:-1]]) + tail = addrs[-1][0] + if body: + msg = "Unable to connect to port {0} on {1} or {2}" + else: + msg = "Unable to connect to port {0} on {2}" + super(NoValidConnectionsError, self).__init__( + None, # stand-in for errno + msg.format(addrs[0][1], body, tail) + ) + self.errors = errors + + def __reduce__(self): + return (self.__class__, (self.errors, )) diff --git a/paramiko/transport.py b/paramiko/transport.py index 07fc8e47..75f3ef75 100644 --- a/paramiko/transport.py +++ b/paramiko/transport.py @@ -20,13 +20,14 @@ Core protocol implementation """ +from __future__ import print_function import os import socket import sys import threading import time import weakref -from hashlib import md5, sha1 +from hashlib import md5, sha1, sha256, sha512 import paramiko from paramiko import util @@ -47,14 +48,14 @@ from paramiko.common import xffffffff, cMSG_CHANNEL_OPEN, cMSG_IGNORE, \ MAX_WINDOW_SIZE, DEFAULT_WINDOW_SIZE, DEFAULT_MAX_PACKET_SIZE from paramiko.compress import ZlibCompressor, ZlibDecompressor from paramiko.dsskey import DSSKey -from paramiko.kex_gex import KexGex +from paramiko.kex_gex import KexGex, KexGexSHA256 from paramiko.kex_group1 import KexGroup1 from paramiko.kex_group14 import KexGroup14 from paramiko.kex_gss import KexGSSGex, KexGSSGroup1, KexGSSGroup14, NullHostKey from paramiko.message import Message from paramiko.packet import Packetizer, NeedRekeyException from paramiko.primes import ModulusPack -from paramiko.py3compat import string_types, long, byte_ord, b +from paramiko.py3compat import string_types, long, byte_ord, b, input, PY2 from paramiko.rsakey import RSAKey from paramiko.ecdsakey import ECDSAKey from paramiko.server import ServerInterface @@ -94,27 +95,109 @@ class Transport (threading.Thread, ClosingContextManager): _PROTO_ID = '2.0' _CLIENT_ID = 'paramiko_%s' % paramiko.__version__ - _preferred_ciphers = ('aes128-ctr', 'aes256-ctr', 'aes128-cbc', 'blowfish-cbc', - 'aes256-cbc', '3des-cbc', 'arcfour128', 'arcfour256') - _preferred_macs = ('hmac-sha1', 'hmac-md5', 'hmac-sha1-96', 'hmac-md5-96') - _preferred_keys = ('ssh-rsa', 'ssh-dss', 'ecdsa-sha2-nistp256') - _preferred_kex = ( 'diffie-hellman-group14-sha1', 'diffie-hellman-group-exchange-sha1' , 'diffie-hellman-group1-sha1') + # These tuples of algorithm identifiers are in preference order; do not + # reorder without reason! + _preferred_ciphers = ( + 'aes128-ctr', + 'aes192-ctr', + 'aes256-ctr', + 'aes128-cbc', + 'blowfish-cbc', + 'aes192-cbc', + 'aes256-cbc', + '3des-cbc', + 'arcfour128', + 'arcfour256', + ) + _preferred_macs = ( + 'hmac-sha2-256', + 'hmac-sha2-512', + 'hmac-md5', + 'hmac-sha1-96', + 'hmac-md5-96', + 'hmac-sha1', + ) + _preferred_keys = ( + 'ssh-rsa', + 'ssh-dss', + 'ecdsa-sha2-nistp256', + ) + _preferred_kex = ( + 'diffie-hellman-group1-sha1', + 'diffie-hellman-group14-sha1', + 'diffie-hellman-group-exchange-sha1', + 'diffie-hellman-group-exchange-sha256', + ) _preferred_compression = ('none',) _cipher_info = { - 'aes128-ctr': {'class': AES, 'mode': AES.MODE_CTR, 'block-size': 16, 'key-size': 16}, - 'aes256-ctr': {'class': AES, 'mode': AES.MODE_CTR, 'block-size': 16, 'key-size': 32}, - 'blowfish-cbc': {'class': Blowfish, 'mode': Blowfish.MODE_CBC, 'block-size': 8, 'key-size': 16}, - 'aes128-cbc': {'class': AES, 'mode': AES.MODE_CBC, 'block-size': 16, 'key-size': 16}, - 'aes256-cbc': {'class': AES, 'mode': AES.MODE_CBC, 'block-size': 16, 'key-size': 32}, - '3des-cbc': {'class': DES3, 'mode': DES3.MODE_CBC, 'block-size': 8, 'key-size': 24}, - 'arcfour128': {'class': ARC4, 'mode': None, 'block-size': 8, 'key-size': 16}, - 'arcfour256': {'class': ARC4, 'mode': None, 'block-size': 8, 'key-size': 32}, + 'aes128-ctr': { + 'class': AES, + 'mode': AES.MODE_CTR, + 'block-size': 16, + 'key-size': 16 + }, + 'aes192-ctr': { + 'class': AES, + 'mode': AES.MODE_CTR, + 'block-size': 16, + 'key-size': 24 + }, + 'aes256-ctr': { + 'class': AES, + 'mode': AES.MODE_CTR, + 'block-size': 16, + 'key-size': 32 + }, + 'blowfish-cbc': { + 'class': Blowfish, + 'mode': Blowfish.MODE_CBC, + 'block-size': 8, + 'key-size': 16 + }, + 'aes128-cbc': { + 'class': AES, + 'mode': AES.MODE_CBC, + 'block-size': 16, + 'key-size': 16 + }, + 'aes192-cbc': { + 'class': AES, + 'mode': AES.MODE_CBC, + 'block-size': 16, + 'key-size': 24 + }, + 'aes256-cbc': { + 'class': AES, + 'mode': AES.MODE_CBC, + 'block-size': 16, + 'key-size': 32 + }, + '3des-cbc': { + 'class': DES3, + 'mode': DES3.MODE_CBC, + 'block-size': 8, + 'key-size': 24 + }, + 'arcfour128': { + 'class': ARC4, + 'mode': None, + 'block-size': 8, + 'key-size': 16 + }, + 'arcfour256': { + 'class': ARC4, + 'mode': None, + 'block-size': 8, + 'key-size': 32 + }, } _mac_info = { 'hmac-sha1': {'class': sha1, 'size': 20}, 'hmac-sha1-96': {'class': sha1, 'size': 12}, + 'hmac-sha2-256': {'class': sha256, 'size': 32}, + 'hmac-sha2-512': {'class': sha512, 'size': 64}, 'hmac-md5': {'class': md5, 'size': 16}, 'hmac-md5-96': {'class': md5, 'size': 12}, } @@ -129,6 +212,7 @@ class Transport (threading.Thread, ClosingContextManager): 'diffie-hellman-group1-sha1': KexGroup1, 'diffie-hellman-group14-sha1': KexGroup14, 'diffie-hellman-group-exchange-sha1': KexGex, + 'diffie-hellman-group-exchange-sha256': KexGexSHA256, 'gss-group1-sha1-toWM5Slw5Ew8Mqkay+al2g==': KexGSSGroup1, 'gss-group14-sha1-toWM5Slw5Ew8Mqkay+al2g==': KexGSSGroup14, 'gss-gex-sha1-toWM5Slw5Ew8Mqkay+al2g==': KexGSSGex @@ -1296,6 +1380,27 @@ class Transport (threading.Thread, ClosingContextManager): self.auth_handler.auth_interactive(username, handler, my_event, submethods) return self.auth_handler.wait_for_response(my_event) + def auth_interactive_dumb(self, username, handler=None, submethods=''): + """ + Autenticate to the server interactively but dumber. + Just print the prompt and / or instructions to stdout and send back + the response. This is good for situations where partial auth is + achieved by key and then the user has to enter a 2fac token. + """ + + if not handler: + def handler(title, instructions, prompt_list): + answers = [] + if title: + print(title.strip()) + if instructions: + print(instructions.strip()) + for prompt,show_input in prompt_list: + print(prompt.strip(),end=' ') + answers.append(input()) + return answers + return self.auth_interactive(username, handler, submethods) + def auth_gssapi_with_mic(self, username, gss_host, gss_deleg_creds): """ Authenticate to the Server using GSS-API / SSPI. @@ -1429,8 +1534,23 @@ class Transport (threading.Thread, ClosingContextManager): def stop_thread(self): self.active = False self.packetizer.close() - while self.is_alive() and (self is not threading.current_thread()): - self.join(10) + if PY2: + # Original join logic; #520 doesn't appear commonly present under + # Python 2. + while self.is_alive() and self is not threading.current_thread(): + self.join(10) + else: + # Keep trying to join() our main thread, quickly, until: + # * We join()ed successfully (self.is_alive() == False) + # * Or it looks like we've hit issue #520 (socket.recv hitting some + # race condition preventing it from timing out correctly), wherein + # our socket and packetizer are both closed (but where we'd + # otherwise be sitting forever on that recv()). + while ( + self.is_alive() and self is not threading.current_thread() + and not self.sock._closed and not self.packetizer.closed + ): + self.join(0.1) ### internals... @@ -1509,13 +1629,23 @@ class Transport (threading.Thread, ClosingContextManager): m.add_bytes(self.H) m.add_byte(b(id)) m.add_bytes(self.session_id) - out = sofar = sha1(m.asbytes()).digest() + # Fallback to SHA1 for kex engines that fail to specify a hex + # algorithm, or for e.g. transport tests that don't run kexinit. + hash_algo = getattr(self.kex_engine, 'hash_algo', None) + hash_select_msg = "kex engine %s specified hash_algo %r" % (self.kex_engine.__class__.__name__, hash_algo) + if hash_algo is None: + hash_algo = sha1 + hash_select_msg += ", falling back to sha1" + if not hasattr(self, '_logged_hash_selection'): + self._log(DEBUG, hash_select_msg) + setattr(self, '_logged_hash_selection', True) + out = sofar = hash_algo(m.asbytes()).digest() while len(out) < nbytes: m = Message() m.add_mpint(self.K) m.add_bytes(self.H) m.add_bytes(sofar) - digest = sha1(m.asbytes()).digest() + digest = hash_algo(m.asbytes()).digest() out += digest sofar += digest return out[:nbytes] @@ -1593,10 +1723,13 @@ class Transport (threading.Thread, ClosingContextManager): try: try: self.packetizer.write_all(b(self.local_version + '\r\n')) + self._log(DEBUG, 'Local version/idstring: %s' % self.local_version) self._check_banner() - # The above is actually very much part of the handshake, but sometimes the banner can be read - # but the machine is not responding, for example when the remote ssh daemon is loaded in to memory - # but we can not read from the disk/spawn a new shell. + # The above is actually very much part of the handshake, but + # sometimes the banner can be read but the machine is not + # responding, for example when the remote ssh daemon is loaded + # in to memory but we can not read from the disk/spawn a new + # shell. # Make sure we can specify a timeout for the initial handshake. # Re-use the banner timeout for now. self.packetizer.start_handshake(self.handshake_timeout) @@ -1698,6 +1831,18 @@ class Transport (threading.Thread, ClosingContextManager): if self.sys.modules is not None: raise + + def _log_agreement(self, which, local, remote): + # Log useful, non-duplicative line re: an agreed-upon algorithm. + # Old code implied algorithms could be asymmetrical (different for + # inbound vs outbound) so we preserve that possibility. + msg = "{0} agreed: ".format(which) + if local == remote: + msg += local + else: + msg += "local={0}, remote={1}".format(local, remote) + self._log(DEBUG, msg) + ### protocol stages def _negotiate_keys(self, m): @@ -1735,6 +1880,7 @@ class Transport (threading.Thread, ClosingContextManager): raise SSHException('Indecipherable protocol version "' + buf + '"') # save this server version string for later self.remote_version = buf + self._log(DEBUG, 'Remote version/idstring: %s' % buf) # pull off any attached comment comment = '' i = buf.find(' ') @@ -1763,10 +1909,12 @@ class Transport (threading.Thread, ClosingContextManager): self.clear_to_send_lock.release() self.in_kex = True if self.server_mode: - if (self._modulus_pack is None) and ('diffie-hellman-group-exchange-sha1' in self._preferred_kex): + mp_required_prefix = 'diffie-hellman-group-exchange-sha' + kex_mp = [k for k in self._preferred_kex if k.startswith(mp_required_prefix)] + if (self._modulus_pack is None) and (len(kex_mp) > 0): # can't do group-exchange if we don't have a pack of potential primes - pkex = list(self.get_security_options().kex) - pkex.remove('diffie-hellman-group-exchange-sha1') + pkex = [k for k in self.get_security_options().kex + if not k.startswith(mp_required_prefix)] self.get_security_options().kex = pkex available_server_keys = list(filter(list(self.server_key_dict.keys()).__contains__, self._preferred_keys)) @@ -1818,15 +1966,24 @@ class Transport (threading.Thread, ClosingContextManager): ' server lang:' + str(server_lang_list) + ' kex follows?' + str(kex_follows)) - # as a server, we pick the first item in the client's list that we support. - # as a client, we pick the first item in our list that the server supports. + # as a server, we pick the first item in the client's list that we + # support. + # as a client, we pick the first item in our list that the server + # supports. if self.server_mode: - agreed_kex = list(filter(self._preferred_kex.__contains__, kex_algo_list)) + agreed_kex = list(filter( + self._preferred_kex.__contains__, + kex_algo_list + )) else: - agreed_kex = list(filter(kex_algo_list.__contains__, self._preferred_kex)) + agreed_kex = list(filter( + kex_algo_list.__contains__, + self._preferred_kex + )) if len(agreed_kex) == 0: raise SSHException('Incompatible ssh peer (no acceptable kex algorithm)') self.kex_engine = self._kex_info[agreed_kex[0]](self) + self._log(DEBUG, "Kex agreed: %s" % agreed_kex[0]) if self.server_mode: available_server_keys = list(filter(list(self.server_key_dict.keys()).__contains__, @@ -1854,7 +2011,9 @@ class Transport (threading.Thread, ClosingContextManager): raise SSHException('Incompatible ssh server (no acceptable ciphers)') self.local_cipher = agreed_local_ciphers[0] self.remote_cipher = agreed_remote_ciphers[0] - self._log(DEBUG, 'Ciphers agreed: local=%s, remote=%s' % (self.local_cipher, self.remote_cipher)) + self._log_agreement( + 'Cipher', local=self.local_cipher, remote=self.remote_cipher + ) if self.server_mode: agreed_remote_macs = list(filter(self._preferred_macs.__contains__, client_mac_algo_list)) @@ -1866,6 +2025,9 @@ class Transport (threading.Thread, ClosingContextManager): raise SSHException('Incompatible ssh server (no acceptable macs)') self.local_mac = agreed_local_macs[0] self.remote_mac = agreed_remote_macs[0] + self._log_agreement( + 'MAC', local=self.local_mac, remote=self.remote_mac + ) if self.server_mode: agreed_remote_compression = list(filter(self._preferred_compression.__contains__, client_compress_algo_list)) @@ -1877,10 +2039,11 @@ class Transport (threading.Thread, ClosingContextManager): raise SSHException('Incompatible ssh server (no acceptable compression) %r %r %r' % (agreed_local_compression, agreed_remote_compression, self._preferred_compression)) self.local_compression = agreed_local_compression[0] self.remote_compression = agreed_remote_compression[0] - - self._log(DEBUG, 'using kex %s; server key type %s; cipher: local %s, remote %s; mac: local %s, remote %s; compression: local %s, remote %s' % - (agreed_kex[0], self.host_key_type, self.local_cipher, self.remote_cipher, self.local_mac, - self.remote_mac, self.local_compression, self.remote_compression)) + self._log_agreement( + 'Compression', + local=self.local_compression, + remote=self.remote_compression + ) # save for computing hash later... # now wait! openssh has a bug (and others might too) where there are @@ -1901,8 +2064,8 @@ class Transport (threading.Thread, ClosingContextManager): engine = self._get_cipher(self.remote_cipher, key_in, IV_in) mac_size = self._mac_info[self.remote_mac]['size'] mac_engine = self._mac_info[self.remote_mac]['class'] - # initial mac keys are done in the hash's natural size (not the potentially truncated - # transmission size) + # initial mac keys are done in the hash's natural size (not the + # potentially truncated transmission size) if self.server_mode: mac_key = self._compute_key('E', mac_engine().digest_size) else: @@ -1928,8 +2091,8 @@ class Transport (threading.Thread, ClosingContextManager): engine = self._get_cipher(self.local_cipher, key_out, IV_out) mac_size = self._mac_info[self.local_mac]['size'] mac_engine = self._mac_info[self.local_mac]['class'] - # initial mac keys are done in the hash's natural size (not the potentially truncated - # transmission size) + # initial mac keys are done in the hash's natural size (not the + # potentially truncated transmission size) if self.server_mode: mac_key = self._compute_key('F', mac_engine().digest_size) else: @@ -1,2 +1,5 @@ [wheel] universal = 1 + +[coverage:run] +omit = paramiko/_winapi.py diff --git a/setup_helper.py b/setup_helper.py index ff6b0e16..9e3834b3 100644 --- a/setup_helper.py +++ b/setup_helper.py @@ -30,9 +30,42 @@ import distutils.archive_util from distutils.dir_util import mkpath from distutils.spawn import spawn - -def make_tarball(base_name, base_dir, compress='gzip', - verbose=False, dry_run=False): +try: + from pwd import getpwnam +except ImportError: + getpwnam = None + +try: + from grp import getgrnam +except ImportError: + getgrnam = None + +def _get_gid(name): + """Returns a gid, given a group name.""" + if getgrnam is None or name is None: + return None + try: + result = getgrnam(name) + except KeyError: + result = None + if result is not None: + return result[2] + return None + +def _get_uid(name): + """Returns an uid, given a user name.""" + if getpwnam is None or name is None: + return None + try: + result = getpwnam(name) + except KeyError: + result = None + if result is not None: + return result[2] + return None + +def make_tarball(base_name, base_dir, compress='gzip', verbose=0, dry_run=0, + owner=None, group=None): """Create a tar file from all the files under 'base_dir'. This file may be compressed. @@ -75,11 +108,30 @@ def make_tarball(base_name, base_dir, compress='gzip', mkpath(os.path.dirname(archive_name), dry_run=dry_run) log.info('Creating tar file %s with mode %s' % (archive_name, mode)) + uid = _get_uid(owner) + gid = _get_gid(group) + + def _set_uid_gid(tarinfo): + if gid is not None: + tarinfo.gid = gid + tarinfo.gname = group + if uid is not None: + tarinfo.uid = uid + tarinfo.uname = owner + return tarinfo + if not dry_run: tar = tarfile.open(archive_name, mode=mode) # This recursively adds everything underneath base_dir - tar.add(base_dir) - tar.close() + try: + try: + # Support for the `filter' parameter was added in Python 2.7, + # earlier versions will raise TypeError. + tar.add(base_dir, filter=_set_uid_gid) + except TypeError: + tar.add(base_dir) + finally: + tar.close() if compress and compress not in tarfile_compress_flag: spawn([compress] + compress_flags[compress] + [archive_name], diff --git a/sites/shared_conf.py b/sites/shared_conf.py index 4a6a5c4e..99fab315 100644 --- a/sites/shared_conf.py +++ b/sites/shared_conf.py @@ -12,7 +12,6 @@ html_theme_options = { 'description': "A Python implementation of SSHv2.", 'github_user': 'paramiko', 'github_repo': 'paramiko', - 'gratipay_user': 'bitprophet', 'analytics_id': 'UA-18486793-2', 'travis_button': True, } diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index dfdaaf44..b7f35878 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,6 +2,32 @@ Changelog ========= +* :release:`1.16.3 <2016-07-25>` +* :bug:`673 (1.16+)` (via :issue:`681`) Fix protocol banner read errors + (``SSHException``) which would occasionally pop up when using + ``ProxyCommand`` gatewaying. Thanks to ``@Depado`` for the initial report and + Paul Kapp for the fix. +* :bug:`774 (1.16+)` Add a ``_closed`` private attribute to + `~paramiko.channel.Channel` objects so that they continue functioning when + used as proxy sockets under Python 3 (e.g. as ``direct-tcpip`` gateways for + other Paramiko connections.) +* :bug:`758 (1.16+)` Apply type definitions to ``_winapi`` module from + `jaraco.windows <https://github.com/jaraco/jaraco.windows>`_ 3.6.1. This + should address issues on Windows platforms that often result in errors like + ``ArgumentError: [...] int too long to convert``. Thanks to ``@swohlerLL`` + for the report and Jason R. Coombs for the patch. +* :release:`1.16.2 <2016-06-21>` +* :bug:`520 (1.16+)` (Partial fix) Fix at least one instance of race condition + driven threading hangs at end of the Python interpreter session. (Includes a + docs update as well - always make sure to ``.close()`` your clients!) +* :bug:`537 (1.16+)` Fix a bug in `BufferedPipe.set_event + <paramiko.buffered_pipe.BufferedPipe.set_event>` which could cause + deadlocks/hangs when one uses `select.select` against + `~paramiko.channel.Channel` objects (or otherwise calls `Channel.fileno + <paramiko.channel.Channel.fileno>` after the channel has closed). Thanks to + Przemysław Strzelczak for the report & reproduction case, and to Krzysztof + Rusek for the fix. +* :release:`1.16.1 <2016-04-28>` * :release:`1.15.5 <2016-04-28>` * :bug:`670` Due to an earlier bugfix, less-specific ``Host`` blocks' ``ProxyCommand`` values were overriding ``ProxyCommand none`` in @@ -9,12 +35,27 @@ Changelog manner (i.e. ``ProxyCommand none`` continues to appear as a total lack of any ``proxycommand`` key in parsed config structures). Thanks to Pat Brisbin for the catch. +* :bug:`676` (via :issue:`677`) Fix a backwards incompatibility issue that + cropped up in `SFTPFile.prefetch <~paramiko.sftp_file.prefetch>` re: the + erroneously non-optional ``file_size`` parameter. Should only affect users + who manually call ``prefetch``. Thanks to ``@stevevanhooser`` for catch & + patch. * :bug:`577` (via :issue:`578`; should also fix :issue:`718`, :issue:`560`) Fix stalled/hung SFTP downloads by cleaning up some threading lock issues. Thanks to Stephen C. Pope for the patch. +* :bug:`716` Fix a Python 3 compatibility issue when handling two-factor + authentication. Thanks to Mateusz Kowalski for the catch & original patch. * :support:`729 backported (>=1.15,<2.0)` Clean up ``setup.py`` to always use ``setuptools``, not doing so was a historical artifact from bygone days. Thanks to Alex Gaynor. +* :bug:`617` (aka `fabric/fabric#1429 + <https://github.com/fabric/fabric/issues/1429>`_; via :issue:`679`; related: + :issue:`678`, :issue:`685`, :issue:`615` & :issue:`616`) Fix up + `~paramiko.ssh_exception.NoValidConnectionsError` so it pickles correctly, + and fix a related Python 3 compatibility issue. Thanks to Rebecca Schlussel + for the report & Marius Gedminas for the patch. +* :bug:`613` (via :issue:`619`) Update to ``jaraco.windows`` 3.4.1 to fix some + errors related to ``ctypes`` on Windows platforms. Credit to Jason R. Coombs. * :support:`621 backported (>=1.15,<2.0)` Annotate some public attributes on `~paramiko.channel.Channel` such as ``.closed``. Thanks to Sergey Vasilyev for the report. @@ -38,6 +79,19 @@ Changelog exception. Patch courtesy of ``@jamercee``. * :support:`636 backported (>=1.15,<2.0)` Clean up and enhance the README (and rename it to ``README.rst`` from just ``README``). Thanks to ``@LucasRMehl``. +* :release:`1.16.0 <2015-11-04>` +* :bug:`194 major` (also :issue:`562`, :issue:`530`, :issue:`576`) Streamline + use of ``stat`` when downloading SFTP files via `SFTPClient.get + <paramiko.sftp_client.SFTPClient.get>`; this avoids triggering bugs in some + off-spec SFTP servers such as IBM Sterling. Thanks to ``@muraleee`` for the + initial report and to Torkil Gustavsen for the patch. +* :feature:`467` (also :issue:`139`, :issue:`412`) Fully enable two-factor + authentication (e.g. when a server requires ``AuthenticationMethods + pubkey,keyboard-interactive``). Thanks to ``@perryjrandall`` for the patch + and to ``@nevins-b`` and Matt Robenolt for additional support. +* :bug:`502 major` Fix 'exec' requests in server mode to use ``get_string`` + instead of ``get_text`` to avoid ``UnicodeDecodeError`` on non-UTF-8 input. + Thanks to Anselm Kruis for the patch & discussion. * :bug:`401` Fix line number reporting in log output regarding invalid ``known_hosts`` line entries. Thanks to Dylan Thacker-Smith for catch & patch. @@ -58,6 +112,23 @@ Changelog * :bug:`565` Don't explode with ``IndexError`` when reading private key files lacking an ``-----END <type> PRIVATE KEY-----`` footer. Patch courtesy of Prasanna Santhanam. +* :feature:`604` Add support for the ``aes192-ctr`` and ``aes192-cbc`` ciphers. + Thanks to Michiel Tiller for noticing it was as easy as tweaking some key + sizes :D +* :feature:`356` (also :issue:`596`, :issue:`365`, :issue:`341`, :issue:`164`, + :issue:`581`, and a bunch of other duplicates besides) Add support for SHA-2 + based key exchange (kex) algorithm ``diffie-hellman-group-exchange-sha256`` + and (H)MAC algorithms ``hmac-sha2-256`` and ``hmac-sha2-512``. + + This change includes tweaks to debug-level logging regarding + algorithm-selection handshakes; the old all-in-one log line is now multiple + easier-to-read, printed-at-handshake-time log lines. + + Thanks to the many people who submitted patches for this functionality and/or + assisted in testing those patches. That list includes but is not limited to, + and in no particular order: Matthias Witte, Dag Wieers, Ash Berlin, Etienne + Perot, Gert van Dijk, ``@GuyShaanan``, Aaron Bieber, ``@cyphase``, and Eric + Brown. * :release:`1.15.3 <2015-10-02>` * :support:`554 backported` Fix inaccuracies in the docstring for the ECDSA key class. Thanks to Jared Hance for the patch. @@ -85,6 +156,13 @@ Changelog which caused ``OverFlowError`` (and other symptoms) in SFTP functionality. Thanks to ``@dboreham`` for leading the troubleshooting charge, and to Scott Maxwell for the final patch. +* :support:`582` Fix some old ``setup.py`` related helper code which was + breaking ``bdist_dumb`` on Mac OS X. Thanks to Peter Odding for the patch. +* :bug:`22 major` Try harder to connect to multiple network families (e.g. IPv4 + vs IPv6) in case of connection issues; this helps with problems such as hosts + which resolve both IPv4 and IPv6 addresses but are only listening on IPv4. + Thanks to Dries Desmet for original report and Torsten Landschoff for the + foundational patchset. * :bug:`402` Check to see if an SSH agent is actually present before trying to forward it to the remote end. This replaces what was usually a useless ``TypeError`` with a human-readable @@ -113,7 +191,7 @@ Changelog use of the ``shlex`` module. Thanks to Yan Kalchevskiy. * :support:`422 backported` Clean up some unused imports. Courtesy of Olle Lundberg. -* :support:`421 backported` Modernize threading calls to user newer API. Thanks +* :support:`421 backported` Modernize threading calls to use newer API. Thanks to Olle Lundberg. * :support:`419 backported` Modernize a bunch of the codebase internals to leverage decorators. Props to ``@beckjake`` for realizing we're no longer on @@ -246,7 +324,7 @@ Changelog Plugaru. * :bug:`-` Fix logging error in sftp_client for filenames containing the '%' character. Thanks to Antoine Brenner. -* :bug:`308` Fix regression in dsskey.py that caused sporadic signature +* :bug:`308` Fix regression in dsskey.py that caused sporadic signature verification failures. Thanks to Chris Rose. * :support:`299` Use deterministic signatures for ECDSA keys for improved security. Thanks to Alex Gaynor. @@ -269,7 +347,7 @@ Changelog * :feature:`16` **Python 3 support!** Our test suite passes under Python 3, and it (& Fabric's test suite) continues to pass under Python 2. **Python 2.5 is no longer supported with this change!** - + The merged code was built on many contributors' efforts, both code & feedback. In no particular order, we thank Daniel Goertzen, Ivan Kolodyazhny, Tomi Pieviläinen, Jason R. Coombs, Jan N. Schulze, ``@Lazik``, Dorian Pula, diff --git a/sites/www/faq.rst b/sites/www/faq.rst index a5d9b383..74b7501e 100644 --- a/sites/www/faq.rst +++ b/sites/www/faq.rst @@ -24,3 +24,13 @@ However, **closed does not imply locked** - affected users can still post comments on such tickets - and **we will always consider actual patch submissions for these issues**, provided they can get +1s from similarly affected users and are proven to not break existing functionality. + +I'm having strange issues with my code hanging at shutdown! +=========================================================== + +Make sure you explicitly ``.close()`` your connection objects (usually +``SSHClient``) if you're having any sort of hang/freeze at shutdown time! + +Doing so isn't strictly necessary 100% of the time, but it is almost always the +right solution if you run into the various corner cases that cause race +conditions, etc. @@ -43,8 +43,9 @@ from tests.test_kex import KexTest from tests.test_packetizer import PacketizerTest from tests.test_auth import AuthTest from tests.test_transport import TransportTest +from tests.test_ssh_exception import NoValidConnectionsErrorTest from tests.test_client import SSHClientTest -from test_client import SSHClientTest +from test_client import SSHClientTest # XXX why shadow the above import? from test_gssapi import GSSAPITest from test_ssh_gss import GSSAuthTest from test_kex_gss import GSSKexTest @@ -156,6 +157,7 @@ def main(): if options.use_transport: suite.addTest(unittest.makeSuite(AuthTest)) suite.addTest(unittest.makeSuite(TransportTest)) + suite.addTest(unittest.makeSuite(NoValidConnectionsErrorTest)) suite.addTest(unittest.makeSuite(SSHClientTest)) if options.use_sftp: suite.addTest(unittest.makeSuite(SFTPTest)) diff --git a/tests/loop.py b/tests/loop.py index 4f5dc163..e805ad96 100644 --- a/tests/loop.py +++ b/tests/loop.py @@ -37,9 +37,11 @@ class LoopSocket (object): self.__cv = threading.Condition(self.__lock) self.__timeout = None self.__mate = None + self._closed = False def close(self): self.__unlink() + self._closed = True try: self.__lock.acquire() self.__in_buffer = bytes() diff --git a/tests/test_client.py b/tests/test_client.py index e080221e..f71efd5a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -75,7 +75,7 @@ class NullServer (paramiko.ServerInterface): return paramiko.OPEN_SUCCEEDED def check_channel_exec_request(self, channel, command): - if command != 'yes': + if command != b'yes': return False return True diff --git a/tests/test_kex.py b/tests/test_kex.py index 56f1b7c7..19804fbf 100644 --- a/tests/test_kex.py +++ b/tests/test_kex.py @@ -26,7 +26,7 @@ import unittest import paramiko.util from paramiko.kex_group1 import KexGroup1 -from paramiko.kex_gex import KexGex +from paramiko.kex_gex import KexGex, KexGexSHA256 from paramiko import Message from paramiko.common import byte_chr @@ -252,3 +252,121 @@ class KexTest (unittest.TestCase): self.assertEqual(H, hexlify(transport._H).upper()) self.assertEqual(x, hexlify(transport._message.asbytes()).upper()) self.assertTrue(transport._activated) + + def test_7_gex_sha256_client(self): + transport = FakeTransport() + transport.server_mode = False + kex = KexGexSHA256(transport) + kex.start_kex() + x = b'22000004000000080000002000' + self.assertEqual(x, hexlify(transport._message.asbytes()).upper()) + self.assertEqual((paramiko.kex_gex._MSG_KEXDH_GEX_GROUP,), transport._expect) + + msg = Message() + msg.add_mpint(FakeModulusPack.P) + msg.add_mpint(FakeModulusPack.G) + msg.rewind() + kex.parse_next(paramiko.kex_gex._MSG_KEXDH_GEX_GROUP, msg) + x = b'20000000807E2DDB1743F3487D6545F04F1C8476092FB912B013626AB5BCEB764257D88BBA64243B9F348DF7B41B8C814A995E00299913503456983FFB9178D3CD79EB6D55522418A8ABF65375872E55938AB99A84A0B5FC8A1ECC66A7C3766E7E0F80B7CE2C9225FC2DD683F4764244B72963BBB383F529DCF0C5D17740B8A2ADBE9208D4' + self.assertEqual(x, hexlify(transport._message.asbytes()).upper()) + self.assertEqual((paramiko.kex_gex._MSG_KEXDH_GEX_REPLY,), transport._expect) + + msg = Message() + msg.add_string('fake-host-key') + msg.add_mpint(69) + msg.add_string('fake-sig') + msg.rewind() + kex.parse_next(paramiko.kex_gex._MSG_KEXDH_GEX_REPLY, msg) + H = b'AD1A9365A67B4496F05594AD1BF656E3CDA0851289A4C1AFF549FEAE50896DF4' + self.assertEqual(self.K, transport._K) + self.assertEqual(H, hexlify(transport._H).upper()) + self.assertEqual((b'fake-host-key', b'fake-sig'), transport._verify) + self.assertTrue(transport._activated) + + def test_8_gex_sha256_old_client(self): + transport = FakeTransport() + transport.server_mode = False + kex = KexGexSHA256(transport) + kex.start_kex(_test_old_style=True) + x = b'1E00000800' + self.assertEqual(x, hexlify(transport._message.asbytes()).upper()) + self.assertEqual((paramiko.kex_gex._MSG_KEXDH_GEX_GROUP,), transport._expect) + + msg = Message() + msg.add_mpint(FakeModulusPack.P) + msg.add_mpint(FakeModulusPack.G) + msg.rewind() + kex.parse_next(paramiko.kex_gex._MSG_KEXDH_GEX_GROUP, msg) + x = b'20000000807E2DDB1743F3487D6545F04F1C8476092FB912B013626AB5BCEB764257D88BBA64243B9F348DF7B41B8C814A995E00299913503456983FFB9178D3CD79EB6D55522418A8ABF65375872E55938AB99A84A0B5FC8A1ECC66A7C3766E7E0F80B7CE2C9225FC2DD683F4764244B72963BBB383F529DCF0C5D17740B8A2ADBE9208D4' + self.assertEqual(x, hexlify(transport._message.asbytes()).upper()) + self.assertEqual((paramiko.kex_gex._MSG_KEXDH_GEX_REPLY,), transport._expect) + + msg = Message() + msg.add_string('fake-host-key') + msg.add_mpint(69) + msg.add_string('fake-sig') + msg.rewind() + kex.parse_next(paramiko.kex_gex._MSG_KEXDH_GEX_REPLY, msg) + H = b'518386608B15891AE5237DEE08DCADDE76A0BCEFCE7F6DB3AD66BC41D256DFE5' + self.assertEqual(self.K, transport._K) + self.assertEqual(H, hexlify(transport._H).upper()) + self.assertEqual((b'fake-host-key', b'fake-sig'), transport._verify) + self.assertTrue(transport._activated) + + def test_9_gex_sha256_server(self): + transport = FakeTransport() + transport.server_mode = True + kex = KexGexSHA256(transport) + kex.start_kex() + self.assertEqual((paramiko.kex_gex._MSG_KEXDH_GEX_REQUEST, paramiko.kex_gex._MSG_KEXDH_GEX_REQUEST_OLD), transport._expect) + + msg = Message() + msg.add_int(1024) + msg.add_int(2048) + msg.add_int(4096) + msg.rewind() + kex.parse_next(paramiko.kex_gex._MSG_KEXDH_GEX_REQUEST, msg) + x = b'1F0000008100FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381FFFFFFFFFFFFFFFF0000000102' + self.assertEqual(x, hexlify(transport._message.asbytes()).upper()) + self.assertEqual((paramiko.kex_gex._MSG_KEXDH_GEX_INIT,), transport._expect) + + msg = Message() + msg.add_mpint(12345) + msg.rewind() + kex.parse_next(paramiko.kex_gex._MSG_KEXDH_GEX_INIT, msg) + K = 67592995013596137876033460028393339951879041140378510871612128162185209509220726296697886624612526735888348020498716482757677848959420073720160491114319163078862905400020959196386947926388406687288901564192071077389283980347784184487280885335302632305026248574716290537036069329724382811853044654824945750581 + H = b'CCAC0497CF0ABA1DBF55E1A3995D17F4CC31824B0E8D95CDF8A06F169D050D80' + x = b'210000000866616B652D6B6579000000807E2DDB1743F3487D6545F04F1C8476092FB912B013626AB5BCEB764257D88BBA64243B9F348DF7B41B8C814A995E00299913503456983FFB9178D3CD79EB6D55522418A8ABF65375872E55938AB99A84A0B5FC8A1ECC66A7C3766E7E0F80B7CE2C9225FC2DD683F4764244B72963BBB383F529DCF0C5D17740B8A2ADBE9208D40000000866616B652D736967' + self.assertEqual(K, transport._K) + self.assertEqual(H, hexlify(transport._H).upper()) + self.assertEqual(x, hexlify(transport._message.asbytes()).upper()) + self.assertTrue(transport._activated) + + def test_10_gex_sha256_server_with_old_client(self): + transport = FakeTransport() + transport.server_mode = True + kex = KexGexSHA256(transport) + kex.start_kex() + self.assertEqual((paramiko.kex_gex._MSG_KEXDH_GEX_REQUEST, paramiko.kex_gex._MSG_KEXDH_GEX_REQUEST_OLD), transport._expect) + + msg = Message() + msg.add_int(2048) + msg.rewind() + kex.parse_next(paramiko.kex_gex._MSG_KEXDH_GEX_REQUEST_OLD, msg) + x = b'1F0000008100FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381FFFFFFFFFFFFFFFF0000000102' + self.assertEqual(x, hexlify(transport._message.asbytes()).upper()) + self.assertEqual((paramiko.kex_gex._MSG_KEXDH_GEX_INIT,), transport._expect) + + msg = Message() + msg.add_mpint(12345) + msg.rewind() + kex.parse_next(paramiko.kex_gex._MSG_KEXDH_GEX_INIT, msg) + K = 67592995013596137876033460028393339951879041140378510871612128162185209509220726296697886624612526735888348020498716482757677848959420073720160491114319163078862905400020959196386947926388406687288901564192071077389283980347784184487280885335302632305026248574716290537036069329724382811853044654824945750581 + H = b'3DDD2AD840AD095E397BA4D0573972DC60F6461FD38A187CACA6615A5BC8ADBB' + x = b'210000000866616B652D6B6579000000807E2DDB1743F3487D6545F04F1C8476092FB912B013626AB5BCEB764257D88BBA64243B9F348DF7B41B8C814A995E00299913503456983FFB9178D3CD79EB6D55522418A8ABF65375872E55938AB99A84A0B5FC8A1ECC66A7C3766E7E0F80B7CE2C9225FC2DD683F4764244B72963BBB383F529DCF0C5D17740B8A2ADBE9208D40000000866616B652D736967' + self.assertEqual(K, transport._K) + self.assertEqual(H, hexlify(transport._H).upper()) + self.assertEqual(x, hexlify(transport._message.asbytes()).upper()) + self.assertTrue(transport._activated) + + diff --git a/tests/test_sftp.py b/tests/test_sftp.py index 6bce1794..ff146ade 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -696,7 +696,8 @@ class SFTPTest (unittest.TestCase): f.readv([(0, 12)]) with sftp.open(FOLDER + '/zero', 'r') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) f.read(100) finally: sftp.unlink(FOLDER + '/zero') diff --git a/tests/test_sftp_big.py b/tests/test_sftp_big.py index abed27b8..cfad5682 100644 --- a/tests/test_sftp_big.py +++ b/tests/test_sftp_big.py @@ -132,7 +132,8 @@ class BigSFTPTest (unittest.TestCase): start = time.time() with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) # read on odd boundaries to make sure the bytes aren't getting scrambled n = 0 @@ -171,7 +172,8 @@ class BigSFTPTest (unittest.TestCase): chunk = 793 for i in range(10): with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) base_offset = (512 * 1024) + 17 * random.randint(1000, 2000) offsets = [base_offset + j * chunk for j in range(100)] # randomly seek around and read them out @@ -245,9 +247,11 @@ class BigSFTPTest (unittest.TestCase): for i in range(10): with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) for n in range(1024): data = f.read(1024) self.assertEqual(data, kblob) @@ -275,7 +279,8 @@ class BigSFTPTest (unittest.TestCase): self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024) with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) data = f.read(1024) self.assertEqual(data, kblob) @@ -353,7 +358,8 @@ class BigSFTPTest (unittest.TestCase): # try to read it too. with sftp.open('%s/hongry.txt' % FOLDER, 'r', 128 * 1024) as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) total = 0 while total < 1024 * 1024: total += len(f.read(32 * 1024)) diff --git a/tests/test_ssh_exception.py b/tests/test_ssh_exception.py new file mode 100644 index 00000000..18f2a97d --- /dev/null +++ b/tests/test_ssh_exception.py @@ -0,0 +1,31 @@ +import pickle +import unittest + +from paramiko.ssh_exception import NoValidConnectionsError + + +class NoValidConnectionsErrorTest (unittest.TestCase): + + def test_pickling(self): + # Regression test for https://github.com/paramiko/paramiko/issues/617 + exc = NoValidConnectionsError({('127.0.0.1', '22'): Exception()}) + new_exc = pickle.loads(pickle.dumps(exc)) + self.assertEqual(type(exc), type(new_exc)) + self.assertEqual(str(exc), str(new_exc)) + self.assertEqual(exc.args, new_exc.args) + + def test_error_message_for_single_host(self): + exc = NoValidConnectionsError({('127.0.0.1', '22'): Exception()}) + assert "Unable to connect to port 22 on 127.0.0.1" in str(exc) + + def test_error_message_for_two_hosts(self): + exc = NoValidConnectionsError({('127.0.0.1', '22'): Exception(), + ('::1', '22'): Exception()}) + assert "Unable to connect to port 22 on 127.0.0.1 or ::1" in str(exc) + + def test_error_message_for_multiple_hosts(self): + exc = NoValidConnectionsError({('127.0.0.1', '22'): Exception(), + ('::1', '22'): Exception(), + ('10.0.0.42', '22'): Exception()}) + exp = "Unable to connect to port 22 on 10.0.0.42, 127.0.0.1 or ::1" + assert exp in str(exc) diff --git a/tests/test_transport.py b/tests/test_transport.py index bf4bac5c..d81ad8f3 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -28,6 +28,7 @@ import socket import time import threading import random +from hashlib import sha1 import unittest from paramiko import Transport, SecurityOptions, ServerInterface, RSAKey, DSSKey, \ @@ -77,7 +78,7 @@ class NullServer (ServerInterface): return OPEN_SUCCEEDED def check_channel_exec_request(self, channel, command): - if command != 'yes': + if command != b'yes': return False return True @@ -251,7 +252,7 @@ class TransportTest(unittest.TestCase): chan = self.tc.open_session() schan = self.ts.accept(1.0) try: - chan.exec_command('no') + chan.exec_command(b'command contains \xfc and is not a valid UTF-8 string') self.assertTrue(False) except SSHException: pass @@ -447,9 +448,11 @@ class TransportTest(unittest.TestCase): bytes = self.tc.packetizer._Packetizer__sent_bytes chan.send('x' * 1024) bytes2 = self.tc.packetizer._Packetizer__sent_bytes + block_size = self.tc._cipher_info[self.tc.local_cipher]['block-size'] + mac_size = self.tc._mac_info[self.tc.local_mac]['size'] # tests show this is actually compressed to *52 bytes*! including packet overhead! nice!! :) self.assertTrue(bytes2 - bytes < 1024) - self.assertEqual(52, bytes2 - bytes) + self.assertEqual(16 + block_size + mac_size, bytes2 - bytes) chan.close() schan.close() @@ -825,3 +828,21 @@ class TransportTest(unittest.TestCase): hostkey=public_host_key, username='slowdive', password='pygmalion') + + def test_M_select_after_close(self): + """ + verify that select works when a channel is already closed. + """ + self.setup_test_server() + chan = self.tc.open_session() + chan.invoke_shell() + schan = self.ts.accept(1.0) + schan.close() + + # give client a moment to receive close notification + time.sleep(0.1) + + r, w, e = select.select([chan], [], [], 0.1) + self.assertEqual([chan], r) + self.assertEqual([], w) + self.assertEqual([], e) |