Merge branch 'master' into switch-to-cryptography

12 files changed, 187 insertions, 65 deletions

diff --git a/README b/README
index 7177ce80..7e848aca 100644
--- a/README
+++ b/README
@@ -5,7 +5,7 @@ paramiko
 
 :Paramiko: Python SSH module
 :Copyright: Copyright (c) 2003-2009  Robey Pointer <robeypointer@gmail.com>
-:Copyright: Copyright (c) 2013-2014  Jeff Forcier <jeff@bitprophet.org>
+:Copyright: Copyright (c) 2013-2015  Jeff Forcier <jeff@bitprophet.org>
 :License: LGPL
 :Homepage: https://github.com/paramiko/paramiko/
 :API docs: http://docs.paramiko.org
diff --git a/paramiko/_version.py b/paramiko/_version.py
index d9f78740..e82b8667 100644
--- a/paramiko/_version.py
+++ b/paramiko/_version.py
@@ -1,2 +1,2 @@
-__version_info__ = (1, 15, 1)
+__version_info__ = (1, 16, 0)
 __version__ = '.'.join(map(str, __version_info__))
diff --git a/paramiko/_winapi.py b/paramiko/_winapi.py
index f48e1890..d6aabf76 100644
--- a/paramiko/_winapi.py
+++ b/paramiko/_winapi.py
@@ -106,7 +106,7 @@ MapViewOfFile.restype = ctypes.wintypes.HANDLE
 
 class MemoryMap(object):
     """
-    A memory map object which can have security attributes overrideden.
+    A memory map object which can have security attributes overridden.
     """
     def __init__(self, name, length, security_attributes=None):
         self.name = name
diff --git a/paramiko/agent.py b/paramiko/agent.py
index a75ac59e..f928881e 100644
--- a/paramiko/agent.py
+++ b/paramiko/agent.py
@@ -32,7 +32,7 @@ from select import select
 from paramiko.common import asbytes, io_sleep
 from paramiko.py3compat import byte_chr
 
-from paramiko.ssh_exception import SSHException
+from paramiko.ssh_exception import SSHException, AuthenticationException
 from paramiko.message import Message
 from paramiko.pkey import PKey
 from paramiko.util import retry_on_signal
@@ -109,9 +109,12 @@ class AgentProxyThread(threading.Thread):
     def run(self):
         try:
             (r, addr) = self.get_connection()
+            # Found that r should be either a socket from the socket library or None
             self.__inr = r
-            self.__addr = addr
+            self.__addr = addr # This should be an IP address as a string? or None
             self._agent.connect()
+            if not isinstance(self._agent, int) and (self._agent._conn is None or not hasattr(self._agent._conn, 'fileno')):
+                raise AuthenticationException("Unable to connect to SSH agent")
             self._communicate()
         except:
             #XXX Not sure what to do here ... raise or pass ?
diff --git a/paramiko/channel.py b/paramiko/channel.py
index 8a97c974..7e39a15b 100644
--- a/paramiko/channel.py
+++ b/paramiko/channel.py
@@ -337,7 +337,7 @@ class Channel (ClosingContextManager):
         further x11 requests can be made from the server to the client,
         when an x11 application is run in a shell session.
 
-        From RFC4254::
+        From :rfc:`4254`::
 
             It is RECOMMENDED that the 'x11 authentication cookie' that is
             sent be a fake, random cookie, and that the cookie be checked and
diff --git a/paramiko/client.py b/paramiko/client.py
index 393e3e09..e10e9da7 100644
--- a/paramiko/client.py
+++ b/paramiko/client.py
@@ -25,6 +25,7 @@ import getpass
 import os
 import socket
 import warnings
+from errno import ECONNREFUSED, EHOSTUNREACH
 
 from paramiko.agent import Agent
 from paramiko.common import DEBUG
@@ -35,7 +36,9 @@ from paramiko.hostkeys import HostKeys
 from paramiko.py3compat import string_types
 from paramiko.resource import ResourceManager
 from paramiko.rsakey import RSAKey
-from paramiko.ssh_exception import SSHException, BadHostKeyException
+from paramiko.ssh_exception import (
+    SSHException, BadHostKeyException, NoValidConnectionsError
+)
 from paramiko.transport import Transport
 from paramiko.util import retry_on_signal, ClosingContextManager
 
@@ -172,10 +175,46 @@ class SSHClient (ClosingContextManager):
         """
         self._policy = policy
 
-    def connect(self, hostname, port=SSH_PORT, username=None, password=None, pkey=None,
-                key_filename=None, timeout=None, allow_agent=True, look_for_keys=True,
-                compress=False, sock=None, gss_auth=False, gss_kex=False,
-                gss_deleg_creds=True, gss_host=None, banner_timeout=None):
+    def _families_and_addresses(self, hostname, port):
+        """
+        Yield pairs of address families and addresses to try for connecting.
+
+        :param str hostname: the server to connect to
+        :param int port: the server port to connect to
+        :returns: Yields an iterable of ``(family, address)`` tuples
+        """
+        guess = True
+        addrinfos = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
+        for (family, socktype, proto, canonname, sockaddr) in addrinfos:
+            if socktype == socket.SOCK_STREAM:
+                yield family, sockaddr
+                guess = False
+
+        # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :(
+        # We only do this if we did not get a single result marked as socktype == SOCK_STREAM.
+        if guess:
+            for family, _, _, _, sockaddr in addrinfos:
+                yield family, sockaddr
+
+    def connect(
+        self,
+        hostname,
+        port=SSH_PORT,
+        username=None,
+        password=None,
+        pkey=None,
+        key_filename=None,
+        timeout=None,
+        allow_agent=True,
+        look_for_keys=True,
+        compress=False,
+        sock=None,
+        gss_auth=False,
+        gss_kex=False,
+        gss_deleg_creds=True,
+        gss_host=None,
+        banner_timeout=None
+    ):
         """
         Connect to an SSH server and authenticate to it.  The server's host key
         is checked against the system host keys (see `load_system_host_keys`)
@@ -206,8 +245,10 @@ class SSHClient (ClosingContextManager):
         :param str key_filename:
             the filename, or list of filenames, of optional private key(s) to
             try for authentication
-        :param float timeout: an optional timeout (in seconds) for the TCP connect
-        :param bool allow_agent: set to False to disable connecting to the SSH agent
+        :param float timeout:
+            an optional timeout (in seconds) for the TCP connect
+        :param bool allow_agent:
+            set to False to disable connecting to the SSH agent
         :param bool look_for_keys:
             set to False to disable searching for discoverable private key
             files in ``~/.ssh/``
@@ -216,9 +257,11 @@ class SSHClient (ClosingContextManager):
             an open socket or socket-like object (such as a `.Channel`) to use
             for communication to the target host
         :param bool gss_auth: ``True`` if you want to use GSS-API authentication
-        :param bool gss_kex: Perform GSS-API Key Exchange and user authentication
+        :param bool gss_kex:
+            Perform GSS-API Key Exchange and user authentication
         :param bool gss_deleg_creds: Delegate GSS-API client credentials or not
-        :param str gss_host: The targets name in the kerberos database. default: hostname
+        :param str gss_host:
+            The targets name in the kerberos database. default: hostname
         :param float banner_timeout: an optional timeout (in seconds) to wait
             for the SSH banner to be presented.
 
@@ -234,21 +277,37 @@ class SSHClient (ClosingContextManager):
             ``gss_deleg_creds`` and ``gss_host`` arguments.
         """
         if not sock:
-            for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM):
-                if socktype == socket.SOCK_STREAM:
-                    af = family
-                    addr = sockaddr
-                    break
-            else:
-                # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :(
-                af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
-            sock = socket.socket(af, socket.SOCK_STREAM)
-            if timeout is not None:
+            errors = {}
+            # Try multiple possible address families (e.g. IPv4 vs IPv6)
+            to_try = list(self._families_and_addresses(hostname, port))
+            for af, addr in to_try:
                 try:
-                    sock.settimeout(timeout)
-                except:
-                    pass
-            retry_on_signal(lambda: sock.connect(addr))
+                    sock = socket.socket(af, socket.SOCK_STREAM)
+                    if timeout is not None:
+                        try:
+                            sock.settimeout(timeout)
+                        except:
+                            pass
+                    retry_on_signal(lambda: sock.connect(addr))
+                    # Break out of the loop on success
+                    break
+                except socket.error as e:
+                    # Raise anything that isn't a straight up connection error
+                    # (such as a resolution error)
+                    if e.errno not in (ECONNREFUSED, EHOSTUNREACH):
+                        raise
+                    # Capture anything else so we know how the run looks once
+                    # iteration is complete. Retain info about which attempt
+                    # this was.
+                    errors[addr] = e
+
+            # Make sure we explode usefully if no address family attempts
+            # succeeded. We've no way of knowing which error is the "right"
+            # one, so we construct a hybrid exception containing all the real
+            # ones, of a subclass that client code should still be watching for
+            # (socket.error)
+            if len(errors) == len(to_try):
+                raise NoValidConnectionsError(errors)
 
         t = self._transport = Transport(sock, gss_kex=gss_kex, gss_deleg_creds=gss_deleg_creds)
         t.use_compression(compress=compress)
diff --git a/paramiko/kex_gss.py b/paramiko/kex_gss.py
index 4e8380ef..d026807c 100644
--- a/paramiko/kex_gss.py
+++ b/paramiko/kex_gss.py
@@ -21,14 +21,15 @@
 
 
 """
-This module provides GSS-API / SSPI Key Exchange as defined in RFC 4462.
+This module provides GSS-API / SSPI Key Exchange as defined in :rfc:`4462`.
 
 .. note:: Credential delegation is not supported in server mode.
 
 .. note::
-    `RFC 4462 Section 2.2 <http://www.ietf.org/rfc/rfc4462.txt>`_ says we are
-    not required to implement GSS-API error messages. Thus, in many methods
-    within this module, if an error occurs an exception will be thrown and the
+    `RFC 4462 Section 2.2
+    <https://tools.ietf.org/html/rfc4462.html#section-2.2>`_ says we are not
+    required to implement GSS-API error messages. Thus, in many methods within
+    this module, if an error occurs an exception will be thrown and the
     connection will be terminated.
 
 .. seealso:: :doc:`/api/ssh_gss`
@@ -55,8 +56,8 @@ c_MSG_KEXGSS_GROUPREQ, c_MSG_KEXGSS_GROUP = [byte_chr(c) for c in range(40, 42)]
 
 class KexGSSGroup1(object):
     """
-    GSS-API / SSPI Authenticated Diffie-Hellman Key Exchange
-    as defined in `RFC 4462 Section 2 <http://www.ietf.org/rfc/rfc4462.txt>`_
+    GSS-API / SSPI Authenticated Diffie-Hellman Key Exchange as defined in `RFC
+    4462 Section 2 <https://tools.ietf.org/html/rfc4462.html#section-2>`_
     """
     # draft-ietf-secsh-transport-09.txt, page 17
     P = 0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381FFFFFFFFFFFFFFFF
@@ -278,8 +279,9 @@ class KexGSSGroup1(object):
 
 class KexGSSGroup14(KexGSSGroup1):
     """
-    GSS-API / SSPI Authenticated Diffie-Hellman Group14 Key Exchange
-    as defined in `RFC 4462 Section 2 <http://www.ietf.org/rfc/rfc4462.txt>`_
+    GSS-API / SSPI Authenticated Diffie-Hellman Group14 Key Exchange as defined
+    in `RFC 4462 Section 2
+    <https://tools.ietf.org/html/rfc4462.html#section-2>`_
     """
     P = 0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AACAA68FFFFFFFFFFFFFFFF
     G = 2
@@ -288,8 +290,8 @@ class KexGSSGroup14(KexGSSGroup1):
 
 class KexGSSGex(object):
     """
-    GSS-API / SSPI Authenticated Diffie-Hellman Group Exchange
-    as defined in `RFC 4462 Section 2 <http://www.ietf.org/rfc/rfc4462.txt>`_
+    GSS-API / SSPI Authenticated Diffie-Hellman Group Exchange as defined in
+    `RFC 4462 Section 2 <https://tools.ietf.org/html/rfc4462.html#section-2>`_
     """
     NAME = "gss-gex-sha1-toWM5Slw5Ew8Mqkay+al2g=="
     min_bits = 1024
@@ -590,8 +592,9 @@ class KexGSSGex(object):
 
 class NullHostKey(object):
     """
-    This class represents the Null Host Key for GSS-API Key Exchange
-    as defined in `RFC 4462 Section 5 <http://www.ietf.org/rfc/rfc4462.txt>`_
+    This class represents the Null Host Key for GSS-API Key Exchange as defined
+    in `RFC 4462 Section 5
+    <https://tools.ietf.org/html/rfc4462.html#section-5>`_
     """
     def __init__(self):
         self.key = ""
diff --git a/paramiko/proxy.py b/paramiko/proxy.py
index 0664ac6e..ca602c4c 100644
--- a/paramiko/proxy.py
+++ b/paramiko/proxy.py
@@ -24,6 +24,7 @@ import signal
 from subprocess import Popen, PIPE
 from select import select
 import socket
+import time
 
 from paramiko.ssh_exception import ProxyCommandFailure
 from paramiko.util import ClosingContextManager
@@ -79,20 +80,24 @@ class ProxyCommand(ClosingContextManager):
         :return: the length of the read content, as an `int`
         """
         try:
-            start = datetime.now()
+            start = time.time()
             while len(self.buffer) < size:
+                select_timeout = None
                 if self.timeout is not None:
-                    elapsed = (datetime.now() - start).microseconds
-                    timeout = self.timeout * 1000 * 1000  # to microseconds
-                    if elapsed >= timeout:
+                    elapsed = (time.time() - start)
+                    if elapsed >= self.timeout:
                         raise socket.timeout()
-                r, w, x = select([self.process.stdout], [], [], 0.0)
+                    select_timeout = self.timeout - elapsed
+
+                r, w, x = select(
+                    [self.process.stdout], [], [], select_timeout)
                 if r and r[0] == self.process.stdout:
-                    b = os.read(self.process.stdout.fileno(), 1)
+                    b = os.read(
+                        self.process.stdout.fileno(), size - len(self.buffer))
                     # Store in class-level buffer for persistence across
                     # timeouts; this makes us act more like a real socket
                     # (where timeouts don't actually drop data.)
-                    self.buffer.append(b)
+                    self.buffer.extend(b)
             result = ''.join(self.buffer)
             self.buffer = []
             return result
diff --git a/paramiko/ssh_exception.py b/paramiko/ssh_exception.py
index b99e42b3..2e09c6d6 100644
--- a/paramiko/ssh_exception.py
+++ b/paramiko/ssh_exception.py
@@ -16,6 +16,8 @@
 # along with Paramiko; if not, write to the Free Software Foundation, Inc.,
 # 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
 
+import socket
+
 
 class SSHException (Exception):
     """
@@ -129,3 +131,39 @@ class ProxyCommandFailure (SSHException):
         self.error = error
         # for unpickling
         self.args = (command, error, )
+
+
+class NoValidConnectionsError(socket.error):
+    """
+    Multiple connection attempts were made and no families succeeded.
+
+    This exception class wraps multiple "real" underlying connection errors,
+    all of which represent failed connection attempts. Because these errors are
+    not guaranteed to all be of the same error type (i.e. different errno,
+    class, message, etc) we expose a single unified error message and a
+    ``None`` errno so that instances of this class match most normal handling
+    of `socket.error` objects.
+    
+    To see the wrapped exception objects, access the ``errors`` attribute.
+    ``errors`` is a dict whose keys are address tuples (e.g. ``('127.0.0.1',
+    22)``) and whose values are the exception encountered trying to connect to
+    that address.
+
+    It is implied/assumed that all the errors given to a single instance of
+    this class are from connecting to the same hostname + port (and thus that
+    the differences are in the resolution of the hostname - e.g. IPv4 vs v6).
+    """
+    def __init__(self, errors):
+        """
+        :param dict errors:
+            The errors dict to store, as described by class docstring.
+        """
+        addrs = errors.keys()
+        body = ', '.join([x[0] for x in addrs[:-1]])
+        tail = addrs[-1][0]
+        msg = "Unable to connect to port {0} on {1} or {2}"
+        super(NoValidConnectionsError, self).__init__(
+            None, # stand-in for errno
+            msg.format(addrs[0][1], body, tail)
+        )
+        self.errors = errors
diff --git a/paramiko/ssh_gss.py b/paramiko/ssh_gss.py
index dfc9ddc8..17b15590 100644
--- a/paramiko/ssh_gss.py
+++ b/paramiko/ssh_gss.py
@@ -20,7 +20,7 @@
 
 
 """
-This module provides GSS-API / SSPI  authentication as defined in RFC 4462.
+This module provides GSS-API / SSPI  authentication as defined in :rfc:`4462`.
 
 .. note:: Credential delegation is not supported in server mode.
 
diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst
index 4e56ad1f..0e8f92c4 100644
--- a/sites/www/changelog.rst
+++ b/sites/www/changelog.rst
@@ -2,6 +2,23 @@
 Changelog
 =========
 
+* :bug:`22 major` Try harder to connect to multiple network families (e.g. IPv4
+  vs IPv6) in case of connection issues; this helps with problems such as hosts
+  which resolve both IPv4 and IPv6 addresses but are only listening on IPv4.
+  Thanks to Dries Desmet for original report and Torsten Landschoff for the
+  foundational patchset.
+* :bug:`402` Check to see if an SSH agent is actually present before trying to
+  forward it to the remote end. This replaces what was usually a useless
+  ``TypeError`` with a human-readable ``AuthenticationError``. Credit to Ken
+  Jordan for the fix and Yvan Marques for original report.
+* :release:`1.15.2 <2014-12-19>`
+* :release:`1.14.2 <2014-12-19>`
+* :release:`1.13.3 <2014-12-19>`
+* :bug:`413` (also :issue:`414`, :issue:`420`, :issue:`454`) Be significantly
+  smarter about polling & timing behavior when running proxy commands, to avoid
+  unnecessary (often 100%!) CPU usage. Major thanks to Jason Dunsmore for
+  report & initial patchset and to Chris Adams & John Morrissey for followup
+  improvements.
 * :bug:`455` Tweak packet size handling to conform better to the OpenSSH RFCs;
   this helps address issues with interactive program cursors. Courtesy of Jeff
   Quast.
@@ -13,14 +30,15 @@ Changelog
   none`` as the lack of a proxy command, instead of as a literal command string
   of ``"none"``. Thanks to Richard Spiers for the catch & Sean Johnson for the
   fix.
-* :support:`431` Replace handrolled ``ssh_config`` parsing code with use of the
-  ``shlex`` module. Thanks to Yan Kalchevskiy.
-* :support:`422` Clean up some unused imports. Courtesy of Olle Lundberg.
-* :support:`421` Modernize threading calls to user newer API. Thanks to Olle
+* :support:`431 backported` Replace handrolled ``ssh_config`` parsing code with
+  use of the ``shlex`` module. Thanks to Yan Kalchevskiy.
+* :support:`422 backported` Clean up some unused imports. Courtesy of Olle
   Lundberg.
-* :support:`419` Modernize a bunch of the codebase internals to leverage
-  decorators. Props to ``@beckjake`` for realizing we're no longer on Python
-  2.2 :D
+* :support:`421 backported` Modernize threading calls to user newer API. Thanks
+  to Olle Lundberg.
+* :support:`419 backported` Modernize a bunch of the codebase internals to
+  leverage decorators. Props to ``@beckjake`` for realizing we're no longer on
+  Python 2.2 :D
 * :bug:`266` Change numbering of `~paramiko.transport.Transport` channels to
   start at 0 instead of 1 for better compatibility with OpenSSH & certain
   server implementations which break on 1-indexed channels. Thanks to
@@ -33,7 +51,7 @@ Changelog
   packets encountered type errors. This is now fixed. Thanks to ``@mjmaenpaa``
   for the catch.
 * :bug:`320` Update our win_pageant module to be Python 3 compatible. Thanks to
-``@sherbang`` and ``@adamkerz`` for the patches.
+  ``@sherbang`` and ``@adamkerz`` for the patches.
 * :release:`1.15.1 <2014-09-22>`
 * :bug:`399` SSH agent forwarding (potentially other functionality as
   well) would hang due to incorrect values passed into the new window size
@@ -94,8 +112,8 @@ Changelog
   async/generator based file listings. Thanks to John Begeman.
 * :support:`378 backported` Minor code cleanup in the SSH config module
   courtesy of Olle Lundberg.
-* :support:`249` Consolidate version information into one spot. Thanks to Gabi
-  Davar for the reminder.
+* :support:`249 backported` Consolidate version information into one spot.
+  Thanks to Gabi Davar for the reminder.
 * :release:`1.14.1 <2014-08-25>`
 * :release:`1.13.2 <2014-08-25>`
 * :bug:`376` Be less aggressive about expanding variables in ``ssh_config``
diff --git a/sites/www/index.rst b/sites/www/index.rst
index fd452fef..340c984b 100644
--- a/sites/www/index.rst
+++ b/sites/www/index.rst
@@ -27,11 +27,7 @@ Please see the sidebar to the left to begin.
 .. rubric:: Footnotes
 
 .. [#]
-    SSH is defined in RFCs
-    `4251 <http://www.rfc-editor.org/rfc/rfc4251.txt>`_,
-    `4252 <http://www.rfc-editor.org/rfc/rfc4252.txt>`_,
-    `4253 <http://www.rfc-editor.org/rfc/rfc4253.txt>`_, and 
-    `4254 <http://www.rfc-editor.org/rfc/rfc4254.txt>`_;
-    the primary working implementation of the protocol is the `OpenSSH project
+    SSH is defined in :rfc:`4251`, :rfc:`4252`, :rfc:`4253` and :rfc:`4254`. The
+    primary working implementation of the protocol is the `OpenSSH project
     <http://openssh.org>`_.  Paramiko implements a large portion of the SSH
     feature set, but there are occasional gaps.