Merge branch '1.11' into 156-int

17 files changed, 153 insertions, 50 deletions

diff --git a/NEWS b/NEWS
index 2bb5341f..e3deb149 100644
--- a/NEWS
+++ b/NEWS
@@ -12,8 +12,37 @@ Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/.
 Releases
 ========
 
-v1.11.0 (DD MM YYYY)
---------------------
+v1.10.4 (27th Sep 2013)
+-----------------------
+
+* #179: Fix a missing variable causing errors when an ssh_config file has a
+  non-default AddressFamily set. Thanks to Ed Marshall & Tomaz Muraus for catch
+  & patch.
+
+v1.11.1 (20th Sep 2013)
+-----------------------
+
+* #162: Clean up HMAC module import to avoid deadlocks in certain uses of
+  SSHClient. Thanks to Gernot Hillier for the catch & suggested
+  fix.
+* #36: Fix the port-forwarding demo to avoid file descriptor errors. Thanks to
+  Jonathan Halcrow for catch & patch.
+* #168: Update config handling to properly handle multiple 'localforward' and
+  'remoteforward' keys. Thanks to Emre Yılmaz for the patch.
+
+v1.10.3 (20th Sep 2013)
+-----------------------
+
+* #162: Clean up HMAC module import to avoid deadlocks in certain uses of
+  SSHClient. Thanks to Gernot Hillier for the catch & suggested
+  fix.
+* #36: Fix the port-forwarding demo to avoid file descriptor errors. Thanks to
+  Jonathan Halcrow for catch & patch.
+* #168: Update config handling to properly handle multiple 'localforward' and
+  'remoteforward' keys. Thanks to Emre Yılmaz for the patch.
+
+v1.11.0 (26th Jul 2013)
+-----------------------
 
 * #98: On Windows, when interacting with the PuTTY PAgeant, Paramiko now
   creates the shared memory map with explicit Security Attributes of the user,
@@ -24,6 +53,18 @@ v1.11.0 (DD MM YYYY)
   dependent on ctypes for constructing appropriate structures and had ctypes
   implementations of all functionality. Thanks to Jason R. Coombs for the
   patch.
+* #87: Ensure updates to `known_hosts` files account for any updates to said
+  files after Paramiko initially read them. (Includes related fix to guard
+  against duplicate entries during subsequent `known_hosts` loads.) Thanks to
+  `@sunweaver` for the contribution.
+
+v1.10.2 (26th Jul 2013)
+-----------------------
+
+* #153, #67: Warn on parse failure when reading known_hosts file. Thanks to
+  `@glasserc` for patch.
+* #146: Indentation fixes for readability. Thanks to Abhinav Upadhyay for catch
+  & patch.
 
 v1.10.1 (5th Apr 2013)
 ----------------------
diff --git a/README b/README
index 68e74347..1899819a 100644
--- a/README
+++ b/README
@@ -8,6 +8,7 @@ paramiko
 :Copyright: Copyright (c) 2013  Jeff Forcier <jeff@bitprophet.org>
 :License: LGPL
 :Homepage: https://github.com/paramiko/paramiko/
+:API docs: http://docs.paramiko.org
 
 
 What
diff --git a/demos/demo.py b/demos/demo.py
index 05524d3c..c21a926e 100755
--- a/demos/demo.py
+++ b/demos/demo.py
@@ -26,7 +26,6 @@ import os
 import select
 import socket
 import sys
-import threading
 import time
 import traceback
 
diff --git a/demos/forward.py b/demos/forward.py
index 4e107855..2a4c4248 100644
--- a/demos/forward.py
+++ b/demos/forward.py
@@ -78,9 +78,11 @@ class Handler (SocketServer.BaseRequestHandler):
                 if len(data) == 0:
                     break
                 self.request.send(data)
+                
+        peername = self.request.getpeername()
         chan.close()
         self.request.close()
-        verbose('Tunnel closed from %r' % (self.request.getpeername(),))
+        verbose('Tunnel closed from %r' % (peername,))
 
 
 def forward_tunnel(local_port, remote_host, remote_port, transport):
diff --git a/dev-requirements.txt b/dev-requirements.txt
new file mode 100644
index 00000000..f706c46f
--- /dev/null
+++ b/dev-requirements.txt
@@ -0,0 +1,2 @@
+tox>=1.4,<1.5
+epydoc>=3.0,<3.1
diff --git a/fabfile.py b/fabfile.py
index 29394f94..7883daba 100644
--- a/fabfile.py
+++ b/fabfile.py
@@ -1,8 +1,10 @@
-from fabric.api import task, sudo, env
+from fabric.api import task, sudo, env, local, hosts
 from fabric.contrib.project import rsync_project
+from fabric.contrib.console import confirm
 
 
 @task
+@hosts("paramiko.org")
 def upload_docs():
     target = "/var/www/paramiko.org"
     staging = "/tmp/paramiko_docs"
@@ -11,3 +13,27 @@ def upload_docs():
     sudo("rm -rf %s/*" % target)
     rsync_project(local_dir='docs/', remote_dir=staging, delete=True)
     sudo("cp -R %s/* %s/" % (staging, target))
+
+@task
+def build_docs():
+    local("epydoc --no-private -o docs/ paramiko")
+
+@task
+def clean():
+    local("rm -rf build dist docs")
+    local("rm -f MANIFEST *.log demos/*.log")
+    local("rm -f paramiko/*.pyc")
+    local("rm -f test.log")
+    local("rm -rf paramiko.egg-info")
+
+@task
+def test():
+    local("python ./test.py")
+
+@task
+def release():
+    confirm("Only hit Enter if you remembered to update the version!")
+    confirm("Also, did you remember to tag your release?")
+    build_docs()
+    local("python setup.py sdist register upload")
+    upload_docs()
diff --git a/paramiko/__init__.py b/paramiko/__init__.py
index 099314ea..4ba42a26 100644
--- a/paramiko/__init__.py
+++ b/paramiko/__init__.py
@@ -46,6 +46,8 @@ Paramiko is written entirely in python (no C or platform-dependent code) and is
 released under the GNU Lesser General Public License (LGPL).
 
 Website: U{https://github.com/paramiko/paramiko/}
+
+Mailing list: U{paramiko@librelist.com<mailto:paramiko@librelist.com>}
 """
 
 import sys
@@ -55,7 +57,7 @@ if sys.version_info < (2, 5):
 
 
 __author__ = "Jeff Forcier <jeff@bitprophet.org>"
-__version__ = "1.10.1"
+__version__ = "1.11.1"
 __license__ = "GNU Lesser General Public License (LGPL)"
 
 
diff --git a/paramiko/agent.py b/paramiko/agent.py
index 1dd30636..d4ff7036 100644
--- a/paramiko/agent.py
+++ b/paramiko/agent.py
@@ -255,11 +255,11 @@ class AgentServerProxy(AgentSSH):
         self.close()
 
     def connect(self):
-         conn_sock = self.__t.open_forward_agent_channel()
-         if conn_sock is None:
-             raise SSHException('lost ssh-agent')
-         conn_sock.set_name('auth-agent')
-         self._connect(conn_sock)
+        conn_sock = self.__t.open_forward_agent_channel()
+        if conn_sock is None:
+            raise SSHException('lost ssh-agent')
+        conn_sock.set_name('auth-agent')
+        self._connect(conn_sock)
 
     def close(self):
         """
diff --git a/paramiko/client.py b/paramiko/client.py
index 5b719581..493d5481 100644
--- a/paramiko/client.py
+++ b/paramiko/client.py
@@ -186,8 +186,13 @@ class SSHClient (object):
 
         @raise IOError: if the file could not be written
         """
+
+        # update local host keys from file (in case other SSH clients
+        # have written to the known_hosts file meanwhile.
+        if self.known_hosts is not None:
+            self.load_host_keys(self.known_hosts)
+
         f = open(filename, 'w')
-        f.write('# SSH host keys collected by paramiko\n')
         for hostname, keys in self._host_keys.iteritems():
             for keytype, key in keys.iteritems():
                 f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
diff --git a/paramiko/config.py b/paramiko/config.py
index e41bae43..520da356 100644
--- a/paramiko/config.py
+++ b/paramiko/config.py
@@ -35,9 +35,10 @@ class LazyFqdn(object):
     Returns the host's fqdn on request as string.
     """
 
-    def __init__(self, config):
+    def __init__(self, config, host=None):
         self.fqdn = None
         self.config = config
+        self.host = host
 
     def __str__(self):
         if self.fqdn is None:
@@ -54,19 +55,27 @@ class LazyFqdn(object):
             fqdn = None
             address_family = self.config.get('addressfamily', 'any').lower()
             if address_family != 'any':
-                family = socket.AF_INET if address_family == 'inet' \
-                    else socket.AF_INET6
-                results = socket.getaddrinfo(host,
-                                             None,
-                                             family,
-                                             socket.SOCK_DGRAM,
-                                             socket.IPPROTO_IP,
-                                             socket.AI_CANONNAME)
-                for res in results:
-                    af, socktype, proto, canonname, sa = res
-                    if canonname and '.' in canonname:
-                        fqdn = canonname
-                        break
+                try:
+                    family = socket.AF_INET if address_family == 'inet' \
+                        else socket.AF_INET6
+                    results = socket.getaddrinfo(
+                        self.host,
+                        None,
+                        family,
+                        socket.SOCK_DGRAM,
+                        socket.IPPROTO_IP,
+                        socket.AI_CANONNAME
+                    )
+                    for res in results:
+                        af, socktype, proto, canonname, sa = res
+                        if canonname and '.' in canonname:
+                            fqdn = canonname
+                            break
+                # giaerror -> socket.getaddrinfo() can't resolve self.host
+                # (which is from socket.gethostname()). Fall back to the
+                # getfqdn() call below.
+                except socket.gaierror:
+                    pass
             # Handle 'any' / unspecified
             if fqdn is None:
                 fqdn = socket.getfqdn()
@@ -126,16 +135,17 @@ class SSHConfig (object):
                 self._config.append(host)
                 value = value.split()
                 host = {key: value, 'config': {}}
-            #identityfile is a special case, since it is allowed to be
+            #identityfile, localforward, remoteforward keys are special cases, since they are allowed to be
             # specified multiple times and they should be tried in order
             # of specification.
-            elif key == 'identityfile':
+            
+            elif key in ['identityfile', 'localforward', 'remoteforward']:
                 if key in host['config']:
-                    host['config']['identityfile'].append(value)
+                    host['config'][key].append(value)
                 else:
-                    host['config']['identityfile'] = [value]
+                    host['config'][key] = [value]
             elif key not in host['config']:
-                    host['config'].update({key: value})
+                host['config'].update({key: value})
         self._config.append(host)
 
     def lookup(self, hostname):
@@ -215,7 +225,7 @@ class SSHConfig (object):
             remoteuser = user
 
         host = socket.gethostname().split('.')[0]
-        fqdn = LazyFqdn(config)
+        fqdn = LazyFqdn(config, host)
         homedir = os.path.expanduser('~')
         replacements = {'controlpath':
                         [
@@ -252,5 +262,5 @@ class SSHConfig (object):
                             config[k][item] = config[k][item].\
                                 replace(find, str(replace))
                     else:
-                            config[k] = config[k].replace(find, str(replace))
+                        config[k] = config[k].replace(find, str(replace))
         return config
diff --git a/paramiko/hostkeys.py b/paramiko/hostkeys.py
index e739312a..f64e6e63 100644
--- a/paramiko/hostkeys.py
+++ b/paramiko/hostkeys.py
@@ -28,6 +28,7 @@ import UserDict
 from paramiko.common import *
 from paramiko.dsskey import DSSKey
 from paramiko.rsakey import RSAKey
+from paramiko.util import get_logger
 
 
 class InvalidHostKey(Exception):
@@ -48,7 +49,7 @@ class HostKeyEntry:
         self.hostnames = hostnames
         self.key = key
 
-    def from_line(cls, line):
+    def from_line(cls, line, lineno=None):
         """
         Parses the given line of text to find the names for the host,
         the type of key, and the key data. The line is expected to be in the
@@ -61,9 +62,12 @@ class HostKeyEntry:
         @param line: a line from an OpenSSH known_hosts file
         @type line: str
         """
+        log = get_logger('paramiko.hostkeys')
         fields = line.split(' ')
         if len(fields) < 3:
             # Bad number of fields
+            log.info("Not enough fields found in known_hosts in line %s (%r)" %
+                     (lineno, line))
             return None
         fields = fields[:3]
 
@@ -78,6 +82,7 @@ class HostKeyEntry:
             elif keytype == 'ssh-dss':
                 key = DSSKey(data=base64.decodestring(key))
             else:
+                log.info("Unable to handle key of type %s" % (keytype,))
                 return None
         except binascii.Error, e:
             raise InvalidHostKey(line, e)
@@ -160,13 +165,18 @@ class HostKeys (UserDict.DictMixin):
         @raise IOError: if there was an error reading the file
         """
         f = open(filename, 'r')
-        for line in f:
+        for lineno, line in enumerate(f):
             line = line.strip()
             if (len(line) == 0) or (line[0] == '#'):
                 continue
-            e = HostKeyEntry.from_line(line)
+            e = HostKeyEntry.from_line(line, lineno)
             if e is not None:
-                self._entries.append(e)
+                _hostnames = e.hostnames
+                for h in _hostnames:
+                    if self.check(h, e.key):
+                        e.hostnames.remove(h)
+                if len(e.hostnames):
+                    self._entries.append(e)
         f.close()
 
     def save(self, filename):
diff --git a/paramiko/packet.py b/paramiko/packet.py
index 2887bde8..99138edc 100644
--- a/paramiko/packet.py
+++ b/paramiko/packet.py
@@ -33,17 +33,13 @@ from paramiko.ssh_exception import SSHException, ProxyCommandFailure
 from paramiko.message import Message
 
 
-got_r_hmac = False
 try:
-    import r_hmac
-    got_r_hmac = True
+    from r_hmac import HMAC
 except ImportError:
-    pass
+    from Crypto.Hash.HMAC import HMAC
+
 def compute_hmac(key, message, digest_class):
-    if got_r_hmac:
-        return r_hmac.HMAC(key, message, digest_class).digest()
-    from Crypto.Hash import HMAC
-    return HMAC.HMAC(key, message, digest_class).digest()
+    return HMAC(key, message, digest_class).digest()
 
 
 class NeedRekeyException (Exception):
diff --git a/paramiko/transport.py b/paramiko/transport.py
index fd63732f..20eff87e 100644
--- a/paramiko/transport.py
+++ b/paramiko/transport.py
@@ -1439,7 +1439,7 @@ class Transport (threading.Thread):
                 break
             self.clear_to_send_lock.release()
             if time.time() > start + self.clear_to_send_timeout:
-              raise SSHException('Key-exchange timed out waiting for key negotiation')
+                raise SSHException('Key-exchange timed out waiting for key negotiation')
         try:
             self._send_message(data)
         finally:
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 75112a23..00000000
--- a/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-pycrypto
-tox
diff --git a/setup.py b/setup.py
index d6caccf7..64484b1c 100644
--- a/setup.py
+++ b/setup.py
@@ -52,7 +52,7 @@ if sys.platform == 'darwin':
 
 
 setup(name = "paramiko",
-      version = "1.10.1",
+      version = "1.11.1",
       description = "SSH2 protocol library",
       author = "Jeff Forcier",
       author_email = "jeff@bitprophet.org",
diff --git a/tests/test_util.py b/tests/test_util.py
index efda9b2f..a528224e 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -329,3 +329,14 @@ IdentityFile id_dsa22
                 paramiko.util.lookup_ssh_host_config(host, config),
                 values
             )
+
+    def test_12_config_addressfamily_and_lazy_fqdn(self):
+        """
+        Ensure the code path honoring non-'all' AddressFamily doesn't asplode
+        """
+        test_config = """
+AddressFamily inet
+IdentityFile something_%l_using_fqdn
+"""
+        config = paramiko.util.parse_ssh_config(cStringIO.StringIO(test_config))
+        assert config.lookup('meh') # will die during lookup() if bug regresses
diff --git a/tox.ini b/tox.ini
index 6cb80012..e2a8dcf4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -2,5 +2,5 @@
 envlist = py25,py26,py27
 
 [testenv]
-commands = pip install --use-mirrors -q -r requirements.txt
+commands = pip install --use-mirrors -q -r dev-requirements.txt
            python test.py