Merge branch 'master' into switch-to-cryptography

11 files changed, 91 insertions, 38 deletions

diff --git a/.travis.yml b/.travis.yml
index cb2a7e8b..55cba46d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,7 +14,7 @@ install:
   - pip install -r dev-requirements.txt
 script:
   # Main tests, w/ coverage!
-  - "inv test --coverage"
+  - inv test --coverage
   # Ensure documentation & invoke pipeline run OK.
   # Run 'docs' first since its objects.inv is referred to by 'www'.
   # Also force warnings to be errors since most of them tend to be actual
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 90cfd477..9e4564a5 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -4,8 +4,8 @@ tox>=1.4,<1.5
 invoke>=0.11.1
 invocations>=0.11.0
 sphinx>=1.1.3
-alabaster>=0.6.1
-releases>=0.5.2
+alabaster>=0.7.5
+releases>=1.0.0
 semantic_version>=2.4,<2.5
 wheel==0.24
 twine==1.5
diff --git a/paramiko/client.py b/paramiko/client.py
index 5a215a81..8d899a15 100644
--- a/paramiko/client.py
+++ b/paramiko/client.py
@@ -256,7 +256,8 @@ class SSHClient (ClosingContextManager):
         :param socket sock:
             an open socket or socket-like object (such as a `.Channel`) to use
             for communication to the target host
-        :param bool gss_auth: ``True`` if you want to use GSS-API authentication
+        :param bool gss_auth:
+            ``True`` if you want to use GSS-API authentication
         :param bool gss_kex:
             Perform GSS-API Key Exchange and user authentication
         :param bool gss_deleg_creds: Delegate GSS-API client credentials or not
@@ -463,7 +464,8 @@ class SSHClient (ClosingContextManager):
         """
         saved_exception = None
         two_factor = False
-        allowed_types = []
+        allowed_types = set()
+        two_factor_types = set(['keyboard-interactive','password'])
 
         # If GSS-API support and GSS-PI Key Exchange was performed, we attempt
         # authentication with gssapi-keyex.
@@ -489,8 +491,8 @@ class SSHClient (ClosingContextManager):
         if pkey is not None:
             try:
                 self._log(DEBUG, 'Trying SSH key %s' % hexlify(pkey.get_fingerprint()))
-                allowed_types = self._transport.auth_publickey(username, pkey)
-                two_factor = (allowed_types == ['password'])
+                allowed_types = set(self._transport.auth_publickey(username, pkey))
+                two_factor = (allowed_types & two_factor_types)
                 if not two_factor:
                     return
             except SSHException as e:
@@ -502,8 +504,8 @@ class SSHClient (ClosingContextManager):
                     try:
                         key = pkey_class.from_private_key_file(key_filename, password)
                         self._log(DEBUG, 'Trying key %s from %s' % (hexlify(key.get_fingerprint()), key_filename))
-                        self._transport.auth_publickey(username, key)
-                        two_factor = (allowed_types == ['password'])
+                        allowed_types = set(self._transport.auth_publickey(username, key))
+                        two_factor = (allowed_types & two_factor_types)
                         if not two_factor:
                             return
                         break
@@ -517,9 +519,9 @@ class SSHClient (ClosingContextManager):
             for key in self._agent.get_keys():
                 try:
                     self._log(DEBUG, 'Trying SSH agent key %s' % hexlify(key.get_fingerprint()))
-                    # for 2-factor auth a successfully auth'd key will result in ['password']
-                    allowed_types = self._transport.auth_publickey(username, key)
-                    two_factor = (allowed_types == ['password'])
+                    # for 2-factor auth a successfully auth'd key password will return an allowed 2fac auth method
+                    allowed_types = set(self._transport.auth_publickey(username, key))
+                    two_factor = (allowed_types & two_factor_types)
                     if not two_factor:
                         return
                     break
@@ -556,8 +558,8 @@ class SSHClient (ClosingContextManager):
                     key = pkey_class.from_private_key_file(filename, password)
                     self._log(DEBUG, 'Trying discovered key %s in %s' % (hexlify(key.get_fingerprint()), filename))
                     # for 2-factor auth a successfully auth'd key will result in ['password']
-                    allowed_types = self._transport.auth_publickey(username, key)
-                    two_factor = (allowed_types == ['password'])
+                    allowed_types = set(self._transport.auth_publickey(username, key))
+                    two_factor = (allowed_types & two_factor_types)
                     if not two_factor:
                         return
                     break
@@ -571,7 +573,11 @@ class SSHClient (ClosingContextManager):
             except SSHException as e:
                 saved_exception = e
         elif two_factor:
-            raise SSHException('Two-factor authentication requires a password')
+            try:
+                self._transport.auth_interactive_dumb(username)
+                return
+            except SSHException as e:
+                saved_exception = e
 
         # if we got an auth-failed exception earlier, re-raise it
         if saved_exception is not None:
diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py
index 55302ffd..57225558 100644
--- a/paramiko/sftp_client.py
+++ b/paramiko/sftp_client.py
@@ -686,9 +686,10 @@ class SFTPClient(BaseSFTP, ClosingContextManager):
 
         .. versionadded:: 1.10
         """
+        file_size = self.stat(remotepath).st_size
         with self.open(remotepath, 'rb') as fr:
-            file_size = self.stat(remotepath).st_size
-            fr.prefetch()
+            fr.prefetch(file_size)
+
             size = 0
             while True:
                 data = fr.read(32768)
@@ -716,7 +717,6 @@ class SFTPClient(BaseSFTP, ClosingContextManager):
         .. versionchanged:: 1.7.4
             Added the ``callback`` param
         """
-        file_size = self.stat(remotepath).st_size
         with open(localpath, 'wb') as fl:
             size = self.getfo(remotepath, fl, callback)
         s = os.stat(localpath)
diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py
index d0a37da3..c5b65488 100644
--- a/paramiko/sftp_file.py
+++ b/paramiko/sftp_file.py
@@ -379,7 +379,7 @@ class SFTPFile (BufferedFile):
         """
         self.pipelined = pipelined
     
-    def prefetch(self):
+    def prefetch(self, file_size):
         """
         Pre-fetch the remaining contents of this file in anticipation of future
         `.read` calls.  If reading the entire file, pre-fetching can
@@ -393,12 +393,11 @@ class SFTPFile (BufferedFile):
 
         .. versionadded:: 1.5.1
         """
-        size = self.stat().st_size
         # queue up async reads for the rest of the file
         chunks = []
         n = self._realpos
-        while n < size:
-            chunk = min(self.MAX_REQUEST_SIZE, size - n)
+        while n < file_size:
+            chunk = min(self.MAX_REQUEST_SIZE, file_size - n)
             chunks.append((n, chunk))
             n += chunk
         if len(chunks) > 0:
diff --git a/paramiko/transport.py b/paramiko/transport.py
index 40cff527..6d0c627a 100644
--- a/paramiko/transport.py
+++ b/paramiko/transport.py
@@ -20,6 +20,7 @@
 Core protocol implementation
 """
 
+from __future__ import print_function
 import os
 import socket
 import sys
@@ -1379,6 +1380,27 @@ class Transport (threading.Thread, ClosingContextManager):
         self.auth_handler.auth_interactive(username, handler, my_event, submethods)
         return self.auth_handler.wait_for_response(my_event)
 
+    def auth_interactive_dumb(self, username, handler=None, submethods=''):
+        """
+        Autenticate to the server interactively but dumber.
+        Just print the prompt and / or instructions to stdout and send back
+        the response. This is good for situations where partial auth is
+        achieved by key and then the user has to enter a 2fac token.
+        """
+
+        if not handler:
+            def handler(title, instructions, prompt_list):
+                answers = []
+                if title:
+                    print(title.strip())
+                if instructions:
+                    print(instructions.strip())
+                for prompt,show_input in prompt_list:
+                    print(prompt.strip(),end=' ')
+                    answers.append(raw_input())
+                return answers
+        return self.auth_interactive(username, handler, submethods)
+
     def auth_gssapi_with_mic(self, username, gss_host, gss_deleg_creds):
         """
         Authenticate to the Server using GSS-API / SSPI.
diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst
index 3310eb32..084d13de 100644
--- a/sites/www/changelog.rst
+++ b/sites/www/changelog.rst
@@ -2,6 +2,16 @@
 Changelog
 =========
 
+* :release:`1.16.0 <2015-11-04>`
+* :bug:`194 major` (also :issue:`562`, :issue:`530`, :issue:`576`) Streamline
+  use of ``stat`` when downloading SFTP files via `SFTPClient.get
+  <paramiko.sftp_client.SFTPClient.get>`; this avoids triggering bugs in some
+  off-spec SFTP servers such as IBM Sterling. Thanks to ``@muraleee`` for the
+  initial report and to Torkil Gustavsen for the patch.
+* :feature:`467` (also :issue:`139`, :issue:`412`) Fully enable two-factor
+  authentication (e.g. when a server requires ``AuthenticationMethods
+  pubkey,keyboard-interactive``). Thanks to ``@perryjrandall`` for the patch
+  and to ``@nevins-b`` and Matt Robenolt for additional support.
 * :bug:`502 major` Fix 'exec' requests in server mode to use ``get_string``
   instead of ``get_text`` to avoid ``UnicodeDecodeError`` on non-UTF-8 input.
   Thanks to Anselm Kruis for the patch & discussion.
@@ -104,7 +114,7 @@ Changelog
   use of the ``shlex`` module. Thanks to Yan Kalchevskiy.
 * :support:`422 backported` Clean up some unused imports. Courtesy of Olle
   Lundberg.
-* :support:`421 backported` Modernize threading calls to user newer API. Thanks
+* :support:`421 backported` Modernize threading calls to use newer API. Thanks
   to Olle Lundberg.
 * :support:`419 backported` Modernize a bunch of the codebase internals to
   leverage decorators. Props to ``@beckjake`` for realizing we're no longer on
diff --git a/tasks.py b/tasks.py
index 3d55a778..d2bed606 100644
--- a/tasks.py
+++ b/tasks.py
@@ -25,7 +25,10 @@ def coverage(ctx):
 
 # Until we stop bundling docs w/ releases. Need to discover use cases first.
 @task
-def release(ctx):
+def release(ctx, sdist=True, wheel=True):
+    """
+    Wraps invocations.packaging.release to add baked-in docs folder.
+    """
     # Build docs first. Use terribad workaround pending invoke #146
     ctx.run("inv docs")
     # Move the built docs into where Epydocs used to live
@@ -34,7 +37,7 @@ def release(ctx):
     # TODO: make it easier to yank out this config val from the docs coll
     copytree('sites/docs/_build', target)
     # Publish
-    publish(ctx)
+    publish(ctx, sdist=sdist, wheel=wheel)
     # Remind
     print("\n\nDon't forget to update RTD's versions page for new minor releases!")
 
diff --git a/tests/test_client.py b/tests/test_client.py
index 54a2fb9b..46246783 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -196,12 +196,18 @@ class SSHClientTest (unittest.TestCase):
             (['dss', 'rsa', 'ecdsa'], ['dss']), # Try ECDSA but fail
             (['rsa', 'ecdsa'], ['ecdsa']), # ECDSA success
         ):
-            self._test_connection(
-                key_filename=[
-                    test_path('test_{0}.key'.format(x)) for x in attempt
-                ],
-                allowed_keys=[types_[x] for x in accept],
-            )
+            try:
+                self._test_connection(
+                    key_filename=[
+                        test_path('test_{0}.key'.format(x)) for x in attempt
+                    ],
+                    allowed_keys=[types_[x] for x in accept],
+                )
+            finally:
+                # Clean up to avoid occasional gc-related deadlocks.
+                # TODO: use nose test generators after nose port
+                self.tearDown()
+                self.setUp()
 
     def test_multiple_key_files_failure(self):
         """
diff --git a/tests/test_sftp.py b/tests/test_sftp.py
index aa450f59..131b8abf 100755
--- a/tests/test_sftp.py
+++ b/tests/test_sftp.py
@@ -696,7 +696,8 @@ class SFTPTest (unittest.TestCase):
                 f.readv([(0, 12)])
 
             with sftp.open(FOLDER + '/zero', 'r') as f:
-                f.prefetch()
+                file_size = f.stat().st_size
+                f.prefetch(file_size)
                 f.read(100)
         finally:
             sftp.unlink(FOLDER + '/zero')
diff --git a/tests/test_sftp_big.py b/tests/test_sftp_big.py
index abed27b8..cfad5682 100644
--- a/tests/test_sftp_big.py
+++ b/tests/test_sftp_big.py
@@ -132,7 +132,8 @@ class BigSFTPTest (unittest.TestCase):
             
             start = time.time()
             with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
-                f.prefetch()
+                file_size = f.stat().st_size
+                f.prefetch(file_size)
 
                 # read on odd boundaries to make sure the bytes aren't getting scrambled
                 n = 0
@@ -171,7 +172,8 @@ class BigSFTPTest (unittest.TestCase):
             chunk = 793
             for i in range(10):
                 with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
-                    f.prefetch()
+                    file_size = f.stat().st_size
+                    f.prefetch(file_size)
                     base_offset = (512 * 1024) + 17 * random.randint(1000, 2000)
                     offsets = [base_offset + j * chunk for j in range(100)]
                     # randomly seek around and read them out
@@ -245,9 +247,11 @@ class BigSFTPTest (unittest.TestCase):
 
             for i in range(10):
                 with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f:
-                    f.prefetch()
+                    file_size = f.stat().st_size
+                    f.prefetch(file_size)
             with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f:
-                f.prefetch()
+                file_size = f.stat().st_size
+                f.prefetch(file_size)
                 for n in range(1024):
                     data = f.read(1024)
                     self.assertEqual(data, kblob)
@@ -275,7 +279,8 @@ class BigSFTPTest (unittest.TestCase):
             self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
 
             with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
-                f.prefetch()
+                file_size = f.stat().st_size
+                f.prefetch(file_size)
                 data = f.read(1024)
                 self.assertEqual(data, kblob)
 
@@ -353,7 +358,8 @@ class BigSFTPTest (unittest.TestCase):
             
             # try to read it too.
             with sftp.open('%s/hongry.txt' % FOLDER, 'r', 128 * 1024) as f:
-                f.prefetch()
+                file_size = f.stat().st_size
+                f.prefetch(file_size)
                 total = 0
                 while total < 1024 * 1024:
                     total += len(f.read(32 * 1024))