diff options
| -rw-r--r-- | .travis.yml | 2 | ||||
| -rw-r--r-- | dev-requirements.txt | 4 | ||||
| -rw-r--r-- | paramiko/client.py | 30 | ||||
| -rw-r--r-- | paramiko/sftp_client.py | 6 | ||||
| -rw-r--r-- | paramiko/sftp_file.py | 7 | ||||
| -rw-r--r-- | paramiko/transport.py | 22 | ||||
| -rw-r--r-- | sites/www/changelog.rst | 12 | ||||
| -rw-r--r-- | tasks.py | 7 | ||||
| -rw-r--r-- | tests/test_client.py | 18 | ||||
| -rwxr-xr-x | tests/test_sftp.py | 3 | ||||
| -rw-r--r-- | tests/test_sftp_big.py | 18 |
11 files changed, 91 insertions, 38 deletions
diff --git a/.travis.yml b/.travis.yml index cb2a7e8b..55cba46d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ install: - pip install -r dev-requirements.txt script: # Main tests, w/ coverage! - - "inv test --coverage" + - inv test --coverage # Ensure documentation & invoke pipeline run OK. # Run 'docs' first since its objects.inv is referred to by 'www'. # Also force warnings to be errors since most of them tend to be actual diff --git a/dev-requirements.txt b/dev-requirements.txt index 90cfd477..9e4564a5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,8 +4,8 @@ tox>=1.4,<1.5 invoke>=0.11.1 invocations>=0.11.0 sphinx>=1.1.3 -alabaster>=0.6.1 -releases>=0.5.2 +alabaster>=0.7.5 +releases>=1.0.0 semantic_version>=2.4,<2.5 wheel==0.24 twine==1.5 diff --git a/paramiko/client.py b/paramiko/client.py index 5a215a81..8d899a15 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -256,7 +256,8 @@ class SSHClient (ClosingContextManager): :param socket sock: an open socket or socket-like object (such as a `.Channel`) to use for communication to the target host - :param bool gss_auth: ``True`` if you want to use GSS-API authentication + :param bool gss_auth: + ``True`` if you want to use GSS-API authentication :param bool gss_kex: Perform GSS-API Key Exchange and user authentication :param bool gss_deleg_creds: Delegate GSS-API client credentials or not @@ -463,7 +464,8 @@ class SSHClient (ClosingContextManager): """ saved_exception = None two_factor = False - allowed_types = [] + allowed_types = set() + two_factor_types = set(['keyboard-interactive','password']) # If GSS-API support and GSS-PI Key Exchange was performed, we attempt # authentication with gssapi-keyex. @@ -489,8 +491,8 @@ class SSHClient (ClosingContextManager): if pkey is not None: try: self._log(DEBUG, 'Trying SSH key %s' % hexlify(pkey.get_fingerprint())) - allowed_types = self._transport.auth_publickey(username, pkey) - two_factor = (allowed_types == ['password']) + allowed_types = set(self._transport.auth_publickey(username, pkey)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return except SSHException as e: @@ -502,8 +504,8 @@ class SSHClient (ClosingContextManager): try: key = pkey_class.from_private_key_file(key_filename, password) self._log(DEBUG, 'Trying key %s from %s' % (hexlify(key.get_fingerprint()), key_filename)) - self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) + allowed_types = set(self._transport.auth_publickey(username, key)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return break @@ -517,9 +519,9 @@ class SSHClient (ClosingContextManager): for key in self._agent.get_keys(): try: self._log(DEBUG, 'Trying SSH agent key %s' % hexlify(key.get_fingerprint())) - # for 2-factor auth a successfully auth'd key will result in ['password'] - allowed_types = self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) + # for 2-factor auth a successfully auth'd key password will return an allowed 2fac auth method + allowed_types = set(self._transport.auth_publickey(username, key)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return break @@ -556,8 +558,8 @@ class SSHClient (ClosingContextManager): key = pkey_class.from_private_key_file(filename, password) self._log(DEBUG, 'Trying discovered key %s in %s' % (hexlify(key.get_fingerprint()), filename)) # for 2-factor auth a successfully auth'd key will result in ['password'] - allowed_types = self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) + allowed_types = set(self._transport.auth_publickey(username, key)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return break @@ -571,7 +573,11 @@ class SSHClient (ClosingContextManager): except SSHException as e: saved_exception = e elif two_factor: - raise SSHException('Two-factor authentication requires a password') + try: + self._transport.auth_interactive_dumb(username) + return + except SSHException as e: + saved_exception = e # if we got an auth-failed exception earlier, re-raise it if saved_exception is not None: diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 55302ffd..57225558 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -686,9 +686,10 @@ class SFTPClient(BaseSFTP, ClosingContextManager): .. versionadded:: 1.10 """ + file_size = self.stat(remotepath).st_size with self.open(remotepath, 'rb') as fr: - file_size = self.stat(remotepath).st_size - fr.prefetch() + fr.prefetch(file_size) + size = 0 while True: data = fr.read(32768) @@ -716,7 +717,6 @@ class SFTPClient(BaseSFTP, ClosingContextManager): .. versionchanged:: 1.7.4 Added the ``callback`` param """ - file_size = self.stat(remotepath).st_size with open(localpath, 'wb') as fl: size = self.getfo(remotepath, fl, callback) s = os.stat(localpath) diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index d0a37da3..c5b65488 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -379,7 +379,7 @@ class SFTPFile (BufferedFile): """ self.pipelined = pipelined - def prefetch(self): + def prefetch(self, file_size): """ Pre-fetch the remaining contents of this file in anticipation of future `.read` calls. If reading the entire file, pre-fetching can @@ -393,12 +393,11 @@ class SFTPFile (BufferedFile): .. versionadded:: 1.5.1 """ - size = self.stat().st_size # queue up async reads for the rest of the file chunks = [] n = self._realpos - while n < size: - chunk = min(self.MAX_REQUEST_SIZE, size - n) + while n < file_size: + chunk = min(self.MAX_REQUEST_SIZE, file_size - n) chunks.append((n, chunk)) n += chunk if len(chunks) > 0: diff --git a/paramiko/transport.py b/paramiko/transport.py index 40cff527..6d0c627a 100644 --- a/paramiko/transport.py +++ b/paramiko/transport.py @@ -20,6 +20,7 @@ Core protocol implementation """ +from __future__ import print_function import os import socket import sys @@ -1379,6 +1380,27 @@ class Transport (threading.Thread, ClosingContextManager): self.auth_handler.auth_interactive(username, handler, my_event, submethods) return self.auth_handler.wait_for_response(my_event) + def auth_interactive_dumb(self, username, handler=None, submethods=''): + """ + Autenticate to the server interactively but dumber. + Just print the prompt and / or instructions to stdout and send back + the response. This is good for situations where partial auth is + achieved by key and then the user has to enter a 2fac token. + """ + + if not handler: + def handler(title, instructions, prompt_list): + answers = [] + if title: + print(title.strip()) + if instructions: + print(instructions.strip()) + for prompt,show_input in prompt_list: + print(prompt.strip(),end=' ') + answers.append(raw_input()) + return answers + return self.auth_interactive(username, handler, submethods) + def auth_gssapi_with_mic(self, username, gss_host, gss_deleg_creds): """ Authenticate to the Server using GSS-API / SSPI. diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 3310eb32..084d13de 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,6 +2,16 @@ Changelog ========= +* :release:`1.16.0 <2015-11-04>` +* :bug:`194 major` (also :issue:`562`, :issue:`530`, :issue:`576`) Streamline + use of ``stat`` when downloading SFTP files via `SFTPClient.get + <paramiko.sftp_client.SFTPClient.get>`; this avoids triggering bugs in some + off-spec SFTP servers such as IBM Sterling. Thanks to ``@muraleee`` for the + initial report and to Torkil Gustavsen for the patch. +* :feature:`467` (also :issue:`139`, :issue:`412`) Fully enable two-factor + authentication (e.g. when a server requires ``AuthenticationMethods + pubkey,keyboard-interactive``). Thanks to ``@perryjrandall`` for the patch + and to ``@nevins-b`` and Matt Robenolt for additional support. * :bug:`502 major` Fix 'exec' requests in server mode to use ``get_string`` instead of ``get_text`` to avoid ``UnicodeDecodeError`` on non-UTF-8 input. Thanks to Anselm Kruis for the patch & discussion. @@ -104,7 +114,7 @@ Changelog use of the ``shlex`` module. Thanks to Yan Kalchevskiy. * :support:`422 backported` Clean up some unused imports. Courtesy of Olle Lundberg. -* :support:`421 backported` Modernize threading calls to user newer API. Thanks +* :support:`421 backported` Modernize threading calls to use newer API. Thanks to Olle Lundberg. * :support:`419 backported` Modernize a bunch of the codebase internals to leverage decorators. Props to ``@beckjake`` for realizing we're no longer on @@ -25,7 +25,10 @@ def coverage(ctx): # Until we stop bundling docs w/ releases. Need to discover use cases first. @task -def release(ctx): +def release(ctx, sdist=True, wheel=True): + """ + Wraps invocations.packaging.release to add baked-in docs folder. + """ # Build docs first. Use terribad workaround pending invoke #146 ctx.run("inv docs") # Move the built docs into where Epydocs used to live @@ -34,7 +37,7 @@ def release(ctx): # TODO: make it easier to yank out this config val from the docs coll copytree('sites/docs/_build', target) # Publish - publish(ctx) + publish(ctx, sdist=sdist, wheel=wheel) # Remind print("\n\nDon't forget to update RTD's versions page for new minor releases!") diff --git a/tests/test_client.py b/tests/test_client.py index 54a2fb9b..46246783 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -196,12 +196,18 @@ class SSHClientTest (unittest.TestCase): (['dss', 'rsa', 'ecdsa'], ['dss']), # Try ECDSA but fail (['rsa', 'ecdsa'], ['ecdsa']), # ECDSA success ): - self._test_connection( - key_filename=[ - test_path('test_{0}.key'.format(x)) for x in attempt - ], - allowed_keys=[types_[x] for x in accept], - ) + try: + self._test_connection( + key_filename=[ + test_path('test_{0}.key'.format(x)) for x in attempt + ], + allowed_keys=[types_[x] for x in accept], + ) + finally: + # Clean up to avoid occasional gc-related deadlocks. + # TODO: use nose test generators after nose port + self.tearDown() + self.setUp() def test_multiple_key_files_failure(self): """ diff --git a/tests/test_sftp.py b/tests/test_sftp.py index aa450f59..131b8abf 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -696,7 +696,8 @@ class SFTPTest (unittest.TestCase): f.readv([(0, 12)]) with sftp.open(FOLDER + '/zero', 'r') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) f.read(100) finally: sftp.unlink(FOLDER + '/zero') diff --git a/tests/test_sftp_big.py b/tests/test_sftp_big.py index abed27b8..cfad5682 100644 --- a/tests/test_sftp_big.py +++ b/tests/test_sftp_big.py @@ -132,7 +132,8 @@ class BigSFTPTest (unittest.TestCase): start = time.time() with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) # read on odd boundaries to make sure the bytes aren't getting scrambled n = 0 @@ -171,7 +172,8 @@ class BigSFTPTest (unittest.TestCase): chunk = 793 for i in range(10): with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) base_offset = (512 * 1024) + 17 * random.randint(1000, 2000) offsets = [base_offset + j * chunk for j in range(100)] # randomly seek around and read them out @@ -245,9 +247,11 @@ class BigSFTPTest (unittest.TestCase): for i in range(10): with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) for n in range(1024): data = f.read(1024) self.assertEqual(data, kblob) @@ -275,7 +279,8 @@ class BigSFTPTest (unittest.TestCase): self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024) with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) data = f.read(1024) self.assertEqual(data, kblob) @@ -353,7 +358,8 @@ class BigSFTPTest (unittest.TestCase): # try to read it too. with sftp.open('%s/hongry.txt' % FOLDER, 'r', 128 * 1024) as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) total = 0 while total < 1024 * 1024: total += len(f.read(32 * 1024)) |