From dae916f7bd6723cee95891778baff51ef45532ee Mon Sep 17 00:00:00 2001 From: Perry Randall Date: Wed, 24 Dec 2014 07:54:06 -0800 Subject: Add more flexible support for two factor authentication. Allow paramiko to partially authenticate and continue by echo'ing the prompt on the remote end and responding. --- paramiko/client.py | 25 +++++++++++++++---------- paramiko/transport.py | 22 ++++++++++++++++++++++ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/paramiko/client.py b/paramiko/client.py index 393e3e09..1ccf0456 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -404,7 +404,8 @@ class SSHClient (ClosingContextManager): """ saved_exception = None two_factor = False - allowed_types = [] + allowed_types = set() + two_factor_types = set(['keyboard-interactive','password']) # If GSS-API support and GSS-PI Key Exchange was performed, we attempt # authentication with gssapi-keyex. @@ -430,8 +431,8 @@ class SSHClient (ClosingContextManager): if pkey is not None: try: self._log(DEBUG, 'Trying SSH key %s' % hexlify(pkey.get_fingerprint())) - allowed_types = self._transport.auth_publickey(username, pkey) - two_factor = (allowed_types == ['password']) + allowed_types = set(self._transport.auth_publickey(username, pkey)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return except SSHException as e: @@ -444,7 +445,7 @@ class SSHClient (ClosingContextManager): key = pkey_class.from_private_key_file(key_filename, password) self._log(DEBUG, 'Trying key %s from %s' % (hexlify(key.get_fingerprint()), key_filename)) self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) + two_factor = (allowed_types & two_factor_types) if not two_factor: return break @@ -458,9 +459,9 @@ class SSHClient (ClosingContextManager): for key in self._agent.get_keys(): try: self._log(DEBUG, 'Trying SSH agent key %s' % hexlify(key.get_fingerprint())) - # for 2-factor auth a successfully auth'd key will result in ['password'] - allowed_types = self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) + # for 2-factor auth a successfully auth'd key password will return an allowed 2fac auth method + allowed_types = set(self._transport.auth_publickey(username, key)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return break @@ -497,8 +498,8 @@ class SSHClient (ClosingContextManager): key = pkey_class.from_private_key_file(filename, password) self._log(DEBUG, 'Trying discovered key %s in %s' % (hexlify(key.get_fingerprint()), filename)) # for 2-factor auth a successfully auth'd key will result in ['password'] - allowed_types = self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) + allowed_types = set(self._transport.auth_publickey(username, key)) + two_factor = (allowed_types & two_factor_types) if not two_factor: return break @@ -512,7 +513,11 @@ class SSHClient (ClosingContextManager): except SSHException as e: saved_exception = e elif two_factor: - raise SSHException('Two-factor authentication requires a password') + try: + self._transport.auth_interactive_dumb(username) + return + except SSHException as e: + saved_exception = e # if we got an auth-failed exception earlier, re-raise it if saved_exception is not None: diff --git a/paramiko/transport.py b/paramiko/transport.py index 36da3043..4fa36191 100644 --- a/paramiko/transport.py +++ b/paramiko/transport.py @@ -20,6 +20,7 @@ Core protocol implementation """ +from __future__ import print_function import os import socket import sys @@ -1284,6 +1285,27 @@ class Transport (threading.Thread, ClosingContextManager): self.auth_handler.auth_interactive(username, handler, my_event, submethods) return self.auth_handler.wait_for_response(my_event) + def auth_interactive_dumb(self, username, handler=None, submethods=''): + """ + Autenticate to the server interactively but dumber. + Just print the prompt and / or instructions to stdout and send back + the response. This is good for situations where partial auth is + achieved by key and then the user has to enter a 2fac token. + """ + + if not handler: + def handler(title, instructions, prompt_list): + answers = [] + if title: + print(title.strip()) + if instructions: + print(instructions.strip()) + for prompt,show_input in prompt_list: + print(prompt.strip(),end=' ') + answers.append(raw_input()) + return answers + return self.auth_interactive(username, handler, submethods) + def auth_gssapi_with_mic(self, username, gss_host, gss_deleg_creds): """ Authenticate to the Server using GSS-API / SSPI. -- cgit v1.2.3 From a671dafb8775e585086600a1fddd364def5aeb20 Mon Sep 17 00:00:00 2001 From: Torkil Gustavsen Date: Thu, 23 Jul 2015 13:22:39 +0200 Subject: prefetch now requires file_size to be passed in as a parameter Calling stat from inside the prefetch-body has led users to receive IOError: The message [] is not extractable. --- paramiko/sftp_client.py | 5 +++-- paramiko/sftp_file.py | 7 +++---- tests/test_sftp.py | 3 ++- tests/test_sftp_big.py | 18 ++++++++++++------ 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 89840eaa..e4c3a37d 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -685,9 +685,10 @@ class SFTPClient(BaseSFTP, ClosingContextManager): .. versionadded:: 1.10 """ + file_size = self.stat(remotepath).st_size with self.open(remotepath, 'rb') as fr: - file_size = self.stat(remotepath).st_size - fr.prefetch() + fr.prefetch(file_size) + size = 0 while True: data = fr.read(32768) diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index d0a37da3..8f73dcbf 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -379,7 +379,7 @@ class SFTPFile (BufferedFile): """ self.pipelined = pipelined - def prefetch(self): + def prefetch(self, file_size=None): """ Pre-fetch the remaining contents of this file in anticipation of future `.read` calls. If reading the entire file, pre-fetching can @@ -393,12 +393,11 @@ class SFTPFile (BufferedFile): .. versionadded:: 1.5.1 """ - size = self.stat().st_size # queue up async reads for the rest of the file chunks = [] n = self._realpos - while n < size: - chunk = min(self.MAX_REQUEST_SIZE, size - n) + while n < file_size: + chunk = min(self.MAX_REQUEST_SIZE, file_size - n) chunks.append((n, chunk)) n += chunk if len(chunks) > 0: diff --git a/tests/test_sftp.py b/tests/test_sftp.py index cb8f7f84..55762c21 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -696,7 +696,8 @@ class SFTPTest (unittest.TestCase): f.readv([(0, 12)]) with sftp.open(FOLDER + '/zero', 'r') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) f.read(100) finally: sftp.unlink(FOLDER + '/zero') diff --git a/tests/test_sftp_big.py b/tests/test_sftp_big.py index abed27b8..cfad5682 100644 --- a/tests/test_sftp_big.py +++ b/tests/test_sftp_big.py @@ -132,7 +132,8 @@ class BigSFTPTest (unittest.TestCase): start = time.time() with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) # read on odd boundaries to make sure the bytes aren't getting scrambled n = 0 @@ -171,7 +172,8 @@ class BigSFTPTest (unittest.TestCase): chunk = 793 for i in range(10): with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) base_offset = (512 * 1024) + 17 * random.randint(1000, 2000) offsets = [base_offset + j * chunk for j in range(100)] # randomly seek around and read them out @@ -245,9 +247,11 @@ class BigSFTPTest (unittest.TestCase): for i in range(10): with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) for n in range(1024): data = f.read(1024) self.assertEqual(data, kblob) @@ -275,7 +279,8 @@ class BigSFTPTest (unittest.TestCase): self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024) with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) data = f.read(1024) self.assertEqual(data, kblob) @@ -353,7 +358,8 @@ class BigSFTPTest (unittest.TestCase): # try to read it too. with sftp.open('%s/hongry.txt' % FOLDER, 'r', 128 * 1024) as f: - f.prefetch() + file_size = f.stat().st_size + f.prefetch(file_size) total = 0 while total < 1024 * 1024: total += len(f.read(32 * 1024)) -- cgit v1.2.3 From 49072f3537a8981e9d448c22481a1d2b92c03643 Mon Sep 17 00:00:00 2001 From: Torkil Gustavsen Date: Thu, 23 Jul 2015 14:29:03 +0200 Subject: prefetch's file_size param is not optional --- paramiko/sftp_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index 8f73dcbf..c5b65488 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -379,7 +379,7 @@ class SFTPFile (BufferedFile): """ self.pipelined = pipelined - def prefetch(self, file_size=None): + def prefetch(self, file_size): """ Pre-fetch the remaining contents of this file in anticipation of future `.read` calls. If reading the entire file, pre-fetching can -- cgit v1.2.3 From b0435808802cf435fd2865b7b8af2326064df82c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 3 Nov 2015 16:58:02 -0800 Subject: 80-col --- paramiko/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paramiko/client.py b/paramiko/client.py index 5a215a81..07175763 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -256,7 +256,8 @@ class SSHClient (ClosingContextManager): :param socket sock: an open socket or socket-like object (such as a `.Channel`) to use for communication to the target host - :param bool gss_auth: ``True`` if you want to use GSS-API authentication + :param bool gss_auth: + ``True`` if you want to use GSS-API authentication :param bool gss_kex: Perform GSS-API Key Exchange and user authentication :param bool gss_deleg_creds: Delegate GSS-API client credentials or not -- cgit v1.2.3 From c9441c4a202dd53d4cb00946943745f580efa084 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 4 Nov 2015 12:44:21 -0800 Subject: Patch up a missed spot re: 2FA plus keys, thanks @mattrobenolt --- paramiko/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paramiko/client.py b/paramiko/client.py index f30aba2f..8d899a15 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -504,7 +504,7 @@ class SSHClient (ClosingContextManager): try: key = pkey_class.from_private_key_file(key_filename, password) self._log(DEBUG, 'Trying key %s from %s' % (hexlify(key.get_fingerprint()), key_filename)) - self._transport.auth_publickey(username, key) + allowed_types = set(self._transport.auth_publickey(username, key)) two_factor = (allowed_types & two_factor_types) if not two_factor: return -- cgit v1.2.3 From 2a99a8c9a4bde66720e9357963ce1896830528a1 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 4 Nov 2015 12:51:16 -0800 Subject: Changelog closes #467 --- sites/www/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 3310eb32..6ea85c45 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +* :feature:`467` (also :issue:`139`, :issue:`412`) Fully enable two-factor + authentication (e.g. when a server requires ``AuthenticationMethods + pubkey,keyboard-interactive``). Thanks to ``@perryjrandall`` for the patch + and to ``@nevins-b`` and Matt Robenolt for additional support. * :bug:`502 major` Fix 'exec' requests in server mode to use ``get_string`` instead of ``get_text`` to avoid ``UnicodeDecodeError`` on non-UTF-8 input. Thanks to Anselm Kruis for the patch & discussion. -- cgit v1.2.3 From 5b077c2022f150a8143a306df0ae8dae6427a4a8 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 4 Nov 2015 14:50:49 -0800 Subject: Remove vestigial extra 'stat' call in SFTPClient.get() Was apparently not removed when getfo() was born. --- paramiko/sftp_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index da6f6e87..57225558 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -717,7 +717,6 @@ class SFTPClient(BaseSFTP, ClosingContextManager): .. versionchanged:: 1.7.4 Added the ``callback`` param """ - file_size = self.stat(remotepath).st_size with open(localpath, 'wb') as fl: size = self.getfo(remotepath, fl, callback) s = os.stat(localpath) -- cgit v1.2.3 From 935711b5a17370494a7b2b8b4587f5466badf1e8 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 4 Nov 2015 14:53:41 -0800 Subject: Changelog closes #194 --- sites/www/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 6ea85c45..304f10a6 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= +* :bug:`194 major` (also :issue:`562`, :issue:`530`, :issue:`576`) Streamline + use of ``stat`` when downloading SFTP files via `SFTPClient.get + `; this avoids triggering bugs in some + off-spec SFTP servers such as IBM Sterling. Thanks to ``@muraleee`` for the + initial report and to Torkil Gustavsen for the patch. * :feature:`467` (also :issue:`139`, :issue:`412`) Fully enable two-factor authentication (e.g. when a server requires ``AuthenticationMethods pubkey,keyboard-interactive``). Thanks to ``@perryjrandall`` for the patch -- cgit v1.2.3 From 1fe8c0de7fc6d6dce3b6ece69be10972480dad8f Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 5 Nov 2015 14:46:29 -0800 Subject: Typo fix --- sites/www/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 304f10a6..9c4ee012 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -113,7 +113,7 @@ Changelog use of the ``shlex`` module. Thanks to Yan Kalchevskiy. * :support:`422 backported` Clean up some unused imports. Courtesy of Olle Lundberg. -* :support:`421 backported` Modernize threading calls to user newer API. Thanks +* :support:`421 backported` Modernize threading calls to use newer API. Thanks to Olle Lundberg. * :support:`419 backported` Modernize a bunch of the codebase internals to leverage decorators. Props to ``@beckjake`` for realizing we're no longer on -- cgit v1.2.3 From 91bb2a1405047ec0920575f3ebbd39e8f7215a21 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 5 Nov 2015 14:48:07 -0800 Subject: Modernize a couple dev-reqs --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 90cfd477..9e4564a5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,8 +4,8 @@ tox>=1.4,<1.5 invoke>=0.11.1 invocations>=0.11.0 sphinx>=1.1.3 -alabaster>=0.6.1 -releases>=0.5.2 +alabaster>=0.7.5 +releases>=1.0.0 semantic_version>=2.4,<2.5 wheel==0.24 twine==1.5 -- cgit v1.2.3 From 96705e26cf9ac7c9c3f6e8cd28e7e408dc5b856a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 5 Nov 2015 14:48:19 -0800 Subject: Cut 1.16 --- sites/www/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 9c4ee012..084d13de 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,6 +2,7 @@ Changelog ========= +* :release:`1.16.0 <2015-11-04>` * :bug:`194 major` (also :issue:`562`, :issue:`530`, :issue:`576`) Streamline use of ``stat`` when downloading SFTP files via `SFTPClient.get `; this avoids triggering bugs in some -- cgit v1.2.3 From cd4073122e1cda1fec4bf0bae7ba0dede6fd3635 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 5 Nov 2015 15:15:51 -0800 Subject: Passthru sdist/wheel options for release task --- tasks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tasks.py b/tasks.py index 3d55a778..d2bed606 100644 --- a/tasks.py +++ b/tasks.py @@ -25,7 +25,10 @@ def coverage(ctx): # Until we stop bundling docs w/ releases. Need to discover use cases first. @task -def release(ctx): +def release(ctx, sdist=True, wheel=True): + """ + Wraps invocations.packaging.release to add baked-in docs folder. + """ # Build docs first. Use terribad workaround pending invoke #146 ctx.run("inv docs") # Move the built docs into where Epydocs used to live @@ -34,7 +37,7 @@ def release(ctx): # TODO: make it easier to yank out this config val from the docs coll copytree('sites/docs/_build', target) # Publish - publish(ctx) + publish(ctx, sdist=sdist, wheel=wheel) # Remind print("\n\nDon't forget to update RTD's versions page for new minor releases!") -- cgit v1.2.3 From e51b24ac8060bda099bcc5ea6c18f96f062aaad8 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 5 Nov 2015 17:19:16 -0800 Subject: Missed a spot re: 3.2 specific crap in Travis config --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c2c20a60..55cba46d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,8 @@ install: - pip install coveralls # For coveralls.io specifically - pip install -r dev-requirements.txt script: - # Main tests, w/ coverage! (but skip coverage on 3.2, coverage.py dropped it) - - "[[ $TRAVIS_PYTHON_VERSION != 3.2 ]] && inv test --coverage || inv test" + # Main tests, w/ coverage! + - inv test --coverage # Ensure documentation & invoke pipeline run OK. # Run 'docs' first since its objects.inv is referred to by 'www'. # Also force warnings to be errors since most of them tend to be actual -- cgit v1.2.3 From a90f247a27eadae138eff5ee78c91c99dbc3596f Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 6 Nov 2015 15:10:44 -0800 Subject: Hacky cleanup of non-gc'd clients in a loopy test. Re #612 --- tests/test_client.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 04cab439..f71efd5a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -193,12 +193,18 @@ class SSHClientTest (unittest.TestCase): (['dss', 'rsa', 'ecdsa'], ['dss']), # Try ECDSA but fail (['rsa', 'ecdsa'], ['ecdsa']), # ECDSA success ): - self._test_connection( - key_filename=[ - test_path('test_{0}.key'.format(x)) for x in attempt - ], - allowed_keys=[types_[x] for x in accept], - ) + try: + self._test_connection( + key_filename=[ + test_path('test_{0}.key'.format(x)) for x in attempt + ], + allowed_keys=[types_[x] for x in accept], + ) + finally: + # Clean up to avoid occasional gc-related deadlocks. + # TODO: use nose test generators after nose port + self.tearDown() + self.setUp() def test_multiple_key_files_failure(self): """ -- cgit v1.2.3 From ba0b12fb09f7334e930fc2b5a02d7e7824695627 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 6 Nov 2015 15:10:44 -0800 Subject: Hacky cleanup of non-gc'd clients in a loopy test. Re #612 --- tests/test_client.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 3d2e75c9..e080221e 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -193,12 +193,18 @@ class SSHClientTest (unittest.TestCase): (['dss', 'rsa', 'ecdsa'], ['dss']), # Try ECDSA but fail (['rsa', 'ecdsa'], ['ecdsa']), # ECDSA success ): - self._test_connection( - key_filename=[ - test_path('test_{0}.key'.format(x)) for x in attempt - ], - allowed_keys=[types_[x] for x in accept], - ) + try: + self._test_connection( + key_filename=[ + test_path('test_{0}.key'.format(x)) for x in attempt + ], + allowed_keys=[types_[x] for x in accept], + ) + finally: + # Clean up to avoid occasional gc-related deadlocks. + # TODO: use nose test generators after nose port + self.tearDown() + self.setUp() def test_multiple_key_files_failure(self): """ -- cgit v1.2.3