diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | .travis.yml | 2 | ||||
| -rw-r--r-- | dev-requirements.txt | 2 | ||||
| -rw-r--r-- | paramiko/sftp_client.py | 2 | ||||
| -rw-r--r-- | paramiko/sftp_file.py | 24 | ||||
| -rw-r--r-- | sites/www/changelog.rst | 21 | ||||
| -rw-r--r-- | tasks.py | 17 |
7 files changed, 45 insertions, 24 deletions
@@ -7,3 +7,4 @@ test.log docs/ !sites/docs _build +.coverage diff --git a/.travis.yml b/.travis.yml index df7c225a..97165c47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: - pip install -r dev-requirements.txt script: # Main tests, with coverage! - - coverage run --source=paramiko test.py --verbose + - invoke coverage # Ensure documentation & invoke pipeline run OK. # Run 'docs' first since its objects.inv is referred to by 'www'. # Also force warnings to be errors since most of them tend to be actual diff --git a/dev-requirements.txt b/dev-requirements.txt index 9fa05df9..89469175 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,4 +7,4 @@ invoke>=0.7.0 invocations>=0.4.4 sphinx>=1.1.3 alabaster>=0.3.0 -releases>=0.2.4 +releases>=0.5.0 diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index d9215743..cf94582c 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -736,7 +736,7 @@ class SFTPClient (BaseSFTP): self._convert_status(msg) return t, msg if fileobj is not type(None): - fileobj._async_response(t, msg) + fileobj._async_response(t, msg, num) if waitfor is None: # just doing a single check break diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index 4ec936d0..a39b1f47 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -20,6 +20,8 @@ L{SFTPFile} """ +from __future__ import with_statement + from binascii import hexlify from collections import deque import socket @@ -53,7 +55,8 @@ class SFTPFile (BufferedFile): self._prefetching = False self._prefetch_done = False self._prefetch_data = {} - self._prefetch_reads = [] + self._prefetch_extents = {} + self._prefetch_lock = threading.Lock() self._saved_exception = None self._reqs = deque() @@ -91,7 +94,7 @@ class SFTPFile (BufferedFile): pass def _data_in_prefetch_requests(self, offset, size): - k = [i for i in self._prefetch_reads if i[0] <= offset] + k = [x for x in self._prefetch_extents.values() if x[0] <= offset] if len(k) == 0: return False k.sort(lambda x, y: cmp(x[0], y[0])) @@ -447,7 +450,6 @@ class SFTPFile (BufferedFile): def _start_prefetch(self, chunks): self._prefetching = True self._prefetch_done = False - self._prefetch_reads.extend(chunks) t = threading.Thread(target=self._prefetch_thread, args=(chunks,)) t.setDaemon(True) @@ -457,9 +459,11 @@ class SFTPFile (BufferedFile): # do these read requests in a temporary thread because there may be # a lot of them, so it may block. for offset, length in chunks: - self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length)) + with self._prefetch_lock: + num = self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length)) + self._prefetch_extents[num] = (offset, length) - def _async_response(self, t, msg): + def _async_response(self, t, msg, num): if t == CMD_STATUS: # save exception and re-raise it on next file operation try: @@ -470,10 +474,12 @@ class SFTPFile (BufferedFile): if t != CMD_DATA: raise SFTPError('Expected data') data = msg.get_string() - offset, length = self._prefetch_reads.pop(0) - self._prefetch_data[offset] = data - if len(self._prefetch_reads) == 0: - self._prefetch_done = True + with self._prefetch_lock: + offset, length = self._prefetch_extents[num] + self._prefetch_data[offset] = data + del self._prefetch_extents[num] + if len(self._prefetch_extents) == 0: + self._prefetch_done = True def _check_exception(self): "if there's a saved exception, raise & clear it" diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 1b4684ac..47946a27 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,20 +2,23 @@ Changelog ========= -* :bug:`193` (and its attentant PRs :issue:`230` & :issue:`253`): Fix SSH agent +* :bug:`34` (PR :issue:`35`) Fix SFTP prefetching incompatibility with some + SFTP servers regarding request/response ordering. Thanks to Richard + Kettlewell for catch & patch. +* :bug:`193` (and its attentant PRs :issue:`230` & :issue:`253`) Fix SSH agent problems present on Windows. Thanks to David Hobbs for initial report and to Aarni Koskela & Olle Lundberg for the patches. * :release:`1.12.1 <2014-01-08>` -* :release:`1.11.3 <2014-01-08>` 176 -* :release:`1.10.5 <2014-01-08>` 176 -* :bug:`225` Note ecdsa requirement in README. Thanks to Amaury Rodriguez for - the catch. +* :release:`1.11.3 <2014-01-08>` +* :release:`1.10.5 <2014-01-08>` +* :bug:`225 (1.12+)` Note ecdsa requirement in README. Thanks to Amaury + Rodriguez for the catch. * :bug:`176` Fix AttributeError bugs in known_hosts file (re)loading. Thanks to Nathan Scowcroft for the patch & Martin Blumenstingl for the initial test case. * :release:`1.12.0 <2013-09-27>` * :release:`1.11.2 <2013-09-27>` -* :release:`1.10.4 <2013-09-27>` 199, 200, 179 +* :release:`1.10.4 <2013-09-27>` * :feature:`152` Add tentative support for ECDSA keys. *This adds the ecdsa module as a new dependency of Paramiko.* The module is available at [warner/python-ecdsa on Github](https://github.com/warner/python-ecdsa) and @@ -28,9 +31,9 @@ Changelog * :feature:`136` Add server-side support for the SSH protocol's 'env' command. Thanks to Benjamin Pollack for the patch. -* :bug:`156` Fix potential deadlock condition when using Channel objects as - sockets (e.g. when using SSH gatewaying). Thanks to Steven Noonan and Frank - Arnold for catch & patch. +* :bug:`156 (1.11+)` Fix potential deadlock condition when using Channel + objects as sockets (e.g. when using SSH gatewaying). Thanks to Steven Noonan + and Frank Arnold for catch & patch. * :bug:`179` Fix a missing variable causing errors when an ssh_config file has a non-default AddressFamily set. Thanks to Ed Marshall & Tomaz Muraus for catch & patch. @@ -1,7 +1,7 @@ from os.path import join -from invoke import Collection -from invocations import docs as _docs, testing +from invoke import Collection, ctask as task +from invocations import docs as _docs d = 'sites' @@ -20,4 +20,15 @@ www = Collection.from_module(_docs, name='www', config={ 'sphinx.target': join(path, '_build'), }) -ns = Collection(testing.test, docs=docs, www=www) + +# Until we move to spec-based testing +@task +def test(ctx): + ctx.run("python test.py --verbose") + +@task +def coverage(ctx): + ctx.run("coverage run --source=paramiko test.py --verbose") + + +ns = Collection(test, coverage, docs=docs, www=www) |