Merge branch '1.16' into 1.17

3 files changed, 28 insertions, 17 deletions

diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py
index daaae3ef..0df94389 100644
--- a/paramiko/sftp_client.py
+++ b/paramiko/sftp_client.py
@@ -748,10 +748,10 @@ class SFTPClient(BaseSFTP, ClosingContextManager):
                     raise Exception('unknown type for %r type %r' % (item, type(item)))
             num = self.request_number
             self._expecting[num] = fileobj
-            self._send_packet(t, msg)
             self.request_number += 1
         finally:
             self._lock.release()
+        self._send_packet(t, msg)
         return num
 
     def _read_response(self, waitfor=None):
@@ -762,15 +762,19 @@ class SFTPClient(BaseSFTP, ClosingContextManager):
                 raise SSHException('Server connection dropped: %s' % str(e))
             msg = Message(data)
             num = msg.get_int()
-            if num not in self._expecting:
-                # might be response for a file that was closed before responses came back
-                self._log(DEBUG, 'Unexpected response #%d' % (num,))
-                if waitfor is None:
-                    # just doing a single check
-                    break
-                continue
-            fileobj = self._expecting[num]
-            del self._expecting[num]
+            self._lock.acquire()
+            try:
+                if num not in self._expecting:
+                    # might be response for a file that was closed before responses came back
+                    self._log(DEBUG, 'Unexpected response #%d' % (num,))
+                    if waitfor is None:
+                        # just doing a single check
+                        break
+                    continue
+                fileobj = self._expecting[num]
+                del self._expecting[num]
+            finally:
+                self._lock.release()
             if num == waitfor:
                 # synchronous
                 if t == CMD_STATUS:
diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py
index 3b584dbe..f90fa068 100644
--- a/paramiko/sftp_file.py
+++ b/paramiko/sftp_file.py
@@ -469,8 +469,8 @@ class SFTPFile (BufferedFile):
         # do these read requests in a temporary thread because there may be
         # a lot of them, so it may block.
         for offset, length in chunks:
+            num = self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length))
             with self._prefetch_lock:
-                num = self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length))
                 self._prefetch_extents[num] = (offset, length)
 
     def _async_response(self, t, msg, num):
@@ -484,12 +484,16 @@ class SFTPFile (BufferedFile):
         if t != CMD_DATA:
             raise SFTPError('Expected data')
         data = msg.get_string()
-        with self._prefetch_lock:
-            offset, length = self._prefetch_extents[num]
-            self._prefetch_data[offset] = data
-            del self._prefetch_extents[num]
-            if len(self._prefetch_extents) == 0:
-                self._prefetch_done = True
+        while True:
+            with self._prefetch_lock:
+                # spin if in race with _prefetch_thread
+                if num in self._prefetch_extents:
+                    offset, length = self._prefetch_extents[num]
+                    self._prefetch_data[offset] = data
+                    del self._prefetch_extents[num]
+                    if len(self._prefetch_extents) == 0:
+                        self._prefetch_done = True
+                    break
 
     def _check_exception(self):
         """if there's a saved exception, raise & clear it"""
diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst
index 8caaba22..5c11a052 100644
--- a/sites/www/changelog.rst
+++ b/sites/www/changelog.rst
@@ -3,6 +3,9 @@ Changelog
 =========
 
 * :release:`1.17.0 <2016-04-25>`
+* :bug:`577` (via :issue:`578`; should also fix :issue:`718`, :issue:`560`) Fix
+  stalled/hung SFTP downloads by cleaning up some threading lock issues. Thanks
+  to Stephen C. Pope for the patch.
 * :bug:`716` Fix a Python 3 compatibility issue when handling two-factor
   authentication. Thanks to Mateusz Kowalski for the catch & original patch.
 * :support:`729 backported` Clean up ``setup.py`` to always use ``setuptools``,