Get sftp-big tests apparently passing.

Very slowly. Pretty sure we will want to retain module-level fixtures for that stuff. heh.

2 files changed, 78 insertions, 119 deletions

diff --git a/tests/test_sftp.py b/tests/test_sftp.py
index d3760bf7..432d3e40 100755
--- a/tests/test_sftp.py
+++ b/tests/test_sftp.py
@@ -94,65 +94,36 @@ unicode_folder = eval(compile(r"u'\u00fcnic\u00f8de'" if PY2 else r"'\u00fcnic\u
 utf8_folder = b'/\xc3\xbcnic\xc3\xb8\x64\x65'
 
 
-def get_sftp():
-    global sftp
-    return sftp
+# TODO: turn into a pytest fixture; consider making it module or session-global
+# to mimic old behavior (though that still feels unclean to me...)
+def make_loopback_sftp():
+    """
+    Set up an in-memory SFTP server.
+
+    :returns:
+        A 2-tuple of the resulting SFTPClient (for tests that just care about a
+        'default' client) and Transport (for testing instantiation _of_
+        SFTPClient itself, which can take an existing transport object.)
+    """
+    socks = LoopSocket()
+    sockc = LoopSocket()
+    sockc.link(socks)
+    tc = paramiko.Transport(sockc)
+    ts = paramiko.Transport(socks)
+
+    host_key = paramiko.RSAKey.from_private_key_file(_support('test_rsa.key'))
+    ts.add_server_key(host_key)
+    event = threading.Event()
+    server = StubServer()
+    ts.set_subsystem_handler('sftp', paramiko.SFTPServer, StubSFTPServer)
+    ts.start_server(event, server)
+    tc.connect(username='slowdive', password='pygmalion')
+    event.wait(1.0)
+
+    return paramiko.SFTP.from_transport(tc), tc
 
 
 class SFTPTest (unittest.TestCase):
-    @staticmethod
-    def init(hostname, username, keyfile, passwd):
-        global sftp, tc
-
-        t = paramiko.Transport(hostname)
-        tc = t
-        try:
-            key = paramiko.RSAKey.from_private_key_file(keyfile, passwd)
-        except paramiko.PasswordRequiredException:
-            sys.stderr.write('\n\nparamiko.RSAKey.from_private_key_file REQUIRES PASSWORD.\n')
-            sys.stderr.write('You have two options:\n')
-            sys.stderr.write('* Use the "-K" option to point to a different (non-password-protected)\n')
-            sys.stderr.write('  private key file.\n')
-            sys.stderr.write('* Use the "-P" option to provide the password needed to unlock this private\n')
-            sys.stderr.write('  key.\n')
-            sys.stderr.write('\n')
-            sys.exit(1)
-        try:
-            t.connect(username=username, pkey=key)
-        except paramiko.SSHException:
-            t.close()
-            sys.stderr.write('\n\nparamiko.Transport.connect FAILED.\n')
-            sys.stderr.write('There are several possible reasons why it might fail so quickly:\n\n')
-            sys.stderr.write('* The host to connect to (%s) is not a valid SSH server.\n' % hostname)
-            sys.stderr.write('  (Use the "-H" option to change the host.)\n')
-            sys.stderr.write('* The username to auth as (%s) is invalid.\n' % username)
-            sys.stderr.write('  (Use the "-U" option to change the username.)\n')
-            sys.stderr.write('* The private key given (%s) is not accepted by the server.\n' % keyfile)
-            sys.stderr.write('  (Use the "-K" option to provide a different key file.)\n')
-            sys.stderr.write('\n')
-            sys.exit(1)
-        sftp = paramiko.SFTP.from_transport(t)
-
-    @staticmethod
-    def init_loopback():
-        global sftp, tc
-
-        socks = LoopSocket()
-        sockc = LoopSocket()
-        sockc.link(socks)
-        tc = paramiko.Transport(sockc)
-        ts = paramiko.Transport(socks)
-
-        host_key = paramiko.RSAKey.from_private_key_file(_support('test_rsa.key'))
-        ts.add_server_key(host_key)
-        event = threading.Event()
-        server = StubServer()
-        ts.set_subsystem_handler('sftp', paramiko.SFTPServer, StubSFTPServer)
-        ts.start_server(event, server)
-        tc.connect(username='slowdive', password='pygmalion')
-        event.wait(1.0)
-
-        sftp = paramiko.SFTP.from_transport(tc)
 
     @staticmethod
     def set_big_file_test(onoff):
@@ -160,6 +131,11 @@ class SFTPTest (unittest.TestCase):
         g_big_file_test = onoff
 
     def setUp(self):
+        # TODO: I think this was created to allow working around litter from
+        # broken previous test runs, and/or to allow concurrent testing? Either
+        # way I hate it and it should die ASAP in favor of pytest fixtures or
+        # similar things that do try/finally.
+        # TODO: also, for god's sake remove the global in favor of self.
         global FOLDER
         for i in range(1000):
             FOLDER = FOLDER[:-3] + '%03d' % i
diff --git a/tests/test_sftp_big.py b/tests/test_sftp_big.py
index 580ba64e..ef12b05c 100644
--- a/tests/test_sftp_big.py
+++ b/tests/test_sftp_big.py
@@ -32,7 +32,7 @@ import unittest
 
 from paramiko.common import o660
 
-from .test_sftp import get_sftp
+from .test_sftp import make_loopback_sftp
 
 
 FOLDER = os.environ.get('TEST_FOLDER', 'temp-testing000')
@@ -42,43 +42,42 @@ class BigSFTPTest (unittest.TestCase):
 
     def setUp(self):
         global FOLDER
-        sftp = get_sftp()
+        self.sftp, _ = make_loopback_sftp()
+        # TODO: same TODOs as in test_sftp.py re: not doing this awful crap
         for i in range(1000):
             FOLDER = FOLDER[:-3] + '%03d' % i
             try:
-                sftp.mkdir(FOLDER)
+                self.sftp.mkdir(FOLDER)
                 break
             except (IOError, OSError):
                 pass
 
     def tearDown(self):
-        sftp = get_sftp()
-        sftp.rmdir(FOLDER)
+        self.sftp.rmdir(FOLDER)
 
     def test_1_lots_of_files(self):
         """
         create a bunch of files over the same session.
         """
-        sftp = get_sftp()
         numfiles = 100
         try:
             for i in range(numfiles):
-                with sftp.open('%s/file%d.txt' % (FOLDER, i), 'w', 1) as f:
+                with self.sftp.open('%s/file%d.txt' % (FOLDER, i), 'w', 1) as f:
                     f.write('this is file #%d.\n' % i)
-                sftp.chmod('%s/file%d.txt' % (FOLDER, i), o660)
+                self.sftp.chmod('%s/file%d.txt' % (FOLDER, i), o660)
 
             # now make sure every file is there, by creating a list of filenmes
             # and reading them in random order.
             numlist = list(range(numfiles))
             while len(numlist) > 0:
                 r = numlist[random.randint(0, len(numlist) - 1)]
-                with sftp.open('%s/file%d.txt' % (FOLDER, r)) as f:
+                with self.sftp.open('%s/file%d.txt' % (FOLDER, r)) as f:
                     self.assertEqual(f.readline(), 'this is file #%d.\n' % r)
                 numlist.remove(r)
         finally:
             for i in range(numfiles):
                 try:
-                    sftp.remove('%s/file%d.txt' % (FOLDER, i))
+                    self.sftp.remove('%s/file%d.txt' % (FOLDER, i))
                 except:
                     pass
 
@@ -86,23 +85,22 @@ class BigSFTPTest (unittest.TestCase):
         """
         write a 1MB file with no buffering.
         """
-        sftp = get_sftp()
         kblob = (1024 * b'x')
         start = time.time()
         try:
-            with sftp.open('%s/hongry.txt' % FOLDER, 'w') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'w') as f:
                 for n in range(1024):
                     f.write(kblob)
                     if n % 128 == 0:
                         sys.stderr.write('.')
             sys.stderr.write(' ')
 
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertEqual(self.sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
             end = time.time()
             sys.stderr.write('%ds ' % round(end - start))
             
             start = time.time()
-            with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'r') as f:
                 for n in range(1024):
                     data = f.read(1024)
                     self.assertEqual(data, kblob)
@@ -110,17 +108,16 @@ class BigSFTPTest (unittest.TestCase):
             end = time.time()
             sys.stderr.write('%ds ' % round(end - start))
         finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
+            self.sftp.remove('%s/hongry.txt' % FOLDER)
 
     def test_3_big_file_pipelined(self):
         """
         write a 1MB file, with no linefeeds, using pipelining.
         """
-        sftp = get_sftp()
         kblob = bytes().join([struct.pack('>H', n) for n in range(512)])
         start = time.time()
         try:
-            with sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
                 f.set_pipelined(True)
                 for n in range(1024):
                     f.write(kblob)
@@ -128,12 +125,12 @@ class BigSFTPTest (unittest.TestCase):
                         sys.stderr.write('.')
             sys.stderr.write(' ')
 
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertEqual(self.sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
             end = time.time()
             sys.stderr.write('%ds ' % round(end - start))
             
             start = time.time()
-            with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
                 file_size = f.stat().st_size
                 f.prefetch(file_size)
 
@@ -153,13 +150,12 @@ class BigSFTPTest (unittest.TestCase):
             end = time.time()
             sys.stderr.write('%ds ' % round(end - start))
         finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
+            self.sftp.remove('%s/hongry.txt' % FOLDER)
 
     def test_4_prefetch_seek(self):
-        sftp = get_sftp()
         kblob = bytes().join([struct.pack('>H', n) for n in range(512)])
         try:
-            with sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
                 f.set_pipelined(True)
                 for n in range(1024):
                     f.write(kblob)
@@ -167,13 +163,13 @@ class BigSFTPTest (unittest.TestCase):
                         sys.stderr.write('.')
             sys.stderr.write(' ')
             
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertEqual(self.sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
             
             start = time.time()
             k2blob = kblob + kblob
             chunk = 793
             for i in range(10):
-                with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
+                with self.sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
                     file_size = f.stat().st_size
                     f.prefetch(file_size)
                     base_offset = (512 * 1024) + 17 * random.randint(1000, 2000)
@@ -190,13 +186,12 @@ class BigSFTPTest (unittest.TestCase):
             end = time.time()
             sys.stderr.write('%ds ' % round(end - start))
         finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
+            self.sftp.remove('%s/hongry.txt' % FOLDER)
 
     def test_5_readv_seek(self):
-        sftp = get_sftp()
         kblob = bytes().join([struct.pack('>H', n) for n in range(512)])
         try:
-            with sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
                 f.set_pipelined(True)
                 for n in range(1024):
                     f.write(kblob)
@@ -204,13 +199,13 @@ class BigSFTPTest (unittest.TestCase):
                         sys.stderr.write('.')
             sys.stderr.write(' ')
 
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertEqual(self.sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
 
             start = time.time()
             k2blob = kblob + kblob
             chunk = 793
             for i in range(10):
-                with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
+                with self.sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
                     base_offset = (512 * 1024) + 17 * random.randint(1000, 2000)
                     # make a bunch of offsets and put them in random order
                     offsets = [base_offset + j * chunk for j in range(100)]
@@ -227,17 +222,16 @@ class BigSFTPTest (unittest.TestCase):
             end = time.time()
             sys.stderr.write('%ds ' % round(end - start))
         finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
+            self.sftp.remove('%s/hongry.txt' % FOLDER)
 
     def test_6_lots_of_prefetching(self):
         """
         prefetch a 1MB file a bunch of times, discarding the file object
         without using it, to verify that paramiko doesn't get confused.
         """
-        sftp = get_sftp()
         kblob = (1024 * b'x')
         try:
-            with sftp.open('%s/hongry.txt' % FOLDER, 'w') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'w') as f:
                 f.set_pipelined(True)
                 for n in range(1024):
                     f.write(kblob)
@@ -245,13 +239,13 @@ class BigSFTPTest (unittest.TestCase):
                         sys.stderr.write('.')
             sys.stderr.write(' ')
 
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertEqual(self.sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
 
             for i in range(10):
-                with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f:
+                with self.sftp.open('%s/hongry.txt' % FOLDER, 'r') as f:
                     file_size = f.stat().st_size
                     f.prefetch(file_size)
-            with sftp.open('%s/hongry.txt' % FOLDER, 'r') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'r') as f:
                 file_size = f.stat().st_size
                 f.prefetch(file_size)
                 for n in range(1024):
@@ -261,16 +255,15 @@ class BigSFTPTest (unittest.TestCase):
                         sys.stderr.write('.')
             sys.stderr.write(' ')
         finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
+            self.sftp.remove('%s/hongry.txt' % FOLDER)
     
     def test_7_prefetch_readv(self):
         """
         verify that prefetch and readv don't conflict with each other.
         """
-        sftp = get_sftp()
         kblob = bytes().join([struct.pack('>H', n) for n in range(512)])
         try:
-            with sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
                 f.set_pipelined(True)
                 for n in range(1024):
                     f.write(kblob)
@@ -278,9 +271,9 @@ class BigSFTPTest (unittest.TestCase):
                         sys.stderr.write('.')
             sys.stderr.write(' ')
             
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertEqual(self.sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
 
-            with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
                 file_size = f.stat().st_size
                 f.prefetch(file_size)
                 data = f.read(1024)
@@ -298,17 +291,16 @@ class BigSFTPTest (unittest.TestCase):
 
             sys.stderr.write(' ')
         finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
+            self.sftp.remove('%s/hongry.txt' % FOLDER)
     
     def test_8_large_readv(self):
         """
         verify that a very large readv is broken up correctly and still
         returned as a single blob.
         """
-        sftp = get_sftp()
         kblob = bytes().join([struct.pack('>H', n) for n in range(512)])
         try:
-            with sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'wb') as f:
                 f.set_pipelined(True)
                 for n in range(1024):
                     f.write(kblob)
@@ -316,9 +308,9 @@ class BigSFTPTest (unittest.TestCase):
                         sys.stderr.write('.')
             sys.stderr.write(' ')
 
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertEqual(self.sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
             
-            with sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'rb') as f:
                 data = list(f.readv([(23 * 1024, 128 * 1024)]))
                 self.assertEqual(1, len(data))
                 data = data[0]
@@ -326,52 +318,43 @@ class BigSFTPTest (unittest.TestCase):
             
             sys.stderr.write(' ')
         finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
+            self.sftp.remove('%s/hongry.txt' % FOLDER)
     
     def test_9_big_file_big_buffer(self):
         """
         write a 1MB file, with no linefeeds, and a big buffer.
         """
-        sftp = get_sftp()
         mblob = (1024 * 1024 * 'x')
         try:
-            with sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024) as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024) as f:
                 f.write(mblob)
 
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertEqual(self.sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
         finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
+            self.sftp.remove('%s/hongry.txt' % FOLDER)
     
     def test_A_big_file_renegotiate(self):
         """
         write a 1MB file, forcing key renegotiation in the middle.
         """
-        sftp = get_sftp()
-        t = sftp.sock.get_transport()
+        t = self.sftp.sock.get_transport()
         t.packetizer.REKEY_BYTES = 512 * 1024
         k32blob = (32 * 1024 * 'x')
         try:
-            with sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024) as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024) as f:
                 for i in range(32):
                     f.write(k32blob)
 
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertEqual(self.sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
             self.assertNotEqual(t.H, t.session_id)
             
             # try to read it too.
-            with sftp.open('%s/hongry.txt' % FOLDER, 'r', 128 * 1024) as f:
+            with self.sftp.open('%s/hongry.txt' % FOLDER, 'r', 128 * 1024) as f:
                 file_size = f.stat().st_size
                 f.prefetch(file_size)
                 total = 0
                 while total < 1024 * 1024:
                     total += len(f.read(32 * 1024))
         finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
+            self.sftp.remove('%s/hongry.txt' % FOLDER)
             t.packetizer.REKEY_BYTES = pow(2, 30)
-
-
-if __name__ == '__main__':
-    from tests.test_sftp import SFTPTest
-    SFTPTest.init_loopback()
-    from unittest import main
-    main()