From 3fce8abf68f386d18f2fad9f086e0d436af57b3a Mon Sep 17 00:00:00 2001 From: Antoine Brenner Date: Wed, 16 Apr 2014 21:58:03 +0200 Subject: BufferedFile.read() now returns byte strings instead of text strings It is the right thing to do since we have no idea what encoding the file is in, or even if the file is text data. BufferedFile.readline() is unchanged and returns text strings assuming the file is utf-8 encoded. This should fix the following issue: http://comments.gmane.org/gmane.comp.sysutils.backup.obnam/252 Antoine Brenner Conflicts: sites/www/changelog.rst --- tests/test_file.py | 45 +++++++++++++++++++++++++-------------------- tests/test_sftp.py | 10 +++++----- tests/test_sftp_big.py | 4 ++-- 3 files changed, 32 insertions(+), 27 deletions(-) (limited to 'tests') diff --git a/tests/test_file.py b/tests/test_file.py index e11d7fd5..c6edd7af 100755 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -53,7 +53,7 @@ class BufferedFileTest (unittest.TestCase): def test_1_simple(self): f = LoopbackFile('r') try: - f.write('hi') + f.write(b'hi') self.assertTrue(False, 'no exception on write to read-only file') except: pass @@ -69,7 +69,7 @@ class BufferedFileTest (unittest.TestCase): def test_2_readline(self): f = LoopbackFile('r+U') - f.write('First line.\nSecond line.\r\nThird line.\nFinal line non-terminated.') + f.write(b'First line.\nSecond line.\r\nThird line.\nFinal line non-terminated.') self.assertEqual(f.readline(), 'First line.\n') # universal newline mode should convert this linefeed: self.assertEqual(f.readline(), 'Second line.\n') @@ -93,9 +93,9 @@ class BufferedFileTest (unittest.TestCase): try to trick the linefeed detector. """ f = LoopbackFile('r+U') - f.write('First line.\r') + f.write(b'First line.\r') self.assertEqual(f.readline(), 'First line.\n') - f.write('\nSecond.\r\n') + f.write(b'\nSecond.\r\n') self.assertEqual(f.readline(), 'Second.\n') f.close() self.assertEqual(f.newlines, crlf) @@ -105,7 +105,7 @@ class BufferedFileTest (unittest.TestCase): verify that write buffering is on. """ f = LoopbackFile('r+', 1) - f.write('Complete line.\nIncomplete line.') + f.write(b'Complete line.\nIncomplete line.') self.assertEqual(f.readline(), 'Complete line.\n') self.assertEqual(f.readline(), '') f.write('..\n') @@ -118,12 +118,12 @@ class BufferedFileTest (unittest.TestCase): """ f = LoopbackFile('r+', 512) f.write('Not\nquite\n512 bytes.\n') - self.assertEqual(f.read(1), '') + self.assertEqual(f.read(1), b'') f.flush() - self.assertEqual(f.read(5), 'Not\nq') - self.assertEqual(f.read(10), 'uite\n512 b') - self.assertEqual(f.read(9), 'ytes.\n') - self.assertEqual(f.read(3), '') + self.assertEqual(f.read(5), b'Not\nq') + self.assertEqual(f.read(10), b'uite\n512 b') + self.assertEqual(f.read(9), b'ytes.\n') + self.assertEqual(f.read(3), b'') f.close() def test_6_buffering(self): @@ -131,12 +131,12 @@ class BufferedFileTest (unittest.TestCase): verify that flushing happens automatically on buffer crossing. """ f = LoopbackFile('r+', 16) - f.write('Too small.') - self.assertEqual(f.read(4), '') - f.write(' ') - self.assertEqual(f.read(4), '') - f.write('Enough.') - self.assertEqual(f.read(20), 'Too small. Enough.') + f.write(b'Too small.') + self.assertEqual(f.read(4), b'') + f.write(b' ') + self.assertEqual(f.read(4), b'') + f.write(b'Enough.') + self.assertEqual(f.read(20), b'Too small. Enough.') f.close() def test_7_read_all(self): @@ -144,9 +144,14 @@ class BufferedFileTest (unittest.TestCase): verify that read(-1) returns everything left in the file. """ f = LoopbackFile('r+', 16) - f.write('The first thing you need to do is open your eyes. ') - f.write('Then, you need to close them again.\n') + f.write(b'The first thing you need to do is open your eyes. ') + f.write(b'Then, you need to close them again.\n') s = f.read(-1) - self.assertEqual(s, 'The first thing you need to do is open your eyes. Then, you ' + - 'need to close them again.\n') + self.assertEqual(s, b'The first thing you need to do is open your eyes. Then, you ' + + b'need to close them again.\n') f.close() + +if __name__ == '__main__': + from unittest import main + main() + diff --git a/tests/test_sftp.py b/tests/test_sftp.py index e0534eb0..720b8215 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -405,7 +405,7 @@ class SFTPTest (unittest.TestCase): self.assertEqual(sftp.stat(FOLDER + '/testing.txt').st_size, 13) with sftp.open(FOLDER + '/testing.txt', 'r') as f: data = f.read(20) - self.assertEqual(data, 'hello kiddy.\n') + self.assertEqual(data, b'hello kiddy.\n') finally: sftp.remove(FOLDER + '/testing.txt') @@ -466,8 +466,8 @@ class SFTPTest (unittest.TestCase): f.write('?\n') with sftp.open(FOLDER + '/happy.txt', 'r') as f: - self.assertEqual(f.readline(), 'full line?\n') - self.assertEqual(f.read(7), 'partial') + self.assertEqual(f.readline(), u'full line?\n') + self.assertEqual(f.read(7), b'partial') finally: try: sftp.remove(FOLDER + '/happy.txt') @@ -662,8 +662,8 @@ class SFTPTest (unittest.TestCase): fd, localname = mkstemp() os.close(fd) - text = 'All I wanted was a plastic bunny rabbit.\n' - with open(localname, 'w') as f: + text = b'All I wanted was a plastic bunny rabbit.\n' + with open(localname, 'wb') as f: f.write(text) saved_progress = [] diff --git a/tests/test_sftp_big.py b/tests/test_sftp_big.py index 521fbdc8..abed27b8 100644 --- a/tests/test_sftp_big.py +++ b/tests/test_sftp_big.py @@ -85,7 +85,7 @@ class BigSFTPTest (unittest.TestCase): write a 1MB file with no buffering. """ sftp = get_sftp() - kblob = (1024 * 'x') + kblob = (1024 * b'x') start = time.time() try: with sftp.open('%s/hongry.txt' % FOLDER, 'w') as f: @@ -231,7 +231,7 @@ class BigSFTPTest (unittest.TestCase): without using it, to verify that paramiko doesn't get confused. """ sftp = get_sftp() - kblob = (1024 * 'x') + kblob = (1024 * b'x') try: with sftp.open('%s/hongry.txt' % FOLDER, 'w') as f: f.set_pipelined(True) -- cgit v1.2.3 From 6f4c159b052eae52def9fde0ddcbb7c864ef6592 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 24 Apr 2014 10:25:37 -0700 Subject: Merge updated a01e449 from al-tonio --- paramiko/file.py | 8 +++++--- tests/test_sftp.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/paramiko/file.py b/paramiko/file.py index 70243e40..856cc101 100644 --- a/paramiko/file.py +++ b/paramiko/file.py @@ -124,9 +124,11 @@ class BufferedFile (object): file first). If the ``size`` argument is negative or omitted, read all the remaining data in the file. - ``'b'`` mode flag is ignored (``self.FLAG_BINARY`` in ``self._flags``), - because SSH treats all files as binary, since we have no idea what - encoding the file is in, or even if the file is text data. + .. note:: + ``'b'`` mode flag is ignored (``self.FLAG_BINARY`` in + ``self._flags``), because SSH treats all files as binary, since we + have no idea what encoding the file is in, or even if the file is + text data. :param int size: maximum number of bytes to read :return: diff --git a/tests/test_sftp.py b/tests/test_sftp.py index 720b8215..c70d0cde 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -67,6 +67,18 @@ liver insulin receptors. Their sensitivity to insulin is, however, similarly decreased compared with chicken. ''' + +# Here is how unicode characters are encoded over 1 to 6 bytes in utf-8 +# U-00000000 - U-0000007F: 0xxxxxxx +# U-00000080 - U-000007FF: 110xxxxx 10xxxxxx +# U-00000800 - U-0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx +# U-00010000 - U-001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx +# U-00200000 - U-03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx +# U-04000000 - U-7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx +# Note that: hex(int('11000011',2)) == '0xc3' +# Thus, the following 2-bytes sequence is not valid utf8: "invalid continuation byte" +NON_UTF8_DATA = b'\xC3\xC3' + FOLDER = os.environ.get('TEST_FOLDER', 'temp-testing000') sftp = None @@ -466,7 +478,7 @@ class SFTPTest (unittest.TestCase): f.write('?\n') with sftp.open(FOLDER + '/happy.txt', 'r') as f: - self.assertEqual(f.readline(), u'full line?\n') + self.assertEqual(f.readline(), u('full line?\n')) self.assertEqual(f.read(7), b'partial') finally: try: @@ -747,6 +759,23 @@ class SFTPTest (unittest.TestCase): sftp.remove(FOLDER + '/test%file') + def test_O_non_utf8_data(self): + """Test write() and read() of non utf8 data""" + try: + with sftp.open('%s/nonutf8data' % FOLDER, 'w') as f: + f.write(NON_UTF8_DATA) + with sftp.open('%s/nonutf8data' % FOLDER, 'r') as f: + data = f.read() + self.assertEqual(data, NON_UTF8_DATA) + with sftp.open('%s/nonutf8data' % FOLDER, 'wb') as f: + f.write(NON_UTF8_DATA) + with sftp.open('%s/nonutf8data' % FOLDER, 'rb') as f: + data = f.read() + self.assertEqual(data, NON_UTF8_DATA) + finally: + sftp.remove('%s/nonutf8data' % FOLDER) + + if __name__ == '__main__': SFTPTest.init_loopback() # logging is required by test_N_file_with_percent -- cgit v1.2.3 From c7c1a24e3023a45cf6713e553c176e42a71a6d3d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 24 Apr 2014 10:26:33 -0700 Subject: Fix some trailing whitespace --- paramiko/file.py | 2 +- tests/test_sftp.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/paramiko/file.py b/paramiko/file.py index 856cc101..3ebcfa39 100644 --- a/paramiko/file.py +++ b/paramiko/file.py @@ -124,7 +124,7 @@ class BufferedFile (object): file first). If the ``size`` argument is negative or omitted, read all the remaining data in the file. - .. note:: + .. note:: ``'b'`` mode flag is ignored (``self.FLAG_BINARY`` in ``self._flags``), because SSH treats all files as binary, since we have no idea what encoding the file is in, or even if the file is diff --git a/tests/test_sftp.py b/tests/test_sftp.py index c70d0cde..2b6aa3b6 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -69,8 +69,8 @@ decreased compared with chicken. # Here is how unicode characters are encoded over 1 to 6 bytes in utf-8 -# U-00000000 - U-0000007F: 0xxxxxxx -# U-00000080 - U-000007FF: 110xxxxx 10xxxxxx +# U-00000000 - U-0000007F: 0xxxxxxx +# U-00000080 - U-000007FF: 110xxxxx 10xxxxxx # U-00000800 - U-0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx # U-00010000 - U-001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx # U-00200000 - U-03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx -- cgit v1.2.3 From c3ba66b90e9da0cce9c88fbc45894fde492edfd0 Mon Sep 17 00:00:00 2001 From: Jelmer Vernooij Date: Sat, 5 Jul 2014 23:51:38 +0200 Subject: Support passing in "buffer" objects again where bytestrings are expected. This fixes bzr's use of paramiko. Fixes issue #343/#285. --- paramiko/py3compat.py | 4 ++++ tests/test_file.py | 10 ++++++++++ 2 files changed, 14 insertions(+) (limited to 'tests') diff --git a/paramiko/py3compat.py b/paramiko/py3compat.py index 8842b988..57c096b2 100644 --- a/paramiko/py3compat.py +++ b/paramiko/py3compat.py @@ -39,6 +39,8 @@ if PY2: return s elif isinstance(s, unicode): return s.encode(encoding) + elif isinstance(s, buffer): + return s else: raise TypeError("Expected unicode or bytes, got %r" % s) @@ -49,6 +51,8 @@ if PY2: return s.decode(encoding) elif isinstance(s, unicode): return s + elif isinstance(s, buffer): + return s.decode(encoding) else: raise TypeError("Expected unicode or bytes, got %r" % s) diff --git a/tests/test_file.py b/tests/test_file.py index c6edd7af..22a34aca 100755 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -23,6 +23,7 @@ Some unit tests for the BufferedFile abstraction. import unittest from paramiko.file import BufferedFile from paramiko.common import linefeed_byte, crlf, cr_byte +import sys class LoopbackFile (BufferedFile): @@ -151,6 +152,15 @@ class BufferedFileTest (unittest.TestCase): b'need to close them again.\n') f.close() + def test_8_buffering(self): + """ + verify that buffered objects can be written + """ + if sys.version_info[0] == 2: + f = LoopbackFile('r+', 16) + f.write(buffer(b'Too small.')) + f.close() + if __name__ == '__main__': from unittest import main main() -- cgit v1.2.3 From 12e752d11aa22ee6ba959115725e386ca779c1cb Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 25 Aug 2014 22:44:54 -0700 Subject: Rework re #239 to work off post-1.13 codebase. Closes #239 --- paramiko/config.py | 2 +- sites/www/changelog.rst | 2 ++ tests/test_util.py | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/paramiko/config.py b/paramiko/config.py index 77fa13d7..f650ee24 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -55,7 +55,7 @@ class SSHConfig (object): """ host = {"host": ['*'], "config": {}} for line in file_obj: - line = line.rstrip('\n').lstrip() + line = line.rstrip('\r\n').lstrip() if (line == '') or (line[0] == '#'): continue if '=' in line: diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index cdd513f4..903e6378 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,6 +2,8 @@ Changelog ========= +* :bug:`239` Add Windows-style CRLF support to SSH config file parsing. Props + to Christopher Swenson. * :support:`229` Fix a couple of incorrectly-copied docstrings' ``.. versionadded::`` RST directives. Thanks to Aarni Koskela for the catch. * :support:`169 backported` Minor refactor of diff --git a/tests/test_util.py b/tests/test_util.py index 6bde4045..ecf8db72 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -338,3 +338,9 @@ IdentityFile something_%l_using_fqdn """ config = paramiko.util.parse_ssh_config(StringIO(test_config)) assert config.lookup('meh') # will die during lookup() if bug regresses + + def test_13_config_dos_crlf_succeeds(self): + config_file = StringIO("host abcqwerty\r\nHostName 127.0.0.1\r\n") + config = paramiko.SSHConfig() + config.parse(config_file) + self.assertEqual(config.lookup("abcqwerty")["hostname"], "127.0.0.1") -- cgit v1.2.3 From f4de3307b1133c0b207f5af40227c64a8cb5389d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 5 Sep 2014 11:16:52 -0700 Subject: 80-col tweaks --- paramiko/sftp_client.py | 14 +++++++++----- tests/test_sftp.py | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 39f08fbc..ee25aa6a 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -222,8 +222,9 @@ class SFTPClient(BaseSFTP): nums = list() while True: try: - # Send out a bunch of readdir requests so that we can read the responses later on - # Section 6.7 of the SSH file transfer RFC explains this + # Send out a bunch of readdir requests so that we can read the + # responses later on Section 6.7 of the SSH file transfer RFC + # explains this # http://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt for i in range(read_ahead_requests): num = self._async_request(type(None), CMD_READDIR, handle) @@ -232,8 +233,10 @@ class SFTPClient(BaseSFTP): # For each of our sent requests # Read and parse the corresponding packets - # If we're at the end of our queued requests, then fire off some more requests - # Exit the loop when we've reached the end of the directory handle + # If we're at the end of our queued requests, then fire off + # some more requests + # Exit the loop when we've reached the end of the directory + # handle for num in nums: t, pkt_data = self._read_packet() msg = Message(pkt_data) @@ -245,7 +248,8 @@ class SFTPClient(BaseSFTP): for i in range(count): filename = msg.get_string() longname = msg.get_string() - attr = SFTPAttributes._from_msg(msg, filename, longname) + attr = SFTPAttributes._from_msg( + msg, filename, longname) if (filename != '.') and (filename != '..'): yield attr diff --git a/tests/test_sftp.py b/tests/test_sftp.py index 2b6aa3b6..26929bdb 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -279,8 +279,8 @@ class SFTPTest (unittest.TestCase): def test_7_listdir(self): """ - verify that a folder can be created, a bunch of files can be placed in it, - and those files show up in sftp.listdir. + verify that a folder can be created, a bunch of files can be placed in + it, and those files show up in sftp.listdir. """ try: sftp.open(FOLDER + '/duck.txt', 'w').close() -- cgit v1.2.3 From e5fc6a6ecc064f6b2b9862a9405b27f40385f821 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 5 Sep 2014 11:38:27 -0700 Subject: Add quick test re #131 --- tests/test_sftp.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'tests') diff --git a/tests/test_sftp.py b/tests/test_sftp.py index 26929bdb..1ae9781d 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -298,6 +298,26 @@ class SFTPTest (unittest.TestCase): sftp.remove(FOLDER + '/fish.txt') sftp.remove(FOLDER + '/tertiary.py') + def test_7_5_listdir_iter(self): + """ + listdir_iter version of above test + """ + try: + sftp.open(FOLDER + '/duck.txt', 'w').close() + sftp.open(FOLDER + '/fish.txt', 'w').close() + sftp.open(FOLDER + '/tertiary.py', 'w').close() + + x = [x.filename for x in sftp.listdir_iter(FOLDER)] + self.assertEqual(len(x), 3) + self.assertTrue('duck.txt' in x) + self.assertTrue('fish.txt' in x) + self.assertTrue('tertiary.py' in x) + self.assertTrue('random' not in x) + finally: + sftp.remove(FOLDER + '/duck.txt') + sftp.remove(FOLDER + '/fish.txt') + sftp.remove(FOLDER + '/tertiary.py') + def test_8_setstat(self): """ verify that the setstat functions (chown, chmod, utime, truncate) work. -- cgit v1.2.3