Merge branch 'master' into gssapi-py3-support

Conflicts:
	dev-requirements.txt
	sites/www/changelog.rst

10 files changed, 103 insertions, 40 deletions

diff --git a/README b/README
index 31cc94e0..94aa3a9c 100644
--- a/README
+++ b/README
@@ -35,7 +35,7 @@ Requirements
 ------------
 
   - Python 2.6 or better <http://www.python.org/> - this includes Python
-    3.3 and higher as well.
+    3.2 and higher as well.
   - pycrypto 2.1 or better <https://www.dlitz.net/software/pycrypto/>
   - ecdsa 0.9 or better <https://pypi.python.org/pypi/ecdsa>
 
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 5207903f..e9052898 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -5,5 +5,6 @@ tox>=1.4,<1.5
 invoke>=0.7.0
 invocations>=0.5.0
 sphinx>=1.1.3
-alabaster>=0.4.0
-releases>=0.5.2
\ No newline at end of file
+alabaster>=0.6.0
+releases>=0.5.2
+wheel==0.23.0
diff --git a/paramiko/__init__.py b/paramiko/__init__.py
index f4ce937b..6d133c4b 100644
--- a/paramiko/__init__.py
+++ b/paramiko/__init__.py
@@ -23,7 +23,7 @@ if sys.version_info < (2, 6):
 
 
 __author__ = "Jeff Forcier <jeff@bitprophet.org>"
-__version__ = "1.13.0"
+__version__ = "1.14.0"
 __version_info__ = tuple([ int(d) for d in __version__.split(".") ])
 __license__ = "GNU Lesser General Public License (LGPL)"
 
diff --git a/paramiko/file.py b/paramiko/file.py
index f57aa79f..2238f0bf 100644
--- a/paramiko/file.py
+++ b/paramiko/file.py
@@ -124,9 +124,15 @@ class BufferedFile (object):
         file first).  If the ``size`` argument is negative or omitted, read all
         the remaining data in the file.
 
+        .. note::
+            ``'b'`` mode flag is ignored (``self.FLAG_BINARY`` in
+            ``self._flags``), because SSH treats all files as binary, since we
+            have no idea what encoding the file is in, or even if the file is
+            text data.
+
         :param int size: maximum number of bytes to read
         :return:
-            data read from the file (as a `str`), or an empty string if EOF was
+            data read from the file (as bytes), or an empty string if EOF was
             encountered immediately
         """
         if self._closed:
@@ -148,12 +154,12 @@ class BufferedFile (object):
                 result += new_data
                 self._realpos += len(new_data)
                 self._pos += len(new_data)
-            return result if self._flags & self.FLAG_BINARY else u(result)
+            return result 
         if size <= len(self._rbuffer):
             result = self._rbuffer[:size]
             self._rbuffer = self._rbuffer[size:]
             self._pos += len(result)
-            return result if self._flags & self.FLAG_BINARY else u(result)
+            return result 
         while len(self._rbuffer) < size:
             read_size = size - len(self._rbuffer)
             if self._flags & self.FLAG_BUFFERED:
@@ -169,7 +175,7 @@ class BufferedFile (object):
         result = self._rbuffer[:size]
         self._rbuffer = self._rbuffer[size:]
         self._pos += len(result)
-        return result if self._flags & self.FLAG_BINARY else u(result)
+        return result 
 
     def readline(self, size=None):
         """
@@ -186,8 +192,12 @@ class BufferedFile (object):
 
         :param int size: maximum length of returned string.
         :return:
-            next line of the file (`str`), or an empty string if the end of the
+            next line of the file, or an empty string if the end of the
             file has been reached.
+
+            If the file was opened in binary (``'b'``) mode: bytes are returned
+            Else: the encoding of the file is assumed to be UTF-8 and character
+            strings (`str`) are returned
         """
         # it's almost silly how complex this function is.
         if self._closed:
@@ -277,7 +287,8 @@ class BufferedFile (object):
         Set the file's current position, like stdio's ``fseek``.  Not all file
         objects support seeking.
 
-        .. note:: If a file is opened in append mode (``'a'`` or ``'a+'``), any seek
+        .. note::
+            If a file is opened in append mode (``'a'`` or ``'a+'``), any seek
             operations will be undone at the next write (as the file position
             will move back to the end of the file).
         
diff --git a/setup.py b/setup.py
index 4a858326..c0f1e579 100644
--- a/setup.py
+++ b/setup.py
@@ -56,7 +56,7 @@ if sys.platform == 'darwin':
 
 setup(
     name = "paramiko",
-    version = "1.13.0",
+    version = "1.14.0",
     description = "SSH2 protocol library",
     long_description = longdesc,
     author = "Jeff Forcier",
diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst
index 0680eb38..0de410c7 100644
--- a/sites/www/changelog.rst
+++ b/sites/www/changelog.rst
@@ -3,6 +3,23 @@ Changelog
 =========
 
 * :feature:`250` GSS-API / SSPI authenticated Diffie-Hellman Key Exchange and user authentication.
+* :release:`1.14.0 <2014-05-07>`
+* :release:`1.13.1 <2014-05-07>`
+* :release:`1.12.4 <2014-05-07>`
+* :release:`1.11.6 <2014-05-07>`
+* :bug:`-` `paramiko.file.BufferedFile.read` incorrectly returned text strings
+  after the Python 3 migration, despite bytes being more appropriate for file
+  contents (which may be binary or of an unknown encoding.) This has been
+  addressed.
+
+  .. note::
+      `paramiko.file.BufferedFile.readline` continues to return strings, not
+      bytes, as "lines" only make sense for textual data. It assumes UTF-8 by
+      default.
+
+  This should fix `this issue raised on the Obnam mailing list
+  <http://comments.gmane.org/gmane.comp.sysutils.backup.obnam/252>`_.  Thanks
+  to Antoine Brenner for the patch.
 * :bug:`-` Added self.args for exception classes. Used for unpickling. Related
   to (`Fabric #986 <https://github.com/fabric/fabric/issues/986>`_, `Fabric
   #714 <https://github.com/fabric/fabric/issues/714>`_). Thanks to Alex
diff --git a/sites/www/installing.rst b/sites/www/installing.rst
index 74c5c6e8..a28ce6cd 100644
--- a/sites/www/installing.rst
+++ b/sites/www/installing.rst
@@ -20,8 +20,8 @@ We currently support **Python 2.6, 2.7 and 3.3** (Python **3.2** should also
 work but has a less-strong compatibility guarantee from us.) Users on Python
 2.5 or older are urged to upgrade.
 
-Paramiko has two dependencies: the pure-Python ECDSA module `ecdsa`, and the
-PyCrypto C extension. `ecdsa` is easily installable from wherever you
+Paramiko has two dependencies: the pure-Python ECDSA module ``ecdsa``, and the
+PyCrypto C extension. ``ecdsa`` is easily installable from wherever you
 obtained Paramiko's package; PyCrypto may require more work. Read on for
 details.
 
diff --git a/tests/test_file.py b/tests/test_file.py
index e11d7fd5..c6edd7af 100755
--- a/tests/test_file.py
+++ b/tests/test_file.py
@@ -53,7 +53,7 @@ class BufferedFileTest (unittest.TestCase):
     def test_1_simple(self):
         f = LoopbackFile('r')
         try:
-            f.write('hi')
+            f.write(b'hi')
             self.assertTrue(False, 'no exception on write to read-only file')
         except:
             pass
@@ -69,7 +69,7 @@ class BufferedFileTest (unittest.TestCase):
 
     def test_2_readline(self):
         f = LoopbackFile('r+U')
-        f.write('First line.\nSecond line.\r\nThird line.\nFinal line non-terminated.')
+        f.write(b'First line.\nSecond line.\r\nThird line.\nFinal line non-terminated.')
         self.assertEqual(f.readline(), 'First line.\n')
         # universal newline mode should convert this linefeed:
         self.assertEqual(f.readline(), 'Second line.\n')
@@ -93,9 +93,9 @@ class BufferedFileTest (unittest.TestCase):
         try to trick the linefeed detector.
         """
         f = LoopbackFile('r+U')
-        f.write('First line.\r')
+        f.write(b'First line.\r')
         self.assertEqual(f.readline(), 'First line.\n')
-        f.write('\nSecond.\r\n')
+        f.write(b'\nSecond.\r\n')
         self.assertEqual(f.readline(), 'Second.\n')
         f.close()
         self.assertEqual(f.newlines, crlf)
@@ -105,7 +105,7 @@ class BufferedFileTest (unittest.TestCase):
         verify that write buffering is on.
         """
         f = LoopbackFile('r+', 1)
-        f.write('Complete line.\nIncomplete line.')
+        f.write(b'Complete line.\nIncomplete line.')
         self.assertEqual(f.readline(), 'Complete line.\n')
         self.assertEqual(f.readline(), '')
         f.write('..\n')
@@ -118,12 +118,12 @@ class BufferedFileTest (unittest.TestCase):
         """
         f = LoopbackFile('r+', 512)
         f.write('Not\nquite\n512 bytes.\n')
-        self.assertEqual(f.read(1), '')
+        self.assertEqual(f.read(1), b'')
         f.flush()
-        self.assertEqual(f.read(5), 'Not\nq')
-        self.assertEqual(f.read(10), 'uite\n512 b')
-        self.assertEqual(f.read(9), 'ytes.\n')
-        self.assertEqual(f.read(3), '')
+        self.assertEqual(f.read(5), b'Not\nq')
+        self.assertEqual(f.read(10), b'uite\n512 b')
+        self.assertEqual(f.read(9), b'ytes.\n')
+        self.assertEqual(f.read(3), b'')
         f.close()
 
     def test_6_buffering(self):
@@ -131,12 +131,12 @@ class BufferedFileTest (unittest.TestCase):
         verify that flushing happens automatically on buffer crossing.
         """
         f = LoopbackFile('r+', 16)
-        f.write('Too small.')
-        self.assertEqual(f.read(4), '')
-        f.write('  ')
-        self.assertEqual(f.read(4), '')
-        f.write('Enough.')
-        self.assertEqual(f.read(20), 'Too small.  Enough.')
+        f.write(b'Too small.')
+        self.assertEqual(f.read(4), b'')
+        f.write(b'  ')
+        self.assertEqual(f.read(4), b'')
+        f.write(b'Enough.')
+        self.assertEqual(f.read(20), b'Too small.  Enough.')
         f.close()
 
     def test_7_read_all(self):
@@ -144,9 +144,14 @@ class BufferedFileTest (unittest.TestCase):
         verify that read(-1) returns everything left in the file.
         """
         f = LoopbackFile('r+', 16)
-        f.write('The first thing you need to do is open your eyes. ')
-        f.write('Then, you need to close them again.\n')
+        f.write(b'The first thing you need to do is open your eyes. ')
+        f.write(b'Then, you need to close them again.\n')
         s = f.read(-1)
-        self.assertEqual(s, 'The first thing you need to do is open your eyes. Then, you ' +
-                         'need to close them again.\n')
+        self.assertEqual(s, b'The first thing you need to do is open your eyes. Then, you ' +
+                            b'need to close them again.\n')
         f.close()
+
+if __name__ == '__main__':
+    from unittest import main
+    main()
+
diff --git a/tests/test_sftp.py b/tests/test_sftp.py
index e0534eb0..2b6aa3b6 100755
--- a/tests/test_sftp.py
+++ b/tests/test_sftp.py
@@ -67,6 +67,18 @@ liver insulin receptors. Their sensitivity to insulin is, however, similarly
 decreased compared with chicken.
 '''
 
+
+# Here is how unicode characters are encoded over 1 to 6 bytes in utf-8
+# U-00000000 - U-0000007F: 0xxxxxxx
+# U-00000080 - U-000007FF: 110xxxxx 10xxxxxx
+# U-00000800 - U-0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx
+# U-00010000 - U-001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+# U-00200000 - U-03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
+# U-04000000 - U-7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
+# Note that: hex(int('11000011',2)) == '0xc3'
+# Thus, the following 2-bytes sequence is not valid utf8: "invalid continuation byte"
+NON_UTF8_DATA = b'\xC3\xC3'
+
 FOLDER = os.environ.get('TEST_FOLDER', 'temp-testing000')
 
 sftp = None
@@ -405,7 +417,7 @@ class SFTPTest (unittest.TestCase):
             self.assertEqual(sftp.stat(FOLDER + '/testing.txt').st_size, 13)
             with sftp.open(FOLDER + '/testing.txt', 'r') as f:
                 data = f.read(20)
-            self.assertEqual(data, 'hello kiddy.\n')
+            self.assertEqual(data, b'hello kiddy.\n')
         finally:
             sftp.remove(FOLDER + '/testing.txt')
 
@@ -466,8 +478,8 @@ class SFTPTest (unittest.TestCase):
                 f.write('?\n')
 
             with sftp.open(FOLDER + '/happy.txt', 'r') as f:
-                self.assertEqual(f.readline(), 'full line?\n')
-                self.assertEqual(f.read(7), 'partial')
+                self.assertEqual(f.readline(), u('full line?\n'))
+                self.assertEqual(f.read(7), b'partial')
         finally:
             try:
                 sftp.remove(FOLDER + '/happy.txt')
@@ -662,8 +674,8 @@ class SFTPTest (unittest.TestCase):
 
         fd, localname = mkstemp()
         os.close(fd)
-        text = 'All I wanted was a plastic bunny rabbit.\n'
-        with open(localname, 'w') as f:
+        text = b'All I wanted was a plastic bunny rabbit.\n'
+        with open(localname, 'wb') as f:
             f.write(text)
         saved_progress = []
 
@@ -747,6 +759,23 @@ class SFTPTest (unittest.TestCase):
             sftp.remove(FOLDER + '/test%file')
 
 
+    def test_O_non_utf8_data(self):
+        """Test write() and read() of non utf8 data"""
+        try:
+            with sftp.open('%s/nonutf8data' % FOLDER, 'w') as f:
+                f.write(NON_UTF8_DATA)
+            with sftp.open('%s/nonutf8data' % FOLDER, 'r') as f:
+                data = f.read()
+            self.assertEqual(data, NON_UTF8_DATA)
+            with sftp.open('%s/nonutf8data' % FOLDER, 'wb') as f:
+                f.write(NON_UTF8_DATA)
+            with sftp.open('%s/nonutf8data' % FOLDER, 'rb') as f:
+                data = f.read()
+            self.assertEqual(data, NON_UTF8_DATA)
+        finally:
+            sftp.remove('%s/nonutf8data' % FOLDER)
+
+
 if __name__ == '__main__':
     SFTPTest.init_loopback()
     # logging is required by test_N_file_with_percent
diff --git a/tests/test_sftp_big.py b/tests/test_sftp_big.py
index 521fbdc8..abed27b8 100644
--- a/tests/test_sftp_big.py
+++ b/tests/test_sftp_big.py
@@ -85,7 +85,7 @@ class BigSFTPTest (unittest.TestCase):
         write a 1MB file with no buffering.
         """
         sftp = get_sftp()
-        kblob = (1024 * 'x')
+        kblob = (1024 * b'x')
         start = time.time()
         try:
             with sftp.open('%s/hongry.txt' % FOLDER, 'w') as f:
@@ -231,7 +231,7 @@ class BigSFTPTest (unittest.TestCase):
         without using it, to verify that paramiko doesn't get confused.
         """
         sftp = get_sftp()
-        kblob = (1024 * 'x')
+        kblob = (1024 * b'x')
         try:
             with sftp.open('%s/hongry.txt' % FOLDER, 'w') as f:
                 f.set_pipelined(True)