diff options
| -rw-r--r-- | .github/ISSUE_TEMPLATE/10_support_request.yml | 113 | ||||
| -rw-r--r-- | .github/ISSUE_TEMPLATE/20_bug_report.yml | 120 | ||||
| -rw-r--r-- | .github/ISSUE_TEMPLATE/30_feature_request.yml | 73 | ||||
| -rw-r--r-- | .github/ISSUE_TEMPLATE/config.yml | 5 | ||||
| -rw-r--r-- | NEWS | 429 | ||||
| -rw-r--r-- | NOTES | 13 | ||||
| -rw-r--r-- | paramiko/_version.py | 2 | ||||
| -rw-r--r-- | paramiko/config.py | 33 | ||||
| -rw-r--r-- | paramiko/sftp_client.py | 43 | ||||
| -rw-r--r-- | paramiko/sftp_file.py | 40 | ||||
| -rw-r--r-- | sites/docs/api/config.rst | 6 | ||||
| -rw-r--r-- | sites/www/changelog.rst | 14 | ||||
| -rw-r--r-- | sites/www/contributing.rst | 3 | ||||
| -rw-r--r-- | tests/_util.py | 23 | ||||
| -rw-r--r-- | tests/configs/match-final | 14 | ||||
| -rw-r--r-- | tests/test_config.py | 16 | ||||
| -rw-r--r-- | tests/test_sftp_big.py | 51 |
17 files changed, 529 insertions, 469 deletions
diff --git a/.github/ISSUE_TEMPLATE/10_support_request.yml b/.github/ISSUE_TEMPLATE/10_support_request.yml new file mode 100644 index 00000000..1da1016e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/10_support_request.yml @@ -0,0 +1,113 @@ +name: Support Request +description: | + Use this template when you're having trouble using paramiko. +title: "[SUPPORT] - <title>" +labels: ["Support"] + +body: + - type: markdown + attributes: + value: | + Thanks for using paramiko! We're sorry you're having trouble making it work the way you want. Please provide the information below and describe the problem you're having and we'll do our best to help. + + - type: dropdown + id: usage_posture + attributes: + label: Are you using paramiko as a client or server? + multiple: false + options: + - Client + - Server + - Both + - Not sure + validations: + required: true + + - type: dropdown + id: features + attributes: + label: What feature(s) aren't working right? + description: Select as many as are relevant + multiple: true + options: + - SSH + - SFTP + - Keys/auth + - known_hosts + - sshconfig + - Exception handling + - Something else + validations: + required: true + + - type: input + id: paramiko_version + attributes: + label: What version(s) of paramiko are you using? + description: | + Find out with `$ python -c "import paramiko; print(paramiko.__version__)"` + placeholder: | + Example: 3.1.0 + validations: + required: true + + - type: input + id: python_version + attributes: + label: What version(s) of Python are you using? + description: | + Find out with `$ python -V` + placeholder: | + Example: 3.11.3 + validations: + required: true + + - type: input + id: os_info + attributes: + label: What operating system and version are you using? + placeholder: | + Example: WSL on Windows 11; or MacOS Mojave; or Ubuntu 22.10 + validations: + required: true + + - type: input + id: server_info + attributes: + label: If you're connecting as a client, which SSH server are you connecting to? + description: | + Leave this blank if you're not sure. + placeholder: | + Example: OpenSSH x.y; or Teleport vNN + + - type: input + id: integrated_tool + attributes: + label: If you're using paramiko as part of another tool, which tool/version? + placeholder: | + Examples: Fabric, Ansible, sftputil + + - type: textarea + id: intended_use + attributes: + label: What are you trying to do with paramiko? + description: | + Please describe in words what you are trying to do. + validations: + required: true + + - type: textarea + id: problem_details + attributes: + label: How are you trying to do it, and what's happening instead? + description: | + Include code snippets and a description of the expected output, and be as detailed as possible. If possible, try to reduce your code examples to a minimal example that reproduces the problem/behavior. + validations: + required: true + + - type: textarea + id: more_info + attributes: + label: Anything else? + description: | + Please provide any additional information that might help us find a solution for you. diff --git a/.github/ISSUE_TEMPLATE/20_bug_report.yml b/.github/ISSUE_TEMPLATE/20_bug_report.yml new file mode 100644 index 00000000..82f90663 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/20_bug_report.yml @@ -0,0 +1,120 @@ +name: Bug Report +description: | + Use this template when paramiko appears to be doing something wrong. +title: "[BUG] - <title>" +labels: ["Bug"] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to file a bug report! + + - type: dropdown + id: usage_posture + attributes: + label: Are you using paramiko as a client or server? + multiple: false + options: + - Client + - Server + - Both + - Exception handling + - Not sure + validations: + required: true + + - type: dropdown + id: features + attributes: + label: What feature(s) aren't working right? + description: Select as many as are relevant + multiple: true + options: + - SSH + - SFTP + - Keys/auth + - known_hosts + - sshconfig + - Something else + validations: + required: true + + - type: input + id: paramiko_version + attributes: + label: What version(s) of paramiko are you using? + description: | + Find out with `$ python -c "import paramiko; print(paramiko.__version__)"` + placeholder: | + Example: 3.1.0 + validations: + required: true + + - type: input + id: python_version + attributes: + label: What version(s) of Python are you using? + description: | + Find out with `$ python -V` + placeholder: | + Example: 3.11.3 + validations: + required: true + + - type: input + id: os_info + attributes: + label: What operating system and version are you using? + placeholder: | + Example: WSL on Windows 11; or MacOS Mojave; or Ubuntu 22.10 + validations: + required: true + + - type: input + id: server_info + attributes: + label: If you're connecting as a client, which SSH server are you connecting to? + description: | + Leave this blank if you're not sure. + placeholder: | + Example: OpenSSH x.y; or Teleport vNN + + - type: input + id: integrated_tool + attributes: + label: If you're using paramiko as part of another tool, which tool/version? + placeholder: | + Examples: Fabric, Ansible, sftputil + + - type: textarea + id: desired_behavior + attributes: + label: Expected/desired behavior + description: | + Please describe what you are trying to do with paramiko. Include code snippets and be as detailed as possible. + validations: + required: true + + - type: textarea + id: actual_behavior + attributes: + label: Actual behavior + description: | + What is paramiko doing instead? + validations: + required: true + + - type: textarea + id: repro + attributes: + label: How to reproduce + description: | + If possible, please provide a minimal code example that reproduces the bug. + + - type: textarea + id: more_info + attributes: + label: Anything else? + description: | + Please provide any additional information that might help us find and fix the bug. diff --git a/.github/ISSUE_TEMPLATE/30_feature_request.yml b/.github/ISSUE_TEMPLATE/30_feature_request.yml new file mode 100644 index 00000000..ac3bef27 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/30_feature_request.yml @@ -0,0 +1,73 @@ +name: Feature Request +description: | + Use this template to request addition of a new paramiko feature. +title: "[FEAT] - <title>" +labels: ["Feature"] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to let us know what you'd like added to paramiko! + + - type: dropdown + id: usage_posture + attributes: + label: Is this feature for paramiko acting as a client or a server? + multiple: false + options: + - Client + - Server + - Both + - Not sure + validations: + required: true + + - type: dropdown + id: features + attributes: + label: What functionality does this feature request relate to? + description: Select as many as are relevant + multiple: true + options: + - SSH + - SFTP + - Keys/auth + - known_hosts + - sshconfig + - Exception handling + - Something else + validations: + required: true + + - type: input + id: server_info + attributes: + label: For client-side features, does this relate to a specific type of SSH server? + description: | + Leave this blank if you're not sure, or if you're requesting a server-side feature. + placeholder: | + Example: OpenSSH x.y; or Teleport vNN + + - type: input + id: integrated_tool + attributes: + label: If you're using paramiko as part of another tool, which tool/version? + placeholder: | + Examples: Fabric, Ansible, sftputil + + - type: textarea + id: desired_behavior + attributes: + label: Desired behavior + description: | + Please describe what you you would like paramiko to be able to do. If possible, include pseudocode or mock code snippets to illustrate the desired behavior, and be as detailed as possible. + validations: + required: true + + - type: textarea + id: more_info + attributes: + label: Anything else? + description: | + Please provide any additional information that would be helpful to provide context for your requested feature. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..717686ad --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: "Blank Issue Template" + url: https://github.com/paramiko/paramiko/issues/new + about: "Use this as a last resort, if none of the other issue types fit."
\ No newline at end of file @@ -1,429 +0,0 @@ - -==== -NEWS -==== - -Highlights of what's new in each release. - -Issues noted as "'ssh' #NN" can be found at https://github.com/bitprophet/ssh/. - -Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/. - - -**PLEASE NOTE:** For changes in 1.10.x and newer releases, please see -www.paramiko.org's changelog page, or the source file, sites/www/changelog.rst - - -Releases -======== - -v1.9.0 (6th Nov 2012) ---------------------- - -* #97 (with a little #93): Improve config parsing of `ProxyCommand` directives - and provide a wrapper class to allow subprocess-driven proxy commands to be - used as `sock=` arguments for `SSHClient.connect`. -* #77: Allow `SSHClient.connect()` to take an explicit `sock` parameter - overriding creation of an internal, implicit socket object. -* Thanks in no particular order to Erwin Bolwidt, Oskari Saarenmaa, Steven - Noonan, Vladimir Lazarenko, Lincoln de Sousa, Valentino Volonghi, Olle - Lundberg, and Github user `@acrish` for the various and sundry patches - leading to the above changes. - -v1.8.1 (6th Nov 2012) ---------------------- - -* #90: Ensure that callbacks handed to `SFTPClient.get()` always fire at least - once, even for zero-length files downloaded. Thanks to Github user `@enB` for - the catch. -* #85: Paramiko's test suite overrides - `unittest.TestCase.assertTrue/assertFalse` to provide these modern assertions - to Python 2.2/2.3, which lacked them. However on newer Pythons such as 2.7, - this now causes deprecation warnings. The overrides have been patched to only - execute when necessary. Thanks to `@Arfrever` for catch & patch. - - -v1.8.0 (3rd Oct 2012) ---------------------- - -* #17 ('ssh' 28): Fix spurious `NoneType has no attribute 'error'` and similar - exceptions that crop up on interpreter exit. -* 'ssh' 32: Raise a more useful error explaining which `known_hosts` key line was - problematic, when encountering `binascii` issues decoding known host keys. - Thanks to `@thomasvs` for catch & patch. -* 'ssh' 33: Bring `ssh_config` parsing more in line with OpenSSH spec, re: order of - setting overrides by `Host` specifiers. Specifically, the overrides now go by - file order instead of automatically sorting by `Host` value length. In - addition, the first value found per config key (e.g. `Port`, `User` etc) - wins, instead of the last. Thanks to Jan Brauer for the contribution. -* 'ssh' 36: Support new server two-factor authentication option - (`RequiredAuthentications2`), at least re: combining key-based & password - auth. Thanks to Github user `bninja`. -* 'ssh' 11: When raising an exception for hosts not listed in - `known_hosts` (when `RejectPolicy` is in effect) the exception message was - confusing/vague. This has been improved somewhat. Thanks to Cal Leeming for - highlighting the issue. -* 'ssh' 40: Fixed up & expanded EINTR signal handling. Thanks to Douglas Turk. -* 'ssh' 15: Implemented parameter substitution in SSHConfig, matching the - implementation of `ssh_config(5)`. Thanks to Olle Lundberg for the patch. -* 'ssh' 24: Switch some internal type checking to use `isinstance` to help prevent - problems with client libraries using subclasses of builtin types. Thanks to - Alex Morega for the patch. -* Fabric #562: Agent forwarding would error out (with `Authentication response - too long`) or freeze, when more than one remote connection to the local agent - was active at the same time. This has been fixed. Thanks to Steven McDonald - for assisting in troubleshooting/patching, and to GitHub user `@lynxis` for - providing the final version of the patch. -* 'ssh' 5: Moved a `fcntl` import closer to where it's used to help avoid - `ImportError` problems on Windows platforms. Thanks to Jason Coombs for the - catch + suggested fix. -* 'ssh' 4: Updated implementation of WinPageant integration to work on 64-bit - Windows. Thanks again to Jason Coombs for the patch. -* Added an IO loop sleep() call to avoid needless CPU usage when agent - forwarding is in use. -* Handful of internal tweaks to version number storage. -* Updated `setup.py` with `==dev` install URL for `pip` users. -* Updated `setup.py` to account for packaging problems in PyCrypto 2.4.0 -* Added an extra `atfork()` call to help prevent spurious RNG errors when - running under high parallel (multiprocess) load. -* Merge PR #28: https://github.com/paramiko/paramiko/pull/28 which adds a - ssh-keygen like demo module. (Sofian Brabez) - -v1.7.7.2 16may12 ----------------- - * Merge pull request #63: https://github.com/paramiko/paramiko/pull/63 which - fixes exceptions that occur when re-keying over fast connections. (Dwayne - Litzenberger) - -v1.7.7.1 (George) 21may11 -------------------------- - * Make the verification phase of SFTP.put optional (Larry Wright) - * Patches to fix AIX support (anonymous) - * Patch from Michele Bertoldi to allow compression to be turned on in the - client constructor. - * Patch from Shad Sharma to raise an exception if the transport isn't active - when you try to open a new channel. - * Stop leaking file descriptors in the SSH agent (John Adams) - * More fixes for Windows address family support (Andrew Bennetts) - * Use Crypto.Random rather than Crypto.Util.RandomPool - (Gary van der Merwe, #271791) - * Support for openssl keys (tehfink) - * Fix multi-process support by calling Random.atfork (sugarc0de) - -v1.7.6 (Fanny) 1nov09 ---------------------- - * fixed bugs 411099 (sftp chdir isn't unicode-safe), 363163 & 411910 (more - IPv6 problems on windows), 413850 (race when server closes the channel), - 426925 (support port numbers in host keys) - -v1.7.5 (Ernest) 19jul09 ------------------------ - * added support for ARC4 cipher and CTR block chaining (Denis Bernard) - * made transport threads daemonize, to fix python 2.6 atexit behavior - * support unicode hostnames, and IP6 addresses (Maxime Ripard, Shikhar - Bhushan) - * various small bug fixes - -v1.7.4 (Desmond) 06jul08 ------------------------- - * more randpool fixes for windows, from Dwayne Litzenberger - (NOTE: this may require a pycrypto upgrade on windows) - * fix potential deadlock during key exchange (Dwayne Litzenberger) - * remove MFC dependency from windows (Mark Hammond) - * added some optional API improvements for SFTPClient get() and put() - -v1.7.3 (Clara) 23mar08 ----------------------- - * SSHClient can be asked not to use an SSH agent now, and not to search - for private keys - * added WarningPolicy option for SSHClient (warn, but allow, on unknown - server keys) - * added Channel.exit_status_ready to poll if a channel has received an - exit status yet - * new demo for reverse port forwarding - * (bug 177117) fix UTF-8 passwords - * (bug 189466) fix typo in osrandom.py - * (bug 191657) potentially fix a race at channel shutdown - * (bug 192749) document that SSHClient.connect may raise socket.error - * (bug 193779) translate EOFError into AuthException during authentication - * (bug 200416) don't create a new logger object for each channel - -v1.7.2 (Basil) 21jan08 ----------------------- - * (bug 137219) catch EINTR and handle correctly - * (bug 157205) fix select() to trigger on stderr for a channel too - * added SSHClient.get_transport() - * added Channel.send_ready() - * added direct-tcpip forwarding [patch from david guerizec] - * fixed the PRNG to be more secure on windows and in cases where fork() is - called [patch from dwayne litzenberger] - -v1.7.1 (Amy) 10jun07 --------------------- - * windows SSH agent support can use the 'ctypes' module now if 'win32all' is - not available [patch from alexander belchenko] - * SFTPClient.listdir_attr() now preserves the 'longname' field [patch from - wesley augur] - * SFTPClient.get_channel() API added - * SSHClient constructor takes an optional 'timeout' parameter [patch from - james bardin] - -v1.7 (zubat) 18feb07 --------------------- - * added x11 channel support (patch from david guerizec) - * added reverse port forwarding support - * (bug 75370) raise an exception when contacting a broken SFTP server - * (bug 80295) SSHClient shouldn't expand the user directory twice when reading - RSA/DSS keys - * (bug 82383) typo in DSS key in SSHClient - * (bug 83523) python 2.5 warning when encoding a file's modification time - * if connecting to an SSH agent fails, silently fallback instead of raising - an exception - -v1.6.4 (yanma) 19nov06 ----------------------- - * fix setup.py on osx (oops!) - * (bug 69330) check for the existence of RSA/DSA keys before trying to open - them in SFTPClient - * (bug 69222) catch EAGAIN in socket code to workaround a bug in recent - Linux 2.6 kernels - * (bug 70398) improve dict emulation in HostKeys objects - * try harder to make sure all worker threads are joined on Transport.close() - -v1.6.3 (xatu) 14oct06 ---------------------- - * fixed bug where HostKeys.__setitem__ wouldn't always do the right thing - * fixed bug in SFTPClient.chdir and SFTPAttributes.__str__ [patch from - mike barber] - * try harder not to raise EOFError from within SFTPClient - * fixed bug where a thread waiting in accept() could block forever if the - transport dies [patch from mike looijmans] - -v1.6.2 (weedle) 16aug06 ------------------------ - * added support for "old" group-exchange server mode, for compatibility - with the windows putty client - * fixed some more interactions with SFTP file readv() and prefetch() - * when saving the known_hosts file, preserve the original order [patch from - warren young] - * fix a couple of broken lines when exporting classes (bug 55946) - -v1.6.1 (vulpix) 10jul06 ------------------------ - * more unit tests fixed for windows/cygwin (thanks to alexander belchenko) - * a couple of fixes related to exceptions leaking out of SFTPClient - * added ability to set items in HostKeys via __setitem__ - * HostKeys now retains order and has a save() method - * added PKey.write_private_key and PKey.from_private_key - -v1.6 (umbreon) 10may06 ----------------------- - * pageant support on Windows thanks to john arbash meinel and todd whiteman - * fixed unit tests to work under windows and cygwin (thanks to alexander - belchenko for debugging) - * various bugfixes/tweaks to SFTP file prefetch - * added SSHClient for a higher-level API - * SFTP readv() now yields results as it gets them - * several APIs changed to throw an exception instead of "False" on failure - -v1.5.4 (tentacool) 11mar06 --------------------------- - * fixed HostKeys to more correctly emulate a python dict - * fixed a bug where file read buffering was too aggressive - * improved prefetching so that out-of-order reads still use the prefetch - buffer - * added experimental SFTPFile.readv() call - * more unit tests - -v1.5.3 (squirtle) 19feb06 -------------------------- - * a few performance enhancements - * added HostKeys, for dealing with openssh style "known_hosts" files, and - added support for hashed hostnames - * added Transport.atfork() for dealing with forked children - * added SFTPClient.truncate, SFTPFile.chmod, SFTPFile.chown, SFTPFile.utime, - and SFTPFile.truncate - * improved windows demos [patch from mike looijmans], added an sftp demo, and - moved demos to the demos/ folder - * fixed a few interoperability bugs - * cleaned up logging a bit - * fixed a bug where EOF on a Channel might not be detected by select [found - by thomas steinacher] - * fixed python 2.4-ism that crept in [patch by jan hudec] - * fixed a few reference loops that could have interacted badly with the python - garbage collector - * fixed a bunch of pychecker warnings, some of which were bugs - -v1.5.2 (rhydon) 04dec05 ------------------------ - * compression support (opt-in via Transport.use_compression) - * sftp files may be opened with mode flag 'x' for O_EXCL (exclusive-open) - behavior, which has no direct python equivalent - * added experimental util functions for parsing openssh config files - * fixed a few bugs (and potential deadlocks) with key renegotiation - * fixed a bug that caused SFTPFile.prefetch to occasionally lock up - * fixed an sftp bug which affected van dyke sftp servers - * fixed the behavior of select()ing on a closed channel, such that it will - always trigger as readable - -v1.5.1 (quilava) 31oct05 ------------------------- - * SFTPFile.prefetch() added to dramatically speed up downloads (automatically - turned on in SFTPClient.get()) - * fixed bug where garbage-collected Channels could trigger the Transport to - close the session (reported by gordon good) - * fixed a deadlock in rekeying (reported by wendell wood) - * fixed some windows bugs and SFTPAttributes.__str__() (reported by grzegorz - makarewicz) - * better sftp error reporting by adding fake "errno" info to IOErrors - -v1.5 (paras) 02oct05 --------------------- - * added support for "keyboard-interactive" authentication - * added mode (on by default) where password authentication will try to - fallback to "keyboard-interactive" if it's supported - * added pipelining to SFTPFile.write and SFTPClient.put - * fixed bug with SFTPFile.close() not guarding against being called more - than once (thanks to Nathaniel Smith) - * fixed broken 'a' flag in SFTPClient.file() (thanks to Nathaniel Smith) - * fixed up epydocs to look nicer - * reorganized auth_transport into auth_handler, which seems to be a cleaner - separation - * demo scripts fixed to have a better chance of loading the host keys - correctly on windows/cygwin - -v1.4 (oddish) 17jul05 ---------------------- - * added SSH-agent support (for posix) from john rochester - * added chdir() and getcwd() to SFTPClient, to emulate a "working directory" - * added get() and put() to SFTPClient, to emulate ftp whole-file transfers - * added check() to SFTPFile (a file hashing protocol extension) - * fixed Channels and SFTPFiles (among others) to auto-close when GC'd - * fixed Channel.fileno() for Windows, this time really - * don't log socket errors as "unknown exception" - * some misc. backward-compatible API improvements (like allowing - Transport.start_client() and start_server() to be called in a blocking way) - -v1.3.1 (nidoran) 28jun05 ------------------------- - * added SFTPClient.close() - * fixed up some outdated documentation - * made SFTPClient.file() an alias for open() - * added Transport.open_sftp_client() for convenience - * refactored packetizing out of Transport - * fixed bug (reported by alain s.) where connecting to a non-SSH host could - cause paramiko to freeze up - * fixed Channel.fileno() for Windows (again) - * some more unit tests - -v1.3 (marowak) 09apr05 ----------------------- - * fixed a bug where packets larger than about 12KB would cause the session - to die on all platforms except osx - * added a potential workaround for windows to let Channel.fileno() (and - therefore the select module) work! - * changed API for subsystem handlers (sorry!) to pass more info and make it - easier to write a functional SFTP server - -v1.2 (lapras) 28feb05 ---------------------- - * added SFTPClient.listdir_attr() for fetching a list of files and their - attributes in one call - * added Channel.recv_exit_status() and Channel.send_exit_status() for - manipulating the exit status of a command from either client or server - mode - * moved check_global_request into ServerInterface, where it should've been - all along (oops) - * SFTPHandle's default implementations are fleshed out more - * made logging a bit more consistent, and started logging thread ids - * fixed a few race conditions, one of which would sometimes cause a Transport - to fail to start on slow machines - * more unit tests - -v1.1 (kabuto) 12dec04 ---------------------- - * server-side SFTP support - * added support for stderr streams on client & server channels - * added a new distinct exception for failed client authentication - when caused by the server rejecting that *type* of auth - * added support for multi-part authentication - * fixed bug where get_username() wasn't working in server mode - -v1.0 (jigglypuff) 06nov04 -------------------------- - * fixed bug that broke server-mode authentication by private key - * fixed bug where closing a Channel could end up killing the entire - Transport - * actually include demo_windows.py this time (oops!) - * fixed recently-introduced bug in group-exchange key negotiation that - would generate the wrong hash (and therefore fail the initial handshake) - * server-mode subsystem handler is a bit more flexible - -v0.9 (ivysaur) 22oct04 ----------------------- - * new ServerInterface class for implementing server policy, so it's no - longer necessary to subclass Transport or Channel -- server code will - need to be updated to follow this new API! (see demo_server.py) - * some bugfixes for re-keying an active session - * Transport.get_security_options() allows fine-tuned control over the - crypto negotiation on a new session - * Transport.connect() takes a single hostkey object now instead of two - string parameters - * the Channel request methods (like 'exec_command') now return True on - success or False on failure - * added a mechanism for providing subsystems in server mode (and a new - class to be subclassed: SubsystemHandler) - * renamed SFTP -> SFTPClient (but left an alias for existing code) - * added SFTPClient.normalize() to resolve paths on the server - * fleshed out the API a bit more for SFTPClient and private keys - * a bunch of new unit tests! - -v0.9 (horsea) 27jun04 ---------------------- - * fixed a lockup that could happen if the channel was closed while the - send window was full - * better checking of maximum packet sizes - * better line buffering for file objects - * now chops sftp requests into smaller packets for some older servers - * more sftp unit tests - -v0.9 (gyarados) 31may04 ------------------------ - * Transport.open_channel() -- supports local & remote port forwarding now - * now imports UTF-8 encodings explicitly as a hint to "freeze" utilities - * no longer rejects older SFTP servers - * default packet size bumped to 8kB - * fixed deadlock in closing a channel - * Transport.connect() -- fixed bug where it would always fail when given a - host key to verify - -v0.9 (fearow) 23apr04 ---------------------- - * Transport.send_ignore() -- send random ignored bytes - * RSAKey/DSSKey added from_private_key_file() as a factory constructor; - write_private_key_file() & generate() to create and save ssh2 keys; - get_base64() to retrieve the exported public key - * Transport added global_request() [client] and check_global_request() - [server] - * Transport.get_remove_server_key() now returns a PKey object instead of a - tuple of strings - * Transport.get_username() -- return the username you auth'd as [client] - * Transport.set_keepalive() -- makes paramiko send periodic junk packets - to the remote host, to keep the session active - * python 2.2 support (thanks to Roger Binns) - * misc. bug fixes - -v0.9 (eevee) 08mar04 --------------------- - -v0.9 (doduo) 04jan04 --------------------- - -v0.1 (charmander) 10nov03 -------------------------- - -v0.1 (bulbasaur) 18sep03 ------------------------- - -v0.1 (aerodactyl) 13sep03 -------------------------- @@ -1,13 +0,0 @@ - - +-------------------+ +-----------------+ -(Socket)InputStream ---> | ssh2 transport | <===> | ssh2 channel | -(Socket)OutputStream --> | (auth, pipe) | N | (buffer) | - +-------------------+ +-----------------+ - @ feeder thread | | - - read InputStream | +-> InputStream - - feed into channel +---> OutputStream - buffers - -SIS <-- @ --> (parse, find chan) --> ssh2 chan: buffer <-- SSHInputStream -SSHOutputStream --> ssh2 chan --> ssh2 transport --> SOS [no thread] - diff --git a/paramiko/_version.py b/paramiko/_version.py index a008f109..3184d18d 100644 --- a/paramiko/_version.py +++ b/paramiko/_version.py @@ -1,2 +1,2 @@ -__version_info__ = (3, 2, 0) +__version_info__ = (3, 3, 1) __version__ = ".".join(map(str, __version_info__)) diff --git a/paramiko/config.py b/paramiko/config.py index 48bcb101..8ab55c64 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -218,6 +218,8 @@ class SSHConfig: Added canonicalization support. .. versionchanged:: 2.7 Added ``Match`` support. + .. versionchanged:: 3.3 + Added ``Match final`` support. """ # First pass options = self._lookup(hostname=hostname) @@ -235,10 +237,16 @@ class SSHConfig: hostname = self.canonicalize(hostname, options, domains) # Overwrite HostName again here (this is also what OpenSSH does) options["hostname"] = hostname - options = self._lookup(hostname, options, canonical=True) + options = self._lookup( + hostname, options, canonical=True, final=True + ) + else: + options = self._lookup( + hostname, options, canonical=False, final=True + ) return options - def _lookup(self, hostname, options=None, canonical=False): + def _lookup(self, hostname, options=None, canonical=False, final=False): # Init if options is None: options = SSHConfigDict() @@ -248,7 +256,11 @@ class SSHConfig: if not ( self._pattern_matches(context.get("host", []), hostname) or self._does_match( - context.get("matches", []), hostname, canonical, options + context.get("matches", []), + hostname, + canonical, + final, + options, ) ): continue @@ -263,9 +275,10 @@ class SSHConfig: options[key].extend( x for x in value if x not in options[key] ) - # Expand variables in resulting values (besides 'Match exec' which was - # already handled above) - options = self._expand_variables(options, hostname) + if final: + # Expand variables in resulting values + # (besides 'Match exec' which was already handled above) + options = self._expand_variables(options, hostname) return options def canonicalize(self, hostname, options, domains): @@ -336,7 +349,9 @@ class SSHConfig: match = True return match - def _does_match(self, match_list, target_hostname, canonical, options): + def _does_match( + self, match_list, target_hostname, canonical, final, options + ): matched = [] candidates = match_list[:] local_username = getpass.getuser() @@ -353,6 +368,8 @@ class SSHConfig: if type_ == "canonical": if self._should_fail(canonical, candidate): return False + if type_ == "final": + passed = final # The parse step ensures we only see this by itself or after # canonical, so it's also an easy hard pass. (No negation here as # that would be uh, pretty weird?) @@ -511,7 +528,7 @@ class SSHConfig: type_ = type_[1:] match["type"] = type_ # all/canonical have no params (everything else does) - if type_ in ("all", "canonical"): + if type_ in ("all", "canonical", "final"): matches.append(match) continue if not tokens: diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 31ac1292..066cd83f 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -758,7 +758,14 @@ class SFTPClient(BaseSFTP, ClosingContextManager): with open(localpath, "rb") as fl: return self.putfo(fl, remotepath, file_size, callback, confirm) - def getfo(self, remotepath, fl, callback=None, prefetch=True): + def getfo( + self, + remotepath, + fl, + callback=None, + prefetch=True, + max_concurrent_prefetch_requests=None, + ): """ Copy a remote file (``remotepath``) from the SFTP server and write to an open file or file-like object, ``fl``. Any exception raised by @@ -773,21 +780,34 @@ class SFTPClient(BaseSFTP, ClosingContextManager): the bytes transferred so far and the total bytes to be transferred :param bool prefetch: controls whether prefetching is performed (default: True) + :param int max_concurrent_prefetch_requests: + The maximum number of concurrent read requests to prefetch. See + `.SFTPClient.get` (its ``max_concurrent_prefetch_requests`` param) + for details. :return: the `number <int>` of bytes written to the opened file object .. versionadded:: 1.10 .. versionchanged:: 2.8 Added the ``prefetch`` keyword argument. + .. versionchanged:: 3.3 + Added ``max_concurrent_prefetch_requests``. """ file_size = self.stat(remotepath).st_size with self.open(remotepath, "rb") as fr: if prefetch: - fr.prefetch(file_size) + fr.prefetch(file_size, max_concurrent_prefetch_requests) return self._transfer_with_callback( reader=fr, writer=fl, file_size=file_size, callback=callback ) - def get(self, remotepath, localpath, callback=None, prefetch=True): + def get( + self, + remotepath, + localpath, + callback=None, + prefetch=True, + max_concurrent_prefetch_requests=None, + ): """ Copy a remote file (``remotepath``) from the SFTP server to the local host as ``localpath``. Any exception raised by operations will be @@ -800,15 +820,30 @@ class SFTPClient(BaseSFTP, ClosingContextManager): the bytes transferred so far and the total bytes to be transferred :param bool prefetch: controls whether prefetching is performed (default: True) + :param int max_concurrent_prefetch_requests: + The maximum number of concurrent read requests to prefetch. + When this is ``None`` (the default), do not limit the number of + concurrent prefetch requests. Note: OpenSSH's sftp internally + imposes a limit of 64 concurrent requests, while Paramiko imposes + no limit by default; consider setting a limit if a file can be + successfully received with sftp but hangs with Paramiko. .. versionadded:: 1.4 .. versionchanged:: 1.7.4 Added the ``callback`` param .. versionchanged:: 2.8 Added the ``prefetch`` keyword argument. + .. versionchanged:: 3.3 + Added ``max_concurrent_prefetch_requests``. """ with open(localpath, "wb") as fl: - size = self.getfo(remotepath, fl, callback, prefetch) + size = self.getfo( + remotepath, + fl, + callback, + prefetch, + max_concurrent_prefetch_requests, + ) s = os.stat(localpath) if s.st_size != size: raise IOError( diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index 9a0a6b34..c74695e0 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -26,7 +26,7 @@ from collections import deque import socket import threading import time -from paramiko.common import DEBUG +from paramiko.common import DEBUG, io_sleep from paramiko.file import BufferedFile from paramiko.util import u @@ -435,7 +435,7 @@ class SFTPFile(BufferedFile): """ self.pipelined = pipelined - def prefetch(self, file_size=None): + def prefetch(self, file_size=None, max_concurrent_requests=None): """ Pre-fetch the remaining contents of this file in anticipation of future `.read` calls. If reading the entire file, pre-fetching can @@ -454,6 +454,10 @@ class SFTPFile(BufferedFile): <https://github.com/paramiko/paramiko/pull/562>`_); as a workaround, one may call `stat` explicitly and pass its value in via this parameter. + :param int max_concurrent_requests: + The maximum number of concurrent read requests to prefetch. See + `.SFTPClient.get` (its ``max_concurrent_prefetch_requests`` param) + for details. .. versionadded:: 1.5.1 .. versionchanged:: 1.16.0 @@ -461,6 +465,8 @@ class SFTPFile(BufferedFile): .. versionchanged:: 1.16.1 The ``file_size`` parameter was made optional for backwards compatibility. + .. versionchanged:: 3.3 + Added ``max_concurrent_requests``. """ if file_size is None: file_size = self.stat().st_size @@ -473,9 +479,9 @@ class SFTPFile(BufferedFile): chunks.append((n, chunk)) n += chunk if len(chunks) > 0: - self._start_prefetch(chunks) + self._start_prefetch(chunks, max_concurrent_requests) - def readv(self, chunks): + def readv(self, chunks, max_concurrent_prefetch_requests=None): """ Read a set of blocks from the file by (offset, length). This is more efficient than doing a series of `.seek` and `.read` calls, since the @@ -485,9 +491,15 @@ class SFTPFile(BufferedFile): :param chunks: a list of ``(offset, length)`` tuples indicating which sections of the file to read + :param int max_concurrent_prefetch_requests: + The maximum number of concurrent read requests to prefetch. See + `.SFTPClient.get` (its ``max_concurrent_prefetch_requests`` param) + for details. :return: a list of blocks read, in the same order as in ``chunks`` .. versionadded:: 1.5.4 + .. versionchanged:: 3.3 + Added ``max_concurrent_prefetch_requests``. """ self.sftp._log( DEBUG, "readv({}, {!r})".format(hexlify(self.handle), chunks) @@ -508,7 +520,7 @@ class SFTPFile(BufferedFile): offset += chunk_size size -= chunk_size - self._start_prefetch(read_chunks) + self._start_prefetch(read_chunks, max_concurrent_prefetch_requests) # now we can just devolve to a bunch of read()s :) for x in chunks: self.seek(x[0]) @@ -522,18 +534,30 @@ class SFTPFile(BufferedFile): except: return 0 - def _start_prefetch(self, chunks): + def _start_prefetch(self, chunks, max_concurrent_requests=None): self._prefetching = True self._prefetch_done = False - t = threading.Thread(target=self._prefetch_thread, args=(chunks,)) + t = threading.Thread( + target=self._prefetch_thread, + args=(chunks, max_concurrent_requests), + ) t.daemon = True t.start() - def _prefetch_thread(self, chunks): + def _prefetch_thread(self, chunks, max_concurrent_requests): # do these read requests in a temporary thread because there may be # a lot of them, so it may block. for offset, length in chunks: + # Limit the number of concurrent requests in a busy-loop + if max_concurrent_requests is not None: + while True: + with self._prefetch_lock: + pf_len = len(self._prefetch_extents) + if pf_len < max_concurrent_requests: + break + time.sleep(io_sleep) + num = self.sftp._async_request( self, CMD_READ, self.handle, int64(offset), int(length) ) diff --git a/sites/docs/api/config.rst b/sites/docs/api/config.rst index d42de8ac..9015a77c 100644 --- a/sites/docs/api/config.rst +++ b/sites/docs/api/config.rst @@ -61,7 +61,9 @@ Paramiko releases) are included. A keyword by itself means no known departures. - ``Host`` - ``HostName``: used in ``%h`` :ref:`token expansion <TOKENS>` -- ``Match``: fully supported, with the following caveats: +- ``Match``: supports the keywords ``all``, ``canonical``, ``exec``, ``final``, + ``host``, ``localuser``, ``originalhost``, and ``user``, with the following + caveats: - You must have the optional dependency Invoke installed; see :ref:`the installation docs <paramiko-itself>` (in brief: install @@ -72,6 +74,8 @@ Paramiko releases) are included. A keyword by itself means no known departures. but has no knowledge about connection-time usernames. .. versionadded:: 2.7 + .. versionchanged:: 3.3 + Added support for the ``final`` keyword. - ``Port``: supplies potential values for ``%p`` :ref:`token expansion <TOKENS>`. diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 2deb6998..45df830c 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -5,6 +5,20 @@ Changelog - :bug:`-` Tweak ``ext-info-(c|s)`` detection during KEXINIT protocol phase; the original implementation made assumptions based on an OpenSSH implementation detail. +- :release:`3.3.1 <2023-07-28>` +- :bug:`-` Cleaned up some very old root level files, mostly just to exercise + some of our doc build and release machinery. This changelog entry + intentionally left blank! ``nothing-to-see-here-move-along.gif`` +- :release:`3.3.0 <2023-07-28>` +- :feature:`1907` (solves :issue:`1992`) Add support and tests for ``Match + final …`` (frequently used in ProxyJump configurations to exclude the jump + host) to our :ref:`SSH config parser <ssh-config-support>`. Patch by + ``@commonism``. +- :feature:`2058` (solves :issue:`1587` and possibly others) Add an explicit + ``max_concurrent_prefetch_requests`` argument to + `paramiko.client.SSHClient.get` and `paramiko.client.SSHClient.getfo`, + allowing users to limit the number of concurrent requests used during + prefetch. Patch by ``@kschoelhorn``, with a test by ``@bwinston-sdp``. - :release:`3.2.0 <2023-05-25>` - :bug:`- major` Fixed a very sneaky bug found at the apparently rarely-traveled intersection of ``RSA-SHA2`` keys, certificates, SSH agents, diff --git a/sites/www/contributing.rst b/sites/www/contributing.rst index a44414e8..9cf0f34b 100644 --- a/sites/www/contributing.rst +++ b/sites/www/contributing.rst @@ -18,8 +18,7 @@ Please see `this project-agnostic contribution guide <http://contribution-guide.org>`_ - we follow it explicitly. Again, our code repository and bug tracker is `on Github`_. -Our current changelog is located in ``sites/www/changelog.rst`` - the top -level files like ``ChangeLog.*`` and ``NEWS`` are historical only. +Our changelog is located in ``sites/www/changelog.rst``. .. _paramiko/paramiko: diff --git a/tests/_util.py b/tests/_util.py index aeee96ea..acc06852 100644 --- a/tests/_util.py +++ b/tests/_util.py @@ -7,7 +7,7 @@ import socket import struct import sys import unittest -from time import sleep +import time import threading import pytest @@ -303,7 +303,7 @@ class TestServer(ServerInterface): if username == "bad-server": raise Exception("Ack!") if username == "unresponsive-server": - sleep(5) + time.sleep(5) return AUTH_SUCCESSFUL return AUTH_FAILED @@ -442,3 +442,22 @@ def server( ts.close() socks.close() sockc.close() + + +def wait_until(condition, *, timeout=2): + """ + Wait until `condition()` no longer raises an `AssertionError` or until + `timeout` seconds have passed, which causes a `TimeoutError` to be raised. + """ + deadline = time.time() + timeout + + while True: + try: + condition() + except AssertionError as e: + if time.time() > deadline: + timeout_message = f"Condition not reached after {timeout}s" + raise TimeoutError(timeout_message) from e + else: + return + time.sleep(0.01) diff --git a/tests/configs/match-final b/tests/configs/match-final new file mode 100644 index 00000000..21e927fc --- /dev/null +++ b/tests/configs/match-final @@ -0,0 +1,14 @@ +Host jump + HostName jump.example.org + Port 1003 + +Host finally + HostName finally.example.org + Port 1001 + +Host default-port + HostName default-port.example.org + +Match final host "*.example.org" !host jump.example.org + ProxyJump jump + Port 1002 diff --git a/tests/test_config.py b/tests/test_config.py index d2395965..1e623e0a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1030,3 +1030,19 @@ class TestComplexMatching: # !canonical in a config that is canonicalized - does NOT match result = load_config("match-canonical-yes").lookup("www") assert result["user"] == "hidden" + + +class TestFinalMatching(object): + def test_finally(self): + result = load_config("match-final").lookup("finally") + assert result["proxyjump"] == "jump" + assert result["port"] == "1001" + + def test_default_port(self): + result = load_config("match-final").lookup("default-port") + assert result["proxyjump"] == "jump" + assert result["port"] == "1002" + + def test_negated(self): + result = load_config("match-final").lookup("jump") + assert result["port"] == "1003" diff --git a/tests/test_sftp_big.py b/tests/test_sftp_big.py index acfe71e3..7d1110c3 100644 --- a/tests/test_sftp_big.py +++ b/tests/test_sftp_big.py @@ -30,7 +30,7 @@ import time from paramiko.common import o660 -from ._util import slow +from ._util import slow, wait_until @slow @@ -365,3 +365,52 @@ class TestBigSFTP: finally: sftp.remove(f"{sftp.FOLDER}/hongry.txt") t.packetizer.REKEY_BYTES = pow(2, 30) + + def test_prefetch_limit(self, sftp): + """ + write a 1MB file and prefetch with a limit + """ + kblob = 1024 * b"x" + start = time.time() + + def expect_prefetch_extents(file, expected_extents): + with file._prefetch_lock: + assert len(file._prefetch_extents) == expected_extents + + try: + with sftp.open(f"{sftp.FOLDER}/hongry.txt", "w") as f: + for n in range(1024): + f.write(kblob) + if n % 128 == 0: + sys.stderr.write(".") + sys.stderr.write(" ") + + assert ( + sftp.stat(f"{sftp.FOLDER}/hongry.txt").st_size == 1024 * 1024 + ) + end = time.time() + sys.stderr.write(f"{round(end - start)}s") + + # read with prefetch, no limit + # expecting 32 requests (32k * 32 == 1M) + with sftp.open(f"{sftp.FOLDER}/hongry.txt", "rb") as f: + file_size = f.stat().st_size + f.prefetch(file_size) + wait_until(lambda: expect_prefetch_extents(f, 32)) + + # read with prefetch, limiting to 5 simultaneous requests + with sftp.open(f"{sftp.FOLDER}/hongry.txt", "rb") as f: + file_size = f.stat().st_size + f.prefetch(file_size, 5) + wait_until(lambda: expect_prefetch_extents(f, 5)) + for n in range(1024): + with f._prefetch_lock: + assert len(f._prefetch_extents) <= 5 + data = f.read(1024) + assert data == kblob + + if n % 128 == 0: + sys.stderr.write(".") + + finally: + sftp.remove(f"{sftp.FOLDER}/hongry.txt") |