Skip to content

urllib: improve bytes handling #9167

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 23, 2022
Merged

urllib: improve bytes handling #9167

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
348a9df
urllib: improve bytes handling
JelleZijlstra Nov 12, 2022
827e749
Update stdlib/urllib/parse.pyi
JelleZijlstra Nov 12, 2022
2d56651
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions stdlib/urllib/parse.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ if sys.version_info >= (3, 9):
else:
def unquote(string: str, encoding: str = ..., errors: str = ...) -> str: ...

def unquote_to_bytes(string: str | bytes) -> bytes: ...
def unquote_to_bytes(string: str | bytes | bytearray) -> bytes: ...
def unquote_plus(string: str, encoding: str = ..., errors: str = ...) -> str: ...
@overload
def urldefrag(url: str) -> DefragResult: ...
Expand All @@ -168,11 +168,22 @@ def urljoin(base: AnyStr, url: AnyStr | None, allow_fragments: bool = ...) -> An
@overload
def urlparse(url: str, scheme: str | None = ..., allow_fragments: bool = ...) -> ParseResult: ...
@overload
def urlparse(url: bytes | bytearray | None, scheme: bytes | None = ..., allow_fragments: bool = ...) -> ParseResultBytes: ...
def urlparse(
url: bytes | bytearray | None, scheme: bytes | bytearray | None = ..., allow_fragments: bool = ...
) -> ParseResultBytes: ...
@overload
def urlsplit(url: str, scheme: str | None = ..., allow_fragments: bool = ...) -> SplitResult: ...
@overload
def urlsplit(url: bytes | bytearray | None, scheme: bytes | None = ..., allow_fragments: bool = ...) -> SplitResultBytes: ...

if sys.version_info >= (3, 11):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

python/cpython#25798 removed bytearray support.

@overload
def urlsplit(url: bytes | None, scheme: bytes | None = ..., allow_fragments: bool = ...) -> SplitResultBytes: ...

else:
@overload
def urlsplit(
url: bytes | bytearray | None, scheme: bytes | bytearray | None = ..., allow_fragments: bool = ...
) -> SplitResultBytes: ...

@overload
def urlunparse(
components: tuple[AnyStr | None, AnyStr | None, AnyStr | None, AnyStr | None, AnyStr | None, AnyStr | None]
Expand Down
48 changes: 26 additions & 22 deletions stdlib/urllib/request.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import ssl
import sys
from _typeshed import StrOrBytesPath, SupportsRead
from _typeshed import ReadableBuffer, StrOrBytesPath, SupportsRead
from collections.abc import Callable, Iterable, Mapping, MutableMapping, Sequence
from email.message import Message
from http.client import HTTPConnection, HTTPMessage, HTTPResponse
Expand Down Expand Up @@ -50,7 +50,7 @@ __all__ = [

_T = TypeVar("_T")
_UrlopenRet: TypeAlias = Any
_DataType: TypeAlias = bytes | SupportsRead[bytes] | Iterable[bytes] | None
_DataType: TypeAlias = ReadableBuffer | SupportsRead[bytes] | Iterable[bytes] | None

def urlopen(
url: str | Request,
Expand Down Expand Up @@ -214,7 +214,7 @@ class AbstractDigestAuthHandler:
def get_cnonce(self, nonce: str) -> str: ...
def get_authorization(self, req: Request, chal: Mapping[str, str]) -> str: ...
def get_algorithm_impls(self, algorithm: str) -> tuple[Callable[[str], str], Callable[[str, str], str]]: ...
def get_entity_digest(self, data: bytes | None, chal: Mapping[str, str]) -> str | None: ...
def get_entity_digest(self, data: ReadableBuffer | None, chal: Mapping[str, str]) -> str | None: ...

class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
auth_header: ClassVar[str] # undocumented
Expand Down Expand Up @@ -301,14 +301,14 @@ def urlcleanup() -> None: ...
class URLopener:
version: ClassVar[str]
def __init__(self, proxies: dict[str, str] | None = ..., **x509: str) -> None: ...
def open(self, fullurl: str, data: bytes | None = ...) -> _UrlopenRet: ...
def open_unknown(self, fullurl: str, data: bytes | None = ...) -> _UrlopenRet: ...
def open(self, fullurl: str, data: ReadableBuffer | None = ...) -> _UrlopenRet: ...
def open_unknown(self, fullurl: str, data: ReadableBuffer | None = ...) -> _UrlopenRet: ...
def retrieve(
self,
url: str,
filename: str | None = ...,
reporthook: Callable[[int, int, int], object] | None = ...,
data: bytes | None = ...,
data: ReadableBuffer | None = ...,
) -> tuple[str, Message | None]: ...
def addheader(self, *args: tuple[str, str]) -> None: ... # undocumented
def cleanup(self) -> None: ... # undocumented
Expand All @@ -319,32 +319,32 @@ class URLopener:
def http_error_default(
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage
) -> _UrlopenRet: ... # undocumented
def open_data(self, url: str, data: bytes | None = ...) -> addinfourl: ... # undocumented
def open_data(self, url: str, data: ReadableBuffer | None = ...) -> addinfourl: ... # undocumented
def open_file(self, url: str) -> addinfourl: ... # undocumented
def open_ftp(self, url: str) -> addinfourl: ... # undocumented
def open_http(self, url: str, data: bytes | None = ...) -> _UrlopenRet: ... # undocumented
def open_https(self, url: str, data: bytes | None = ...) -> _UrlopenRet: ... # undocumented
def open_http(self, url: str, data: ReadableBuffer | None = ...) -> _UrlopenRet: ... # undocumented
def open_https(self, url: str, data: ReadableBuffer | None = ...) -> _UrlopenRet: ... # undocumented
def open_local_file(self, url: str) -> addinfourl: ... # undocumented
def open_unknown_proxy(self, proxy: str, fullurl: str, data: bytes | None = ...) -> None: ... # undocumented
def open_unknown_proxy(self, proxy: str, fullurl: str, data: ReadableBuffer | None = ...) -> None: ... # undocumented

class FancyURLopener(URLopener):
def prompt_user_passwd(self, host: str, realm: str) -> tuple[str, str]: ...
def get_user_passwd(self, host: str, realm: str, clear_cache: int = ...) -> tuple[str, str]: ... # undocumented
def http_error_301(
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: bytes | None = ...
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = ...
) -> _UrlopenRet | addinfourl | None: ... # undocumented
def http_error_302(
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: bytes | None = ...
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = ...
) -> _UrlopenRet | addinfourl | None: ... # undocumented
def http_error_303(
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: bytes | None = ...
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = ...
) -> _UrlopenRet | addinfourl | None: ... # undocumented
def http_error_307(
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: bytes | None = ...
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = ...
) -> _UrlopenRet | addinfourl | None: ... # undocumented
if sys.version_info >= (3, 11):
def http_error_308(
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: bytes | None = ...
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = ...
) -> _UrlopenRet | addinfourl | None: ... # undocumented

def http_error_401(
Expand All @@ -354,7 +354,7 @@ class FancyURLopener(URLopener):
errcode: int,
errmsg: str,
headers: HTTPMessage,
data: bytes | None = ...,
data: ReadableBuffer | None = ...,
retry: bool = ...,
) -> _UrlopenRet | None: ... # undocumented
def http_error_407(
Expand All @@ -364,20 +364,24 @@ class FancyURLopener(URLopener):
errcode: int,
errmsg: str,
headers: HTTPMessage,
data: bytes | None = ...,
data: ReadableBuffer | None = ...,
retry: bool = ...,
) -> _UrlopenRet | None: ... # undocumented
def http_error_default(
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage
) -> addinfourl: ... # undocumented
def redirect_internal(
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: bytes | None
self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None
) -> _UrlopenRet | None: ... # undocumented
def retry_http_basic_auth(
self, url: str, realm: str, data: ReadableBuffer | None = ...
) -> _UrlopenRet | None: ... # undocumented
def retry_https_basic_auth(
self, url: str, realm: str, data: ReadableBuffer | None = ...
) -> _UrlopenRet | None: ... # undocumented
def retry_http_basic_auth(self, url: str, realm: str, data: bytes | None = ...) -> _UrlopenRet | None: ... # undocumented
def retry_https_basic_auth(self, url: str, realm: str, data: bytes | None = ...) -> _UrlopenRet | None: ... # undocumented
def retry_proxy_http_basic_auth(
self, url: str, realm: str, data: bytes | None = ...
self, url: str, realm: str, data: ReadableBuffer | None = ...
) -> _UrlopenRet | None: ... # undocumented
def retry_proxy_https_basic_auth(
self, url: str, realm: str, data: bytes | None = ...
self, url: str, realm: str, data: ReadableBuffer | None = ...
) -> _UrlopenRet | None: ... # undocumented