From 4badcd80138d74d91bb91f469472468ddab1ff90 Mon Sep 17 00:00:00 2001 From: Moist-Cat Date: Thu, 2 Jul 2026 19:38:45 -0400 Subject: [PATCH 01/12] Add support for HTTP/2 This implementation is backwards compatible, functional, but still incomplete. --- aiohttp/client.py | 34 +- aiohttp/connector.py | 11 +- aiohttp/http2/connection.py | 664 ++++++++++++++++++++++++++++++++++++ aiohttp/http2/errors.py | 2 + aiohttp/http2/response.py | 120 +++++++ aiohttp/http2/settings.py | 65 ++++ aiohttp/http2/stream.py | 132 +++++++ aiohttp/http_protocol.py | 52 +++ aiohttp/http_writer.py | 3 +- pyproject.toml | 1 + tests/http2/test_http2.py | 539 +++++++++++++++++++++++++++++ 11 files changed, 1610 insertions(+), 13 deletions(-) create mode 100644 aiohttp/http2/connection.py create mode 100644 aiohttp/http2/errors.py create mode 100644 aiohttp/http2/response.py create mode 100644 aiohttp/http2/settings.py create mode 100644 aiohttp/http2/stream.py create mode 100644 aiohttp/http_protocol.py create mode 100644 tests/http2/test_http2.py diff --git a/aiohttp/client.py b/aiohttp/client.py index 809fa63ed24..8bb3ae571c5 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -236,18 +236,34 @@ async def _connect_and_send_request(req: ClientRequest) -> ClientResponse: except asyncio.TimeoutError as exc: raise ConnectionTimeoutError(f"Connection timeout to host {req.url}") from exc - assert conn.protocol is not None - conn.protocol.set_response_params(**req._response_params) + resp = None + started = False try: - resp = await req._send(conn) - try: + assert conn.protocol is not None + + ssl_object = conn.protocol.transport.get_extra_info("ssl_object") + alpn_protocol = ( + ssl_object.selected_alpn_protocol() if ssl_object else "http/1.1" + ) + + # backwards compatibility + if alpn_protocol == "h2": + resp = await conn.protocol.send( + req.method, req.url, list(req.headers.items()), b"" + ) + else: + conn.protocol.set_response_params(**req._response_params) + resp = await req._send(conn) await resp.start(conn) - except BaseException: + # h3 not implemented + + started = True + finally: + if resp is not None and not started: resp.close() - raise - except BaseException: - conn.close() - raise + conn.close() + if resp is None: + conn.close() return resp diff --git a/aiohttp/connector.py b/aiohttp/connector.py index c62d216648d..93df3567007 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -51,6 +51,7 @@ set_exception, set_result, ) +from .http_protocol import HttpDispatcherProtocol from .log import client_logger from .resolver import DefaultResolver @@ -292,7 +293,7 @@ def __init__( ] = defaultdict(OrderedDict) self._loop = loop - self._factory = functools.partial(ResponseHandler, loop=loop) + self._factory = functools.partial(HttpDispatcherProtocol, loop=loop) # start keep-alive connection cleanup task self._cleanup_handle: asyncio.TimerHandle | None = None @@ -856,7 +857,12 @@ def _make_ssl_context(verified: bool) -> SSLContext: sslcontext.verify_mode = ssl.CERT_NONE sslcontext.options |= ssl.OP_NO_COMPRESSION sslcontext.set_default_verify_paths() - sslcontext.set_alpn_protocols(("http/1.1",)) + sslcontext.set_alpn_protocols( + ( + "h2", + "http/1.1", + ) + ) return sslcontext @@ -1495,7 +1501,6 @@ async def _create_direct_connection( bad_peer = sock.getpeername() aiohappyeyeballs.remove_addr_infos(addr_infos, bad_peer) continue - return transp, proto assert last_exc is not None raise last_exc diff --git a/aiohttp/http2/connection.py b/aiohttp/http2/connection.py new file mode 100644 index 00000000000..d650f5be12d --- /dev/null +++ b/aiohttp/http2/connection.py @@ -0,0 +1,664 @@ +""" +aiohttp.http2.protocol +====================== + +Complete HTTP/2 client implementation (RFC 7540) for asyncio. + +This module provides: +- Frame-level binary wire protocol with debug logging. +- HPACK compression/decompression (using the `hpack` library). +- Full stream state machine (idle -> open -> half‑closed -> closed). +- Multiplexed connection handling (concurrent streams, flow control). +- Server settings tracking (MAX_CONCURRENT_STREAMS, INITIAL_WINDOW_SIZE, etc.). +- Integration point for aiohttp's TCPConnector via an asyncio.Protocol subclass. + +Dependencies: +- asyncio +- struct +- logging +- enum +- hpack (install with `pip install hpack`) +- collections.defaultdict + +Usage: + Connector replaces `ResponseHandler` with `Http2Protocol`. +""" + +import asyncio +import struct +import logging +from enum import IntEnum, IntFlag, auto +from collections import defaultdict +from typing import Optional, Dict, Set, Tuple, List, Any, Union + +from hpack import Encoder, Decoder + +from .settings import ( + DEFAULT_SETTINGS, + Setting, + FrameType, + FlagData, + FlagHeaders, + FlagSettings, + FlagPing, +) +from .stream import Stream, StreamState +from .response import Http2Response + +# ---------------------------------------------------------------------- +# Logging – plaintext wire‑format emission for debugging +# ---------------------------------------------------------------------- +logger = logging.getLogger("aiohttp.http2.connection") +logger.setLevel(logging.DEBUG) + +FRAME_HEADER_LENGTH = 9 # 9 octects +STREAM_ID_MASK = 0x7FFFFFFF # to avoid setting the reserved bit to 1 + + +# ---------------------------------------------------------------------- +# Connection‑level management +# ---------------------------------------------------------------------- +class Http2Connection: + """Manages a single HTTP/2 connection. + + Handles: + - Connection preface and SETTINGS handshake. + - Frame parsing and dispatch. + - HPACK encoding/decoding. + - Stream multiplexing and flow control. + - Server settings tracking. + """ + + def __init__( + self, transport: asyncio.Transport, loop: asyncio.AbstractEventLoop + ) -> None: + self._transport = transport + self._loop = loop + + # HPACK + self.hpack_encoder = Encoder() + self.hpack_decoder = Decoder() + + # Settings + self.remote_settings: Dict[Setting, int] = DEFAULT_SETTINGS.copy() + self.local_settings: Dict[Setting, int] = DEFAULT_SETTINGS.copy() + + # Flow control + self.session_outbound_window = 65535 # initial flow control (RFC 7540, 6.9.1) + self.session_inbound_window = 65535 + self._flow_control_updated = asyncio.Event() + self._flow_control_updated.set() # initially writable + + # Streams + self.streams: Dict[int, Stream] = {} + self.next_stream_id = 1 # client streams are odd + self.max_concurrent_streams = DEFAULT_SETTINGS[Setting.MAX_CONCURRENT_STREAMS] + self._pending_streams: List[asyncio.Future[Stream]] = [] + self._last_peer_stream_id = ( + 0 # highest server‑initiated stream (even, unused for client) + ) + + # Frame buffers + self._frame_buffer = bytearray() + + # GOAWAY state + self._goaway_received = False + self._goaway_sent = False + self._last_stream_id = 0 + self._error_code = 0 + + # Closed streams cleanup + self._closed_streams: Set[int] = set() + + # -------------------- Transport callbacks -------------------- + def data_received(self, data: bytes) -> None: + """Assemble frames from the byte stream and dispatch them.""" + self._frame_buffer.extend(data) + # Consume complete frames while enough bytes for the header exist + while len(self._frame_buffer) >= FRAME_HEADER_LENGTH: + # Parse 24-bit length, 8-bit type, 8-bit flags, 32-bit stream ID + # (RFC 9113 4.1) + # "!I H B B" + length = ( + self._frame_buffer[0] << 16 + | self._frame_buffer[1] << 8 + | self._frame_buffer[2] + ) + frame_type_val = self._frame_buffer[3] + flags = self._frame_buffer[4] + stream_id = struct.unpack("!I", self._frame_buffer[5:9])[0] & STREAM_ID_MASK + + if len(self._frame_buffer) < FRAME_HEADER_LENGTH + length: + break # incomplete frame; wait for more data + + payload = bytes( + self._frame_buffer[FRAME_HEADER_LENGTH : FRAME_HEADER_LENGTH + length] + ) + del self._frame_buffer[: FRAME_HEADER_LENGTH + length] + + # invalid frames cause a value error + if frame_type_val <= 9 and frame_type_val >= 0: + logger.debug( + "<- %s stream=%d flags=0x%02x len=%d", + FrameType(frame_type_val).name, + stream_id, + flags, + length, + ) + + try: + self._dispatch_frame(frame_type_val, flags, stream_id, payload) + except ConnectionError as exc: + logger.error("Frame dispatch error: %s", exc, exc_info=True) + self._protocol_error() + + def eof_received(self) -> bool: + logger.debug("EOF received from server") + self._transport.close() + return False + + def connection_lost(self, exc: Optional[Exception]) -> None: + logger.debug(f"Connection lost: {exc}") + # Cancel all pending streams (including those in the queue) + for stream in list(self.streams.values()): + if not stream.response_future.done(): + stream.response_future.set_exception(ConnectionError("Connection lost")) + for fut in self._pending_streams: + fut.set_exception(ConnectionError("Connection lost")) + self.streams.clear() + + # -------------------- Frame dispatch -------------------- + def _dispatch_frame( + self, frame_type: int, flags: int, stream_id: int, payload: bytes + ) -> None: + if frame_type == FrameType.DATA: + self._handle_data_frame(flags, stream_id, payload) + elif frame_type == FrameType.HEADERS: + self._handle_headers_frame(flags, stream_id, payload) + elif frame_type == FrameType.PRIORITY: + self._handle_priority_frame(stream_id, payload) + elif frame_type == FrameType.RST_STREAM: + self._handle_rst_stream_frame(flags, stream_id, payload) + elif frame_type == FrameType.SETTINGS: + self._handle_settings_frame(flags, stream_id, payload) + elif frame_type == FrameType.PUSH_PROMISE: + # Client cannot receive push promises -- ignore or GOAWAY + logger.warning("Received PUSH_PROMISE; ignoring (no push support)") + elif frame_type == FrameType.PING: + self._handle_ping_frame(flags, stream_id, payload) + elif frame_type == FrameType.GOAWAY: + self._handle_goaway_frame(flags, stream_id, payload) + elif frame_type == FrameType.WINDOW_UPDATE: + self._handle_window_update_frame(flags, stream_id, payload) + elif frame_type == FrameType.CONTINUATION: + self._handle_continuation_frame(flags, stream_id, payload) + else: + logger.warning(f"Ignoring unknown frame type {frame_type}") + + # ---------- Individual frame handlers ---------- + def _handle_data_frame(self, flags: int, stream_id: int, payload: bytes) -> None: + stream = self.streams.get(stream_id) + if stream is None: + if stream_id > self._last_peer_stream_id: + self._send_rst_stream(stream_id, 1) # PROTOCOL_ERROR + return + + pad_length = 0 + pos = 0 + if flags & FlagData.PADDED: + pad_length = payload[0] + pos = 1 + + data = payload[pos : len(payload) - pad_length] + end_stream = bool(flags & FlagData.END_STREAM) + + # Update session flow control + self.session_inbound_window -= len(data) + if self.session_inbound_window < 32768: + self._send_window_update(0, 65535 - self.session_inbound_window) + self.session_inbound_window = 65535 + + stream.receive_data(data, end_stream) + + def _handle_headers_frame(self, flags: int, stream_id: int, payload: bytes) -> None: + if flags & FlagHeaders.PRIORITY: + # Exclusive flag + stream dependency + weight + priority_data = payload[:5] + payload = payload[5:] + + # Decode headers with HPACK + try: + headers = self.hpack_decoder.decode(payload) + except Exception as exc: + logger.error(f"HPACK decode error: {exc}") + self._send_rst_stream(stream_id, 1) # PROTOCOL_ERROR + return + + end_stream = bool(flags & FlagHeaders.END_STREAM) + + stream = self.streams.get(stream_id) + + if stream is None: + logger.error("Unknown stream_id: %d", stream) + else: + stream.receive_headers(headers, end_stream) + + def _handle_priority_frame(self, stream_id: int, payload: bytes) -> None: + # it's deprecated in the most recent RFC + pass + + def _handle_rst_stream_frame( + self, flags: int, stream_id: int, payload: bytes + ) -> None: + del flags # rst doesn't use flags + + error_code = struct.unpack("!I", payload)[0] + stream = self.streams.get(stream_id) + if stream: + stream.transition(StreamState.CLOSED) + if not stream.response_future.done(): + stream.response_future.set_exception( + RuntimeError(f"Stream reset by server (code={error_code})") + ) + self._close_stream(stream) + + def _handle_settings_frame( + self, flags: int, stream_id: int, payload: bytes + ) -> None: + """ + Process SETTINGS frame (6.5) + """ + if flags & FlagSettings.ACK: + logger.debug("Received SETTINGS ACK") + return # Our settings were acknowledged + if stream_id != 0: + logger.error( + "SETTING frame received after the first stream in violation of the protocol standard (RFC-9113, 3.4)" + ) + self._protocol_error() + return + + if len(payload) % 6 != 0: + logger.error("SETTINGS payload length not a multiple of 6") + self._protocol_error() + return + + # Parse key‑value pairs + for i in range(0, len(payload), 6): + identifier, value = struct.unpack("!H I", payload[i : i + 6]) + setting = Setting(identifier) + old_value = self.remote_settings.get(setting, value) + self.remote_settings[setting] = value + logger.info(f"Server SETTINGS: {setting.name} = {value}") + + # React to certain settings + if setting == Setting.INITIAL_WINDOW_SIZE and value != old_value: + delta = value - old_value + for s in self.streams.values(): + s.outbound_window += delta + # self.session_outbound_window += delta + elif setting == Setting.MAX_CONCURRENT_STREAMS: + self.max_concurrent_streams = value + self._maybe_unblock_streams() + elif setting == Setting.HEADER_TABLE_SIZE: + self.hpack_encoder.header_table_size = value + + # Acknowledge settings + self._send_settings_ack() + + def _handle_ping_frame(self, flags: int, stream_id: int, payload: bytes) -> None: + if stream_id != 0: + self._protocol_error() + return + if flags & FlagPing.ACK: + logger.debug("Received PING ACK") + else: + # Respond with ACK + logger.debug("Received PING, sending ACK") + self._send_ping(ack=True, opaque_data=payload) + + def _handle_goaway_frame(self, flags: int, stream_id: int, payload: bytes) -> None: + del flags, stream_id # interface + + self._goaway_received = True + last_stream_id, error_code = struct.unpack("!I I", payload[:8]) + extra = payload[8:] + self._last_stream_id = last_stream_id + self._error_code = error_code + logger.info( + f"GOAWAY received: last_stream={last_stream_id}, error={error_code}, extra={extra}" + ) + # Cancel streams with higher IDs + for sid, stream in list(self.streams.items()): + if sid > last_stream_id: + if not stream.response_future.done(): + stream.response_future.set_exception( + ConnectionError("GOAWAY received") + ) + self._close_stream(stream) + + def _handle_window_update_frame( + self, flags: int, stream_id: int, payload: bytes + ) -> None: + increment = struct.unpack("!I", payload)[0] + if stream_id == 0: + # Session window update + self.session_outbound_window += increment + else: + stream = self.streams.get(stream_id) + if stream: + stream.outbound_window += increment + # Wake up any writer waiting for flow control + self._flow_control_updated.set() + + def _handle_continuation_frame( + self, flags: int, stream_id: int, payload: bytes + ) -> None: + # Not implemented; would require accumulating headers across frames. + logger.warning("CONTINUATION frame ignored (not implemented)") + + # -------------------- Frame sending helpers -------------------- + def _send_frame( + self, frame_type: FrameType, flags: int, stream_id: int, payload: bytes = b"" + ) -> None: + length = len(payload) & 0x00FFFFFF # 24 bits -> 3 bytes + # header = struct.pack("!I H B B", length, frame_type, flags, stream_id) + + header = struct.pack("!I", length)[ + 1: + ] + struct.pack( # drop the first (most‑significant) byte → 3 bytes + "!B B I", frame_type, flags, stream_id + ) + + logger.debug( + f"-> FRAME type={frame_type.name:>15} flags=0x{flags:02x} " + f"stream_id={stream_id:<5} length={len(payload)}" + ) + self._transport.write(header + payload) + + def _send_settings_ack(self) -> None: + self._send_frame(FrameType.SETTINGS, FlagSettings.ACK, 0) + + def _send_ping(self, ack: bool = False, opaque_data: bytes = b"\x00" * 8) -> None: + flags = FlagPing.ACK if ack else 0 + self._send_frame(FrameType.PING, flags, 0, opaque_data) + + def _send_goaway(self, last_stream_id: int, error_code: int) -> None: + payload = struct.pack("!I I", last_stream_id, error_code) + self._send_frame(FrameType.GOAWAY, 0, 0, payload) + self._goaway_sent = True + + def _send_rst_stream(self, stream_id: int, error_code: int) -> None: + payload = struct.pack("!I", error_code) + self._send_frame(FrameType.RST_STREAM, 0, stream_id, payload) + + def _send_window_update(self, stream_id: int, increment: int) -> None: + payload = struct.pack("!I", increment) + self._send_frame(FrameType.WINDOW_UPDATE, 0, stream_id, payload) + + # -------------------- Stream lifecycle -------------------- + def _close_stream(self, stream: Stream) -> None: + self.streams.pop(stream.stream_id, None) + self._closed_streams.add(stream.stream_id) + # Release stream concurrency slot + self._maybe_unblock_streams() + + def _maybe_unblock_streams(self) -> None: + """Create streams from pending requests if concurrency allows.""" + while self._pending_streams and len(self.streams) < self.max_concurrent_streams: + fut = self._pending_streams.pop(0) + if not fut.done(): + stream = self._create_stream_internal() + fut.set_result(stream) + + def _create_stream_internal(self) -> Stream: + sid = self.next_stream_id + self.next_stream_id += 2 # next client stream + stream = Stream(sid, self, self._loop) + self.streams[sid] = stream + return stream + + async def create_stream(self) -> Stream: + """Return a new client stream, waiting if concurrency limit is reached.""" + if self._goaway_sent or self._goaway_received: + raise ConnectionError("Connection is shutting down") + if len(self.streams) < self.max_concurrent_streams: + return self._create_stream_internal() + # Queue the request + fut = self._loop.create_future() + self._pending_streams.append(fut) + return await fut + + # -------------------- Request sending -------------------- + async def send_data( + self, stream: Stream, data: bytes, end_stream: bool = True + ) -> None: + """Asynchronously send DATA frames, respecting flow control windows.""" + max_frame_size = self.remote_settings[Setting.MAX_FRAME_SIZE] + offset = 0 + total = len(data) + + while offset < total: + # Wait until both session and stream windows have capacity + while stream.outbound_window <= 0 or self.session_outbound_window <= 0: + self._flow_control_updated.clear() + await self._flow_control_updated.wait() + + chunk_size = min( + max_frame_size, + stream.outbound_window, + self.session_outbound_window, + total - offset, + ) + flags = 0 + if offset + chunk_size >= total and end_stream: + flags |= FlagData.END_STREAM + + if stream.state == StreamState.OPEN: + stream.transition(StreamState.HALF_CLOSED_LOCAL) + elif stream.state == StreamState.HALF_CLOSED_REMOTE: + stream.transition(StreamState.CLOSED) + self._close_stream(stream) + # else: error + + + self._send_frame( + FrameType.DATA, + flags, + stream.stream_id, + data[offset : offset + chunk_size], + ) + stream.outbound_window -= chunk_size + self.session_outbound_window -= chunk_size + offset += chunk_size + + # there is space available + if self.session_outbound_window and stream.outbound_window: + self._flow_control_updated.set() + + async def send_request( + self, + stream: Stream, + method: str, + url: str, + headers: List[Tuple[str, str]], + body: Optional[bytes] = None, + ) -> None: + """Send HEADERS and optional DATA frames for a stream.""" + # Build pseudo‑headers + req_headers = [ + (":method", method), + (":path", url.path), + (":scheme", url.scheme), + (":authority", url.host), + ] + # 8.2. HTTP Fields + # Field names MUST be converted to lowercase when constructing an HTTP/2 message. + req_headers.extend([(h[0].lower(), h[1]) for h in headers]) + + hdrs = self.hpack_encoder.encode(req_headers) + end_stream = body is None or len(body) == 0 + flags = FlagHeaders.END_HEADERS + + # stream transitions + stream.transition(StreamState.OPEN) + if end_stream: + flags |= FlagHeaders.END_STREAM + stream.transition(StreamState.HALF_CLOSED_LOCAL) + + self._send_frame(FrameType.HEADERS, flags, stream.stream_id, hdrs) + + if body: + await self.send_data(stream, body, end_stream=True) + + # -------------------- Connection lifecycle -------------------- + def initiate_connection(self) -> None: + """Send the connection preface and initial SETTINGS.""" + # Connection preface (RFC 7540 §3.5) + self._transport.write(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") + + # Send initial SETTINGS (our preferences) + # We can advertise supported settings, e.g., enable push = 0 + settings_payload = struct.pack( + "!H I", Setting.ENABLE_PUSH, 0 # disable server push + ) + self._send_frame(FrameType.SETTINGS, 0, 0, settings_payload) + + # Update local HPACK table size if needed + self.hpack_encoder.header_table_size = self.local_settings[ + Setting.HEADER_TABLE_SIZE + ] + + logger.debug("Connection preface and initial SETTINGS sent") + + def _protocol_error(self) -> None: + self._send_goaway(0, 1) # PROTOCOL_ERROR + self._transport.close() + + # -------------------- Shutdown -------------------- + def close(self) -> None: + """Perform graceful shutdown.""" + if not self._goaway_sent: + # GOAWAY could be sent before we create the first stream + self._send_goaway(self._last_stream_id or max(0, self.next_stream_id - 2), 0) + # Wait a moment for GOAWAY to be sent, then close transport + self._transport.close() + + # Compatibility methods for aiohttp connector + @property + def should_close(self) -> bool: + return self._goaway_sent or self._goaway_received + + def is_connected(self) -> bool: + return not self._transport.is_closing() + + @property + def closed(self) -> asyncio.Future: + return self._loop.create_future() # XXX not properly implemented + + +# ---------------------------------------------------------------------- +# Protocol wrapper for asyncio.Transport (connector integration) +# ---------------------------------------------------------------------- +class Http2Protocol(asyncio.Protocol): + """asyncio.Protocol subclass that bridges transport and Http2Connection. + + This replaces aiohttp's ResponseHandler in the connector. + """ + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._connection: Optional[Http2Connection] = None + self._closed_future = loop.create_future() # resolves when connection closes + self.transport: Optional[asyncio.Transport] = None + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + self.transport = transport # type: ignore[assignment] + self._connection = Http2Connection(self.transport, self._loop) + self._connection.initiate_connection() + + def data_received(self, data: bytes) -> None: + if self._connection: + self._connection.data_received(data) + + def eof_received(self) -> bool: + if self._connection: + self._connection.eof_received() + return False + + def connection_lost(self, exc: Optional[Exception]) -> None: + if self._connection: + self._connection.connection_lost(exc) + if not self._closed_future.done(): + self._closed_future.set_result(None) + + # Compatibility with connector's expectations + @property + def should_close(self) -> bool: + return self._connection.should_close if self._connection else False + + def is_connected(self) -> bool: + return self._connection.is_connected() if self._connection else False + + @property + def closed(self) -> asyncio.Future: + return self._closed_future + + # Public API for creating streams + async def create_stream(self) -> Stream: + if self._connection is None: + raise RuntimeError("Connection not established") + return await self._connection.create_stream() + + def send_request( + self, + stream: Stream, + method: str, + url: str, + headers: List[Tuple[str, str]], + body: Optional[bytes] = None, + ) -> None: + if self._connection is None: + raise RuntimeError("Connection not established") + self._connection.send_request(stream, method, url, headers, body) + + async def send( + self, + method: str, + url: Any, # yarl.URL or str (but the protocol expects a URL object later) + headers: List[Tuple[str, str]], + body: Optional[bytes] = None, + ) -> Http2Response: + """Send an HTTP/2 request and return a compatible response. + + The response object will carry `url`, `method`, and `_connection` + so that aiohttp’s `ClientSession._request` can work without modification. + """ + if self._connection is None: + raise RuntimeError("Connection not established") + + # Obtain a stream from the pool (blocks only if concurrency limit reached) + stream = await self._connection.create_stream() + + # The internal connection still expects a URL object with .host, .path etc. + # Here we assume `url` is a yarl.URL (the session always passes a yarl.URL). + # If you receive a string, convert it: url = URL(url) + await self._connection.send_request(stream, method, url, headers, body) + + # Wait for the response future to be set by the stream when complete + _, resp_headers, resp_body = await stream.response_future + + # Build the full response object + response = Http2Response( + headers=resp_headers, + body=resp_body, + method=method, + url=url, + ) + + return response + + def close(self) -> None: + if self._connection: + self._connection.close() + self.transport = None diff --git a/aiohttp/http2/errors.py b/aiohttp/http2/errors.py new file mode 100644 index 00000000000..339578b44f4 --- /dev/null +++ b/aiohttp/http2/errors.py @@ -0,0 +1,2 @@ +class ProtocolError(Exception): + pass diff --git a/aiohttp/http2/response.py b/aiohttp/http2/response.py new file mode 100644 index 00000000000..8679b409bfb --- /dev/null +++ b/aiohttp/http2/response.py @@ -0,0 +1,120 @@ +from http.cookies import SimpleCookie +import json +from multidict import CIMultiDict +from typing import List, Tuple, Optional, NamedTuple, Any + +from ..http_writer import HttpVersion2 + + +class Http2Response: + """A fully aiohttp.ClientResponse-compatible response for HTTP/2.""" + + def __init__( + self, + headers: List[Tuple[str, str]], + body: bytes, + *, + method: Optional[str] = None, + url: Optional[Any] = None, + ) -> None: + self.reason = "" # HTTP/2 doesn't carry a reason phrase + self._body = body + self.url = url + self.method = method + self.history: List[Any] = [] # for redirects + + # Headers as case-insensitive multi-dict (mimics aiohttp's CIMultiDict) + self.headers: CIMultiDict = CIMultiDict(headers) + # no status error implies a server side error + self.status = int(self.headers.get(":status", 500)) + + # Cookie jar integration + self._cookies: Optional[SimpleCookie] = None + + # HTTP version pseudo-attribute (aiohttp expects a namedtuple-like object) + #self.version = NamedTuple("HttpVersion", major=int, minor=int)(2, 0) + self.version = HttpVersion2 + + # not implemented + self._raw_cookie_headers = None + self.connection = None + + # Connection back-reference (set by the protocol) + self._connection: Optional["Http2Protocol"] = None + + # ---------------------------------------------------------------- + # Body access (synchronous: entire body is already in memory) + # ---------------------------------------------------------------- + async def read(self) -> bytes: + """Return the response body.""" + return self._body + + @property + def body(self): + return self._body + + async def text(self, encoding: str = "utf-8") -> str: + """Decode the body to a string.""" + return self._body.decode(encoding) + + async def json(self, **kwargs) -> Any: + """Parse JSON body.""" + return json.loads(self._body, **kwargs) + + # ---------------------------------------------------------------- + # Cookies + # ---------------------------------------------------------------- + @property + def cookies(self) -> SimpleCookie: + """Parse 'Set-Cookie' headers and return a SimpleCookie.""" + if self._cookies is None: + self._cookies = SimpleCookie() + # self.headers is a CIMultiDict – use getall() to obtain all values + for raw in self.headers.getall("set-cookie", []): + self._cookies.load(raw) + return self._cookies + + # ---------------------------------------------------------------- + # Status helpers + # ---------------------------------------------------------------- + @property + def ok(self) -> bool: + """True if status is < 400.""" + return 200 <= self.status < 400 + + def raise_for_status(self) -> None: + """Raise an HTTPError for 4xx/5xx responses.""" + if not self.ok: + from aiohttp.client_exceptions import ClientResponseError + + raise ClientResponseError( + request_info=None, # simplified + history=self.history, + status=self.status, + message=f"{self.status}, message='{self.reason}'", + headers=self.headers, + ) + + # ---------------------------------------------------------------- + # Connection release (stream-level cleanup) + # ---------------------------------------------------------------- + async def release(self) -> None: + """Release the HTTP/2 stream back to the connection. + + In HTTP/2 the stream is already closed once the full response is + received. This method is a no-op but required for aiohttp + compatibility. + """ + pass # nothing to do; the stream has ended + + def close(self): + pass + + # ---------------------------------------------------------------- + # Context manager support (optional, often used with 'async with') + # ---------------------------------------------------------------- + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + pass diff --git a/aiohttp/http2/settings.py b/aiohttp/http2/settings.py new file mode 100644 index 00000000000..e85c9ec7d1d --- /dev/null +++ b/aiohttp/http2/settings.py @@ -0,0 +1,65 @@ +from enum import IntEnum, IntFlag + + +# ---------------------------------------------------------------------- +# HTTP/2 Frame Definitions (RFC 7540, 4) +# ---------------------------------------------------------------------- +class FrameType(IntEnum): + DATA = 0x0 + HEADERS = 0x1 + PRIORITY = 0x2 + RST_STREAM = 0x3 + SETTINGS = 0x4 + PUSH_PROMISE = 0x5 + PING = 0x6 + GOAWAY = 0x7 + WINDOW_UPDATE = 0x8 + CONTINUATION = 0x9 + + +class FlagData(IntFlag): + END_STREAM = 0x1 + PADDED = 0x8 + + +class FlagHeaders(IntFlag): + END_STREAM = 0x1 + END_HEADERS = 0x4 + PADDED = 0x8 + PRIORITY = 0x20 + + +class FlagSettings(IntFlag): + ACK = 0x1 + + +class FlagPing(IntFlag): + ACK = 0x1 + + +# Known settings parameters +class Setting(IntEnum): + HEADER_TABLE_SIZE = 0x1 + ENABLE_PUSH = 0x2 + MAX_CONCURRENT_STREAMS = 0x3 + INITIAL_WINDOW_SIZE = 0x4 + MAX_FRAME_SIZE = 0x5 + MAX_HEADER_LIST_SIZE = 0x6 + # this is mostly for web sockets + # via proxies + # which is not supported to the date + SETTINGS_ENABLE_CONNECT_PROTOCOL = 0x8 + # does nothing as we don't implement this deprecated + # implementation + NO_RFC7540_PRIORITIES = 0x9 + + +# Default values (RFC 7540, 6.5.2) +DEFAULT_SETTINGS = { + Setting.HEADER_TABLE_SIZE: 4096, + Setting.ENABLE_PUSH: 1, + Setting.MAX_CONCURRENT_STREAMS: 2**32 - 1, # effectively unlimited + Setting.INITIAL_WINDOW_SIZE: 65535, + Setting.MAX_FRAME_SIZE: 16384, + Setting.MAX_HEADER_LIST_SIZE: 2**32 - 1, +} diff --git a/aiohttp/http2/stream.py b/aiohttp/http2/stream.py new file mode 100644 index 00000000000..4442a72b564 --- /dev/null +++ b/aiohttp/http2/stream.py @@ -0,0 +1,132 @@ +import asyncio +from enum import IntEnum +from typing import List, Tuple, Optional + +from .settings import Setting +from .errors import ProtocolError + + +# ---------------------------------------------------------------------- +# Stream State Machine (RFC 7540 5.1) +# ---------------------------------------------------------------------- +class StreamState(IntEnum): + IDLE = 0 + RESERVED_LOCAL = 1 + RESERVED_REMOTE = 2 + OPEN = 3 + HALF_CLOSED_LOCAL = 4 + HALF_CLOSED_REMOTE = 5 + CLOSED = 6 + + +# Valid transitions (RFC 7540 Figure 2) +# Added missing RESERVED_REMOTE from IDLE +VALID_TRANSITIONS = { + StreamState.IDLE: { + StreamState.OPEN, + StreamState.RESERVED_LOCAL, + StreamState.RESERVED_REMOTE, + }, + StreamState.RESERVED_LOCAL: {StreamState.HALF_CLOSED_REMOTE, StreamState.CLOSED}, + StreamState.RESERVED_REMOTE: {StreamState.HALF_CLOSED_LOCAL, StreamState.CLOSED}, + StreamState.OPEN: {StreamState.HALF_CLOSED_LOCAL, StreamState.HALF_CLOSED_REMOTE}, + StreamState.HALF_CLOSED_LOCAL: {StreamState.CLOSED}, + StreamState.HALF_CLOSED_REMOTE: {StreamState.CLOSED}, + StreamState.CLOSED: set(), # terminal state +} + + +# ---------------------------------------------------------------------- +# Stream object – multiplexed stream handle +# ---------------------------------------------------------------------- +class Stream: + """A single HTTP/2 stream. + + Manages state, flow control, and provides a future for the response. + """ + + __slots__ = ( + "stream_id", + "state", + "conn", + "outbound_window", + "inbound_window", + "response_future", + "request_body", + "response_headers", + "response_data", + "closed_event", + ) + + def __init__(self, stream_id: int, conn: "Http2Connection", loop) -> None: + self.stream_id = stream_id + self.state = StreamState.IDLE + self.conn = conn + + # Flow‑control windows (stream‑level only; session window managed by connection) + self.outbound_window = conn.remote_settings[Setting.INITIAL_WINDOW_SIZE] + self.inbound_window = conn.local_settings[Setting.INITIAL_WINDOW_SIZE] + + self.response_future: asyncio.Future[ + Tuple[int, List[Tuple[str, str]], bytes] + ] = loop.create_future() + + self.response_headers: Optional[List[Tuple[str, str]]] = None + self.response_data = bytearray() + + # Event notified when the stream enters CLOSED state + self.closed_event = asyncio.Event() + + def transition(self, new_state: StreamState) -> None: + if new_state not in VALID_TRANSITIONS[self.state] and new_state != StreamState.CLOSED: + raise ProtocolError( + f"Invalid stream state transition {self.state.name} -> {new_state.name}" + ) + self.state = new_state + if new_state == StreamState.CLOSED: + self.closed_event.set() + + # ------------------------------------------------------------------ + # Data and header reception – state transitions are now **conditional** + # on the current state, matching RFC 7540/9113 exactly. + # ------------------------------------------------------------------ + def receive_data(self, data: bytes, end_stream: bool) -> None: + """Process incoming DATA frame payload.""" + self.inbound_window -= len(data) + self.response_data.extend(data) + + if end_stream: + # Correct transition depending on current state + if self.state == StreamState.OPEN: + self.transition(StreamState.HALF_CLOSED_REMOTE) + elif self.state == StreamState.HALF_CLOSED_LOCAL: + self.transition(StreamState.CLOSED) + self.conn._close_stream(self) # clean up connection map + else: + raise ProtocolError( + f"Unexpected stream state {self.state.name} for END_STREAM" + ) + + # Deliver the full response once headers have been received + if self.response_headers is not None and not self.response_future.done(): + self.response_future.set_result( + (self.stream_id, self.response_headers, bytes(self.response_data)) + ) + + def receive_headers(self, headers: List[Tuple[str, str]], end_stream: bool) -> None: + """Process incoming HEADERS frame payload.""" + self.response_headers = headers + + if end_stream: + if self.state == StreamState.OPEN: + self.transition(StreamState.HALF_CLOSED_REMOTE) + elif self.state == StreamState.HALF_CLOSED_LOCAL: + self.transition(StreamState.CLOSED) + self.conn._close_stream(self) + else: + raise ProtocolError( + f"Unexpected stream state {self.state.name} for END_STREAM on headers" + ) + # The response is complete (no body) + self.response_future.set_result((self.stream_id, headers, b"")) + # else: stream remains in OPEN (or HALF_CLOSED_LOCAL), headers stored for later delivery. diff --git a/aiohttp/http_protocol.py b/aiohttp/http_protocol.py new file mode 100644 index 00000000000..e612612a0ca --- /dev/null +++ b/aiohttp/http_protocol.py @@ -0,0 +1,52 @@ +# aiohttp/http_protocol.py (new file) +import asyncio +from typing import Optional + +from .client_proto import ResponseHandler +from .http2.connection import Http2Protocol + + +class HttpDispatcherProtocol(asyncio.Protocol): + """Protocol that switches between HTTP/1.1 and HTTP/2 based on ALPN.""" + + __slots__ = ("_loop", "_transport", "_handler") + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._transport: Optional[asyncio.Transport] = None + self._handler: Optional[asyncio.Protocol] = None + + # ---- Transport callbacks forwarded to the real handler ---- + def connection_made(self, transport: asyncio.BaseTransport) -> None: + if self._handler: + return self._handler.connection_made(transport) + + self._transport = transport # type: ignore[assignment] + + # Determine ALPN after TLS is established + ssl_object = transport.get_extra_info("ssl_object") + alpn_protocol = ( + ssl_object.selected_alpn_protocol() if ssl_object else "http/1.1" + ) + + if alpn_protocol == "h2": + self._handler = Http2Protocol(self._loop) + else: + self._handler = ResponseHandler(self._loop) + + # Hand the real transport to the handler. The handler will now own + # all incoming data and callbacks. + self._handler.connection_made(transport) + + def __getattribute__(self, name): + if not name.startswith("__") and name not in {"connection_made", "__getattribute__", "_handler", "_transport", "_loop"}: + return getattr(self._handler, name) + return super().__getattribute__(name) + + def __setattr__(self, name, value): + if name not in {"_handler", "_transport", "_loop"}: + return self._handler.__setattr__(name, value) + return super().__setattr__(name, value) + + def __delattr__(self, name): + return self._handler.__delattr__(name) diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index a1168cfdebb..06cc44f5008 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -3,7 +3,7 @@ import asyncio import re import sys -from typing import ( # noqa +from typing import ( TYPE_CHECKING, Any, Awaitable, @@ -44,6 +44,7 @@ class HttpVersion(NamedTuple): HttpVersion10 = HttpVersion(1, 0) HttpVersion11 = HttpVersion(1, 1) +HttpVersion2 = HttpVersion(2, 0) _T_OnChunkSent = Optional[ diff --git a/pyproject.toml b/pyproject.toml index 0c27cc88bb5..a82ee886371 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "propcache >= 0.2.0", "typing_extensions >= 4.4 ; python_version < '3.13'", "yarl >= 1.17.0, < 2.0", + "hpack >= 4.2.0" ] dynamic = [ "version", diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py new file mode 100644 index 00000000000..ca6072ca83d --- /dev/null +++ b/tests/http2/test_http2.py @@ -0,0 +1,539 @@ +""" +Test suite for aiohttp.http2.protocol + +Categories: +- integration: against a real httpbin server (skipped, requires network) +- unit / protocol: black‑box frame‑level RFC compliance +- unit / misc: race conditions, deadlocks, edge cases +- unit / interface: high‑level features (JSON, redirects, compression, etc.) +""" + +import asyncio +import struct +import json +import gzip +from typing import List, Tuple, Optional +from unittest.mock import MagicMock, AsyncMock, patch + +import pytest +from hpack import Encoder, Decoder + +import aiohttp +from aiohttp.http2.errors import ProtocolError +from aiohttp.http2.stream import Stream, StreamState +from aiohttp.http2.settings import ( + DEFAULT_SETTINGS, + Setting, + FrameType, + FlagSettings, + FlagHeaders, + FlagData, + FlagPing, +) +from aiohttp.http2.response import Http2Response +from aiohttp.http2.connection import Http2Connection, Http2Protocol + + +# ---------------------------------------------------------------------- +# Helper: minimal URL mock +# ---------------------------------------------------------------------- +def url_mock(path="/"): + """Create a simple URL-like object expected by the implementation.""" + return type("URL", (), {"scheme": "https", "host": "example.com", "path": path}) + +# ---------------------------------------------------------------------- +# Fixtures +# ---------------------------------------------------------------------- +@pytest.fixture +def mock_transport(): + """Return a mock asyncio.Transport that records writes.""" + t = MagicMock(spec=asyncio.Transport) + t.is_closing.return_value = False + t.write = MagicMock() + t.close = MagicMock() + return t + + +@pytest.fixture +def event_loop(): + """Create a new event loop for each test.""" + loop = asyncio.new_event_loop() + try: + yield loop + finally: + loop.close() + + +@pytest.fixture +#async def connection(mock_transport, event_loop): +async def connection(mock_transport): + """Set up Http2Connection with mock transport, send preface, and clear write log.""" + event_loop = asyncio.get_running_loop() + + conn = Http2Connection(mock_transport, event_loop) + conn.initiate_connection() + mock_transport.write.reset_mock() # discard preface + initial SETTINGS + return conn, mock_transport + + +@pytest.fixture +#async def protocol(mock_transport, event_loop): +async def protocol(mock_transport): + """Create Http2Protocol and simulate connection_made.""" + event_loop = asyncio.get_running_loop() + proto = Http2Protocol(event_loop) + proto.connection_made(mock_transport) + mock_transport.write.reset_mock() + return proto, mock_transport + + +# ---------------------------------------------------------------------- +# Frame construction helpers +# ---------------------------------------------------------------------- +# one could argue that we could add these static methods to the implementation +def frame_header(length: int, ftype: FrameType, flags: int, stream_id: int) -> bytes: + # 24-bit length (3 bytes) + type + flags + stream_id + return struct.pack("!I", length)[1:] + struct.pack( + "!B B I", ftype, flags, stream_id + ) + + +def build_settings_frame( + settings_pairs: List[Tuple[Setting, int]] = None, ack: bool = False +) -> bytes: + payload = b"" + if not ack and settings_pairs: + for setting_id, value in settings_pairs: + payload += struct.pack("!H I", setting_id, value) + flags = FlagSettings.ACK if ack else 0 + return frame_header(len(payload), FrameType.SETTINGS, flags, 0) + payload + + +def build_headers_frame( + stream_id: int, + headers: List[Tuple[str, str]], + end_headers: bool = True, + end_stream: bool = False, + priority: Optional[bytes] = None, +) -> bytes: + encoder = Encoder() + header_block = encoder.encode(headers) + flags = 0 + if end_headers: + flags |= FlagHeaders.END_HEADERS + if end_stream: + flags |= FlagHeaders.END_STREAM + if priority is not None: + flags |= FlagHeaders.PRIORITY + header_block = priority + header_block + return ( + frame_header(len(header_block), FrameType.HEADERS, flags, stream_id) + + header_block + ) + + +def build_data_frame( + stream_id: int, data: bytes, end_stream: bool = False, pad: bool = False +) -> bytes: + payload = data + flags = 0 + if end_stream: + flags |= FlagData.END_STREAM + if pad: + pad_len = 1 # minimal padding for test + payload = bytes([pad_len]) + data + b"\x00" * pad_len + flags |= FlagData.PADDED + return frame_header(len(payload), FrameType.DATA, flags, stream_id) + payload + + +def build_rst_stream(stream_id: int, error_code: int = 0) -> bytes: + payload = struct.pack("!I", error_code) + return frame_header(4, FrameType.RST_STREAM, 0, stream_id) + payload + + +def build_goaway(last_stream_id: int, error_code: int, extra: bytes = b"") -> bytes: + payload = struct.pack("!I I", last_stream_id, error_code) + extra + return frame_header(len(payload), FrameType.GOAWAY, 0, 0) + payload + + +def build_window_update(stream_id: int, increment: int) -> bytes: + payload = struct.pack("!I", increment) + return frame_header(4, FrameType.WINDOW_UPDATE, 0, stream_id) + payload + + +def build_ping(ack: bool = False, opaque: bytes = b"\x00" * 8) -> bytes: + flags = FlagPing.ACK if ack else 0 + return frame_header(8, FrameType.PING, flags, 0) + opaque + + +# ---------------------------------------------------------------------- +# Integration tests (skipped by default) +# ---------------------------------------------------------------------- +@pytest.mark.skip(reason="Requires internet connection and httpbin") +@pytest.mark.asyncio +async def test_integration_httpbin_get(): + """Real HTTP/2 request to httpbin.org (requires a server supporting h2).""" + async with aiohttp.ClientSession() as session: + # Force HTTP/2 if possible; this test assumes the connector is configured + # to use Http2Protocol when ALPN negotiates h2. + async with session.get("https://httpbin.org/get") as resp: + assert resp.status == 200 + data = await resp.json() + assert "headers" in data + + +@pytest.mark.skip(reason="Requires internet connection and httpbin") +@pytest.mark.asyncio +async def test_integration_httpbin_post(): + import aiohttp + + async with aiohttp.ClientSession() as session: + async with session.post( + "https://httpbin.org/post", json={"key": "value"} + ) as resp: + assert resp.status == 200 + data = await resp.json() + assert data["json"] == {"key": "value"} + + +# ====================================================================== +# UNIT TESTS +# ====================================================================== + + +# ---------------------------------------------------------------------- +# 1. Protocol compliance (black‑box, frame‑by‑frame) +# ---------------------------------------------------------------------- +class TestProtocolCompliance: + @pytest.mark.asyncio + async def test_receive_settings_updates_remote_and_acks( + self, connection, mock_transport + ): + conn, transport = connection + # Send server SETTINGS (HEADER_TABLE_SIZE=8192, MAX_CONCURRENT_STREAMS=50) + frame = build_settings_frame( + [ + (Setting.HEADER_TABLE_SIZE, 8192), + (Setting.MAX_CONCURRENT_STREAMS, 50), + ] + ) + conn.data_received(frame) + assert conn.remote_settings[Setting.HEADER_TABLE_SIZE] == 8192 + assert conn.remote_settings[Setting.MAX_CONCURRENT_STREAMS] == 50 + # Must have sent an ACK + assert any( + call[0][0][3:4] == FrameType.SETTINGS.to_bytes(1, "big") + and call[0][0][4] & FlagSettings.ACK + for call in transport.write.call_args_list + ) + + @pytest.mark.asyncio + async def test_receive_headers_for_new_stream(self, connection, mock_transport): + conn, _ = connection + # Create a stream from client side first (simulate request sent) + stream = await conn.create_stream() + await conn.send_request(stream, "GET", url_mock(), []) + stream.state = StreamState.OPEN # skip real send, just set state + + # Send response HEADERS with END_STREAM + headers = [(":status", "200"), ("content-type", "text/plain")] + frame = build_headers_frame(stream.stream_id, headers, end_stream=True) + conn.data_received(frame) + + # Verify stream received response + assert stream.response_future.done() + sid, resp_headers, body = stream.response_future.result() + assert sid == stream.stream_id + assert dict(resp_headers)[":status"] == "200" + assert body == b"" + + @pytest.mark.asyncio + async def test_receive_data_flow_control(self, connection, mock_transport): + conn, transport = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN # assume request already sent + + # Send DATA with some bytes + data = b"hello" + frame = build_data_frame(stream.stream_id, data, end_stream=False) + initial_window = conn.session_inbound_window + conn.data_received(frame) + assert conn.session_inbound_window == initial_window - len(data) + # Should trigger WINDOW_UPDATE when below threshold (32768) + # Because initial window is 65535 and we just consumed 5, still above threshold + assert not any( + b"WINDOW_UPDATE" in call.args[0] + for call in transport.write.call_args_list + ) + + # Send more data to drop below 32768 + big_data = b"x" * 40000 + frame2 = build_data_frame(stream.stream_id, big_data, end_stream=False) + conn.data_received(frame2) + # Now session window should have triggered an update + updates = [ + call.args[0] + for call in transport.write.call_args_list + if FrameType.WINDOW_UPDATE.to_bytes(1, "big") in call.args[0] + ] + assert len(updates) >= 1 + + @pytest.mark.asyncio + async def test_rst_stream_handling(self, connection): + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + + frame = build_rst_stream(stream.stream_id, error_code=0x8) # CANCEL + conn.data_received(frame) + + assert stream.state == StreamState.CLOSED + assert stream.response_future.done() + with pytest.raises(RuntimeError): + stream.response_future.result() + assert stream.stream_id not in conn.streams + + @pytest.mark.asyncio + async def test_goaway_cancels_higher_streams(self, connection): + conn, _ = connection + # create three streams (1,3,5) + s1 = await conn.create_stream() + s3 = await conn.create_stream() + s5 = await conn.create_stream() + s1.state = s3.state = s5.state = StreamState.OPEN + + # GOAWAY with last_stream_id = 3 + frame = build_goaway(last_stream_id=3, error_code=0) + conn.data_received(frame) + + # s1 (1) and s3 (3) should be unaffected, s5 (5) cancelled + assert s1.stream_id in conn.streams + assert s3.stream_id in conn.streams + assert s5.stream_id not in conn.streams + assert s5.response_future.exception() is not None + + @pytest.mark.asyncio + async def test_ping_ack(self, connection, mock_transport): + conn, transport = connection + frame = build_ping(ack=False, opaque=b"12345678") + conn.data_received(frame) + # Expect ACK sent back with same data + acks = [] + for call in transport.write.call_args_list: + arg = call.args[0] + if FrameType.PING.to_bytes(1, "big") in arg and arg[4] & FlagPing.ACK: + acks.append(arg) + assert len(acks) == 1 + assert b"12345678" in acks[0] + + @pytest.mark.asyncio + async def test_max_concurrent_streams_blocking(self, connection): + conn, _ = connection + conn.max_concurrent_streams = 1 + s1 = await conn.create_stream() + # second stream should block + create_task = asyncio.ensure_future(conn.create_stream()) + await asyncio.sleep(0.01) + assert not create_task.done() + # close s1 to release slot + conn._close_stream(s1) + s2 = await create_task + assert s2.stream_id > s1.stream_id + assert len(conn.streams) == 1 + + @pytest.mark.asyncio + async def test_unknown_frame_ignored(self, connection): + conn, _ = connection + # send frame type 0x1a (unused) + frame = frame_header(0, 0x1A, 0, 0) + conn.data_received(frame) + # Should not raise, connection stays intact + assert not conn._goaway_sent + + @pytest.mark.asyncio + async def test_bad_hpack_triggers_protocol_error(self, connection): + conn, transport = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + # corrupted headers + payload = b"\xff\xff\xff" + frame = ( + frame_header( + len(payload), + FrameType.HEADERS, + FlagHeaders.END_HEADERS, + stream.stream_id, + ) + + payload + ) + conn.data_received(frame) + # Should have sent RST_STREAM and/or GOAWAY + rst = any( + call.args[0][3:4] == FrameType.RST_STREAM.to_bytes(1, "big") + for call in transport.write.call_args_list + ) + goaway = any( + call.args[0][3:4] == FrameType.GOAWAY.to_bytes(1, "big") + for call in transport.write.call_args_list + ) + assert rst or goaway + + +# ---------------------------------------------------------------------- +# 2. Miscellaneous tests (race conditions, deadlocks, edge cases) +# ---------------------------------------------------------------------- +class TestMiscellaneous: + @pytest.mark.asyncio + async def test_concurrent_send_data_does_not_deadlock( + self, connection, mock_transport + ): + """Multiple tasks sending data on the same stream should not deadlock.""" + conn, transport = connection + stream = await conn.create_stream() + # Set window large enough + conn.session_outbound_window = 1_000_000 + stream.outbound_window = 1_000_000 + + async def send_chunk(): + await conn.send_data(stream, b"x" * 100, end_stream=False) + + tasks = [asyncio.create_task(send_chunk()) for _ in range(5)] + await asyncio.gather(*tasks, return_exceptions=True) + # All writes should complete eventually + assert transport.write.call_count >= 5 + + @pytest.mark.asyncio + async def test_window_update_wakes_all_waiters(self, connection, mock_transport): + """When window is zero, multiple blocked tasks resume on WINDOW_UPDATE.""" + conn, transport = connection + stream = await conn.create_stream() + conn.session_outbound_window = 0 + stream.outbound_window = 0 + + async def blocked_send(): + await conn.send_data(stream, b"hello", end_stream=False) + + task1 = asyncio.create_task(blocked_send()) + task2 = asyncio.create_task(blocked_send()) + await asyncio.sleep(0.01) # both waiting + + # Simulate WINDOW_UPDATE that opens 10 bytes + frame = build_window_update(0, 10) # session + conn.data_received(frame) + frame2 = build_window_update(stream.stream_id, 10) + conn.data_received(frame2) + await asyncio.sleep(0.01) + assert task1.done() + assert task2.done() + + @pytest.mark.asyncio + async def test_stream_cancelled_before_response(self, connection): + """Pending create_stream futures are cancelled on connection loss.""" + conn, _ = connection + conn.max_concurrent_streams = 1 + s1 = await conn.create_stream() + # Queue a second stream + fut = asyncio.ensure_future(conn.create_stream()) + await asyncio.sleep(0.01) + assert not fut.done() + # Simulate connection loss + conn.connection_lost(ConnectionError("test")) + await asyncio.sleep(0.01) + assert fut.done() + with pytest.raises(ConnectionError): + fut.result() + + @pytest.mark.asyncio + async def test_close_stream_on_rst_without_headers(self, connection): + """RST_STREAM before headers deliver must resolve future with error.""" + conn, _ = connection + stream = await conn.create_stream() + frame = build_rst_stream(stream.stream_id, error_code=0x8) + conn.data_received(frame) + assert stream.response_future.done() + with pytest.raises(RuntimeError): + stream.response_future.result() + + +# ---------------------------------------------------------------------- +# 3. Interface tests (high‑level features via Http2Protocol.send) +# ---------------------------------------------------------------------- +class TestHighLevelInterface: + @pytest.mark.asyncio + async def test_json_response(self, protocol, mock_transport): + """send() returns a Http2Response with correct JSON body.""" + proto, transport = protocol + + # Start a request using the public API + async def do_request(): + # url_mock() returns a simple URL object with required attributes + return await proto.send( + "GET", url_mock("/json"), headers=[("accept", "application/json")] + ) + + task = asyncio.create_task(do_request()) + # Let it run until it creates a stream and sends HEADERS + await asyncio.sleep(0.01) + # Now simulate server response + # Find the stream ID from the HEADERS frame written + write_calls = [call[0][0] for call in transport.write.call_args_list] + # The HEADERS frame will contain stream ID (first client stream = 1) + # Construct response HEADERS + DATA + headers = [(":status", "200"), ("content-type", "application/json")] + body = b'{"key": "value"}' + resp_frame = build_headers_frame( + 1, headers, end_stream=False + ) + build_data_frame(1, body, end_stream=True) + proto.data_received(resp_frame) + response = await task + assert response.status == 200 + assert response.body == body + assert json.loads(response.body) == {"key": "value"} + + @pytest.mark.asyncio + async def test_redirect_response(self, protocol, mock_transport): + """302 redirect headers are correctly returned.""" + proto, transport = protocol + task = asyncio.create_task(proto.send("GET", url_mock("/redirect"), headers=[])) + await asyncio.sleep(0.01) + headers = [(":status", "302"), ("location", "/new")] + resp_frame = build_headers_frame(1, headers, end_stream=True) + proto.data_received(resp_frame) + resp = await task + assert resp.status == 302 + assert dict(resp.headers).get("location") == "/new" + + @pytest.mark.asyncio + async def test_compressed_body_delivery(self, protocol, mock_transport): + """Response with content-encoding: gzip delivers raw compressed bytes.""" + proto, transport = protocol + task = asyncio.create_task(proto.send("GET", url_mock("/gzip"), headers=[])) + await asyncio.sleep(0.01) + raw_data = gzip.compress(b"uncompressed") + headers = [(":status", "200"), ("content-encoding", "gzip")] + frames = build_headers_frame(1, headers, end_stream=False) + build_data_frame( + 1, raw_data, end_stream=True + ) + proto.data_received(frames) + resp = await task + assert resp.body == raw_data + # Higher layer (aiohttp) will handle decompression + + async def test_multiple_requests_concurrently(self, protocol, mock_transport): + """Multiple send() calls create distinct streams and receive responses.""" + proto, transport = protocol + tasks = [] + for i in range(2): + tasks.append(asyncio.create_task(proto.send("GET", url_mock(f"/r{i}"), []))) + await asyncio.sleep(0.01) + + # Stream 1 and 3 should have been created + # Respond to each + resp1 = build_headers_frame(1, [(":status", "200")], end_stream=True) + resp3 = build_headers_frame(3, [(":status", "201")], end_stream=True) + proto.data_received(resp1) + proto.data_received(resp3) + + r1, r2 = await asyncio.gather(*tasks) + assert r1.status == 200 + assert r2.status == 201 From 8d6ffe23054843aa5f06594178af53249a4275c1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 2 Jul 2026 23:57:57 +0000 Subject: [PATCH 02/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/http2/connection.py | 23 ++++++++++++----------- aiohttp/http2/response.py | 7 ++++--- aiohttp/http2/stream.py | 9 ++++++--- aiohttp/http_protocol.py | 12 +++++++++--- tests/http2/test_http2.py | 32 ++++++++++++++++---------------- 5 files changed, 47 insertions(+), 36 deletions(-) diff --git a/aiohttp/http2/connection.py b/aiohttp/http2/connection.py index d650f5be12d..daf972d95b2 100644 --- a/aiohttp/http2/connection.py +++ b/aiohttp/http2/connection.py @@ -25,25 +25,25 @@ """ import asyncio -import struct import logging -from enum import IntEnum, IntFlag, auto +import struct from collections import defaultdict -from typing import Optional, Dict, Set, Tuple, List, Any, Union +from enum import IntEnum, IntFlag, auto +from typing import Any, Dict, List, Optional, Set, Tuple, Union -from hpack import Encoder, Decoder +from hpack import Decoder, Encoder +from .response import Http2Response from .settings import ( DEFAULT_SETTINGS, - Setting, - FrameType, FlagData, FlagHeaders, - FlagSettings, FlagPing, + FlagSettings, + FrameType, + Setting, ) from .stream import Stream, StreamState -from .response import Http2Response # ---------------------------------------------------------------------- # Logging – plaintext wire‑format emission for debugging @@ -461,7 +461,6 @@ async def send_data( self._close_stream(stream) # else: error - self._send_frame( FrameType.DATA, flags, @@ -540,7 +539,9 @@ def close(self) -> None: """Perform graceful shutdown.""" if not self._goaway_sent: # GOAWAY could be sent before we create the first stream - self._send_goaway(self._last_stream_id or max(0, self.next_stream_id - 2), 0) + self._send_goaway( + self._last_stream_id or max(0, self.next_stream_id - 2), 0 + ) # Wait a moment for GOAWAY to be sent, then close transport self._transport.close() @@ -625,7 +626,7 @@ def send_request( async def send( self, method: str, - url: Any, # yarl.URL or str (but the protocol expects a URL object later) + url: Any, # yarl.URL or str (but the protocol expects a URL object later) headers: List[Tuple[str, str]], body: Optional[bytes] = None, ) -> Http2Response: diff --git a/aiohttp/http2/response.py b/aiohttp/http2/response.py index 8679b409bfb..78e3c941c13 100644 --- a/aiohttp/http2/response.py +++ b/aiohttp/http2/response.py @@ -1,7 +1,8 @@ -from http.cookies import SimpleCookie import json +from http.cookies import SimpleCookie +from typing import Any, List, NamedTuple, Optional, Tuple + from multidict import CIMultiDict -from typing import List, Tuple, Optional, NamedTuple, Any from ..http_writer import HttpVersion2 @@ -32,7 +33,7 @@ def __init__( self._cookies: Optional[SimpleCookie] = None # HTTP version pseudo-attribute (aiohttp expects a namedtuple-like object) - #self.version = NamedTuple("HttpVersion", major=int, minor=int)(2, 0) + # self.version = NamedTuple("HttpVersion", major=int, minor=int)(2, 0) self.version = HttpVersion2 # not implemented diff --git a/aiohttp/http2/stream.py b/aiohttp/http2/stream.py index 4442a72b564..be374f5d79b 100644 --- a/aiohttp/http2/stream.py +++ b/aiohttp/http2/stream.py @@ -1,9 +1,9 @@ import asyncio from enum import IntEnum -from typing import List, Tuple, Optional +from typing import List, Optional, Tuple -from .settings import Setting from .errors import ProtocolError +from .settings import Setting # ---------------------------------------------------------------------- @@ -78,7 +78,10 @@ def __init__(self, stream_id: int, conn: "Http2Connection", loop) -> None: self.closed_event = asyncio.Event() def transition(self, new_state: StreamState) -> None: - if new_state not in VALID_TRANSITIONS[self.state] and new_state != StreamState.CLOSED: + if ( + new_state not in VALID_TRANSITIONS[self.state] + and new_state != StreamState.CLOSED + ): raise ProtocolError( f"Invalid stream state transition {self.state.name} -> {new_state.name}" ) diff --git a/aiohttp/http_protocol.py b/aiohttp/http_protocol.py index e612612a0ca..32061964ffc 100644 --- a/aiohttp/http_protocol.py +++ b/aiohttp/http_protocol.py @@ -39,14 +39,20 @@ def connection_made(self, transport: asyncio.BaseTransport) -> None: self._handler.connection_made(transport) def __getattribute__(self, name): - if not name.startswith("__") and name not in {"connection_made", "__getattribute__", "_handler", "_transport", "_loop"}: + if not name.startswith("__") and name not in { + "connection_made", + "__getattribute__", + "_handler", + "_transport", + "_loop", + }: return getattr(self._handler, name) return super().__getattribute__(name) - + def __setattr__(self, name, value): if name not in {"_handler", "_transport", "_loop"}: return self._handler.__setattr__(name, value) return super().__setattr__(name, value) - + def __delattr__(self, name): return self._handler.__delattr__(name) diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index ca6072ca83d..cf695a29e99 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -9,29 +9,29 @@ """ import asyncio -import struct -import json import gzip -from typing import List, Tuple, Optional -from unittest.mock import MagicMock, AsyncMock, patch +import json +import struct +from typing import List, Optional, Tuple +from unittest.mock import AsyncMock, MagicMock, patch import pytest -from hpack import Encoder, Decoder +from hpack import Decoder, Encoder import aiohttp +from aiohttp.http2.connection import Http2Connection, Http2Protocol from aiohttp.http2.errors import ProtocolError -from aiohttp.http2.stream import Stream, StreamState +from aiohttp.http2.response import Http2Response from aiohttp.http2.settings import ( DEFAULT_SETTINGS, - Setting, - FrameType, - FlagSettings, - FlagHeaders, FlagData, + FlagHeaders, FlagPing, + FlagSettings, + FrameType, + Setting, ) -from aiohttp.http2.response import Http2Response -from aiohttp.http2.connection import Http2Connection, Http2Protocol +from aiohttp.http2.stream import Stream, StreamState # ---------------------------------------------------------------------- @@ -41,6 +41,7 @@ def url_mock(path="/"): """Create a simple URL-like object expected by the implementation.""" return type("URL", (), {"scheme": "https", "host": "example.com", "path": path}) + # ---------------------------------------------------------------------- # Fixtures # ---------------------------------------------------------------------- @@ -65,7 +66,7 @@ def event_loop(): @pytest.fixture -#async def connection(mock_transport, event_loop): +# async def connection(mock_transport, event_loop): async def connection(mock_transport): """Set up Http2Connection with mock transport, send preface, and clear write log.""" event_loop = asyncio.get_running_loop() @@ -77,7 +78,7 @@ async def connection(mock_transport): @pytest.fixture -#async def protocol(mock_transport, event_loop): +# async def protocol(mock_transport, event_loop): async def protocol(mock_transport): """Create Http2Protocol and simulate connection_made.""" event_loop = asyncio.get_running_loop() @@ -262,8 +263,7 @@ async def test_receive_data_flow_control(self, connection, mock_transport): # Should trigger WINDOW_UPDATE when below threshold (32768) # Because initial window is 65535 and we just consumed 5, still above threshold assert not any( - b"WINDOW_UPDATE" in call.args[0] - for call in transport.write.call_args_list + b"WINDOW_UPDATE" in call.args[0] for call in transport.write.call_args_list ) # Send more data to drop below 32768 From 982ad3abde804a31e7c5e001caffc513c780dfd9 Mon Sep 17 00:00:00 2001 From: Moist-Cat Date: Fri, 3 Jul 2026 22:27:59 -0400 Subject: [PATCH 03/12] Lint code and add test --- aiohttp/http2/connection.py | 27 ++++++++------------ aiohttp/http2/response.py | 4 +-- aiohttp/http2/stream.py | 2 +- requirements/runtime-deps.in | 1 + tests/http2/test_http2.py | 49 ++++++++---------------------------- 5 files changed, 25 insertions(+), 58 deletions(-) diff --git a/aiohttp/http2/connection.py b/aiohttp/http2/connection.py index daf972d95b2..9e6f1c93817 100644 --- a/aiohttp/http2/connection.py +++ b/aiohttp/http2/connection.py @@ -1,7 +1,4 @@ """ -aiohttp.http2.protocol -====================== - Complete HTTP/2 client implementation (RFC 7540) for asyncio. This module provides: @@ -27,9 +24,7 @@ import asyncio import logging import struct -from collections import defaultdict -from enum import IntEnum, IntFlag, auto -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple from hpack import Decoder, Encoder @@ -51,7 +46,7 @@ logger = logging.getLogger("aiohttp.http2.connection") logger.setLevel(logging.DEBUG) -FRAME_HEADER_LENGTH = 9 # 9 octects +FRAME_HEADER_LENGTH = 9 # 9 octets STREAM_ID_MASK = 0x7FFFFFFF # to avoid setting the reserved bit to 1 @@ -175,15 +170,17 @@ def _dispatch_frame( self._handle_data_frame(flags, stream_id, payload) elif frame_type == FrameType.HEADERS: self._handle_headers_frame(flags, stream_id, payload) - elif frame_type == FrameType.PRIORITY: + elif frame_type in { + FrameType.PRIORITY, + FrameType.PUSH_PROMISE, + FrameType.CONTINUATION, + }: + logger.warning("%d frame ignored (not implemented)", frame_type) self._handle_priority_frame(stream_id, payload) elif frame_type == FrameType.RST_STREAM: self._handle_rst_stream_frame(flags, stream_id, payload) elif frame_type == FrameType.SETTINGS: self._handle_settings_frame(flags, stream_id, payload) - elif frame_type == FrameType.PUSH_PROMISE: - # Client cannot receive push promises -- ignore or GOAWAY - logger.warning("Received PUSH_PROMISE; ignoring (no push support)") elif frame_type == FrameType.PING: self._handle_ping_frame(flags, stream_id, payload) elif frame_type == FrameType.GOAWAY: @@ -193,7 +190,7 @@ def _dispatch_frame( elif frame_type == FrameType.CONTINUATION: self._handle_continuation_frame(flags, stream_id, payload) else: - logger.warning(f"Ignoring unknown frame type {frame_type}") + logger.warning("Ignoring unknown frame type %d", frame_type) # ---------- Individual frame handlers ---------- def _handle_data_frame(self, flags: int, stream_id: int, payload: bytes) -> None: @@ -223,7 +220,7 @@ def _handle_data_frame(self, flags: int, stream_id: int, payload: bytes) -> None def _handle_headers_frame(self, flags: int, stream_id: int, payload: bytes) -> None: if flags & FlagHeaders.PRIORITY: # Exclusive flag + stream dependency + weight - priority_data = payload[:5] + # exclude priority data payload = payload[5:] # Decode headers with HPACK @@ -265,9 +262,7 @@ def _handle_rst_stream_frame( def _handle_settings_frame( self, flags: int, stream_id: int, payload: bytes ) -> None: - """ - Process SETTINGS frame (6.5) - """ + """Process SETTINGS frame (6.5)""" if flags & FlagSettings.ACK: logger.debug("Received SETTINGS ACK") return # Our settings were acknowledged diff --git a/aiohttp/http2/response.py b/aiohttp/http2/response.py index 78e3c941c13..f1df7abf3c8 100644 --- a/aiohttp/http2/response.py +++ b/aiohttp/http2/response.py @@ -1,6 +1,6 @@ import json from http.cookies import SimpleCookie -from typing import Any, List, NamedTuple, Optional, Tuple +from typing import Any, List, Optional, Tuple from multidict import CIMultiDict @@ -41,7 +41,7 @@ def __init__( self.connection = None # Connection back-reference (set by the protocol) - self._connection: Optional["Http2Protocol"] = None + self._connection = None # ---------------------------------------------------------------- # Body access (synchronous: entire body is already in memory) diff --git a/aiohttp/http2/stream.py b/aiohttp/http2/stream.py index be374f5d79b..2d84491d184 100644 --- a/aiohttp/http2/stream.py +++ b/aiohttp/http2/stream.py @@ -58,7 +58,7 @@ class Stream: "closed_event", ) - def __init__(self, stream_id: int, conn: "Http2Connection", loop) -> None: + def __init__(self, stream_id: int, conn, loop) -> None: self.stream_id = stream_id self.state = StreamState.IDLE self.conn = conn diff --git a/requirements/runtime-deps.in b/requirements/runtime-deps.in index d70fc5a9dbc..52b6be17191 100644 --- a/requirements/runtime-deps.in +++ b/requirements/runtime-deps.in @@ -8,6 +8,7 @@ backports.zstd; platform_python_implementation == 'CPython' and python_version < Brotli >= 1.2; platform_python_implementation == 'CPython' and sys_platform != 'android' and sys_platform != 'ios' brotlicffi >= 1.2; platform_python_implementation != 'CPython' frozenlist >= 1.1.1 +hpack >= 4.2.0 multidict >=4.5, < 7.0 propcache >= 0.2.0 typing_extensions >= 4.4 ; python_version < '3.13' diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index cf695a29e99..a550c187235 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -13,17 +13,14 @@ import json import struct from typing import List, Optional, Tuple -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock import pytest -from hpack import Decoder, Encoder +from hpack import Encoder import aiohttp from aiohttp.http2.connection import Http2Connection, Http2Protocol -from aiohttp.http2.errors import ProtocolError -from aiohttp.http2.response import Http2Response from aiohttp.http2.settings import ( - DEFAULT_SETTINGS, FlagData, FlagHeaders, FlagPing, @@ -31,7 +28,7 @@ FrameType, Setting, ) -from aiohttp.http2.stream import Stream, StreamState +from aiohttp.http2.stream import StreamState # ---------------------------------------------------------------------- @@ -167,34 +164,13 @@ def build_ping(ack: bool = False, opaque: bytes = b"\x00" * 8) -> bytes: return frame_header(8, FrameType.PING, flags, 0) + opaque -# ---------------------------------------------------------------------- -# Integration tests (skipped by default) -# ---------------------------------------------------------------------- -@pytest.mark.skip(reason="Requires internet connection and httpbin") -@pytest.mark.asyncio -async def test_integration_httpbin_get(): - """Real HTTP/2 request to httpbin.org (requires a server supporting h2).""" - async with aiohttp.ClientSession() as session: - # Force HTTP/2 if possible; this test assumes the connector is configured - # to use Http2Protocol when ALPN negotiates h2. - async with session.get("https://httpbin.org/get") as resp: - assert resp.status == 200 - data = await resp.json() - assert "headers" in data - - -@pytest.mark.skip(reason="Requires internet connection and httpbin") -@pytest.mark.asyncio -async def test_integration_httpbin_post(): - import aiohttp - async with aiohttp.ClientSession() as session: - async with session.post( - "https://httpbin.org/post", json={"key": "value"} - ) as resp: - assert resp.status == 200 - data = await resp.json() - assert data["json"] == {"key": "value"} +@pytest.mark.asyncio +async def test_incomplete_frame(connection): + connection, _ = connection + frame = b"111111111" + connection.data_received(frame) + assert connection._frame_buffer == frame # ====================================================================== @@ -431,7 +407,7 @@ async def test_stream_cancelled_before_response(self, connection): """Pending create_stream futures are cancelled on connection loss.""" conn, _ = connection conn.max_concurrent_streams = 1 - s1 = await conn.create_stream() + await conn.create_stream() # Queue a second stream fut = asyncio.ensure_future(conn.create_stream()) await asyncio.sleep(0.01) @@ -474,11 +450,6 @@ async def do_request(): task = asyncio.create_task(do_request()) # Let it run until it creates a stream and sends HEADERS await asyncio.sleep(0.01) - # Now simulate server response - # Find the stream ID from the HEADERS frame written - write_calls = [call[0][0] for call in transport.write.call_args_list] - # The HEADERS frame will contain stream ID (first client stream = 1) - # Construct response HEADERS + DATA headers = [(":status", "200"), ("content-type", "application/json")] body = b'{"key": "value"}' resp_frame = build_headers_frame( From 2ecae5a920c599153bdfafb2bd1141c1de4b24fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 4 Jul 2026 02:28:59 +0000 Subject: [PATCH 04/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/http2/test_http2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index a550c187235..fe80222beb2 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -164,7 +164,6 @@ def build_ping(ack: bool = False, opaque: bytes = b"\x00" * 8) -> bytes: return frame_header(8, FrameType.PING, flags, 0) + opaque - @pytest.mark.asyncio async def test_incomplete_frame(connection): connection, _ = connection From 73bfa162ece87d8d26d3a2d7d8d21ee792ef3436 Mon Sep 17 00:00:00 2001 From: Moist-Cat Date: Sat, 4 Jul 2026 11:46:22 -0400 Subject: [PATCH 05/12] Fix lint errors --- aiohttp/helpers.py | 1 - aiohttp/http_writer.py | 2 -- docs/conf.py | 2 +- tests/http2/test_http2.py | 3 +-- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 1a45a060ff1..6748183f53e 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -47,7 +47,6 @@ from . import hdrs from .log import client_logger -from .typedefs import PathLike # noqa if sys.version_info >= (3, 11): import asyncio as async_timeout diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index 06cc44f5008..e648b8b2462 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -5,11 +5,9 @@ import sys from typing import ( TYPE_CHECKING, - Any, Awaitable, Callable, Iterable, - List, NamedTuple, Optional, Union, diff --git a/docs/conf.py b/docs/conf.py index a3254645cfb..9cf25a6082e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -66,7 +66,7 @@ try: - import sphinxcontrib.spelling # noqa + import sphinxcontrib.spelling extensions.append("sphinxcontrib.spelling") except ImportError: diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index fe80222beb2..8f653f77445 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -1,5 +1,5 @@ """ -Test suite for aiohttp.http2.protocol +Test suite for aiohttp.http2 Categories: - integration: against a real httpbin server (skipped, requires network) @@ -18,7 +18,6 @@ import pytest from hpack import Encoder -import aiohttp from aiohttp.http2.connection import Http2Connection, Http2Protocol from aiohttp.http2.settings import ( FlagData, From 9bd6ff9b5bb942105907c4cf483352767b248e27 Mon Sep 17 00:00:00 2001 From: Moist-Cat Date: Sat, 4 Jul 2026 21:52:13 -0400 Subject: [PATCH 06/12] Add more tests. It was necessary to add a semaphore to ensure the requests connect sequentially to the hosts and reuse connections when necessary. HTTP/2 uses a single connection per host. --- aiohttp/client.py | 32 ++- aiohttp/connector.py | 6 + aiohttp/http2/connection.py | 21 +- aiohttp/http2/response.py | 10 +- aiohttp/http2/stream.py | 17 +- tests/http2/test_http2.py | 343 ++++++++++++++++++++++++++ tests/http2/test_http2_integration.py | 214 ++++++++++++++++ 7 files changed, 604 insertions(+), 39 deletions(-) create mode 100644 tests/http2/test_http2_integration.py diff --git a/aiohttp/client.py b/aiohttp/client.py index 8bb3ae571c5..282f90510d9 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -7,8 +7,10 @@ import json import os import sys +import time import traceback import warnings +from collections import deque from collections.abc import ( Awaitable, Callable, @@ -232,25 +234,37 @@ async def _connect_and_send_request(req: ClientRequest) -> ClientResponse: connector = req._session._connector assert connector is not None try: - conn = await connector.connect(req, traces=req._traces, timeout=req._timeout) + async with connector.sem: + # at most just one + conn = await connector.connect( + req, traces=req._traces, timeout=req._timeout + ) except asyncio.TimeoutError as exc: raise ConnectionTimeoutError(f"Connection timeout to host {req.url}") from exc + assert conn.protocol is not None + + ssl_object = conn.protocol.transport.get_extra_info("ssl_object") + alpn_protocol = ssl_object.selected_alpn_protocol() if ssl_object else "http/1.1" + resp = None started = False - try: - assert conn.protocol is not None - - ssl_object = conn.protocol.transport.get_extra_info("ssl_object") - alpn_protocol = ( - ssl_object.selected_alpn_protocol() if ssl_object else "http/1.1" - ) + if alpn_protocol == "h2" and not connector._conns[conn._key]: + connector._conns[conn._key] = deque([(conn._protocol, time.monotonic())]) + try: # backwards compatibility if alpn_protocol == "h2": + body = await req.body.as_bytes() resp = await conn.protocol.send( - req.method, req.url, list(req.headers.items()), b"" + req.method, + req.url, + list(req.headers.items()), + body, ) + + # we are done with the connection + conn._protocol = None else: conn.protocol.set_response_params(**req._response_params) resp = await req._send(conn) diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 93df3567007..7b5cc8d612a 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -320,6 +320,12 @@ def __init__( self._placeholder_future.set_result(None) self._cleanup_closed() + # Semaphore for HTTP/2 connections + # avoids duplicate connections to the + # same host + # (HTTP/2 doesn't need connection pooling to send multiple requests) + self.sem = asyncio.Semaphore(1) + def __del__(self, _warnings: Any = warnings) -> None: if self._closed: return diff --git a/aiohttp/http2/connection.py b/aiohttp/http2/connection.py index 9e6f1c93817..4265e246512 100644 --- a/aiohttp/http2/connection.py +++ b/aiohttp/http2/connection.py @@ -149,7 +149,7 @@ def data_received(self, data: bytes) -> None: def eof_received(self) -> bool: logger.debug("EOF received from server") - self._transport.close() + self.close() return False def connection_lost(self, exc: Optional[Exception]) -> None: @@ -236,7 +236,7 @@ def _handle_headers_frame(self, flags: int, stream_id: int, payload: bytes) -> N stream = self.streams.get(stream_id) if stream is None: - logger.error("Unknown stream_id: %d", stream) + logger.error("Unknown stream_id: %d", stream_id) else: stream.receive_headers(headers, end_stream) @@ -532,15 +532,8 @@ def _protocol_error(self) -> None: # -------------------- Shutdown -------------------- def close(self) -> None: """Perform graceful shutdown.""" - if not self._goaway_sent: - # GOAWAY could be sent before we create the first stream - self._send_goaway( - self._last_stream_id or max(0, self.next_stream_id - 2), 0 - ) - # Wait a moment for GOAWAY to be sent, then close transport self._transport.close() - # Compatibility methods for aiohttp connector @property def should_close(self) -> bool: return self._goaway_sent or self._goaway_received @@ -548,10 +541,6 @@ def should_close(self) -> bool: def is_connected(self) -> bool: return not self._transport.is_closing() - @property - def closed(self) -> asyncio.Future: - return self._loop.create_future() # XXX not properly implemented - # ---------------------------------------------------------------------- # Protocol wrapper for asyncio.Transport (connector integration) @@ -636,9 +625,6 @@ async def send( # Obtain a stream from the pool (blocks only if concurrency limit reached) stream = await self._connection.create_stream() - # The internal connection still expects a URL object with .host, .path etc. - # Here we assume `url` is a yarl.URL (the session always passes a yarl.URL). - # If you receive a string, convert it: url = URL(url) await self._connection.send_request(stream, method, url, headers, body) # Wait for the response future to be set by the stream when complete @@ -658,3 +644,6 @@ def close(self) -> None: if self._connection: self._connection.close() self.transport = None + + def abort(self) -> None: + self.close() diff --git a/aiohttp/http2/response.py b/aiohttp/http2/response.py index f1df7abf3c8..988e3f0a599 100644 --- a/aiohttp/http2/response.py +++ b/aiohttp/http2/response.py @@ -33,16 +33,11 @@ def __init__( self._cookies: Optional[SimpleCookie] = None # HTTP version pseudo-attribute (aiohttp expects a namedtuple-like object) - # self.version = NamedTuple("HttpVersion", major=int, minor=int)(2, 0) self.version = HttpVersion2 - # not implemented self._raw_cookie_headers = None self.connection = None - # Connection back-reference (set by the protocol) - self._connection = None - # ---------------------------------------------------------------- # Body access (synchronous: entire body is already in memory) # ---------------------------------------------------------------- @@ -99,7 +94,7 @@ def raise_for_status(self) -> None: # ---------------------------------------------------------------- # Connection release (stream-level cleanup) # ---------------------------------------------------------------- - async def release(self) -> None: + def release(self) -> None: """Release the HTTP/2 stream back to the connection. In HTTP/2 the stream is already closed once the full response is @@ -109,7 +104,8 @@ async def release(self) -> None: pass # nothing to do; the stream has ended def close(self): - pass + if self.connection: + self.connection.close() # ---------------------------------------------------------------- # Context manager support (optional, often used with 'async with') diff --git a/aiohttp/http2/stream.py b/aiohttp/http2/stream.py index 2d84491d184..876ed802190 100644 --- a/aiohttp/http2/stream.py +++ b/aiohttp/http2/stream.py @@ -110,11 +110,14 @@ def receive_data(self, data: bytes, end_stream: bool) -> None: f"Unexpected stream state {self.state.name} for END_STREAM" ) - # Deliver the full response once headers have been received - if self.response_headers is not None and not self.response_future.done(): - self.response_future.set_result( - (self.stream_id, self.response_headers, bytes(self.response_data)) - ) + self.maybe_deliver_response() + + def maybe_deliver_response(self): + # Deliver the full response once headers have been received + if self.response_headers is not None and not self.response_future.done(): + self.response_future.set_result( + (self.stream_id, self.response_headers, bytes(self.response_data)) + ) def receive_headers(self, headers: List[Tuple[str, str]], end_stream: bool) -> None: """Process incoming HEADERS frame payload.""" @@ -130,6 +133,6 @@ def receive_headers(self, headers: List[Tuple[str, str]], end_stream: bool) -> N raise ProtocolError( f"Unexpected stream state {self.state.name} for END_STREAM on headers" ) - # The response is complete (no body) - self.response_future.set_result((self.stream_id, headers, b"")) + self.maybe_deliver_response() + # else: stream remains in OPEN (or HALF_CLOSED_LOCAL), headers stored for later delivery. diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index 8f653f77445..be9a895c80a 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -19,6 +19,8 @@ from hpack import Encoder from aiohttp.http2.connection import Http2Connection, Http2Protocol +from aiohttp.http2.errors import ProtocolError +from aiohttp.http2.response import Http2Response from aiohttp.http2.settings import ( FlagData, FlagHeaders, @@ -352,6 +354,212 @@ async def test_bad_hpack_triggers_protocol_error(self, connection): ) assert rst or goaway + @pytest.mark.asyncio + async def test_data_frame_with_padding(self, connection): + """Cover DATA frame with PADDED flag.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + # padded data: pad length 1, data 'x', zero padding byte + frame = build_data_frame(stream.stream_id, b"x", pad=True) + conn.data_received(frame) + assert stream.response_data == b"x" + + @pytest.mark.asyncio + async def test_headers_frame_with_priority(self, connection): + """Cover HEADERS frame with PRIORITY flag.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + # priority block: exclusive (1 byte) + dependency (4 bytes) + weight (1 byte) + priority_data = b"\x00\x00\x00\x00\x10" + headers = [(":status", "200")] + frame = build_headers_frame( + stream.stream_id, + headers, + end_headers=True, + end_stream=True, + priority=priority_data, + ) + conn.data_received(frame) + assert stream.response_future.done() + assert stream.response_headers is not None + + @pytest.mark.asyncio + async def test_rst_stream_for_unknown_stream(self, connection): + """RST_STREAM on an unknown stream ID is silently ignored.""" + conn, _ = connection + frame = build_rst_stream(999, error_code=0) + conn.data_received(frame) # must not raise + + @pytest.mark.asyncio + async def test_rst_stream_when_future_done(self, connection): + """RST_STREAM when response future already completed (else branch of line 256).""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + stream.response_future.set_result((stream.stream_id, [], b"")) + frame = build_rst_stream(stream.stream_id, error_code=0) + conn.data_received(frame) + assert stream.state == StreamState.CLOSED + + @pytest.mark.asyncio + async def test_receive_settings_ack(self, connection): + """Receive SETTINGS ACK (should be a no‑op).""" + conn, _ = connection + frame = build_settings_frame(ack=True) + conn.data_received(frame) # no crash + + @pytest.mark.asyncio + async def test_settings_on_nonzero_stream(self, connection, mock_transport): + """SETTINGS frame on stream_id != 0 triggers GOAWAY.""" + conn, transport = connection + frame = frame_header(0, FrameType.SETTINGS, 0, 5) # stream 5 + conn.data_received(frame) + assert any( + call[0][0][3:4] == FrameType.GOAWAY.to_bytes(1, "big") + for call in transport.write.call_args_list + ) + + @pytest.mark.asyncio + async def test_settings_invalid_payload_length(self, connection, mock_transport): + """SETTINGS payload not a multiple of 6 triggers protocol error.""" + conn, transport = connection + payload = b"\x00\x01\x02" # 3 bytes + frame = frame_header(len(payload), FrameType.SETTINGS, 0, 0) + payload + conn.data_received(frame) + assert any( + call[0][0][3:4] == FrameType.GOAWAY.to_bytes(1, "big") + for call in transport.write.call_args_list + ) + + @pytest.mark.asyncio + async def test_settings_initial_window_size_update(self, connection): + """INITIAL_WINDOW_SIZE setting updates stream windows.""" + conn, _ = connection + stream = await conn.create_stream() + old_window = stream.outbound_window + frame = build_settings_frame([(Setting.INITIAL_WINDOW_SIZE, 131072)]) + conn.data_received(frame) + assert stream.outbound_window == old_window + (131072 - 65535) + + @pytest.mark.asyncio + async def test_settings_header_table_size(self, connection): + """HEADER_TABLE_SIZE setting is processed.""" + conn, _ = connection + frame = build_settings_frame([(Setting.HEADER_TABLE_SIZE, 4096)]) + conn.data_received(frame) + # internal effect on encoder, just confirm no crash + + @pytest.mark.asyncio + async def test_ping_on_nonzero_stream(self, connection, mock_transport): + """PING on non‑zero stream triggers GOAWAY.""" + conn, transport = connection + frame = frame_header(8, FrameType.PING, 0, 1) + b"\x00" * 8 + conn.data_received(frame) + assert any( + call[0][0][3:4] == FrameType.GOAWAY.to_bytes(1, "big") + for call in transport.write.call_args_list + ) + + @pytest.mark.asyncio + async def test_receive_ping_ack(self, connection): + """Receiving a PING ACK is logged and does not cause another response.""" + conn, _ = connection + frame = build_ping(ack=True, opaque=b"12345678") + conn.data_received(frame) # no crash, no additional PING sent + + @pytest.mark.asyncio + async def test_goaway_when_future_already_done(self, connection): + """GOAWAY should not fail when a stream's future is already complete.""" + conn, _ = connection + s1 = await conn.create_stream() + s3 = await conn.create_stream() + s1.state = s3.state = StreamState.OPEN + s1.response_future.set_result((s1.stream_id, [], b"")) + frame = build_goaway(last_stream_id=1, error_code=0) + conn.data_received(frame) + assert s1.stream_id in conn.streams + assert s3.stream_id not in conn.streams + + @pytest.mark.asyncio + async def test_window_update_for_unknown_stream(self, connection): + """WINDOW_UPDATE on unknown stream must be ignored (no crash).""" + conn, _ = connection + frame = build_window_update(123, 100) + conn.data_received(frame) + assert conn.session_outbound_window == 65535 # unchanged + + @pytest.mark.asyncio + async def test_continuation_frame_ignored(self, connection): + """CONTINUATION frame is ignored with a warning.""" + conn, _ = connection + frame = frame_header(0, FrameType.CONTINUATION, 0, 1) + conn.data_received(frame) # no crash + + @pytest.mark.asyncio + async def test_unknown_frame_type_ignored(self, connection): + """Unknown frame type (>9) is ignored.""" + conn, _ = connection + frame = frame_header(0, 0x1A, 0, 0) + conn.data_received(frame) # no crash + + @pytest.mark.asyncio + async def test_send_data_end_stream_last_chunk(self, connection, mock_transport): + """send_data with end_stream=True sets END_STREAM on the last chunk.""" + conn, transport = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + conn.session_outbound_window = 100 + stream.outbound_window = 100 + await conn.send_data(stream, b"x" * 10, end_stream=True) + data_frames = [ + call[0][0] + for call in transport.write.call_args_list + if FrameType.DATA.to_bytes(1, "big") in call[0][0] + ] + assert len(data_frames) == 1 + assert data_frames[0][4] & FlagData.END_STREAM + + @pytest.mark.asyncio + async def test_send_request_with_body(self, connection, mock_transport): + """send_request with body sends HEADERS (no END_STREAM) followed by DATA.""" + conn, transport = connection + stream = await conn.create_stream() + await conn.send_request(stream, "POST", url_mock("/"), [], body=b"hello") + sent = [ + FrameType(call[0][0][3]).name + for call in transport.write.call_args_list + if call[0][0][3] in (FrameType.HEADERS, FrameType.DATA) + ] + assert sent == ["HEADERS", "DATA"] + + @pytest.mark.asyncio + async def test_create_stream_after_goaway(self, connection): + """create_stream raises ConnectionError when GOAWAY has been sent.""" + conn, _ = connection + conn._goaway_sent = True + with pytest.raises(ConnectionError): + await conn.create_stream() + + @pytest.mark.asyncio + async def test_maybe_unblock_streams_done_future(self, connection): + """_maybe_unblock_streams skips futures that are already done.""" + conn, _ = connection + fut = conn._loop.create_future() + fut.set_result(None) + conn._pending_streams.append(fut) + conn._maybe_unblock_streams() + assert len(conn.streams) == 0 # no new stream created + + @pytest.mark.asyncio + async def test_connection_lost_done_futures(self, connection): + """connection_lost must handle streams whose futures are already done.""" + conn, _ = connection + stream = await conn.create_stream() + stream.response_future.set_result((stream.stream_id, [], b"")) + conn.connection_lost(None) # no exception + # ---------------------------------------------------------------------- # 2. Miscellaneous tests (race conditions, deadlocks, edge cases) @@ -428,6 +636,13 @@ async def test_close_stream_on_rst_without_headers(self, connection): with pytest.raises(RuntimeError): stream.response_future.result() + @pytest.mark.asyncio + async def test_data_received_without_connection(self, protocol): + """Http2Protocol.data_received is a no‑op before connection_made.""" + proto, _ = protocol + proto._connection = None + proto.data_received(b"anything") # must not raise + # ---------------------------------------------------------------------- # 3. Interface tests (high‑level features via Http2Protocol.send) @@ -506,3 +721,131 @@ async def test_multiple_requests_concurrently(self, protocol, mock_transport): r1, r2 = await asyncio.gather(*tasks) assert r1.status == 200 assert r2.status == 201 + + @pytest.mark.asyncio + async def test_response_all_methods(self): + """Cover Http2Response.read, .text, .json, .cookies, .raise_for_status, .release, .close, context manager.""" + headers = [ + (":status", "200"), + ("set-cookie", "a=b"), + ("content-type", "application/json"), + ] + body = b'{"ok":true}' + resp = Http2Response(headers, body, method="GET", url=url_mock("/")) + assert resp.status == 200 + assert await resp.read() == body + assert await resp.text() == '{"ok":true}' + assert await resp.json() == {"ok": True} + assert "a" in resp.cookies + resp.raise_for_status() # 200 is ok + resp.release() + resp.close() + async with resp: + pass + + @pytest.mark.asyncio + async def test_response_raise_for_status_error(self): + """Http2Response.raise_for_status raises for 4xx.""" + headers = [(":status", "404")] + resp = Http2Response(headers, b"", method="GET", url=url_mock("/")) + from aiohttp.client_exceptions import ClientResponseError + + with pytest.raises(ClientResponseError): + resp.raise_for_status() + + @pytest.mark.asyncio + async def test_send_with_body_through_protocol(self, protocol, mock_transport): + """Full send() with a body completes successfully.""" + proto, transport = protocol + url = url_mock("/upload") + task = asyncio.create_task(proto.send("POST", url, [], body=b"data")) + await asyncio.sleep(0.01) + resp_frame = build_headers_frame(1, [(":status", "200")], end_stream=True) + proto.data_received(resp_frame) + response = await task + assert response.status == 200 + + +class TestStreamStateMachine: + @pytest.mark.asyncio + async def test_invalid_transition_raises(self, connection): + """Invalid state transition raises ProtocolError.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + with pytest.raises(ProtocolError): + stream.transition(StreamState.RESERVED_LOCAL) + + @pytest.mark.asyncio + async def test_receive_data_end_stream_half_closed_local(self, connection): + """DATA END_STREAM when HALF_CLOSED_LOCAL -> CLOSED and stream removed.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.HALF_CLOSED_LOCAL + stream.response_headers = [(":status", "200")] + stream.receive_data(b"body", end_stream=True) + assert stream.state == StreamState.CLOSED + assert stream.stream_id not in conn.streams + + @pytest.mark.asyncio + async def test_receive_data_end_stream_invalid_state(self, connection): + """DATA END_STREAM in CLOSED state raises ProtocolError.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.CLOSED + with pytest.raises(ProtocolError): + stream.receive_data(b"x", end_stream=True) + + @pytest.mark.asyncio + async def test_receive_headers_end_stream_half_closed_local(self, connection): + """HEADERS END_STREAM when HALF_CLOSED_LOCAL -> CLOSED and stream removed.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.HALF_CLOSED_LOCAL + stream.receive_headers([(":status", "200")], end_stream=True) + assert stream.state == StreamState.CLOSED + assert stream.stream_id not in conn.streams + + @pytest.mark.asyncio + async def test_receive_headers_end_stream_invalid_state(self, connection): + """HEADERS END_STREAM in CLOSED state raises ProtocolError.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.CLOSED + with pytest.raises(ProtocolError): + stream.receive_headers([(":status", "200")], end_stream=True) + + @pytest.mark.asyncio + async def test_data_stream_before_headers(self, connection): + """DATA before headers does NOT set future until headers arrive.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + # double end stream is invalid + stream.receive_data(b"body", end_stream=False) + assert not stream.response_future.done() + stream.receive_headers([(":status", "200")], end_stream=True) + assert stream.response_future.done() + _, headers, body = stream.response_future.result() + assert body == b"body" + + @pytest.mark.asyncio + async def test_future_already_done_data_end_stream(self, connection): + """receive_data with END_STREAM does not double‑set an already done future.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + stream.response_headers = [(":status", "200")] + stream.response_future.set_result( + (stream.stream_id, stream.response_headers, b"") + ) + stream.receive_data(b"more", end_stream=True) # no exception + + @pytest.mark.asyncio + async def test_future_already_done_headers_end_stream(self, connection): + """receive_headers with END_STREAM does not double‑set an already done future.""" + conn, _ = connection + stream = await conn.create_stream() + stream.state = StreamState.OPEN + stream.response_future.set_result((stream.stream_id, [], b"")) + stream.receive_headers([(":status", "200")], end_stream=True) # no exception diff --git a/tests/http2/test_http2_integration.py b/tests/http2/test_http2_integration.py new file mode 100644 index 00000000000..049a6748924 --- /dev/null +++ b/tests/http2/test_http2_integration.py @@ -0,0 +1,214 @@ +""" +Tests for aiohttp’s public HTTP/2 client API. + +Divided into: +- Outgoing frame correctness (what aiohttp writes to the transport). +- Incoming response parsing (what aiohttp returns when frames are fed). + +Both rely on a mocked transport that simulates ALPN negotiation of "h2" +and a custom connector that plugs in your Http2Protocol. +""" + +import asyncio +import struct +from typing import List, Tuple +from unittest.mock import MagicMock, patch + +import pytest +import aiohttp +from aiohttp.connector import TCPConnector +from test_http2 import ( + Http2Protocol, + build_headers_frame, + build_data_frame, + build_goaway, +) +from aiohttp.http2.settings import FrameType, FlagHeaders, FlagData + + +# ---------------------------------------------------------------------- +# Mock transport – records writes and lies about ALPN +# ---------------------------------------------------------------------- +class MockH2Transport(asyncio.Transport): + def __init__(self, extra_info=None): + super().__init__() + self.written = bytearray() + self._closing = False + self._extra = extra_info or {} + + def write(self, data): + self.written.extend(data) + + def close(self): + self._closing = True + + def is_closing(self): + return self._closing + + def get_extra_info(self, name, default=None): + if name == "ssl_object": + return self._extra.get("ssl_object", MagicMock()) + return self._extra.get(name, default) + + +# ---------------------------------------------------------------------- +# Custom connector – always returns Http2Protocol for h2 connections +# ---------------------------------------------------------------------- +class H2TestConnector(TCPConnector): + def _get_protocol(self, loop): + # Return the class; aiohttp will instantiate it + return Http2Protocol + + async def close(self): + self._closed = True + + +# ---------------------------------------------------------------------- +# Fixture: session + mock transport + captured protocol +# ---------------------------------------------------------------------- +@pytest.fixture +async def h2_client(): + """Create a ClientSession that uses our Http2Protocol over a mock transport.""" + loop = asyncio.get_running_loop() + + # Mock SSL object that tells aiohttp we’ve negotiated h2 + mock_ssl = MagicMock() + mock_ssl.selected_alpn_protocol.return_value = "h2" + transport = MockH2Transport(extra_info={"ssl_object": mock_ssl}) + + protocol_instance = None + + async def fake_create_connection(protocol_factory, *args, **kwargs): + nonlocal protocol_instance + protocol_instance = protocol_factory() # Http2Protocol() + protocol_instance.connection_made(transport) + transport._protocol = protocol_instance + return transport, protocol_instance + + connector = H2TestConnector() + connector._wrap_create_connection = fake_create_connection + async with aiohttp.ClientSession(connector=connector) as session: + yield session, transport, protocol_instance + + +CEASE = build_goaway(0, 1) +URL = "https://127.3.3.3" + + +class TestIncomingResponses: + @pytest.mark.asyncio + async def test_get_200_response(self, h2_client): + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) # request sent + + # Feed a minimal 200 response + hframe = build_headers_frame(1, [(":status", "200")], end_stream=True) + proto = transport._protocol + proto.data_received(hframe) + + resp = await task + assert resp.status == 200 + assert await resp.read() == b"" + + @pytest.mark.asyncio + async def test_response_with_body(self, h2_client): + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + # Send HEADERS (no END_STREAM) then DATA with body + hframe = build_headers_frame(1, [(":status", "200")], end_stream=False) + dframe = build_data_frame(1, b"Hello, h2!", end_stream=True) + proto = transport._protocol + proto.data_received(hframe) + proto.data_received(dframe) + + resp = await task + assert resp.status == 200 + assert await resp.text() == "Hello, h2!" + + @pytest.mark.asyncio + async def test_json_response(self, h2_client): + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + headers = [(":status", "200"), ("content-type", "application/json")] + body = b'{"key":"value"}' + proto = transport._protocol + proto.data_received(build_headers_frame(1, headers, end_stream=False)) + proto.data_received(build_data_frame(1, body, end_stream=True)) + + resp = await task + assert await resp.json() == {"key": "value"} + + @pytest.mark.asyncio + async def test_response_cookies(self, h2_client): + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + headers = [(":status", "200"), ("set-cookie", "session=abc123; Path=/")] + proto = transport._protocol + proto.data_received(build_headers_frame(1, headers, end_stream=True)) + + resp = await task + assert "session" in resp.cookies + assert resp.cookies["session"].value == "abc123" + + @pytest.mark.asyncio + async def test_concurrent_requests_mux(self, h2_client): + session, transport, _ = h2_client + t1 = asyncio.create_task(session.get(URL)) + t2 = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + # Stream 1 gets response, stream 3 gets response + proto = transport._protocol + proto.data_received( + build_headers_frame(1, [(":status", "200")], end_stream=True) + ) + proto.data_received( + build_headers_frame(3, [(":status", "201")], end_stream=True) + ) + + r1, r2 = await asyncio.gather(t1, t2) + assert r1.status == 200 + assert r2.status == 201 + + @pytest.mark.asyncio + async def test_redirect_headers(self, h2_client): + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + headers = [(":status", "302"), ("location", "/new")] + proto = transport._protocol + proto.data_received(build_headers_frame(1, headers, end_stream=True)) + + await asyncio.sleep(0.01) + + headers = [(":status", "200")] + proto.data_received(build_headers_frame(3, headers, end_stream=True)) + + resp = await task + first = resp._history[0] + assert first.status == 302 + assert first.headers.get("location") == "/new" + + assert resp.status == 200 + + @pytest.mark.asyncio + async def test_error_response_raises(self, h2_client): + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + proto = transport._protocol + proto.data_received( + build_headers_frame(1, [(":status", "404")], end_stream=True) + ) + resp = await task + with pytest.raises(aiohttp.ClientResponseError): + resp.raise_for_status() From 91b46d8162fb20b7dc8cb3e94562ba83d2c2f9d2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 5 Jul 2026 01:54:28 +0000 Subject: [PATCH 07/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/http2/test_http2_integration.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/http2/test_http2_integration.py b/tests/http2/test_http2_integration.py index 049a6748924..8958ee8d4d8 100644 --- a/tests/http2/test_http2_integration.py +++ b/tests/http2/test_http2_integration.py @@ -15,15 +15,16 @@ from unittest.mock import MagicMock, patch import pytest -import aiohttp -from aiohttp.connector import TCPConnector from test_http2 import ( Http2Protocol, - build_headers_frame, build_data_frame, build_goaway, + build_headers_frame, ) -from aiohttp.http2.settings import FrameType, FlagHeaders, FlagData + +import aiohttp +from aiohttp.connector import TCPConnector +from aiohttp.http2.settings import FlagData, FlagHeaders, FrameType # ---------------------------------------------------------------------- From 0b361d364a9b7840c644fb693c564f8ec8a6bd1f Mon Sep 17 00:00:00 2001 From: Moist-Cat Date: Sat, 4 Jul 2026 23:32:18 -0400 Subject: [PATCH 08/12] Fix type issues --- aiohttp/client.py | 5 +- aiohttp/connector.py | 5 +- aiohttp/http2/connection.py | 83 +++-- aiohttp/http2/response.py | 63 ++-- aiohttp/http2/settings.py | 10 +- aiohttp/http2/stream.py | 49 +-- aiohttp/http_protocol.py | 13 +- tests/http2/test_http2.py | 424 +++++++++++++++++++++----- tests/http2/test_http2_integration.py | 215 ------------- 9 files changed, 461 insertions(+), 406 deletions(-) delete mode 100644 tests/http2/test_http2_integration.py diff --git a/aiohttp/client.py b/aiohttp/client.py index 282f90510d9..eb2ad26d2ab 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -243,6 +243,7 @@ async def _connect_and_send_request(req: ClientRequest) -> ClientResponse: raise ConnectionTimeoutError(f"Connection timeout to host {req.url}") from exc assert conn.protocol is not None + assert conn.protocol.transport is not None ssl_object = conn.protocol.transport.get_extra_info("ssl_object") alpn_protocol = ssl_object.selected_alpn_protocol() if ssl_object else "http/1.1" @@ -251,12 +252,12 @@ async def _connect_and_send_request(req: ClientRequest) -> ClientResponse: started = False if alpn_protocol == "h2" and not connector._conns[conn._key]: - connector._conns[conn._key] = deque([(conn._protocol, time.monotonic())]) + connector._conns[conn._key] = deque([(conn.protocol, time.monotonic())]) try: # backwards compatibility if alpn_protocol == "h2": body = await req.body.as_bytes() - resp = await conn.protocol.send( + resp = await conn.protocol.send( # type: ignore[attr-defined] req.method, req.url, list(req.headers.items()), diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 7b5cc8d612a..120882b34c4 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1415,7 +1415,8 @@ async def _start_tls_connection( tls_transport ) # Kick the state machine of the new TLS protocol - return tls_transport, tls_proto + # HACK use the correct type + return tls_transport, tls_proto # type: ignore[return-value] def _convert_hosts_to_addr_infos( self, hosts: list[ResolveResult] @@ -1638,7 +1639,7 @@ async def _create_connection( raise raise UnixClientConnectorError(self.path, req.connection_key, exc) from exc - return proto + return proto # type: ignore[return-value] class NamedPipeConnector(BaseConnector): diff --git a/aiohttp/http2/connection.py b/aiohttp/http2/connection.py index 4265e246512..bbada75b5ec 100644 --- a/aiohttp/http2/connection.py +++ b/aiohttp/http2/connection.py @@ -71,36 +71,40 @@ def __init__( self._loop = loop # HPACK - self.hpack_encoder = Encoder() - self.hpack_decoder = Decoder() + self.hpack_encoder: Encoder = Encoder() + self.hpack_decoder: Decoder = Decoder() # Settings self.remote_settings: Dict[Setting, int] = DEFAULT_SETTINGS.copy() self.local_settings: Dict[Setting, int] = DEFAULT_SETTINGS.copy() # Flow control - self.session_outbound_window = 65535 # initial flow control (RFC 7540, 6.9.1) - self.session_inbound_window = 65535 - self._flow_control_updated = asyncio.Event() + self.session_outbound_window: int = ( + 65535 # initial flow control (RFC 7540, 6.9.1) + ) + self.session_inbound_window: int = 65535 + self._flow_control_updated: asyncio.Event = asyncio.Event() self._flow_control_updated.set() # initially writable # Streams self.streams: Dict[int, Stream] = {} - self.next_stream_id = 1 # client streams are odd - self.max_concurrent_streams = DEFAULT_SETTINGS[Setting.MAX_CONCURRENT_STREAMS] + self.next_stream_id: int = 1 # client streams are odd + self.max_concurrent_streams: int = DEFAULT_SETTINGS[ + Setting.MAX_CONCURRENT_STREAMS + ] self._pending_streams: List[asyncio.Future[Stream]] = [] - self._last_peer_stream_id = ( + self._last_peer_stream_id: int = ( 0 # highest server‑initiated stream (even, unused for client) ) # Frame buffers - self._frame_buffer = bytearray() + self._frame_buffer: bytearray = bytearray() # GOAWAY state - self._goaway_received = False - self._goaway_sent = False - self._last_stream_id = 0 - self._error_code = 0 + self._goaway_received: bool = False + self._goaway_sent: bool = False + self._last_stream_id: int = 0 + self._error_code: int = 0 # Closed streams cleanup self._closed_streams: Set[int] = set() @@ -112,8 +116,6 @@ def data_received(self, data: bytes) -> None: # Consume complete frames while enough bytes for the header exist while len(self._frame_buffer) >= FRAME_HEADER_LENGTH: # Parse 24-bit length, 8-bit type, 8-bit flags, 32-bit stream ID - # (RFC 9113 4.1) - # "!I H B B" length = ( self._frame_buffer[0] << 16 | self._frame_buffer[1] << 8 @@ -132,7 +134,7 @@ def data_received(self, data: bytes) -> None: del self._frame_buffer[: FRAME_HEADER_LENGTH + length] # invalid frames cause a value error - if frame_type_val <= 9 and frame_type_val >= 0: + if 0 <= frame_type_val <= 9: logger.debug( "<- %s stream=%d flags=0x%02x len=%d", FrameType(frame_type_val).name, @@ -321,7 +323,10 @@ def _handle_goaway_frame(self, flags: int, stream_id: int, payload: bytes) -> No self._last_stream_id = last_stream_id self._error_code = error_code logger.info( - f"GOAWAY received: last_stream={last_stream_id}, error={error_code}, extra={extra}" + "GOAWAY received: last_stream=%d, error=%d, extra=%s", + last_stream_id, + error_code, + extra.decode(), ) # Cancel streams with higher IDs for sid, stream in list(self.streams.items()): @@ -354,11 +359,13 @@ def _handle_continuation_frame( # -------------------- Frame sending helpers -------------------- def _send_frame( - self, frame_type: FrameType, flags: int, stream_id: int, payload: bytes = b"" + self, + frame_type: FrameType, + flags: int, + stream_id: int, + payload: bytes = b"", ) -> None: length = len(payload) & 0x00FFFFFF # 24 bits -> 3 bytes - # header = struct.pack("!I H B B", length, frame_type, flags, stream_id) - header = struct.pack("!I", length)[ 1: ] + struct.pack( # drop the first (most‑significant) byte → 3 bytes @@ -420,7 +427,7 @@ async def create_stream(self) -> Stream: if len(self.streams) < self.max_concurrent_streams: return self._create_stream_internal() # Queue the request - fut = self._loop.create_future() + fut: asyncio.Future[Stream] = self._loop.create_future() self._pending_streams.append(fut) return await fut @@ -474,7 +481,7 @@ async def send_request( self, stream: Stream, method: str, - url: str, + url: Any, # Usually a yarl.URL, but abstracted for mocking headers: List[Tuple[str, str]], body: Optional[bytes] = None, ) -> None: @@ -486,8 +493,6 @@ async def send_request( (":scheme", url.scheme), (":authority", url.host), ] - # 8.2. HTTP Fields - # Field names MUST be converted to lowercase when constructing an HTTP/2 message. req_headers.extend([(h[0].lower(), h[1]) for h in headers]) hdrs = self.hpack_encoder.encode(req_headers) @@ -512,7 +517,6 @@ def initiate_connection(self) -> None: self._transport.write(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") # Send initial SETTINGS (our preferences) - # We can advertise supported settings, e.g., enable push = 0 settings_payload = struct.pack( "!H I", Setting.ENABLE_PUSH, 0 # disable server push ) @@ -554,12 +558,12 @@ class Http2Protocol(asyncio.Protocol): def __init__(self, loop: asyncio.AbstractEventLoop) -> None: self._loop = loop self._connection: Optional[Http2Connection] = None - self._closed_future = loop.create_future() # resolves when connection closes + self._closed_future: asyncio.Future[None] = loop.create_future() self.transport: Optional[asyncio.Transport] = None def connection_made(self, transport: asyncio.BaseTransport) -> None: self.transport = transport # type: ignore[assignment] - self._connection = Http2Connection(self.transport, self._loop) + self._connection = Http2Connection(self.transport, self._loop) # type: ignore[arg-type] self._connection.initiate_connection() def data_received(self, data: bytes) -> None: @@ -586,7 +590,7 @@ def is_connected(self) -> bool: return self._connection.is_connected() if self._connection else False @property - def closed(self) -> asyncio.Future: + def closed(self) -> asyncio.Future[None]: return self._closed_future # Public API for creating streams @@ -595,34 +599,18 @@ async def create_stream(self) -> Stream: raise RuntimeError("Connection not established") return await self._connection.create_stream() - def send_request( - self, - stream: Stream, - method: str, - url: str, - headers: List[Tuple[str, str]], - body: Optional[bytes] = None, - ) -> None: - if self._connection is None: - raise RuntimeError("Connection not established") - self._connection.send_request(stream, method, url, headers, body) - async def send( self, method: str, - url: Any, # yarl.URL or str (but the protocol expects a URL object later) + url: Any, headers: List[Tuple[str, str]], body: Optional[bytes] = None, ) -> Http2Response: - """Send an HTTP/2 request and return a compatible response. - - The response object will carry `url`, `method`, and `_connection` - so that aiohttp’s `ClientSession._request` can work without modification. - """ + """Send an HTTP/2 request and return a compatible response.""" if self._connection is None: raise RuntimeError("Connection not established") - # Obtain a stream from the pool (blocks only if concurrency limit reached) + # Obtain a stream from the pool stream = await self._connection.create_stream() await self._connection.send_request(stream, method, url, headers, body) @@ -630,7 +618,6 @@ async def send( # Wait for the response future to be set by the stream when complete _, resp_headers, resp_body = await stream.response_future - # Build the full response object response = Http2Response( headers=resp_headers, body=resp_body, diff --git a/aiohttp/http2/response.py b/aiohttp/http2/response.py index 988e3f0a599..d73d56f4487 100644 --- a/aiohttp/http2/response.py +++ b/aiohttp/http2/response.py @@ -1,9 +1,13 @@ import json from http.cookies import SimpleCookie -from typing import Any, List, Optional, Tuple +from typing import Any, Iterable, List, Optional, Tuple +from hpack import HeaderTuple from multidict import CIMultiDict +from aiohttp.client_exceptions import ClientResponseError +from aiohttp.client_reqrep import RequestInfo + from ..http_writer import HttpVersion2 @@ -12,31 +16,32 @@ class Http2Response: def __init__( self, - headers: List[Tuple[str, str]], + headers: Iterable[HeaderTuple] | Iterable[Tuple[str, str]], body: bytes, *, method: Optional[str] = None, url: Optional[Any] = None, ) -> None: - self.reason = "" # HTTP/2 doesn't carry a reason phrase - self._body = body - self.url = url - self.method = method - self.history: List[Any] = [] # for redirects - - # Headers as case-insensitive multi-dict (mimics aiohttp's CIMultiDict) - self.headers: CIMultiDict = CIMultiDict(headers) + self.reason: str = "" # HTTP/2 doesn't carry a reason phrase + self._body: bytes = body + self.url: Optional[Any] = url + self.method: Optional[str] = method + # redirects + self._history: List["Http2Response"] = [] + + # Headers as case-insensitive multi-dict + self.headers: CIMultiDict[str] = CIMultiDict(headers) # type: ignore[arg-type] # no status error implies a server side error - self.status = int(self.headers.get(":status", 500)) + self.status: int = int(self.headers.get(":status", 500)) # Cookie jar integration self._cookies: Optional[SimpleCookie] = None - # HTTP version pseudo-attribute (aiohttp expects a namedtuple-like object) + # HTTP version pseudo-attribute self.version = HttpVersion2 - self._raw_cookie_headers = None - self.connection = None + self._raw_cookie_headers: Optional[List[Tuple[str, str]]] = None + self.connection: Optional[Any] = None # ---------------------------------------------------------------- # Body access (synchronous: entire body is already in memory) @@ -46,14 +51,14 @@ async def read(self) -> bytes: return self._body @property - def body(self): + def body(self) -> bytes: return self._body async def text(self, encoding: str = "utf-8") -> str: """Decode the body to a string.""" return self._body.decode(encoding) - async def json(self, **kwargs) -> Any: + async def json(self, **kwargs: Any) -> Any: """Parse JSON body.""" return json.loads(self._body, **kwargs) @@ -65,7 +70,6 @@ def cookies(self) -> SimpleCookie: """Parse 'Set-Cookie' headers and return a SimpleCookie.""" if self._cookies is None: self._cookies = SimpleCookie() - # self.headers is a CIMultiDict – use getall() to obtain all values for raw in self.headers.getall("set-cookie", []): self._cookies.load(raw) return self._cookies @@ -81,11 +85,11 @@ def ok(self) -> bool: def raise_for_status(self) -> None: """Raise an HTTPError for 4xx/5xx responses.""" if not self.ok: - from aiohttp.client_exceptions import ClientResponseError - raise ClientResponseError( - request_info=None, # simplified - history=self.history, + request_info=RequestInfo( + url=self.url, method=self.method, headers=self.headers # type: ignore[arg-type] + ), + history=self._history, # type: ignore[arg-type] status=self.status, message=f"{self.status}, message='{self.reason}'", headers=self.headers, @@ -95,23 +99,20 @@ def raise_for_status(self) -> None: # Connection release (stream-level cleanup) # ---------------------------------------------------------------- def release(self) -> None: - """Release the HTTP/2 stream back to the connection. - - In HTTP/2 the stream is already closed once the full response is - received. This method is a no-op but required for aiohttp - compatibility. - """ + """Release the HTTP/2 stream back to the connection.""" pass # nothing to do; the stream has ended - def close(self): + def close(self) -> None: if self.connection: self.connection.close() # ---------------------------------------------------------------- - # Context manager support (optional, often used with 'async with') + # Context manager support # ---------------------------------------------------------------- - async def __aenter__(self): + async def __aenter__(self) -> "Http2Response": return self - async def __aexit__(self, exc_type, exc, tb): + async def __aexit__( + self, exc_type: Optional[type], exc: Optional[BaseException], tb: Any + ) -> None: pass diff --git a/aiohttp/http2/settings.py b/aiohttp/http2/settings.py index e85c9ec7d1d..c9c5b303a3c 100644 --- a/aiohttp/http2/settings.py +++ b/aiohttp/http2/settings.py @@ -1,4 +1,5 @@ from enum import IntEnum, IntFlag +from typing import Dict # ---------------------------------------------------------------------- @@ -45,20 +46,15 @@ class Setting(IntEnum): INITIAL_WINDOW_SIZE = 0x4 MAX_FRAME_SIZE = 0x5 MAX_HEADER_LIST_SIZE = 0x6 - # this is mostly for web sockets - # via proxies - # which is not supported to the date SETTINGS_ENABLE_CONNECT_PROTOCOL = 0x8 - # does nothing as we don't implement this deprecated - # implementation NO_RFC7540_PRIORITIES = 0x9 # Default values (RFC 7540, 6.5.2) -DEFAULT_SETTINGS = { +DEFAULT_SETTINGS: Dict[Setting, int] = { Setting.HEADER_TABLE_SIZE: 4096, Setting.ENABLE_PUSH: 1, - Setting.MAX_CONCURRENT_STREAMS: 2**32 - 1, # effectively unlimited + Setting.MAX_CONCURRENT_STREAMS: 2**32 - 1, Setting.INITIAL_WINDOW_SIZE: 65535, Setting.MAX_FRAME_SIZE: 16384, Setting.MAX_HEADER_LIST_SIZE: 2**32 - 1, diff --git a/aiohttp/http2/stream.py b/aiohttp/http2/stream.py index 876ed802190..f6dd4194587 100644 --- a/aiohttp/http2/stream.py +++ b/aiohttp/http2/stream.py @@ -1,10 +1,15 @@ import asyncio from enum import IntEnum -from typing import List, Optional, Tuple +from typing import TYPE_CHECKING, Dict, Iterable, Optional, Set, Tuple + +from hpack import HeaderTuple from .errors import ProtocolError from .settings import Setting +if TYPE_CHECKING: + from .connection import Http2Connection + # ---------------------------------------------------------------------- # Stream State Machine (RFC 7540 5.1) @@ -20,8 +25,7 @@ class StreamState(IntEnum): # Valid transitions (RFC 7540 Figure 2) -# Added missing RESERVED_REMOTE from IDLE -VALID_TRANSITIONS = { +VALID_TRANSITIONS: Dict[StreamState, Set[StreamState]] = { StreamState.IDLE: { StreamState.OPEN, StreamState.RESERVED_LOCAL, @@ -32,7 +36,7 @@ class StreamState(IntEnum): StreamState.OPEN: {StreamState.HALF_CLOSED_LOCAL, StreamState.HALF_CLOSED_REMOTE}, StreamState.HALF_CLOSED_LOCAL: {StreamState.CLOSED}, StreamState.HALF_CLOSED_REMOTE: {StreamState.CLOSED}, - StreamState.CLOSED: set(), # terminal state + StreamState.CLOSED: set(), } @@ -52,30 +56,33 @@ class Stream: "outbound_window", "inbound_window", "response_future", - "request_body", "response_headers", "response_data", "closed_event", ) - def __init__(self, stream_id: int, conn, loop) -> None: + def __init__( + self, stream_id: int, conn: "Http2Connection", loop: asyncio.AbstractEventLoop + ) -> None: self.stream_id = stream_id self.state = StreamState.IDLE self.conn = conn - # Flow‑control windows (stream‑level only; session window managed by connection) - self.outbound_window = conn.remote_settings[Setting.INITIAL_WINDOW_SIZE] - self.inbound_window = conn.local_settings[Setting.INITIAL_WINDOW_SIZE] + # Flow‑control windows (stream‑level only) + self.outbound_window: int = conn.remote_settings[Setting.INITIAL_WINDOW_SIZE] + self.inbound_window: int = conn.local_settings[Setting.INITIAL_WINDOW_SIZE] self.response_future: asyncio.Future[ - Tuple[int, List[Tuple[str, str]], bytes] + Tuple[int, Iterable[HeaderTuple] | Iterable[Tuple[str, str]], bytes] ] = loop.create_future() - self.response_headers: Optional[List[Tuple[str, str]]] = None - self.response_data = bytearray() + self.response_headers: Optional[ + Iterable[HeaderTuple] | Iterable[Tuple[str, str]] + ] = None + self.response_data: bytearray = bytearray() # Event notified when the stream enters CLOSED state - self.closed_event = asyncio.Event() + self.closed_event: asyncio.Event = asyncio.Event() def transition(self, new_state: StreamState) -> None: if ( @@ -90,8 +97,7 @@ def transition(self, new_state: StreamState) -> None: self.closed_event.set() # ------------------------------------------------------------------ - # Data and header reception – state transitions are now **conditional** - # on the current state, matching RFC 7540/9113 exactly. + # Data and header reception # ------------------------------------------------------------------ def receive_data(self, data: bytes, end_stream: bool) -> None: """Process incoming DATA frame payload.""" @@ -99,12 +105,11 @@ def receive_data(self, data: bytes, end_stream: bool) -> None: self.response_data.extend(data) if end_stream: - # Correct transition depending on current state if self.state == StreamState.OPEN: self.transition(StreamState.HALF_CLOSED_REMOTE) elif self.state == StreamState.HALF_CLOSED_LOCAL: self.transition(StreamState.CLOSED) - self.conn._close_stream(self) # clean up connection map + self.conn._close_stream(self) else: raise ProtocolError( f"Unexpected stream state {self.state.name} for END_STREAM" @@ -112,14 +117,18 @@ def receive_data(self, data: bytes, end_stream: bool) -> None: self.maybe_deliver_response() - def maybe_deliver_response(self): + def maybe_deliver_response(self) -> None: # Deliver the full response once headers have been received if self.response_headers is not None and not self.response_future.done(): self.response_future.set_result( (self.stream_id, self.response_headers, bytes(self.response_data)) ) - def receive_headers(self, headers: List[Tuple[str, str]], end_stream: bool) -> None: + def receive_headers( + self, + headers: Iterable[HeaderTuple] | Iterable[Tuple[str, str]], + end_stream: bool, + ) -> None: """Process incoming HEADERS frame payload.""" self.response_headers = headers @@ -134,5 +143,3 @@ def receive_headers(self, headers: List[Tuple[str, str]], end_stream: bool) -> N f"Unexpected stream state {self.state.name} for END_STREAM on headers" ) self.maybe_deliver_response() - - # else: stream remains in OPEN (or HALF_CLOSED_LOCAL), headers stored for later delivery. diff --git a/aiohttp/http_protocol.py b/aiohttp/http_protocol.py index 32061964ffc..596f23bd48f 100644 --- a/aiohttp/http_protocol.py +++ b/aiohttp/http_protocol.py @@ -1,6 +1,5 @@ -# aiohttp/http_protocol.py (new file) import asyncio -from typing import Optional +from typing import Any, Optional from .client_proto import ResponseHandler from .http2.connection import Http2Protocol @@ -24,8 +23,8 @@ def connection_made(self, transport: asyncio.BaseTransport) -> None: self._transport = transport # type: ignore[assignment] # Determine ALPN after TLS is established - ssl_object = transport.get_extra_info("ssl_object") - alpn_protocol = ( + ssl_object: Any = transport.get_extra_info("ssl_object") + alpn_protocol: str = ( ssl_object.selected_alpn_protocol() if ssl_object else "http/1.1" ) @@ -38,7 +37,7 @@ def connection_made(self, transport: asyncio.BaseTransport) -> None: # all incoming data and callbacks. self._handler.connection_made(transport) - def __getattribute__(self, name): + def __getattribute__(self, name: str) -> Any: if not name.startswith("__") and name not in { "connection_made", "__getattribute__", @@ -49,10 +48,10 @@ def __getattribute__(self, name): return getattr(self._handler, name) return super().__getattribute__(name) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: if name not in {"_handler", "_transport", "_loop"}: return self._handler.__setattr__(name, value) return super().__setattr__(name, value) - def __delattr__(self, name): + def __delattr__(self, name: str) -> None: return self._handler.__delattr__(name) diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index be9a895c80a..160965dd915 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -12,12 +12,14 @@ import gzip import json import struct -from typing import List, Optional, Tuple +from typing import Any, Dict, Generator, List, Optional, Tuple from unittest.mock import MagicMock import pytest from hpack import Encoder +import aiohttp +from aiohttp.connector import TCPConnector from aiohttp.http2.connection import Http2Connection, Http2Protocol from aiohttp.http2.errors import ProtocolError from aiohttp.http2.response import Http2Response @@ -35,7 +37,7 @@ # ---------------------------------------------------------------------- # Helper: minimal URL mock # ---------------------------------------------------------------------- -def url_mock(path="/"): +def url_mock(path: str = "/") -> Any: """Create a simple URL-like object expected by the implementation.""" return type("URL", (), {"scheme": "https", "host": "example.com", "path": path}) @@ -44,7 +46,7 @@ def url_mock(path="/"): # Fixtures # ---------------------------------------------------------------------- @pytest.fixture -def mock_transport(): +def mock_transport() -> MagicMock: """Return a mock asyncio.Transport that records writes.""" t = MagicMock(spec=asyncio.Transport) t.is_closing.return_value = False @@ -53,8 +55,8 @@ def mock_transport(): return t -@pytest.fixture -def event_loop(): +@pytest.fixture(scope="session") +def event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]: """Create a new event loop for each test.""" loop = asyncio.new_event_loop() try: @@ -64,8 +66,7 @@ def event_loop(): @pytest.fixture -# async def connection(mock_transport, event_loop): -async def connection(mock_transport): +async def connection(mock_transport: MagicMock) -> Tuple[Http2Connection, MagicMock]: """Set up Http2Connection with mock transport, send preface, and clear write log.""" event_loop = asyncio.get_running_loop() @@ -76,8 +77,7 @@ async def connection(mock_transport): @pytest.fixture -# async def protocol(mock_transport, event_loop): -async def protocol(mock_transport): +async def protocol(mock_transport: MagicMock) -> Tuple[Http2Protocol, MagicMock]: """Create Http2Protocol and simulate connection_made.""" event_loop = asyncio.get_running_loop() proto = Http2Protocol(event_loop) @@ -89,8 +89,7 @@ async def protocol(mock_transport): # ---------------------------------------------------------------------- # Frame construction helpers # ---------------------------------------------------------------------- -# one could argue that we could add these static methods to the implementation -def frame_header(length: int, ftype: FrameType, flags: int, stream_id: int) -> bytes: +def frame_header(length: int, ftype: int, flags: int, stream_id: int) -> bytes: # 24-bit length (3 bytes) + type + flags + stream_id return struct.pack("!I", length)[1:] + struct.pack( "!B B I", ftype, flags, stream_id @@ -98,7 +97,7 @@ def frame_header(length: int, ftype: FrameType, flags: int, stream_id: int) -> b def build_settings_frame( - settings_pairs: List[Tuple[Setting, int]] = None, ack: bool = False + settings_pairs: Optional[List[Tuple[Setting, int]]] = None, ack: bool = False ) -> bytes: payload = b"" if not ack and settings_pairs: @@ -166,11 +165,11 @@ def build_ping(ack: bool = False, opaque: bytes = b"\x00" * 8) -> bytes: @pytest.mark.asyncio -async def test_incomplete_frame(connection): - connection, _ = connection +async def test_incomplete_frame(connection: Tuple[Http2Connection, MagicMock]) -> None: + conn, _ = connection frame = b"111111111" - connection.data_received(frame) - assert connection._frame_buffer == frame + conn.data_received(frame) + assert conn._frame_buffer == frame # ====================================================================== @@ -184,8 +183,8 @@ async def test_incomplete_frame(connection): class TestProtocolCompliance: @pytest.mark.asyncio async def test_receive_settings_updates_remote_and_acks( - self, connection, mock_transport - ): + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: conn, transport = connection # Send server SETTINGS (HEADER_TABLE_SIZE=8192, MAX_CONCURRENT_STREAMS=50) frame = build_settings_frame( @@ -205,7 +204,9 @@ async def test_receive_settings_updates_remote_and_acks( ) @pytest.mark.asyncio - async def test_receive_headers_for_new_stream(self, connection, mock_transport): + async def test_receive_headers_for_new_stream( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: conn, _ = connection # Create a stream from client side first (simulate request sent) stream = await conn.create_stream() @@ -225,7 +226,9 @@ async def test_receive_headers_for_new_stream(self, connection, mock_transport): assert body == b"" @pytest.mark.asyncio - async def test_receive_data_flow_control(self, connection, mock_transport): + async def test_receive_data_flow_control( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: conn, transport = connection stream = await conn.create_stream() stream.state = StreamState.OPEN # assume request already sent @@ -255,7 +258,9 @@ async def test_receive_data_flow_control(self, connection, mock_transport): assert len(updates) >= 1 @pytest.mark.asyncio - async def test_rst_stream_handling(self, connection): + async def test_rst_stream_handling( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: conn, _ = connection stream = await conn.create_stream() stream.state = StreamState.OPEN @@ -270,7 +275,9 @@ async def test_rst_stream_handling(self, connection): assert stream.stream_id not in conn.streams @pytest.mark.asyncio - async def test_goaway_cancels_higher_streams(self, connection): + async def test_goaway_cancels_higher_streams( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: conn, _ = connection # create three streams (1,3,5) s1 = await conn.create_stream() @@ -289,12 +296,14 @@ async def test_goaway_cancels_higher_streams(self, connection): assert s5.response_future.exception() is not None @pytest.mark.asyncio - async def test_ping_ack(self, connection, mock_transport): + async def test_ping_ack( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: conn, transport = connection frame = build_ping(ack=False, opaque=b"12345678") conn.data_received(frame) # Expect ACK sent back with same data - acks = [] + acks: List[bytes] = [] for call in transport.write.call_args_list: arg = call.args[0] if FrameType.PING.to_bytes(1, "big") in arg and arg[4] & FlagPing.ACK: @@ -303,7 +312,9 @@ async def test_ping_ack(self, connection, mock_transport): assert b"12345678" in acks[0] @pytest.mark.asyncio - async def test_max_concurrent_streams_blocking(self, connection): + async def test_max_concurrent_streams_blocking( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: conn, _ = connection conn.max_concurrent_streams = 1 s1 = await conn.create_stream() @@ -318,7 +329,9 @@ async def test_max_concurrent_streams_blocking(self, connection): assert len(conn.streams) == 1 @pytest.mark.asyncio - async def test_unknown_frame_ignored(self, connection): + async def test_unknown_frame_ignored( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: conn, _ = connection # send frame type 0x1a (unused) frame = frame_header(0, 0x1A, 0, 0) @@ -327,7 +340,9 @@ async def test_unknown_frame_ignored(self, connection): assert not conn._goaway_sent @pytest.mark.asyncio - async def test_bad_hpack_triggers_protocol_error(self, connection): + async def test_bad_hpack_triggers_protocol_error( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: conn, transport = connection stream = await conn.create_stream() stream.state = StreamState.OPEN @@ -355,7 +370,9 @@ async def test_bad_hpack_triggers_protocol_error(self, connection): assert rst or goaway @pytest.mark.asyncio - async def test_data_frame_with_padding(self, connection): + async def test_data_frame_with_padding( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """Cover DATA frame with PADDED flag.""" conn, _ = connection stream = await conn.create_stream() @@ -366,7 +383,9 @@ async def test_data_frame_with_padding(self, connection): assert stream.response_data == b"x" @pytest.mark.asyncio - async def test_headers_frame_with_priority(self, connection): + async def test_headers_frame_with_priority( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """Cover HEADERS frame with PRIORITY flag.""" conn, _ = connection stream = await conn.create_stream() @@ -386,14 +405,18 @@ async def test_headers_frame_with_priority(self, connection): assert stream.response_headers is not None @pytest.mark.asyncio - async def test_rst_stream_for_unknown_stream(self, connection): + async def test_rst_stream_for_unknown_stream( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """RST_STREAM on an unknown stream ID is silently ignored.""" conn, _ = connection frame = build_rst_stream(999, error_code=0) conn.data_received(frame) # must not raise @pytest.mark.asyncio - async def test_rst_stream_when_future_done(self, connection): + async def test_rst_stream_when_future_done( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """RST_STREAM when response future already completed (else branch of line 256).""" conn, _ = connection stream = await conn.create_stream() @@ -404,14 +427,18 @@ async def test_rst_stream_when_future_done(self, connection): assert stream.state == StreamState.CLOSED @pytest.mark.asyncio - async def test_receive_settings_ack(self, connection): + async def test_receive_settings_ack( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """Receive SETTINGS ACK (should be a no‑op).""" conn, _ = connection frame = build_settings_frame(ack=True) conn.data_received(frame) # no crash @pytest.mark.asyncio - async def test_settings_on_nonzero_stream(self, connection, mock_transport): + async def test_settings_on_nonzero_stream( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: """SETTINGS frame on stream_id != 0 triggers GOAWAY.""" conn, transport = connection frame = frame_header(0, FrameType.SETTINGS, 0, 5) # stream 5 @@ -422,7 +449,9 @@ async def test_settings_on_nonzero_stream(self, connection, mock_transport): ) @pytest.mark.asyncio - async def test_settings_invalid_payload_length(self, connection, mock_transport): + async def test_settings_invalid_payload_length( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: """SETTINGS payload not a multiple of 6 triggers protocol error.""" conn, transport = connection payload = b"\x00\x01\x02" # 3 bytes @@ -434,7 +463,9 @@ async def test_settings_invalid_payload_length(self, connection, mock_transport) ) @pytest.mark.asyncio - async def test_settings_initial_window_size_update(self, connection): + async def test_settings_initial_window_size_update( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """INITIAL_WINDOW_SIZE setting updates stream windows.""" conn, _ = connection stream = await conn.create_stream() @@ -444,7 +475,9 @@ async def test_settings_initial_window_size_update(self, connection): assert stream.outbound_window == old_window + (131072 - 65535) @pytest.mark.asyncio - async def test_settings_header_table_size(self, connection): + async def test_settings_header_table_size( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """HEADER_TABLE_SIZE setting is processed.""" conn, _ = connection frame = build_settings_frame([(Setting.HEADER_TABLE_SIZE, 4096)]) @@ -452,7 +485,9 @@ async def test_settings_header_table_size(self, connection): # internal effect on encoder, just confirm no crash @pytest.mark.asyncio - async def test_ping_on_nonzero_stream(self, connection, mock_transport): + async def test_ping_on_nonzero_stream( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: """PING on non‑zero stream triggers GOAWAY.""" conn, transport = connection frame = frame_header(8, FrameType.PING, 0, 1) + b"\x00" * 8 @@ -463,14 +498,18 @@ async def test_ping_on_nonzero_stream(self, connection, mock_transport): ) @pytest.mark.asyncio - async def test_receive_ping_ack(self, connection): + async def test_receive_ping_ack( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """Receiving a PING ACK is logged and does not cause another response.""" conn, _ = connection frame = build_ping(ack=True, opaque=b"12345678") conn.data_received(frame) # no crash, no additional PING sent @pytest.mark.asyncio - async def test_goaway_when_future_already_done(self, connection): + async def test_goaway_when_future_already_done( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """GOAWAY should not fail when a stream's future is already complete.""" conn, _ = connection s1 = await conn.create_stream() @@ -483,7 +522,9 @@ async def test_goaway_when_future_already_done(self, connection): assert s3.stream_id not in conn.streams @pytest.mark.asyncio - async def test_window_update_for_unknown_stream(self, connection): + async def test_window_update_for_unknown_stream( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """WINDOW_UPDATE on unknown stream must be ignored (no crash).""" conn, _ = connection frame = build_window_update(123, 100) @@ -491,21 +532,27 @@ async def test_window_update_for_unknown_stream(self, connection): assert conn.session_outbound_window == 65535 # unchanged @pytest.mark.asyncio - async def test_continuation_frame_ignored(self, connection): + async def test_continuation_frame_ignored( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """CONTINUATION frame is ignored with a warning.""" conn, _ = connection frame = frame_header(0, FrameType.CONTINUATION, 0, 1) conn.data_received(frame) # no crash @pytest.mark.asyncio - async def test_unknown_frame_type_ignored(self, connection): + async def test_unknown_frame_type_ignored( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """Unknown frame type (>9) is ignored.""" conn, _ = connection frame = frame_header(0, 0x1A, 0, 0) conn.data_received(frame) # no crash @pytest.mark.asyncio - async def test_send_data_end_stream_last_chunk(self, connection, mock_transport): + async def test_send_data_end_stream_last_chunk( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: """send_data with end_stream=True sets END_STREAM on the last chunk.""" conn, transport = connection stream = await conn.create_stream() @@ -522,7 +569,9 @@ async def test_send_data_end_stream_last_chunk(self, connection, mock_transport) assert data_frames[0][4] & FlagData.END_STREAM @pytest.mark.asyncio - async def test_send_request_with_body(self, connection, mock_transport): + async def test_send_request_with_body( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: """send_request with body sends HEADERS (no END_STREAM) followed by DATA.""" conn, transport = connection stream = await conn.create_stream() @@ -535,7 +584,9 @@ async def test_send_request_with_body(self, connection, mock_transport): assert sent == ["HEADERS", "DATA"] @pytest.mark.asyncio - async def test_create_stream_after_goaway(self, connection): + async def test_create_stream_after_goaway( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """create_stream raises ConnectionError when GOAWAY has been sent.""" conn, _ = connection conn._goaway_sent = True @@ -543,7 +594,9 @@ async def test_create_stream_after_goaway(self, connection): await conn.create_stream() @pytest.mark.asyncio - async def test_maybe_unblock_streams_done_future(self, connection): + async def test_maybe_unblock_streams_done_future( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """_maybe_unblock_streams skips futures that are already done.""" conn, _ = connection fut = conn._loop.create_future() @@ -553,7 +606,9 @@ async def test_maybe_unblock_streams_done_future(self, connection): assert len(conn.streams) == 0 # no new stream created @pytest.mark.asyncio - async def test_connection_lost_done_futures(self, connection): + async def test_connection_lost_done_futures( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """connection_lost must handle streams whose futures are already done.""" conn, _ = connection stream = await conn.create_stream() @@ -567,8 +622,8 @@ async def test_connection_lost_done_futures(self, connection): class TestMiscellaneous: @pytest.mark.asyncio async def test_concurrent_send_data_does_not_deadlock( - self, connection, mock_transport - ): + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: """Multiple tasks sending data on the same stream should not deadlock.""" conn, transport = connection stream = await conn.create_stream() @@ -576,7 +631,7 @@ async def test_concurrent_send_data_does_not_deadlock( conn.session_outbound_window = 1_000_000 stream.outbound_window = 1_000_000 - async def send_chunk(): + async def send_chunk() -> None: await conn.send_data(stream, b"x" * 100, end_stream=False) tasks = [asyncio.create_task(send_chunk()) for _ in range(5)] @@ -585,14 +640,16 @@ async def send_chunk(): assert transport.write.call_count >= 5 @pytest.mark.asyncio - async def test_window_update_wakes_all_waiters(self, connection, mock_transport): + async def test_window_update_wakes_all_waiters( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: """When window is zero, multiple blocked tasks resume on WINDOW_UPDATE.""" conn, transport = connection stream = await conn.create_stream() conn.session_outbound_window = 0 stream.outbound_window = 0 - async def blocked_send(): + async def blocked_send() -> None: await conn.send_data(stream, b"hello", end_stream=False) task1 = asyncio.create_task(blocked_send()) @@ -609,7 +666,9 @@ async def blocked_send(): assert task2.done() @pytest.mark.asyncio - async def test_stream_cancelled_before_response(self, connection): + async def test_stream_cancelled_before_response( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """Pending create_stream futures are cancelled on connection loss.""" conn, _ = connection conn.max_concurrent_streams = 1 @@ -626,7 +685,9 @@ async def test_stream_cancelled_before_response(self, connection): fut.result() @pytest.mark.asyncio - async def test_close_stream_on_rst_without_headers(self, connection): + async def test_close_stream_on_rst_without_headers( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """RST_STREAM before headers deliver must resolve future with error.""" conn, _ = connection stream = await conn.create_stream() @@ -637,7 +698,9 @@ async def test_close_stream_on_rst_without_headers(self, connection): stream.response_future.result() @pytest.mark.asyncio - async def test_data_received_without_connection(self, protocol): + async def test_data_received_without_connection( + self, protocol: Tuple[Http2Protocol, MagicMock] + ) -> None: """Http2Protocol.data_received is a no‑op before connection_made.""" proto, _ = protocol proto._connection = None @@ -649,13 +712,13 @@ async def test_data_received_without_connection(self, protocol): # ---------------------------------------------------------------------- class TestHighLevelInterface: @pytest.mark.asyncio - async def test_json_response(self, protocol, mock_transport): + async def test_json_response( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: """send() returns a Http2Response with correct JSON body.""" proto, transport = protocol - # Start a request using the public API - async def do_request(): - # url_mock() returns a simple URL object with required attributes + async def do_request() -> Http2Response: return await proto.send( "GET", url_mock("/json"), headers=[("accept", "application/json")] ) @@ -675,7 +738,9 @@ async def do_request(): assert json.loads(response.body) == {"key": "value"} @pytest.mark.asyncio - async def test_redirect_response(self, protocol, mock_transport): + async def test_redirect_response( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: """302 redirect headers are correctly returned.""" proto, transport = protocol task = asyncio.create_task(proto.send("GET", url_mock("/redirect"), headers=[])) @@ -688,7 +753,9 @@ async def test_redirect_response(self, protocol, mock_transport): assert dict(resp.headers).get("location") == "/new" @pytest.mark.asyncio - async def test_compressed_body_delivery(self, protocol, mock_transport): + async def test_compressed_body_delivery( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: """Response with content-encoding: gzip delivers raw compressed bytes.""" proto, transport = protocol task = asyncio.create_task(proto.send("GET", url_mock("/gzip"), headers=[])) @@ -703,7 +770,10 @@ async def test_compressed_body_delivery(self, protocol, mock_transport): assert resp.body == raw_data # Higher layer (aiohttp) will handle decompression - async def test_multiple_requests_concurrently(self, protocol, mock_transport): + @pytest.mark.asyncio + async def test_multiple_requests_concurrently( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: """Multiple send() calls create distinct streams and receive responses.""" proto, transport = protocol tasks = [] @@ -723,7 +793,7 @@ async def test_multiple_requests_concurrently(self, protocol, mock_transport): assert r2.status == 201 @pytest.mark.asyncio - async def test_response_all_methods(self): + async def test_response_all_methods(self) -> None: """Cover Http2Response.read, .text, .json, .cookies, .raise_for_status, .release, .close, context manager.""" headers = [ (":status", "200"), @@ -744,7 +814,7 @@ async def test_response_all_methods(self): pass @pytest.mark.asyncio - async def test_response_raise_for_status_error(self): + async def test_response_raise_for_status_error(self) -> None: """Http2Response.raise_for_status raises for 4xx.""" headers = [(":status", "404")] resp = Http2Response(headers, b"", method="GET", url=url_mock("/")) @@ -754,7 +824,9 @@ async def test_response_raise_for_status_error(self): resp.raise_for_status() @pytest.mark.asyncio - async def test_send_with_body_through_protocol(self, protocol, mock_transport): + async def test_send_with_body_through_protocol( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: """Full send() with a body completes successfully.""" proto, transport = protocol url = url_mock("/upload") @@ -768,7 +840,9 @@ async def test_send_with_body_through_protocol(self, protocol, mock_transport): class TestStreamStateMachine: @pytest.mark.asyncio - async def test_invalid_transition_raises(self, connection): + async def test_invalid_transition_raises( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """Invalid state transition raises ProtocolError.""" conn, _ = connection stream = await conn.create_stream() @@ -777,7 +851,9 @@ async def test_invalid_transition_raises(self, connection): stream.transition(StreamState.RESERVED_LOCAL) @pytest.mark.asyncio - async def test_receive_data_end_stream_half_closed_local(self, connection): + async def test_receive_data_end_stream_half_closed_local( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """DATA END_STREAM when HALF_CLOSED_LOCAL -> CLOSED and stream removed.""" conn, _ = connection stream = await conn.create_stream() @@ -788,7 +864,9 @@ async def test_receive_data_end_stream_half_closed_local(self, connection): assert stream.stream_id not in conn.streams @pytest.mark.asyncio - async def test_receive_data_end_stream_invalid_state(self, connection): + async def test_receive_data_end_stream_invalid_state( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """DATA END_STREAM in CLOSED state raises ProtocolError.""" conn, _ = connection stream = await conn.create_stream() @@ -797,7 +875,9 @@ async def test_receive_data_end_stream_invalid_state(self, connection): stream.receive_data(b"x", end_stream=True) @pytest.mark.asyncio - async def test_receive_headers_end_stream_half_closed_local(self, connection): + async def test_receive_headers_end_stream_half_closed_local( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """HEADERS END_STREAM when HALF_CLOSED_LOCAL -> CLOSED and stream removed.""" conn, _ = connection stream = await conn.create_stream() @@ -807,7 +887,9 @@ async def test_receive_headers_end_stream_half_closed_local(self, connection): assert stream.stream_id not in conn.streams @pytest.mark.asyncio - async def test_receive_headers_end_stream_invalid_state(self, connection): + async def test_receive_headers_end_stream_invalid_state( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """HEADERS END_STREAM in CLOSED state raises ProtocolError.""" conn, _ = connection stream = await conn.create_stream() @@ -816,7 +898,9 @@ async def test_receive_headers_end_stream_invalid_state(self, connection): stream.receive_headers([(":status", "200")], end_stream=True) @pytest.mark.asyncio - async def test_data_stream_before_headers(self, connection): + async def test_data_stream_before_headers( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """DATA before headers does NOT set future until headers arrive.""" conn, _ = connection stream = await conn.create_stream() @@ -830,7 +914,9 @@ async def test_data_stream_before_headers(self, connection): assert body == b"body" @pytest.mark.asyncio - async def test_future_already_done_data_end_stream(self, connection): + async def test_future_already_done_data_end_stream( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """receive_data with END_STREAM does not double‑set an already done future.""" conn, _ = connection stream = await conn.create_stream() @@ -842,10 +928,202 @@ async def test_future_already_done_data_end_stream(self, connection): stream.receive_data(b"more", end_stream=True) # no exception @pytest.mark.asyncio - async def test_future_already_done_headers_end_stream(self, connection): + async def test_future_already_done_headers_end_stream( + self, connection: Tuple[Http2Connection, MagicMock] + ) -> None: """receive_headers with END_STREAM does not double‑set an already done future.""" conn, _ = connection stream = await conn.create_stream() stream.state = StreamState.OPEN stream.response_future.set_result((stream.stream_id, [], b"")) stream.receive_headers([(":status", "200")], end_stream=True) # no exception + + +# ---------------------------------------------------------------------- +# Mock transport – records writes and lies about ALPN +# ---------------------------------------------------------------------- +class MockH2Transport(asyncio.Transport): + def __init__(self, extra_info: Optional[Dict[str, Any]] = None) -> None: + super().__init__() + self.written = bytearray() + self._closing = False + self._extra = extra_info or {} + self._protocol: Optional[Http2Protocol] = None + + def write(self, data: bytes | bytearray | memoryview) -> None: + self.written.extend(data) + + def close(self) -> None: + self._closing = True + + def is_closing(self) -> bool: + return self._closing + + def get_extra_info(self, name: str, default: Any = None) -> Any: + if name == "ssl_object": + return self._extra.get("ssl_object", MagicMock()) + return self._extra.get(name, default) + + +# ---------------------------------------------------------------------- +# Custom connector – always returns Http2Protocol for h2 connections +# ---------------------------------------------------------------------- +class H2TestConnector(TCPConnector): + def _get_protocol(self, loop: asyncio.AbstractEventLoop) -> type: + # Return the class; aiohttp will instantiate it + return Http2Protocol + + async def close(self, *, abort_ssl: bool = False) -> None: + self._closed = True + return None + + +# ---------------------------------------------------------------------- +# Fixture: session + mock transport + captured protocol +# ---------------------------------------------------------------------- +@pytest.fixture +async def h2_client() -> Any: # returns a generator of (session, transport, protocol) + """Create a ClientSession that uses our Http2Protocol over a mock transport.""" + # Mock SSL object that tells aiohttp we’ve negotiated h2 + mock_ssl = MagicMock() + mock_ssl.selected_alpn_protocol.return_value = "h2" + transport = MockH2Transport(extra_info={"ssl_object": mock_ssl}) + + protocol_instance: Optional[Http2Protocol] = None + + async def fake_create_connection( + protocol_factory: Any, *args: Any, **kwargs: Any + ) -> Tuple[MockH2Transport, Http2Protocol]: + nonlocal protocol_instance + protocol_instance = protocol_factory() # Http2Protocol() + protocol_instance.connection_made(transport) + transport._protocol = protocol_instance + return transport, protocol_instance + + connector = H2TestConnector() + connector._wrap_create_connection = fake_create_connection # type: ignore[assignment] + async with aiohttp.ClientSession(connector=connector) as session: + yield session, transport, protocol_instance + + +CEASE = build_goaway(0, 1) +URL = "https://127.3.3.3" + + +class TestIncomingResponses: + @pytest.mark.asyncio + async def test_get_200_response(self, h2_client: Any) -> None: # type: ignore[misc] + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) # request sent + + # Feed a minimal 200 response + hframe = build_headers_frame(1, [(":status", "200")], end_stream=True) + proto = transport._protocol + proto.data_received(hframe) + + resp = await task + assert resp.status == 200 + assert await resp.read() == b"" + + @pytest.mark.asyncio + async def test_response_with_body(self, h2_client: Any) -> None: # type: ignore[misc] + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + # Send HEADERS (no END_STREAM) then DATA with body + hframe = build_headers_frame(1, [(":status", "200")], end_stream=False) + dframe = build_data_frame(1, b"Hello, h2!", end_stream=True) + proto = transport._protocol + proto.data_received(hframe) + proto.data_received(dframe) + + resp = await task + assert resp.status == 200 + assert await resp.text() == "Hello, h2!" + + @pytest.mark.asyncio + async def test_json_response(self, h2_client: Any) -> None: # type: ignore[misc] + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + headers = [(":status", "200"), ("content-type", "application/json")] + body = b'{"key":"value"}' + proto = transport._protocol + proto.data_received(build_headers_frame(1, headers, end_stream=False)) + proto.data_received(build_data_frame(1, body, end_stream=True)) + + resp = await task + assert await resp.json() == {"key": "value"} + + @pytest.mark.asyncio + async def test_response_cookies(self, h2_client: Any) -> None: # type: ignore[misc] + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + headers = [(":status", "200"), ("set-cookie", "session=abc123; Path=/")] + proto = transport._protocol + proto.data_received(build_headers_frame(1, headers, end_stream=True)) + + resp = await task + assert "session" in resp.cookies + assert resp.cookies["session"].value == "abc123" + + @pytest.mark.asyncio + async def test_concurrent_requests_mux(self, h2_client: Any) -> None: # type: ignore[misc] + session, transport, _ = h2_client + t1 = asyncio.create_task(session.get(URL)) + t2 = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + # Stream 1 gets response, stream 3 gets response + proto = transport._protocol + proto.data_received( + build_headers_frame(1, [(":status", "200")], end_stream=True) + ) + proto.data_received( + build_headers_frame(3, [(":status", "201")], end_stream=True) + ) + + r1, r2 = await asyncio.gather(t1, t2) + assert r1.status == 200 + assert r2.status == 201 + + @pytest.mark.asyncio + async def test_redirect_headers(self, h2_client: Any) -> None: # type: ignore[misc] + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + headers = [(":status", "302"), ("location", "/new")] + proto = transport._protocol + proto.data_received(build_headers_frame(1, headers, end_stream=True)) + + await asyncio.sleep(0.01) + + headers = [(":status", "200")] + proto.data_received(build_headers_frame(3, headers, end_stream=True)) + + resp = await task + first = resp._history[0] + assert first.status == 302 + assert first.headers.get("location") == "/new" + + assert resp.status == 200 + + @pytest.mark.asyncio + async def test_error_response_raises(self, h2_client: Any) -> None: # type: ignore[misc] + session, transport, _ = h2_client + task = asyncio.create_task(session.get(URL)) + await asyncio.sleep(0.01) + + proto = transport._protocol + proto.data_received( + build_headers_frame(1, [(":status", "404")], end_stream=True) + ) + resp = await task + with pytest.raises(aiohttp.ClientResponseError): + resp.raise_for_status() diff --git a/tests/http2/test_http2_integration.py b/tests/http2/test_http2_integration.py deleted file mode 100644 index 8958ee8d4d8..00000000000 --- a/tests/http2/test_http2_integration.py +++ /dev/null @@ -1,215 +0,0 @@ -""" -Tests for aiohttp’s public HTTP/2 client API. - -Divided into: -- Outgoing frame correctness (what aiohttp writes to the transport). -- Incoming response parsing (what aiohttp returns when frames are fed). - -Both rely on a mocked transport that simulates ALPN negotiation of "h2" -and a custom connector that plugs in your Http2Protocol. -""" - -import asyncio -import struct -from typing import List, Tuple -from unittest.mock import MagicMock, patch - -import pytest -from test_http2 import ( - Http2Protocol, - build_data_frame, - build_goaway, - build_headers_frame, -) - -import aiohttp -from aiohttp.connector import TCPConnector -from aiohttp.http2.settings import FlagData, FlagHeaders, FrameType - - -# ---------------------------------------------------------------------- -# Mock transport – records writes and lies about ALPN -# ---------------------------------------------------------------------- -class MockH2Transport(asyncio.Transport): - def __init__(self, extra_info=None): - super().__init__() - self.written = bytearray() - self._closing = False - self._extra = extra_info or {} - - def write(self, data): - self.written.extend(data) - - def close(self): - self._closing = True - - def is_closing(self): - return self._closing - - def get_extra_info(self, name, default=None): - if name == "ssl_object": - return self._extra.get("ssl_object", MagicMock()) - return self._extra.get(name, default) - - -# ---------------------------------------------------------------------- -# Custom connector – always returns Http2Protocol for h2 connections -# ---------------------------------------------------------------------- -class H2TestConnector(TCPConnector): - def _get_protocol(self, loop): - # Return the class; aiohttp will instantiate it - return Http2Protocol - - async def close(self): - self._closed = True - - -# ---------------------------------------------------------------------- -# Fixture: session + mock transport + captured protocol -# ---------------------------------------------------------------------- -@pytest.fixture -async def h2_client(): - """Create a ClientSession that uses our Http2Protocol over a mock transport.""" - loop = asyncio.get_running_loop() - - # Mock SSL object that tells aiohttp we’ve negotiated h2 - mock_ssl = MagicMock() - mock_ssl.selected_alpn_protocol.return_value = "h2" - transport = MockH2Transport(extra_info={"ssl_object": mock_ssl}) - - protocol_instance = None - - async def fake_create_connection(protocol_factory, *args, **kwargs): - nonlocal protocol_instance - protocol_instance = protocol_factory() # Http2Protocol() - protocol_instance.connection_made(transport) - transport._protocol = protocol_instance - return transport, protocol_instance - - connector = H2TestConnector() - connector._wrap_create_connection = fake_create_connection - async with aiohttp.ClientSession(connector=connector) as session: - yield session, transport, protocol_instance - - -CEASE = build_goaway(0, 1) -URL = "https://127.3.3.3" - - -class TestIncomingResponses: - @pytest.mark.asyncio - async def test_get_200_response(self, h2_client): - session, transport, _ = h2_client - task = asyncio.create_task(session.get(URL)) - await asyncio.sleep(0.01) # request sent - - # Feed a minimal 200 response - hframe = build_headers_frame(1, [(":status", "200")], end_stream=True) - proto = transport._protocol - proto.data_received(hframe) - - resp = await task - assert resp.status == 200 - assert await resp.read() == b"" - - @pytest.mark.asyncio - async def test_response_with_body(self, h2_client): - session, transport, _ = h2_client - task = asyncio.create_task(session.get(URL)) - await asyncio.sleep(0.01) - - # Send HEADERS (no END_STREAM) then DATA with body - hframe = build_headers_frame(1, [(":status", "200")], end_stream=False) - dframe = build_data_frame(1, b"Hello, h2!", end_stream=True) - proto = transport._protocol - proto.data_received(hframe) - proto.data_received(dframe) - - resp = await task - assert resp.status == 200 - assert await resp.text() == "Hello, h2!" - - @pytest.mark.asyncio - async def test_json_response(self, h2_client): - session, transport, _ = h2_client - task = asyncio.create_task(session.get(URL)) - await asyncio.sleep(0.01) - - headers = [(":status", "200"), ("content-type", "application/json")] - body = b'{"key":"value"}' - proto = transport._protocol - proto.data_received(build_headers_frame(1, headers, end_stream=False)) - proto.data_received(build_data_frame(1, body, end_stream=True)) - - resp = await task - assert await resp.json() == {"key": "value"} - - @pytest.mark.asyncio - async def test_response_cookies(self, h2_client): - session, transport, _ = h2_client - task = asyncio.create_task(session.get(URL)) - await asyncio.sleep(0.01) - - headers = [(":status", "200"), ("set-cookie", "session=abc123; Path=/")] - proto = transport._protocol - proto.data_received(build_headers_frame(1, headers, end_stream=True)) - - resp = await task - assert "session" in resp.cookies - assert resp.cookies["session"].value == "abc123" - - @pytest.mark.asyncio - async def test_concurrent_requests_mux(self, h2_client): - session, transport, _ = h2_client - t1 = asyncio.create_task(session.get(URL)) - t2 = asyncio.create_task(session.get(URL)) - await asyncio.sleep(0.01) - - # Stream 1 gets response, stream 3 gets response - proto = transport._protocol - proto.data_received( - build_headers_frame(1, [(":status", "200")], end_stream=True) - ) - proto.data_received( - build_headers_frame(3, [(":status", "201")], end_stream=True) - ) - - r1, r2 = await asyncio.gather(t1, t2) - assert r1.status == 200 - assert r2.status == 201 - - @pytest.mark.asyncio - async def test_redirect_headers(self, h2_client): - session, transport, _ = h2_client - task = asyncio.create_task(session.get(URL)) - await asyncio.sleep(0.01) - - headers = [(":status", "302"), ("location", "/new")] - proto = transport._protocol - proto.data_received(build_headers_frame(1, headers, end_stream=True)) - - await asyncio.sleep(0.01) - - headers = [(":status", "200")] - proto.data_received(build_headers_frame(3, headers, end_stream=True)) - - resp = await task - first = resp._history[0] - assert first.status == 302 - assert first.headers.get("location") == "/new" - - assert resp.status == 200 - - @pytest.mark.asyncio - async def test_error_response_raises(self, h2_client): - session, transport, _ = h2_client - task = asyncio.create_task(session.get(URL)) - await asyncio.sleep(0.01) - - proto = transport._protocol - proto.data_received( - build_headers_frame(1, [(":status", "404")], end_stream=True) - ) - resp = await task - with pytest.raises(aiohttp.ClientResponseError): - resp.raise_for_status() From 9ab172f33fe2df1fe67f4735a2ee43ec1cb0f398 Mon Sep 17 00:00:00 2001 From: Moist-Cat Date: Sun, 5 Jul 2026 17:38:34 -0400 Subject: [PATCH 09/12] Fix protocol violations and perform functional tests --- aiohttp/connector.py | 12 ++++++------ aiohttp/http2/connection.py | 26 ++++++++++++++++++++++++-- aiohttp/http2/response.py | 14 +++++++++++--- aiohttp/http2/stream.py | 10 ++++++++++ tests/http2/test_http2.py | 5 ++--- 5 files changed, 53 insertions(+), 14 deletions(-) diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 120882b34c4..294c0d2b7c6 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1,5 +1,6 @@ import asyncio import functools +import os import random import socket import sys @@ -863,12 +864,11 @@ def _make_ssl_context(verified: bool) -> SSLContext: sslcontext.verify_mode = ssl.CERT_NONE sslcontext.options |= ssl.OP_NO_COMPRESSION sslcontext.set_default_verify_paths() - sslcontext.set_alpn_protocols( - ( - "h2", - "http/1.1", - ) - ) + + protocols = ["http/1.1"] + if os.getenv("AIOHTTP_ENABLE_EXPERIMENTAL_PROTOCOLS", False): + protocols += ["h2"] + sslcontext.set_alpn_protocols(tuple(protocols)) return sslcontext diff --git a/aiohttp/http2/connection.py b/aiohttp/http2/connection.py index bbada75b5ec..867d6d91cfc 100644 --- a/aiohttp/http2/connection.py +++ b/aiohttp/http2/connection.py @@ -213,6 +213,7 @@ def _handle_data_frame(self, flags: int, stream_id: int, payload: bytes) -> None # Update session flow control self.session_inbound_window -= len(data) + # hardcoded values if self.session_inbound_window < 32768: self._send_window_update(0, 65535 - self.session_inbound_window) self.session_inbound_window = 65535 @@ -283,6 +284,9 @@ def _handle_settings_frame( # Parse key‑value pairs for i in range(0, len(payload), 6): identifier, value = struct.unpack("!H I", payload[i : i + 6]) + if identifier < 0 or identifier > 9: + logger.warning("Unknown setting identifier %d", identifier) + continue setting = Setting(identifier) old_value = self.remote_settings.get(setting, value) self.remote_settings[setting] = value @@ -486,14 +490,32 @@ async def send_request( body: Optional[bytes] = None, ) -> None: """Send HEADERS and optional DATA frames for a stream.""" + # :path needs the query as well + path_and_query = url.path + if url.query: + path_and_query += "?" + url.raw_query_string + # Build pseudo‑headers req_headers = [ (":method", method), - (":path", url.path), + (":path", path_and_query), (":scheme", url.scheme), (":authority", url.host), ] - req_headers.extend([(h[0].lower(), h[1]) for h in headers]) + + for name, value in headers: + lname = name.lower() + # HTTP/2 forbids connection-specific headers and the Host header + if lname in ( + "host", + "connection", + "keep-alive", + "proxy-connection", + "transfer-encoding", + "upgrade", + ): + continue + req_headers.append((lname, value)) hdrs = self.hpack_encoder.encode(req_headers) end_stream = body is None or len(body) == 0 diff --git a/aiohttp/http2/response.py b/aiohttp/http2/response.py index d73d56f4487..ba78ddf4f7a 100644 --- a/aiohttp/http2/response.py +++ b/aiohttp/http2/response.py @@ -8,6 +8,7 @@ from aiohttp.client_exceptions import ClientResponseError from aiohttp.client_reqrep import RequestInfo +from ..compression_utils import ZLibDecompressor from ..http_writer import HttpVersion2 @@ -22,6 +23,13 @@ def __init__( method: Optional[str] = None, url: Optional[Any] = None, ) -> None: + # Headers as case-insensitive multi-dict + self.headers: CIMultiDict[str] = CIMultiDict(headers) # type: ignore[arg-type] + encoding = self.headers.get("content-encoding", None) + if encoding in {"gzip", "deflate"}: + comp = ZLibDecompressor(encoding=encoding) + body = comp.decompress_sync(body) + self.reason: str = "" # HTTP/2 doesn't carry a reason phrase self._body: bytes = body self.url: Optional[Any] = url @@ -29,8 +37,6 @@ def __init__( # redirects self._history: List["Http2Response"] = [] - # Headers as case-insensitive multi-dict - self.headers: CIMultiDict[str] = CIMultiDict(headers) # type: ignore[arg-type] # no status error implies a server side error self.status: int = int(self.headers.get(":status", 500)) @@ -40,7 +46,9 @@ def __init__( # HTTP version pseudo-attribute self.version = HttpVersion2 - self._raw_cookie_headers: Optional[List[Tuple[str, str]]] = None + self._raw_cookie_headers: Optional[List[str]] = self.headers.getall( + "set-cookie", [] + ) self.connection: Optional[Any] = None # ---------------------------------------------------------------- diff --git a/aiohttp/http2/stream.py b/aiohttp/http2/stream.py index f6dd4194587..c98722b29bd 100644 --- a/aiohttp/http2/stream.py +++ b/aiohttp/http2/stream.py @@ -59,6 +59,7 @@ class Stream: "response_headers", "response_data", "closed_event", + "_inbound_window_initial", ) def __init__( @@ -71,6 +72,8 @@ def __init__( # Flow‑control windows (stream‑level only) self.outbound_window: int = conn.remote_settings[Setting.INITIAL_WINDOW_SIZE] self.inbound_window: int = conn.local_settings[Setting.INITIAL_WINDOW_SIZE] + # Store the initial inbound window for refilling without hard-coded values + self._inbound_window_initial: int = self.inbound_window self.response_future: asyncio.Future[ Tuple[int, Iterable[HeaderTuple] | Iterable[Tuple[str, str]], bytes] @@ -104,6 +107,13 @@ def receive_data(self, data: bytes, end_stream: bool) -> None: self.inbound_window -= len(data) self.response_data.extend(data) + # --- stream-level flow control refill --- + if self.inbound_window < self._inbound_window_initial // 2: + increment = self._inbound_window_initial - self.inbound_window + self.inbound_window = self._inbound_window_initial + # Use the connection’s helper to send the WINDOW_UPDATE frame + self.conn._send_window_update(self.stream_id, increment) + if end_stream: if self.state == StreamState.OPEN: self.transition(StreamState.HALF_CLOSED_REMOTE) diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index 160965dd915..dd22b013cac 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -39,7 +39,7 @@ # ---------------------------------------------------------------------- def url_mock(path: str = "/") -> Any: """Create a simple URL-like object expected by the implementation.""" - return type("URL", (), {"scheme": "https", "host": "example.com", "path": path}) + return type("URL", (), {"scheme": "https", "host": "example.com", "path": path, "query": None}) # ---------------------------------------------------------------------- @@ -767,8 +767,7 @@ async def test_compressed_body_delivery( ) proto.data_received(frames) resp = await task - assert resp.body == raw_data - # Higher layer (aiohttp) will handle decompression + assert resp.body == b"uncompressed" @pytest.mark.asyncio async def test_multiple_requests_concurrently( From 38b9d3df5d51538e950887785590bdf8752f6b41 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:40:23 +0000 Subject: [PATCH 10/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/http2/test_http2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index dd22b013cac..bb6ce128b81 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -39,7 +39,11 @@ # ---------------------------------------------------------------------- def url_mock(path: str = "/") -> Any: """Create a simple URL-like object expected by the implementation.""" - return type("URL", (), {"scheme": "https", "host": "example.com", "path": path, "query": None}) + return type( + "URL", + (), + {"scheme": "https", "host": "example.com", "path": path, "query": None}, + ) # ---------------------------------------------------------------------- From 538164e1bfb64fd54317e9af7a33d1b6e9acb85f Mon Sep 17 00:00:00 2001 From: Moist-Cat Date: Sun, 5 Jul 2026 18:52:00 -0400 Subject: [PATCH 11/12] More tests --- aiohttp/http2/connection.py | 20 +--- tests/http2/test_http2.py | 222 ++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 19 deletions(-) diff --git a/aiohttp/http2/connection.py b/aiohttp/http2/connection.py index 867d6d91cfc..130c0f4aadd 100644 --- a/aiohttp/http2/connection.py +++ b/aiohttp/http2/connection.py @@ -143,11 +143,7 @@ def data_received(self, data: bytes) -> None: length, ) - try: - self._dispatch_frame(frame_type_val, flags, stream_id, payload) - except ConnectionError as exc: - logger.error("Frame dispatch error: %s", exc, exc_info=True) - self._protocol_error() + self._dispatch_frame(frame_type_val, flags, stream_id, payload) def eof_received(self) -> bool: logger.debug("EOF received from server") @@ -189,8 +185,6 @@ def _dispatch_frame( self._handle_goaway_frame(flags, stream_id, payload) elif frame_type == FrameType.WINDOW_UPDATE: self._handle_window_update_frame(flags, stream_id, payload) - elif frame_type == FrameType.CONTINUATION: - self._handle_continuation_frame(flags, stream_id, payload) else: logger.warning("Ignoring unknown frame type %d", frame_type) @@ -355,12 +349,6 @@ def _handle_window_update_frame( # Wake up any writer waiting for flow control self._flow_control_updated.set() - def _handle_continuation_frame( - self, flags: int, stream_id: int, payload: bytes - ) -> None: - # Not implemented; would require accumulating headers across frames. - logger.warning("CONTINUATION frame ignored (not implemented)") - # -------------------- Frame sending helpers -------------------- def _send_frame( self, @@ -615,12 +603,6 @@ def is_connected(self) -> bool: def closed(self) -> asyncio.Future[None]: return self._closed_future - # Public API for creating streams - async def create_stream(self) -> Stream: - if self._connection is None: - raise RuntimeError("Connection not established") - return await self._connection.create_stream() - async def send( self, method: str, diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index bb6ce128b81..b95a79a1741 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -1130,3 +1130,225 @@ async def test_error_response_raises(self, h2_client: Any) -> None: # type: ign resp = await task with pytest.raises(aiohttp.ClientResponseError): resp.raise_for_status() + + +# ---------------------------------------------------------------------- +# Additional tests for full coverage of missing paths +# ---------------------------------------------------------------------- + + +class TestConnectionEdgeCases: + @pytest.mark.asyncio + async def test_eof_received_calls_close( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """EOF triggers connection close.""" + conn, transport = connection + result = conn.eof_received() + assert result is False + transport.close.assert_called_once() + + @pytest.mark.asyncio + async def test_protocol_eof_received( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: + """Http2Protocol.eof_received delegates to connection and returns False.""" + proto, transport = protocol + # connection is established + assert proto._connection is not None + result = proto.eof_received() + assert result is False + # transport.close should be called via conn.eof_received -> conn.close() + transport.close.assert_called_once() + + @pytest.mark.asyncio + async def test_data_frame_unknown_stream_above_last_peer( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """DATA frame for unknown stream_id > _last_peer_stream_id sends RST_STREAM.""" + conn, transport = connection + # last_peer_stream_id is initially 0 + unknown_id = 5 # > 0 + frame = build_data_frame(unknown_id, b"x", end_stream=False) + conn.data_received(frame) + # Should have sent RST_STREAM with PROTOCOL_ERROR (1) + rst_frames = [ + call.args[0] + for call in transport.write.call_args_list + if FrameType.RST_STREAM.to_bytes(1, "big") in call.args[0] + ] + assert len(rst_frames) == 1 + payload = rst_frames[0][9:] # after 9-byte header + error_code = struct.unpack("!I", payload)[0] + assert error_code == 1 # PROTOCOL_ERROR + + @pytest.mark.asyncio + async def test_data_frame_unknown_stream_not_above_last_peer( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """DATA frame for unknown stream_id <= _last_peer_stream_id is ignored.""" + conn, transport = connection + # artificially raise last_peer_stream_id + conn._last_peer_stream_id = 10 + frame = build_data_frame(5, b"x", end_stream=False) + conn.data_received(frame) + # No RST_STREAM sent + assert not any( + FrameType.RST_STREAM.to_bytes(1, "big") in call.args[0] + for call in transport.write.call_args_list + ) + + @pytest.mark.asyncio + async def test_send_data_end_stream_when_half_closed_remote( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """When stream is HALF_CLOSED_REMOTE, final DATA with END_STREAM closes it.""" + conn, transport = connection + stream = await conn.create_stream() + # Simulate remote half-close + stream.state = StreamState.HALF_CLOSED_REMOTE + conn.session_outbound_window = 1000 + stream.outbound_window = 1000 + await conn.send_data(stream, b"done", end_stream=True) + # Stream should be CLOSED and removed + assert stream.state == StreamState.CLOSED + assert stream.stream_id not in conn.streams + + @pytest.mark.asyncio + async def test_connection_lost_cancels_open_streams( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """connection_lost sets ConnectionError on all unfinished streams.""" + conn, _ = connection + s1 = await conn.create_stream() + s2 = await conn.create_stream() + s1.state = s2.state = StreamState.OPEN + # Both futures not done + conn.connection_lost(ConnectionError("test")) + assert s1.response_future.exception() is not None + assert s2.response_future.exception() is not None + # Also check pending streams cleared + assert len(conn._pending_streams) == 0 + + @pytest.mark.asyncio + async def test_connection_lost_clears_pending_futures( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """Pending create_stream futures are cancelled on connection loss.""" + conn, _ = connection + conn.max_concurrent_streams = 1 + await conn.create_stream() # fills one slot + fut = asyncio.ensure_future(conn.create_stream()) + await asyncio.sleep(0.01) + assert not fut.done() + conn.connection_lost(ConnectionError("test")) + await asyncio.sleep(0.01) + assert fut.done() + with pytest.raises(ConnectionError): + fut.result() + + @pytest.mark.asyncio + async def test_close_transport( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """close() calls transport.close().""" + conn, transport = connection + conn.close() + transport.close.assert_called_once() + + @pytest.mark.asyncio + async def test_should_close_reflects_goaway( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """should_close returns True if GOAWAY sent or received.""" + conn, _ = connection + assert not conn.should_close + conn._goaway_received = True + assert conn.should_close + + @pytest.mark.asyncio + async def test_is_connected( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """is_connected mirrors transport.is_closing().""" + conn, transport = connection + assert conn.is_connected() is True + transport.is_closing.return_value = True + assert conn.is_connected() is False + + @pytest.mark.asyncio + async def test_protocol_close_and_abort( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: + """Http2Protocol.close() and abort() propagate to transport.""" + proto, transport = protocol + proto.close() + transport.close.assert_called_once() + # abort is same as close + proto.abort() + assert transport.close.call_count == 2 + + @pytest.mark.asyncio + async def test_protocol_should_close_no_connection( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: + """should_close returns False when connection is None.""" + proto, _ = protocol + proto._connection = None + assert not proto.should_close + + @pytest.mark.asyncio + async def test_protocol_is_connected_no_connection( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: + """is_connected returns False when connection is None.""" + proto, _ = protocol + proto._connection = None + assert not proto.is_connected() + + @pytest.mark.asyncio + async def test_initiate_connection_sends_preface_and_settings( + self, mock_transport: MagicMock + ) -> None: + """initiate_connection writes HTTP/2 preface and initial SETTINGS.""" + loop = asyncio.get_running_loop() + conn = Http2Connection(mock_transport, loop) + conn.initiate_connection() + written = b"".join(call.args[0] for call in mock_transport.write.call_args_list) + assert b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" in written + assert FrameType.SETTINGS.to_bytes(1, "big") in written + + @pytest.mark.asyncio + async def test_create_stream_protocol_raises_without_connection( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: + """Http2Protocol.create_stream raises RuntimeError if no connection.""" + proto, _ = protocol + proto._connection = None + with pytest.raises(RuntimeError): + await proto.send("get", "", []) + + @pytest.mark.asyncio + async def test_send_protocol_raises_without_connection( + self, protocol: Tuple[Http2Protocol, MagicMock], mock_transport: MagicMock + ) -> None: + """Http2Protocol.send raises RuntimeError if no connection.""" + proto, _ = protocol + proto._connection = None + with pytest.raises(RuntimeError): + await proto.send("GET", url_mock(), []) + + @pytest.mark.asyncio + async def test_send_data_flow_control_releases_waiters( + self, connection: Tuple[Http2Connection, MagicMock], mock_transport: MagicMock + ) -> None: + """After a send, if window space remains, _flow_control_updated is set.""" + conn, transport = connection + stream = await conn.create_stream() + conn.session_outbound_window = 100 + stream.outbound_window = 100 + # set event cleared before wait + conn._flow_control_updated.clear() + await conn.send_data(stream, b"x" * 10, end_stream=False) + # After send, there is still window space, so event should be set + assert conn._flow_control_updated.is_set() From 2ce9b416c91185b384905feda860bc6bd2485ef3 Mon Sep 17 00:00:00 2001 From: Moist-Cat Date: Sun, 5 Jul 2026 20:30:03 -0400 Subject: [PATCH 12/12] Misc changes --- aiohttp/http2/connection.py | 16 +++++++--------- tests/http2/test_http2.py | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/aiohttp/http2/connection.py b/aiohttp/http2/connection.py index 130c0f4aadd..f6b8b9c3b9f 100644 --- a/aiohttp/http2/connection.py +++ b/aiohttp/http2/connection.py @@ -44,7 +44,7 @@ # Logging – plaintext wire‑format emission for debugging # ---------------------------------------------------------------------- logger = logging.getLogger("aiohttp.http2.connection") -logger.setLevel(logging.DEBUG) +# logger.setLevel(logging.DEBUG) FRAME_HEADER_LENGTH = 9 # 9 octets STREAM_ID_MASK = 0x7FFFFFFF # to avoid setting the reserved bit to 1 @@ -174,7 +174,6 @@ def _dispatch_frame( FrameType.CONTINUATION, }: logger.warning("%d frame ignored (not implemented)", frame_type) - self._handle_priority_frame(stream_id, payload) elif frame_type == FrameType.RST_STREAM: self._handle_rst_stream_frame(flags, stream_id, payload) elif frame_type == FrameType.SETTINGS: @@ -202,6 +201,7 @@ def _handle_data_frame(self, flags: int, stream_id: int, payload: bytes) -> None pad_length = payload[0] pos = 1 + # padding might be too long data = payload[pos : len(payload) - pad_length] end_stream = bool(flags & FlagData.END_STREAM) @@ -223,7 +223,7 @@ def _handle_headers_frame(self, flags: int, stream_id: int, payload: bytes) -> N # Decode headers with HPACK try: headers = self.hpack_decoder.decode(payload) - except Exception as exc: + except Exception as exc: # too general? logger.error(f"HPACK decode error: {exc}") self._send_rst_stream(stream_id, 1) # PROTOCOL_ERROR return @@ -237,10 +237,6 @@ def _handle_headers_frame(self, flags: int, stream_id: int, payload: bytes) -> N else: stream.receive_headers(headers, end_stream) - def _handle_priority_frame(self, stream_id: int, payload: bytes) -> None: - # it's deprecated in the most recent RFC - pass - def _handle_rst_stream_frame( self, flags: int, stream_id: int, payload: bytes ) -> None: @@ -288,10 +284,11 @@ def _handle_settings_frame( # React to certain settings if setting == Setting.INITIAL_WINDOW_SIZE and value != old_value: + # might become negative + # send WINDOW_UPDATE delta = value - old_value for s in self.streams.values(): s.outbound_window += delta - # self.session_outbound_window += delta elif setting == Setting.MAX_CONCURRENT_STREAMS: self.max_concurrent_streams = value self._maybe_unblock_streams() @@ -334,6 +331,7 @@ def _handle_goaway_frame(self, flags: int, stream_id: int, payload: bytes) -> No ConnectionError("GOAWAY received") ) self._close_stream(stream) + # clear pending streams? def _handle_window_update_frame( self, flags: int, stream_id: int, payload: bytes @@ -523,7 +521,7 @@ async def send_request( # -------------------- Connection lifecycle -------------------- def initiate_connection(self) -> None: """Send the connection preface and initial SETTINGS.""" - # Connection preface (RFC 7540 §3.5) + # Connection preface (RFC 7540, 3.5) self._transport.write(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") # Send initial SETTINGS (our preferences) diff --git a/tests/http2/test_http2.py b/tests/http2/test_http2.py index b95a79a1741..32ff86c2e59 100644 --- a/tests/http2/test_http2.py +++ b/tests/http2/test_http2.py @@ -1009,7 +1009,7 @@ async def fake_create_connection( yield session, transport, protocol_instance -CEASE = build_goaway(0, 1) +# if it's a real URL it hangs (missing mock?) URL = "https://127.3.3.3"