SolarManager unter Versionsverwaltung

Erster Stand der Hintergrundprozesse, die auf der Synology unter
/volume1/homes/wagner/SolarManager laufen: der Manager selbst, die Sammler
je Geraet, die MQTT-Bruecke, der Wecker und - neu hinzugezogen - der
AutoAction-Runner, der als Hintergrundprozess hierher gehoert und nicht ins
Web-Verzeichnis.

Zugangsdaten stehen nicht mehr im Quelltext, sondern in config.ini, die
nicht mit eingecheckt wird. Vorlage ist config.ini.example, gelesen wird sie
von konfig.py. Betroffen waren solarManager.py (Datenbank und Wattpilot),
zeit.py, gatherWaterData.py, wecker.py und skoda_testdaten.py, das sich das
Passwort bisher aus dem Quelltext eines anderen Moduls herausgesucht hat.

Die Kia-Anbindung ist mit dem Fahrzeug entfallen: kiaTest.py,
gatherCarData.py und hyundai_kia_connect_api sind nicht mehr dabei, ebenso
gatherInverterData.py, auf das nur noch eine auskommentierte Zeile zeigte.

Die mitgelieferten Bibliotheken bleiben im Repository - die NAS hat kein
pip, sie muessen neben den Skripten liegen.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.gitattributes

@@ -0,0 +1,6 @@
+# Alles mit LF im Repository. Die Start-Skripte laufen auf der Synology, und
+# ein CRLF im Shebang macht daraus ein "bad interpreter: no such file or
+# directory" - ein Fehler, der sich schlecht suchen laesst.
+* text=auto eol=lf
+
+*.zip binary

.gitignore

@@ -0,0 +1,30 @@
+# Zugangsdaten - nichts davon gehoert ins Repository
+config.ini
+skoda.conf
+*.conf
+secrets.conf
+
+# Protokolle. wattpilotshell.log ist allein 329 MB gross, die Logs machen
+# 93 Prozent des Verzeichnisses aus.
+*.log
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*~
+
+# Betriebssystem
+.DS_Store
+Thumbs.db
+@eaDir/
+
+# Temporaeres
+*.tmp
+*.bak
+*.backup

Saldierung.txt

@@ -0,0 +1,4 @@
+Bi-directional table update for better meter matching:
+ALTER TABLE `EnergyFlow` ADD `gridPcons` FLOAT NOT NULL AFTER `gridP`, ADD `gridPfeed` FLOAT NOT NULL AFTER `gridPcons`;
+UPDATE EnergyFlow SET gridPcons = gridP WHERE gridP > 0 AND gridPcons = 0;
+UPDATE EnergyFlow SET gridPfeed = -gridP WHERE gridP < 0 AND gridPfeed = 0;

Wattpilot/Wattpilot.py

@@ -0,0 +1,673 @@
+import sys
+sys.path.append("./")
+import websocket
+import json
+import hashlib
+import random
+import threading
+import hmac
+import logging
+
+import base64
+import subprocess
+
+from time import sleep
+from types import SimpleNamespace
+
+
+
+_LOGGER = logging.getLogger(__name__)
+
+	
+
+class LoadMode():
+    """Wrapper Class to represent the Load Mode of the Wattpilot"""
+    DEFAULT=3
+    ECO=4
+    NEXTTRIP=5
+
+
+class Wattpilot(object):
+
+    supported_events = [
+        # Wattpilot events:
+        "wp_auth",
+        "wp_authError",
+        "wp_authSuccess",
+        "wp_clearInverters",
+        "wp_connect",
+        "wp_deltaStatus",
+        "wp_disconnect",
+        "wp_fullStatus",
+        "wp_fullStatus_finished",
+        "wp_hello",
+        "wp_init",
+        "wp_property",
+        "wp_response",
+        "wp_updateInverter",
+        # WebSocketApp events:
+        "ws_close",
+        "ws_error",
+        "ws_message",
+        "ws_open",
+    ]
+    
+    carValues = {}
+    alwValues = {}
+    astValues = {}
+    lmoValues = {}
+    ustValues = {}
+    errValues = {}
+    acsValues = {}
+
+    lmoValues[3] = "Default"
+    lmoValues[4] = "Eco"
+    lmoValues[5] = "Next Trip"
+
+    astValues[0] = "open"
+    astValues[1] = "locked"
+    astValues[2] = "auto"
+
+    carValues[1] = "no car"
+    carValues[2] = "charging"
+    carValues[3] = "ready"
+    carValues[4] = "complete"
+
+    alwValues[0] = False
+    alwValues[1] = True
+
+    ustValues[0] = "Normal"
+    ustValues[1] = "AutoUnlock"
+    ustValues[2] = "AlwaysLock"
+
+    errValues[0] = "Unknown Error"
+    errValues[1] = "Idle"
+    errValues[2] = "Charging"
+    errValues[3] = "Wait Car"
+    errValues[4] = "Complete"
+    errValues[5] = "Error"
+
+    acsValues[0] = "Open"
+    acsValues[1] = "Wait"
+
+
+    @property
+    def allProps(self):
+        """Returns a dictionary with all properties"""
+        return self._allProps
+
+    @property
+    def allPropsInitialized(self):
+        """Returns true, if all properties have been initialized"""
+        return self._allPropsInitialized
+
+    @property
+    def cableType(self):
+        """Returns the Cable Type (Ampere) of the connected cable"""
+        return self._cableType
+
+    @property
+    def frequency(self):
+        """Returns the power frequency"""
+        return self._frequency
+
+    @property
+    def phases(self):
+        """returns the phases"""
+        return self._phases
+    
+    @property
+    def energyCounterSinceStart(self):
+        """Returns used kwh since start of charging"""
+        return self._energyCounterSinceStart
+    
+    @property
+    def errorState(self):
+        """Returns error State"""
+        return self._errorState
+
+    @property
+    def cableLock(self):
+        return self._cableLock
+    
+    @property
+    def energyCounterTotal(self):
+        return self._energyCounterTotal
+
+    @property
+    def serial(self):
+        """Returns the serial number of Wattpilot Device (read only)"""
+        return self._serial
+    @serial.setter
+    def serial(self,value):
+        self._serial = value
+        if (self._password is not None) & (self._serial is not None):
+            self._hashedpassword = base64.b64encode(hashlib.pbkdf2_hmac('sha512',self._password.encode(),self._serial.encode(),100000,256))[:32]
+           
+    @property
+    def name(self):
+        """Returns the name of Wattpilot Device (read only)"""
+        return self._name
+
+
+    @property
+    def hostname(self):
+        """Returns the DNS Hostname of Wattpilot Device (read only)"""
+        return self._hostname
+
+    @property
+    def friendlyName(self):
+        """Returns the friendly name of Wattpilot Device (read only)"""
+        return self._friendlyName
+
+    @property
+    def manufacturer(self):
+        """Returns the Manufacturer of Wattpilot Device (read only)"""
+        return self._manufacturer
+
+    @property
+    def devicetype(self):
+        return self._devicetype
+
+    @property
+    def protocol(self):
+        return self._protocol
+    
+    @property
+    def secured(self):
+        return self._secured
+
+    @property
+    def password(self):
+        return self._password
+    @password.setter
+    def password(self,value):
+        self._password = value
+        if (self._password is not None) & (self._serial is not None):
+            self._hashedpassword = base64.b64encode(hashlib.pbkdf2_hmac('sha512',self._password.encode(),self._serial.encode(),100000,256))[:32]
+
+
+    @property
+    def url(self):
+        return self._url
+    @url.setter
+    def url(self,value):
+        self._url = value
+
+    @property
+    def connected(self):
+        return self._connected
+
+    @property
+    def voltage1(self):
+        return self._voltage1
+
+    @property
+    def voltage2(self):
+        return self._voltage2
+
+    @property
+    def voltage3(self):
+        return self._voltage3
+
+    @property
+    def voltageN(self):
+        return self._voltageN
+
+    @property
+    def amps1(self):
+        return self._amps1
+
+    @property
+    def amps2(self):
+        return self._amps2
+
+    @property
+    def amps3(self):
+        return self._amps3
+
+    @property
+    def power1(self):
+        return self._power1
+
+    @property
+    def power2(self):
+        return self._power2
+
+    @property
+    def power3(self):
+        return self._power3
+
+    @property
+    def powerN(self):
+        return self._powerN
+
+    @property
+    def power(self):
+        return self._power
+
+    @property
+    def version(self):
+        return self._version
+
+    @property
+    def amp(self):
+        return self._amp
+
+    @property
+    def AccessState(self):
+        return self._AccessState
+
+    @property
+    def firmware(self):
+        """Returns the Firmwareversion of Wattpilot Device (read only)"""
+        return self._firmware
+        
+    @property
+    def ftt(self):
+        """Returns the The desired charged time for NextTrip"""
+        return self._ftt
+    	
+    @property
+    def WifiSSID(self):
+        """Returns the SSID of the Wifi network currently connected (read only)"""
+        return self._WifiSSID
+
+    @property
+    def AllowCharging(self):
+        return self._AllowCharging
+
+    @property
+    def mode(self):
+        return self._mode
+
+    @property
+    def carConnected(self):
+        return self._carConnected
+
+    @property
+    def cae(self):
+        """Returns true if Cloud API Access is enabled (read only)"""
+        return self._cae
+
+    @property
+    def cak(self):
+        """Returns the API Key for Cloud API Access (read only)"""
+        return self._cak
+
+
+    def __str__(self):
+        """Returns a String representation of the core Wattpilot attributes"""
+        if self.connected:
+            ret = "Wattpilot: " + str(self.name) + "\n"
+            ret = ret +  "Serial: " + str(self.serial) + "\n"
+            ret = ret +  "Connected: " + str(self.connected) + "\n"
+            ret = ret + "Car Connected: " + str(self.carConnected) + "\n"
+            ret = ret + "Charge Status " + str(self.AllowCharging) + "\n"
+            ret = ret + "Mode: " + str(self.mode) + "\n"
+            ret = ret + "Power: " + str(self.amp) + "\n"
+            ret = ret +  "Charge: " + "%.2f" % self.power + "kW" + " ---- " + str(self.voltage1) + "V/" + str(self.voltage2) + "V/" + str(self.voltage3) + "V" + " -- "
+            ret = ret + "%.2f" % self.amps1 + "A/" + "%.2f" % self.amps2 + "A/" + "%.2f" % self.amps3 + "A" + " -- "
+            ret = ret + "%.2f" % self.power1 + "kW/" + "%.2f" % self.power2 + "kW/" + "%.2f" % self.power3 + "kW" + "\n"
+        else:
+            ret = "Not connected"
+
+        return ret
+    def connect(self):
+        self._wst = threading.Thread(target=self._wsapp.run_forever)
+        self._wst.daemon = True
+        self._wst.start()
+        self.__call_event_handler("wp_connect")
+        _LOGGER.info("Wattpilot connected")
+
+    def disconnect(self):
+        self._wsapp.close()
+        self._connected=False
+        self._auto_reconnect=False # Do not reconnect on explicit disconnect
+        self.__call_event_handler("wp_disconnect")
+        _LOGGER.info("Wattpilot disconnected")
+
+    # Wattpilot Event Handling
+
+    def add_event_handler(self,event_type,callback_fn):
+        if event_type not in self._event_handler:
+            self._event_handler[event_type] = []
+        self._event_handler[event_type].append(callback_fn)
+
+    def remove_event_handler(self,event_type,callback_fn):
+        if event_type in self._event_handler and callback_fn in self._event_handler[event_type]:
+            self._event_handler[event_type].remove(callback_fn)
+
+    def __call_event_handler(self, event_type, *args):
+        _LOGGER.debug(f"Calling event handler for event type '{event_type} ...")
+        for callback_fn in self._event_handler[event_type]:
+            event = {
+                "type": event_type,
+                "wp": self,
+            }
+            callback_fn(event,*args)
+
+
+    def set_power(self,power):
+        self.send_update("amp",power)
+
+    def set_mode(self,mode):
+        self.send_update("lmo",mode)
+
+
+    def send_update(self,name,value):
+        message = {}
+        message["type"]="setValue"
+        self.__requestid = self.__requestid+1
+        message["requestId"]=self.__requestid
+        message["key"]=name
+        message["value"]=value
+        if (self._secured is not None):
+            if  (self._secured > 0):
+                self.__send(message,True)
+            else:
+                self.__send(message)
+        else:
+            self.__send(message)
+
+    def unpairInverter(self,InverterID):
+        message = {}
+        message["type"]="unpairInverter"
+        self.__requestid = self.__requestid+1
+        message["requestId"]=self.__requestid
+        message["inverterId"]=InverterID
+        if (self._secured is not None):
+            if  (self._secured > 0):
+                self.__send(message,True)
+            else:
+                self.__send(message)
+        else:
+            self.__send(message)
+
+    def pairInverter(self,InverterID):
+        message = {}
+        message["type"]="pairInverter"
+        self.__requestid = self.__requestid+1
+        message["requestId"]=self.__requestid
+        message["inverterId"]=InverterID
+        if (self._secured is not None):
+            if  (self._secured > 0):
+                self.__send(message,True)
+            else:
+                self.__send(message)
+        else:
+            self.__send(message)
+
+    def __update_property(self,name,value):
+
+        self._allProps[name] = value
+        if name=="acs":
+            self._AccessState = Wattpilot.acsValues[value]
+
+        if name=="cbl":
+            self._cableType = value
+
+        if name=="fhz":
+            self._frequency = value
+
+        if name=="pha":
+            self._phases = value
+        
+        if name=="wh":
+            self._energyCounterSinceStart = value
+
+        if name=="err":
+            self._errorState = Wattpilot.errValues[value]
+
+        if name=="ust":
+            self._cableLock = Wattpilot.ustValues[value]
+
+        if name=="eto":
+            self._energyCounterTotal = value
+
+        if name=="cae":
+            self._cae = value
+        if name=="cak":
+            self._cak = value
+        if name=="lmo":
+            self._mode = Wattpilot.lmoValues[value]
+        if name=="car":
+            self._carConnected = Wattpilot.carValues[value]
+        if name=="alw":
+            self._AllowCharging = Wattpilot.alwValues[value]
+        if name=="nrg":
+            self._voltage1=value[0]
+            self._voltage2=value[1]
+            self._voltage3=value[2]
+            self._voltageN=value[3]
+            self._amps1=value[4]
+            self._amps2=value[5]
+            self._amps3=value[6]
+            self._power1=value[7]*0.001
+            self._power2=value[8]*0.001
+            self._power3=value[9]*0.001
+            self._powerN=value[10]*0.001
+            self._power=value[11]*0.001
+        if name=="amp":
+            self._amp = value
+        if name=="version":
+            self._version = value
+        if name=="ast":
+            self._AllowCharging = self._astValues[value]
+        if name=="fwv":
+            self._firmware = value
+        if name=="wss":
+            self._WifiSSID=value
+        if name=="ftt":
+            self._ftt=value
+        if name=="upd":
+            if value=="0":
+                self._updateAvailable = False
+            else:
+                self._updateAvailable = True
+        self.__call_event_handler("wp_property",name,value)
+
+    def __on_hello(self,message):
+        _LOGGER.info("Connected to WattPilot Serial %s",message.serial)
+        if hasattr(message,"hostname"):
+            self._name=message.hostname
+        self.serial = message.serial
+        if hasattr(message,"hostname"):
+            self._hostname=message.hostname
+        if hasattr(message,"version"):
+            self._version=message.version
+        self._manufacturer=message.manufacturer
+        self._devicetype=message.devicetype
+        self._protocol=message.protocol
+        if hasattr(message,"secured"):
+            self._secured=message.secured
+        self.__call_event_handler("wp_hello",message)
+
+    def __on_auth(self,wsapp,message):
+        ran = random.randrange(10**80)
+        self._token3 = "%064x" % ran
+        self._token3 = self._token3[:32]
+        hash1 = hashlib.sha256((message.token1.encode()+self._hashedpassword)).hexdigest()
+        hash = hashlib.sha256((self._token3 + message.token2+hash1).encode()).hexdigest()
+        response = {}
+        response["type"] = "auth"
+        response["token3"] = self._token3
+        response["hash"] = hash
+        self.__send(response)
+        self.__call_event_handler("wp_auth",message)
+
+    def __send(self,message,secure=False):
+        # If the  connection to wattpilot is over a unsecure channel (http) all send messages are wrapped in
+        # a "securedMsg" Message which contains the original messageobject and a sha256 HMAC Hashed created
+        # using the password
+        if secure:
+            messageid=message["requestId"]
+            payload=json.dumps(message)
+            h = hmac.new(bytearray(self._hashedpassword), bytearray(payload.encode()), hashlib.sha256 )
+            message={}
+            message["type"]="securedMsg"
+            message["data"]=payload
+            message["requestId"]=str(messageid)+"sm"
+            message["hmac"]=h.hexdigest()
+
+        _LOGGER.debug("Message send: %s",json.dumps(message)  )
+        self._wsapp.send(json.dumps(message))
+
+    def __on_AuthSuccess(self,message):
+        self._connected = True
+        self.__call_event_handler("wp_authSuccess",message)
+        _LOGGER.info("Authentication successful")
+
+    def __on_FullStatus(self,message):
+        props = message.status.__dict__
+        for key in props:
+            self.__update_property(key,props[key])
+        self.__call_event_handler("wp_fullStatus",message)
+        self._allPropsInitialized = not message.partial
+        _LOGGER.info("Status part")
+        if message.partial == False:
+            self.__call_event_handler("wp_fullStatus_finished",message)
+            _LOGGER.info("Status complete")
+
+    def __on_AuthError(self,message):
+        if message.message=="Wrong password":
+            self._wsapp.close()
+            _LOGGER.error("Authentication failed: %s" , message.message)
+        self.__call_event_handler("wp_authError",message)
+
+    def __on_DeltaStatus(self,message):
+        props = message.status.__dict__
+        for key in props:
+            self.__update_property(key,props[key])
+        self.__call_event_handler("wp_deltaStatus",message)
+
+
+    def __on_clearInverters(self,message):
+        self.__call_event_handler("wp_clearInverters",message)
+
+    def __on_updateInverter(self,message):
+        self.__call_event_handler("wp_updateInverter",message)
+
+    def __on_response(self,message):
+        if message.success:
+            if hasattr(message,"status"):
+                props = message.status.__dict__
+                for key in props:
+                    self.__update_property(key,props[key])
+        else:
+            _LOGGER.error("Error Sending Request %s. Message: %s" ,message.requestId,message.message)
+        self.__call_event_handler("wp_response",message)
+
+    def __on_open(self,wsapp):
+        self.__call_event_handler("ws_open",wsapp)
+
+    def __on_error(self,wsapp,err):
+        self.__call_event_handler("ws_error",wsapp,err)
+        _LOGGER.error(f"Error received from WebSocketApp: {err}")
+
+    def __on_close(self,wsapp,code,msg):
+        self._connected=False
+        self.__call_event_handler("ws_close",wsapp,code,msg)
+        if (self._auto_reconnect):
+            sleep(self._reconnect_interval)
+            self._wsapp.run_forever()
+
+    def __on_message(self, wsapp, message):
+        ## called whenever a message through websocket is received
+        _LOGGER.debug("Message received: %s", message)
+        msg=json.loads(message, object_hook=lambda d: SimpleNamespace(**d))
+        self.__call_event_handler("ws_message",message)
+        if (msg.type == 'hello'):  # Hello Message -> Received upon connection before auth
+            self.__on_hello(msg)
+        if (msg.type == 'authRequired'): # Auth Required -> Received after hello 
+            self.__on_auth(wsapp,msg)
+        if (msg.type == 'response'): # Response Message -> Received after sending a update and contains result of update
+            self.__on_response(msg)
+        if (msg.type == 'authSuccess'): # Auth Success -> Received after sending correct authentication message
+            self.__on_AuthSuccess(msg)
+        if (msg.type == 'authError'): # Auth Error -> Received after sending incorrect authentication message (e.g. wrong password)
+            self.__on_AuthError(msg)
+        if (msg.type == 'fullStatus'): # Full Status -> Received after successful connection. Contains all properties of Wattpilot
+            self.__on_FullStatus(msg)
+        if (msg.type == 'deltaStatus'): # Delta Status -> Whenever a property changes a Delta Status is send
+            self.__on_DeltaStatus(msg)
+        if (msg.type == 'clearInverters'): # Unknown
+            self.__on_clearInverters(msg)
+        if (msg.type == 'updateInverter'): # Contains information of connected Photovoltaik inverter / powermeter
+            self.__on_updateInverter(msg)
+
+
+    def __init__(self, ip ,password,serial=None,cloud=False):
+        self._auto_reconnect = True
+        self._reconnect_interval = 30
+        self._websocket_default_timeout = 10
+        self.__requestid = 0
+        self._name = None
+        self._hostname = None
+        self._friendlyName = None
+        self._manufacturer = None
+        self._devicetype = None
+        self._protocol = None
+        self._secured = None
+        self._serial = None
+        self._password = None
+
+        self.password = password
+
+        if(cloud):
+            self._url= "wss://app.wattpilot.io/app/" + serial + "?version=1.2.9"
+        else:
+            self._url = "ws://"+ip+"/ws"
+        _LOGGER.info("URL: %s",self._url);
+        self.serial = None
+        self._connected = False
+        self._allProps={}
+        self._allPropsInitialized=False
+        self._voltage1=None
+        self._voltage2=None
+        self._voltage3=None
+        self._voltageN=None
+        self._amps1=None
+        self._amps2=None
+        self._amps3=None
+        self._power1=None
+        self._power2=None
+        self._power3=None
+        self._powerN=None
+        self._power=None
+        self._version = None
+        self._amp = None
+        self._AccessState = None
+        self._firmware = None
+        self._ftt = None
+        self._WifiSSID = None
+        self._AllowCharging = None
+        self._mode=None
+        self._carConnected=None
+        self._cae=None
+        self._cak=None
+        # Initialize callback lists:
+        self._event_handler = {}
+        for event_type in self.supported_events:
+            self._event_handler[event_type] = []
+
+        self._wst=threading.Thread()
+
+        websocket.setdefaulttimeout(self._websocket_default_timeout)
+        self._wsapp = websocket.WebSocketApp(
+            self.url,
+            on_close=self.__on_close,
+            on_error=self.__on_error,
+            on_message=self.__on_message,
+            on_open=self.__on_open,
+        )
+        self.__call_event_handler("wp_init")
+        _LOGGER.info ("Wattpilot %s initialized",self.serial)
+        
+        
+def wp_handle_events(event, *args):
+    _LOGGER.debug(f"wp_handle_events(event={event},{args})")
+    _LOGGER.debug(f"wp_handle_events(): MQTT client not yet initialized - status publishing skipped.")
+    if event['type'] == 'fullStatus':
+        _LOGGER.debug(f"wp_handle_events(Hz={wp.frequency})")
+    return

aiohttp/__init__.py

@@ -0,0 +1,230 @@
+__version__ = "4.0.0a2.dev0"
+
+from typing import TYPE_CHECKING, Tuple
+
+from . import hdrs
+from .client import (
+    BaseConnector,
+    ClientConnectionError,
+    ClientConnectorCertificateError,
+    ClientConnectorError,
+    ClientConnectorSSLError,
+    ClientError,
+    ClientHttpProxyError,
+    ClientOSError,
+    ClientPayloadError,
+    ClientProxyConnectionError,
+    ClientRequest,
+    ClientResponse,
+    ClientResponseError,
+    ClientSession,
+    ClientSSLError,
+    ClientTimeout,
+    ClientWebSocketResponse,
+    ContentTypeError,
+    Fingerprint,
+    InvalidURL,
+    NamedPipeConnector,
+    RequestInfo,
+    ServerConnectionError,
+    ServerDisconnectedError,
+    ServerFingerprintMismatch,
+    ServerTimeoutError,
+    TCPConnector,
+    TooManyRedirects,
+    UnixConnector,
+    WSServerHandshakeError,
+    request,
+)
+from .cookiejar import CookieJar, DummyCookieJar
+from .formdata import FormData
+from .helpers import BasicAuth, ChainMapProxy, ETag
+from .http import (
+    HttpVersion,
+    HttpVersion10,
+    HttpVersion11,
+    WebSocketError,
+    WSCloseCode,
+    WSMessage,
+    WSMsgType,
+)
+from .multipart import (
+    BadContentDispositionHeader,
+    BadContentDispositionParam,
+    BodyPartReader,
+    MultipartReader,
+    MultipartWriter,
+    content_disposition_filename,
+    parse_content_disposition,
+)
+from .payload import (
+    PAYLOAD_REGISTRY,
+    AsyncIterablePayload,
+    BufferedReaderPayload,
+    BytesIOPayload,
+    BytesPayload,
+    IOBasePayload,
+    JsonPayload,
+    Payload,
+    StringIOPayload,
+    StringPayload,
+    TextIOPayload,
+    get_payload,
+    payload_type,
+)
+from .resolver import AsyncResolver, DefaultResolver, ThreadedResolver
+from .streams import (
+    EMPTY_PAYLOAD,
+    DataQueue,
+    EofStream,
+    FlowControlDataQueue,
+    StreamReader,
+)
+from .tracing import (
+    TraceConfig,
+    TraceConnectionCreateEndParams,
+    TraceConnectionCreateStartParams,
+    TraceConnectionQueuedEndParams,
+    TraceConnectionQueuedStartParams,
+    TraceConnectionReuseconnParams,
+    TraceDnsCacheHitParams,
+    TraceDnsCacheMissParams,
+    TraceDnsResolveHostEndParams,
+    TraceDnsResolveHostStartParams,
+    TraceRequestChunkSentParams,
+    TraceRequestEndParams,
+    TraceRequestExceptionParams,
+    TraceRequestRedirectParams,
+    TraceRequestStartParams,
+    TraceResponseChunkReceivedParams,
+)
+
+if TYPE_CHECKING:  # pragma: no cover
+    # At runtime these are lazy-loaded at the bottom of the file.
+    from .worker import GunicornUVLoopWebWorker, GunicornWebWorker
+
+__all__: Tuple[str, ...] = (
+    "hdrs",
+    # client
+    "BaseConnector",
+    "ClientConnectionError",
+    "ClientConnectorCertificateError",
+    "ClientConnectorError",
+    "ClientConnectorSSLError",
+    "ClientError",
+    "ClientHttpProxyError",
+    "ClientOSError",
+    "ClientPayloadError",
+    "ClientProxyConnectionError",
+    "ClientResponse",
+    "ClientRequest",
+    "ClientResponseError",
+    "ClientSSLError",
+    "ClientSession",
+    "ClientTimeout",
+    "ClientWebSocketResponse",
+    "ContentTypeError",
+    "Fingerprint",
+    "InvalidURL",
+    "RequestInfo",
+    "ServerConnectionError",
+    "ServerDisconnectedError",
+    "ServerFingerprintMismatch",
+    "ServerTimeoutError",
+    "TCPConnector",
+    "TooManyRedirects",
+    "UnixConnector",
+    "NamedPipeConnector",
+    "WSServerHandshakeError",
+    "request",
+    # cookiejar
+    "CookieJar",
+    "DummyCookieJar",
+    # formdata
+    "FormData",
+    # helpers
+    "BasicAuth",
+    "ChainMapProxy",
+    "ETag",
+    # http
+    "HttpVersion",
+    "HttpVersion10",
+    "HttpVersion11",
+    "WSMsgType",
+    "WSCloseCode",
+    "WSMessage",
+    "WebSocketError",
+    # multipart
+    "BadContentDispositionHeader",
+    "BadContentDispositionParam",
+    "BodyPartReader",
+    "MultipartReader",
+    "MultipartWriter",
+    "content_disposition_filename",
+    "parse_content_disposition",
+    # payload
+    "AsyncIterablePayload",
+    "BufferedReaderPayload",
+    "BytesIOPayload",
+    "BytesPayload",
+    "IOBasePayload",
+    "JsonPayload",
+    "PAYLOAD_REGISTRY",
+    "Payload",
+    "StringIOPayload",
+    "StringPayload",
+    "TextIOPayload",
+    "get_payload",
+    "payload_type",
+    # resolver
+    "AsyncResolver",
+    "DefaultResolver",
+    "ThreadedResolver",
+    # streams
+    "DataQueue",
+    "EMPTY_PAYLOAD",
+    "EofStream",
+    "FlowControlDataQueue",
+    "StreamReader",
+    # tracing
+    "TraceConfig",
+    "TraceConnectionCreateEndParams",
+    "TraceConnectionCreateStartParams",
+    "TraceConnectionQueuedEndParams",
+    "TraceConnectionQueuedStartParams",
+    "TraceConnectionReuseconnParams",
+    "TraceDnsCacheHitParams",
+    "TraceDnsCacheMissParams",
+    "TraceDnsResolveHostEndParams",
+    "TraceDnsResolveHostStartParams",
+    "TraceRequestChunkSentParams",
+    "TraceRequestEndParams",
+    "TraceRequestExceptionParams",
+    "TraceRequestRedirectParams",
+    "TraceRequestStartParams",
+    "TraceResponseChunkReceivedParams",
+    # workers (imported lazily with __getattr__)
+    "GunicornUVLoopWebWorker",
+    "GunicornWebWorker",
+)
+
+
+def __dir__() -> Tuple[str, ...]:
+    return __all__ + ("__author__", "__doc__")
+
+
+def __getattr__(name: str) -> object:
+    global GunicornUVLoopWebWorker, GunicornWebWorker
+
+    # Importing gunicorn takes a long time (>100ms), so only import if actually needed.
+    if name in ("GunicornUVLoopWebWorker", "GunicornWebWorker"):
+        try:
+            from .worker import GunicornUVLoopWebWorker as guv, GunicornWebWorker as gw
+        except ImportError:
+            return None
+
+        GunicornUVLoopWebWorker = guv  # type: ignore[misc]
+        GunicornWebWorker = gw  # type: ignore[misc]
+        return guv if name == "GunicornUVLoopWebWorker" else gw
+
+    raise AttributeError(f"module {__name__} has no attribute {name}")

aiohttp/_cparser.pxd

@@ -0,0 +1,158 @@
+from libc.stdint cimport int32_t, uint8_t, uint16_t, uint64_t
+
+
+cdef extern from "../vendor/llhttp/build/llhttp.h":
+
+    struct llhttp__internal_s:
+        int32_t _index
+        void* _span_pos0
+        void* _span_cb0
+        int32_t error
+        const char* reason
+        const char* error_pos
+        void* data
+        void* _current
+        uint64_t content_length
+        uint8_t type
+        uint8_t method
+        uint8_t http_major
+        uint8_t http_minor
+        uint8_t header_state
+        uint8_t lenient_flags
+        uint8_t upgrade
+        uint8_t finish
+        uint16_t flags
+        uint16_t status_code
+        void* settings
+
+    ctypedef llhttp__internal_s llhttp__internal_t
+    ctypedef llhttp__internal_t llhttp_t
+
+    ctypedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length) except -1
+    ctypedef int (*llhttp_cb)(llhttp_t*) except -1
+
+    struct llhttp_settings_s:
+        llhttp_cb      on_message_begin
+        llhttp_data_cb on_url
+        llhttp_data_cb on_status
+        llhttp_data_cb on_header_field
+        llhttp_data_cb on_header_value
+        llhttp_cb      on_headers_complete
+        llhttp_data_cb on_body
+        llhttp_cb      on_message_complete
+        llhttp_cb      on_chunk_header
+        llhttp_cb      on_chunk_complete
+
+        llhttp_cb      on_url_complete
+        llhttp_cb      on_status_complete
+        llhttp_cb      on_header_field_complete
+        llhttp_cb      on_header_value_complete
+
+    ctypedef llhttp_settings_s llhttp_settings_t
+
+    enum llhttp_errno:
+        HPE_OK,
+        HPE_INTERNAL,
+        HPE_STRICT,
+        HPE_LF_EXPECTED,
+        HPE_UNEXPECTED_CONTENT_LENGTH,
+        HPE_CLOSED_CONNECTION,
+        HPE_INVALID_METHOD,
+        HPE_INVALID_URL,
+        HPE_INVALID_CONSTANT,
+        HPE_INVALID_VERSION,
+        HPE_INVALID_HEADER_TOKEN,
+        HPE_INVALID_CONTENT_LENGTH,
+        HPE_INVALID_CHUNK_SIZE,
+        HPE_INVALID_STATUS,
+        HPE_INVALID_EOF_STATE,
+        HPE_INVALID_TRANSFER_ENCODING,
+        HPE_CB_MESSAGE_BEGIN,
+        HPE_CB_HEADERS_COMPLETE,
+        HPE_CB_MESSAGE_COMPLETE,
+        HPE_CB_CHUNK_HEADER,
+        HPE_CB_CHUNK_COMPLETE,
+        HPE_PAUSED,
+        HPE_PAUSED_UPGRADE,
+        HPE_USER
+
+    ctypedef llhttp_errno llhttp_errno_t
+
+    enum llhttp_flags:
+        F_CHUNKED,
+        F_CONTENT_LENGTH
+
+    enum llhttp_type:
+        HTTP_REQUEST,
+        HTTP_RESPONSE,
+        HTTP_BOTH
+
+    enum llhttp_method:
+        HTTP_DELETE,
+        HTTP_GET,
+        HTTP_HEAD,
+        HTTP_POST,
+        HTTP_PUT,
+        HTTP_CONNECT,
+        HTTP_OPTIONS,
+        HTTP_TRACE,
+        HTTP_COPY,
+        HTTP_LOCK,
+        HTTP_MKCOL,
+        HTTP_MOVE,
+        HTTP_PROPFIND,
+        HTTP_PROPPATCH,
+        HTTP_SEARCH,
+        HTTP_UNLOCK,
+        HTTP_BIND,
+        HTTP_REBIND,
+        HTTP_UNBIND,
+        HTTP_ACL,
+        HTTP_REPORT,
+        HTTP_MKACTIVITY,
+        HTTP_CHECKOUT,
+        HTTP_MERGE,
+        HTTP_MSEARCH,
+        HTTP_NOTIFY,
+        HTTP_SUBSCRIBE,
+        HTTP_UNSUBSCRIBE,
+        HTTP_PATCH,
+        HTTP_PURGE,
+        HTTP_MKCALENDAR,
+        HTTP_LINK,
+        HTTP_UNLINK,
+        HTTP_SOURCE,
+        HTTP_PRI,
+        HTTP_DESCRIBE,
+        HTTP_ANNOUNCE,
+        HTTP_SETUP,
+        HTTP_PLAY,
+        HTTP_PAUSE,
+        HTTP_TEARDOWN,
+        HTTP_GET_PARAMETER,
+        HTTP_SET_PARAMETER,
+        HTTP_REDIRECT,
+        HTTP_RECORD,
+        HTTP_FLUSH
+
+    ctypedef llhttp_method llhttp_method_t;
+
+    void llhttp_settings_init(llhttp_settings_t* settings)
+    void llhttp_init(llhttp_t* parser, llhttp_type type,
+                 const llhttp_settings_t* settings)
+
+    llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len)
+
+    int llhttp_should_keep_alive(const llhttp_t* parser)
+
+    void llhttp_resume_after_upgrade(llhttp_t* parser)
+
+    llhttp_errno_t llhttp_get_errno(const llhttp_t* parser)
+    const char* llhttp_get_error_reason(const llhttp_t* parser)
+    const char* llhttp_get_error_pos(const llhttp_t* parser)
+
+    const char* llhttp_method_name(llhttp_method_t method)
+
+    void llhttp_set_lenient_headers(llhttp_t* parser, int enabled)
+    void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled)
+    void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled)

aiohttp/_find_header.h

@@ -0,0 +1,14 @@
+#ifndef _FIND_HEADERS_H
+#define _FIND_HEADERS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int find_header(const char *str, int size);
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif

aiohttp/_find_header.pxd

@@ -0,0 +1,2 @@
+cdef extern from "_find_header.h":
+    int find_header(char *, int)

aiohttp/_helpers.pyi

@@ -0,0 +1,6 @@
+from typing import Any
+
+class reify:
+    def __init__(self, wrapped: Any) -> None: ...
+    def __get__(self, inst: Any, owner: Any) -> Any: ...
+    def __set__(self, inst: Any, value: Any) -> None: ...

aiohttp/_helpers.pyx

@@ -0,0 +1,35 @@
+cdef class reify:
+    """Use as a class method decorator.  It operates almost exactly like
+    the Python `@property` decorator, but it puts the result of the
+    method it decorates into the instance dict after the first call,
+    effectively replacing the function it decorates with an instance
+    variable.  It is, in Python parlance, a data descriptor.
+
+    """
+
+    cdef object wrapped
+    cdef object name
+
+    def __init__(self, wrapped):
+        self.wrapped = wrapped
+        self.name = wrapped.__name__
+
+    @property
+    def __doc__(self):
+        return self.wrapped.__doc__
+
+    def __get__(self, inst, owner):
+        try:
+            try:
+                return inst._cache[self.name]
+            except KeyError:
+                val = self.wrapped(inst)
+                inst._cache[self.name] = val
+                return val
+        except AttributeError:
+            if inst is None:
+                return self
+            raise
+
+    def __set__(self, inst, value):
+        raise AttributeError("reified property is read-only")

aiohttp/_http_parser.pyx

@@ -0,0 +1,844 @@
+#cython: language_level=3
+#
+# Based on https://github.com/MagicStack/httptools
+#
+from __future__ import absolute_import, print_function
+
+from cpython cimport (
+    Py_buffer,
+    PyBUF_SIMPLE,
+    PyBuffer_Release,
+    PyBytes_AsString,
+    PyBytes_AsStringAndSize,
+    PyObject_GetBuffer,
+)
+from cpython.mem cimport PyMem_Free, PyMem_Malloc
+from libc.limits cimport ULLONG_MAX
+from libc.string cimport memcpy
+
+from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiDictProxy
+from yarl import URL as _URL
+
+from aiohttp import hdrs
+from aiohttp.helpers import DEBUG
+
+from .http_exceptions import (
+    BadHttpMessage,
+    BadStatusLine,
+    ContentLengthError,
+    InvalidHeader,
+    InvalidURLError,
+    LineTooLong,
+    PayloadEncodingError,
+    TransferEncodingError,
+)
+from .http_parser import DeflateBuffer as _DeflateBuffer
+from .http_writer import (
+    HttpVersion as _HttpVersion,
+    HttpVersion10 as _HttpVersion10,
+    HttpVersion11 as _HttpVersion11,
+)
+from .streams import EMPTY_PAYLOAD as _EMPTY_PAYLOAD, StreamReader as _StreamReader
+
+cimport cython
+
+from aiohttp cimport _cparser as cparser
+
+include "_headers.pxi"
+
+from aiohttp cimport _find_header
+
+DEF DEFAULT_FREELIST_SIZE = 250
+
+cdef extern from "Python.h":
+    int PyByteArray_Resize(object, Py_ssize_t) except -1
+    Py_ssize_t PyByteArray_Size(object) except -1
+    char* PyByteArray_AsString(object)
+
+__all__ = ('HttpRequestParser', 'HttpResponseParser',
+           'RawRequestMessage', 'RawResponseMessage')
+
+cdef object URL = _URL
+cdef object URL_build = URL.build
+cdef object CIMultiDict = _CIMultiDict
+cdef object CIMultiDictProxy = _CIMultiDictProxy
+cdef object HttpVersion = _HttpVersion
+cdef object HttpVersion10 = _HttpVersion10
+cdef object HttpVersion11 = _HttpVersion11
+cdef object SEC_WEBSOCKET_KEY1 = hdrs.SEC_WEBSOCKET_KEY1
+cdef object CONTENT_ENCODING = hdrs.CONTENT_ENCODING
+cdef object EMPTY_PAYLOAD = _EMPTY_PAYLOAD
+cdef object StreamReader = _StreamReader
+cdef object DeflateBuffer = _DeflateBuffer
+
+
+cdef inline object extend(object buf, const char* at, size_t length):
+    cdef Py_ssize_t s
+    cdef char* ptr
+    s = PyByteArray_Size(buf)
+    PyByteArray_Resize(buf, s + length)
+    ptr = PyByteArray_AsString(buf)
+    memcpy(ptr + s, at, length)
+
+
+DEF METHODS_COUNT = 46;
+
+cdef list _http_method = []
+
+for i in range(METHODS_COUNT):
+    _http_method.append(
+        cparser.llhttp_method_name(<cparser.llhttp_method_t> i).decode('ascii'))
+
+
+cdef inline str http_method_str(int i):
+    if i < METHODS_COUNT:
+        return <str>_http_method[i]
+    else:
+        return "<unknown>"
+
+cdef inline object find_header(bytes raw_header):
+    cdef Py_ssize_t size
+    cdef char *buf
+    cdef int idx
+    PyBytes_AsStringAndSize(raw_header, &buf, &size)
+    idx = _find_header.find_header(buf, size)
+    if idx == -1:
+        return raw_header.decode('utf-8', 'surrogateescape')
+    return headers[idx]
+
+
+@cython.freelist(DEFAULT_FREELIST_SIZE)
+cdef class RawRequestMessage:
+    cdef readonly str method
+    cdef readonly str path
+    cdef readonly object version  # HttpVersion
+    cdef readonly object headers  # CIMultiDict
+    cdef readonly object raw_headers  # tuple
+    cdef readonly object should_close
+    cdef readonly object compression
+    cdef readonly object upgrade
+    cdef readonly object chunked
+    cdef readonly object url  # yarl.URL
+
+    def __init__(self, method, path, version, headers, raw_headers,
+                 should_close, compression, upgrade, chunked, url):
+        self.method = method
+        self.path = path
+        self.version = version
+        self.headers = headers
+        self.raw_headers = raw_headers
+        self.should_close = should_close
+        self.compression = compression
+        self.upgrade = upgrade
+        self.chunked = chunked
+        self.url = url
+
+    def __repr__(self):
+        info = []
+        info.append(("method", self.method))
+        info.append(("path", self.path))
+        info.append(("version", self.version))
+        info.append(("headers", self.headers))
+        info.append(("raw_headers", self.raw_headers))
+        info.append(("should_close", self.should_close))
+        info.append(("compression", self.compression))
+        info.append(("upgrade", self.upgrade))
+        info.append(("chunked", self.chunked))
+        info.append(("url", self.url))
+        sinfo = ', '.join(name + '=' + repr(val) for name, val in info)
+        return '<RawRequestMessage(' + sinfo + ')>'
+
+    def _replace(self, **dct):
+        cdef RawRequestMessage ret
+        ret = _new_request_message(self.method,
+                                   self.path,
+                                   self.version,
+                                   self.headers,
+                                   self.raw_headers,
+                                   self.should_close,
+                                   self.compression,
+                                   self.upgrade,
+                                   self.chunked,
+                                   self.url)
+        if "method" in dct:
+            ret.method = dct["method"]
+        if "path" in dct:
+            ret.path = dct["path"]
+        if "version" in dct:
+            ret.version = dct["version"]
+        if "headers" in dct:
+            ret.headers = dct["headers"]
+        if "raw_headers" in dct:
+            ret.raw_headers = dct["raw_headers"]
+        if "should_close" in dct:
+            ret.should_close = dct["should_close"]
+        if "compression" in dct:
+            ret.compression = dct["compression"]
+        if "upgrade" in dct:
+            ret.upgrade = dct["upgrade"]
+        if "chunked" in dct:
+            ret.chunked = dct["chunked"]
+        if "url" in dct:
+            ret.url = dct["url"]
+        return ret
+
+cdef _new_request_message(str method,
+                           str path,
+                           object version,
+                           object headers,
+                           object raw_headers,
+                           bint should_close,
+                           object compression,
+                           bint upgrade,
+                           bint chunked,
+                           object url):
+    cdef RawRequestMessage ret
+    ret = RawRequestMessage.__new__(RawRequestMessage)
+    ret.method = method
+    ret.path = path
+    ret.version = version
+    ret.headers = headers
+    ret.raw_headers = raw_headers
+    ret.should_close = should_close
+    ret.compression = compression
+    ret.upgrade = upgrade
+    ret.chunked = chunked
+    ret.url = url
+    return ret
+
+
+@cython.freelist(DEFAULT_FREELIST_SIZE)
+cdef class RawResponseMessage:
+    cdef readonly object version  # HttpVersion
+    cdef readonly int code
+    cdef readonly str reason
+    cdef readonly object headers  # CIMultiDict
+    cdef readonly object raw_headers  # tuple
+    cdef readonly object should_close
+    cdef readonly object compression
+    cdef readonly object upgrade
+    cdef readonly object chunked
+
+    def __init__(self, version, code, reason, headers, raw_headers,
+                 should_close, compression, upgrade, chunked):
+        self.version = version
+        self.code = code
+        self.reason = reason
+        self.headers = headers
+        self.raw_headers = raw_headers
+        self.should_close = should_close
+        self.compression = compression
+        self.upgrade = upgrade
+        self.chunked = chunked
+
+    def __repr__(self):
+        info = []
+        info.append(("version", self.version))
+        info.append(("code", self.code))
+        info.append(("reason", self.reason))
+        info.append(("headers", self.headers))
+        info.append(("raw_headers", self.raw_headers))
+        info.append(("should_close", self.should_close))
+        info.append(("compression", self.compression))
+        info.append(("upgrade", self.upgrade))
+        info.append(("chunked", self.chunked))
+        sinfo = ', '.join(name + '=' + repr(val) for name, val in info)
+        return '<RawResponseMessage(' + sinfo + ')>'
+
+
+cdef _new_response_message(object version,
+                           int code,
+                           str reason,
+                           object headers,
+                           object raw_headers,
+                           bint should_close,
+                           object compression,
+                           bint upgrade,
+                           bint chunked):
+    cdef RawResponseMessage ret
+    ret = RawResponseMessage.__new__(RawResponseMessage)
+    ret.version = version
+    ret.code = code
+    ret.reason = reason
+    ret.headers = headers
+    ret.raw_headers = raw_headers
+    ret.should_close = should_close
+    ret.compression = compression
+    ret.upgrade = upgrade
+    ret.chunked = chunked
+    return ret
+
+
+@cython.internal
+cdef class HttpParser:
+
+    cdef:
+        cparser.llhttp_t* _cparser
+        cparser.llhttp_settings_t* _csettings
+
+        bytearray _raw_name
+        bytearray _raw_value
+        bint      _has_value
+
+        object _protocol
+        object _loop
+        object _timer
+
+        size_t _max_line_size
+        size_t _max_field_size
+        size_t _max_headers
+        bint _response_with_body
+        bint _read_until_eof
+
+        bint    _started
+        object  _url
+        bytearray   _buf
+        str     _path
+        str     _reason
+        object  _headers
+        list    _raw_headers
+        bint    _upgraded
+        list    _messages
+        object  _payload
+        bint    _payload_error
+        object  _payload_exception
+        object  _last_error
+        bint    _auto_decompress
+        int     _limit
+
+        str     _content_encoding
+
+        Py_buffer py_buf
+
+    def __cinit__(self):
+        self._cparser = <cparser.llhttp_t*> \
+                                PyMem_Malloc(sizeof(cparser.llhttp_t))
+        if self._cparser is NULL:
+            raise MemoryError()
+
+        self._csettings = <cparser.llhttp_settings_t*> \
+                                PyMem_Malloc(sizeof(cparser.llhttp_settings_t))
+        if self._csettings is NULL:
+            raise MemoryError()
+
+    def __dealloc__(self):
+        PyMem_Free(self._cparser)
+        PyMem_Free(self._csettings)
+
+    cdef _init(
+        self, cparser.llhttp_type mode,
+        object protocol, object loop, int limit,
+        object timer=None,
+        size_t max_line_size=8190, size_t max_headers=32768,
+        size_t max_field_size=8190, payload_exception=None,
+        bint response_with_body=True, bint read_until_eof=False,
+        bint auto_decompress=True,
+    ):
+        cparser.llhttp_settings_init(self._csettings)
+        cparser.llhttp_init(self._cparser, mode, self._csettings)
+        self._cparser.data = <void*>self
+        self._cparser.content_length = 0
+
+        self._protocol = protocol
+        self._loop = loop
+        self._timer = timer
+
+        self._buf = bytearray()
+        self._payload = None
+        self._payload_error = 0
+        self._payload_exception = payload_exception
+        self._messages = []
+
+        self._raw_name = bytearray()
+        self._raw_value = bytearray()
+        self._has_value = False
+
+        self._max_line_size = max_line_size
+        self._max_headers = max_headers
+        self._max_field_size = max_field_size
+        self._response_with_body = response_with_body
+        self._read_until_eof = read_until_eof
+        self._upgraded = False
+        self._auto_decompress = auto_decompress
+        self._content_encoding = None
+
+        self._csettings.on_url = cb_on_url
+        self._csettings.on_status = cb_on_status
+        self._csettings.on_header_field = cb_on_header_field
+        self._csettings.on_header_value = cb_on_header_value
+        self._csettings.on_headers_complete = cb_on_headers_complete
+        self._csettings.on_body = cb_on_body
+        self._csettings.on_message_begin = cb_on_message_begin
+        self._csettings.on_message_complete = cb_on_message_complete
+        self._csettings.on_chunk_header = cb_on_chunk_header
+        self._csettings.on_chunk_complete = cb_on_chunk_complete
+
+        self._last_error = None
+        self._limit = limit
+
+    cdef _process_header(self):
+        if self._raw_name:
+            raw_name = bytes(self._raw_name)
+            raw_value = bytes(self._raw_value)
+
+            name = find_header(raw_name)
+            value = raw_value.decode('utf-8', 'surrogateescape')
+
+            self._headers.add(name, value)
+
+            if name is CONTENT_ENCODING:
+                self._content_encoding = value
+
+            PyByteArray_Resize(self._raw_name, 0)
+            PyByteArray_Resize(self._raw_value, 0)
+            self._has_value = False
+            self._raw_headers.append((raw_name, raw_value))
+
+    cdef _on_header_field(self, char* at, size_t length):
+        cdef Py_ssize_t size
+        cdef char *buf
+        if self._has_value:
+            self._process_header()
+
+        size = PyByteArray_Size(self._raw_name)
+        PyByteArray_Resize(self._raw_name, size + length)
+        buf = PyByteArray_AsString(self._raw_name)
+        memcpy(buf + size, at, length)
+
+    cdef _on_header_value(self, char* at, size_t length):
+        cdef Py_ssize_t size
+        cdef char *buf
+
+        size = PyByteArray_Size(self._raw_value)
+        PyByteArray_Resize(self._raw_value, size + length)
+        buf = PyByteArray_AsString(self._raw_value)
+        memcpy(buf + size, at, length)
+        self._has_value = True
+
+    cdef _on_headers_complete(self):
+        self._process_header()
+
+        method = http_method_str(self._cparser.method)
+        should_close = not cparser.llhttp_should_keep_alive(self._cparser)
+        upgrade = self._cparser.upgrade
+        chunked = self._cparser.flags & cparser.F_CHUNKED
+
+        raw_headers = tuple(self._raw_headers)
+        headers = CIMultiDictProxy(self._headers)
+
+        if upgrade or self._cparser.method == cparser.HTTP_CONNECT:
+            self._upgraded = True
+
+        # do not support old websocket spec
+        if SEC_WEBSOCKET_KEY1 in headers:
+            raise InvalidHeader(SEC_WEBSOCKET_KEY1)
+
+        encoding = None
+        enc = self._content_encoding
+        if enc is not None:
+            self._content_encoding = None
+            enc = enc.lower()
+            if enc in ('gzip', 'deflate', 'br'):
+                encoding = enc
+
+        if self._cparser.type == cparser.HTTP_REQUEST:
+            msg = _new_request_message(
+                method, self._path,
+                self.http_version(), headers, raw_headers,
+                should_close, encoding, upgrade, chunked, self._url)
+        else:
+            msg = _new_response_message(
+                self.http_version(), self._cparser.status_code, self._reason,
+                headers, raw_headers, should_close, encoding,
+                upgrade, chunked)
+
+        if (
+            ULLONG_MAX > self._cparser.content_length > 0 or chunked or
+            self._cparser.method == cparser.HTTP_CONNECT or
+            (self._cparser.status_code >= 199 and
+             self._cparser.content_length == 0 and
+             self._read_until_eof)
+        ):
+            payload = StreamReader(
+                self._protocol, timer=self._timer, loop=self._loop,
+                limit=self._limit)
+        else:
+            payload = EMPTY_PAYLOAD
+
+        self._payload = payload
+        if encoding is not None and self._auto_decompress:
+            self._payload = DeflateBuffer(payload, encoding)
+
+        if not self._response_with_body:
+            payload = EMPTY_PAYLOAD
+
+        self._messages.append((msg, payload))
+
+    cdef _on_message_complete(self):
+        self._payload.feed_eof()
+        self._payload = None
+
+    cdef _on_chunk_header(self):
+        self._payload.begin_http_chunk_receiving()
+
+    cdef _on_chunk_complete(self):
+        self._payload.end_http_chunk_receiving()
+
+    cdef object _on_status_complete(self):
+        pass
+
+    cdef inline http_version(self):
+        cdef cparser.llhttp_t* parser = self._cparser
+
+        if parser.http_major == 1:
+            if parser.http_minor == 0:
+                return HttpVersion10
+            elif parser.http_minor == 1:
+                return HttpVersion11
+
+        return HttpVersion(parser.http_major, parser.http_minor)
+
+    ### Public API ###
+
+    def feed_eof(self):
+        cdef bytes desc
+
+        if self._payload is not None:
+            if self._cparser.flags & cparser.F_CHUNKED:
+                raise TransferEncodingError(
+                    "Not enough data for satisfy transfer length header.")
+            elif self._cparser.flags & cparser.F_CONTENT_LENGTH:
+                raise ContentLengthError(
+                    "Not enough data for satisfy content length header.")
+            elif cparser.llhttp_get_errno(self._cparser) != cparser.HPE_OK:
+                desc = cparser.llhttp_get_error_reason(self._cparser)
+                raise PayloadEncodingError(desc.decode('latin-1'))
+            else:
+                self._payload.feed_eof()
+        elif self._started:
+            self._on_headers_complete()
+            if self._messages:
+                return self._messages[-1][0]
+
+    def feed_data(self, data):
+        cdef:
+            size_t data_len
+            size_t nb
+            cdef cparser.llhttp_errno_t errno
+
+        PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE)
+        data_len = <size_t>self.py_buf.len
+
+        errno = cparser.llhttp_execute(
+            self._cparser,
+            <char*>self.py_buf.buf,
+            data_len)
+
+        if errno is cparser.HPE_PAUSED_UPGRADE:
+            cparser.llhttp_resume_after_upgrade(self._cparser)
+
+            nb = cparser.llhttp_get_error_pos(self._cparser) - <char*>self.py_buf.buf
+
+        PyBuffer_Release(&self.py_buf)
+
+        if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED_UPGRADE):
+            if self._payload_error == 0:
+                if self._last_error is not None:
+                    ex = self._last_error
+                    self._last_error = None
+                else:
+                    after = cparser.llhttp_get_error_pos(self._cparser)
+                    before = data[:after - <char*>self.py_buf.buf]
+                    after_b = after.split(b"\r\n", 1)[0]
+                    before = before.rsplit(b"\r\n", 1)[-1]
+                    data = before + after_b
+                    pointer = " " * (len(repr(before))-1) + "^"
+                    ex = parser_error_from_errno(self._cparser, data, pointer)
+                self._payload = None
+                raise ex
+
+        if self._messages:
+            messages = self._messages
+            self._messages = []
+        else:
+            messages = ()
+
+        if self._upgraded:
+            return messages, True, data[nb:]
+        else:
+            return messages, False, b''
+
+    def set_upgraded(self, val):
+        self._upgraded = val
+
+
+cdef class HttpRequestParser(HttpParser):
+
+    def __init__(
+        self, protocol, loop, int limit, timer=None,
+        size_t max_line_size=8190, size_t max_headers=32768,
+        size_t max_field_size=8190, payload_exception=None,
+        bint response_with_body=True, bint read_until_eof=False,
+        bint auto_decompress=True,
+    ):
+        self._init(cparser.HTTP_REQUEST, protocol, loop, limit, timer,
+                   max_line_size, max_headers, max_field_size,
+                   payload_exception, response_with_body, read_until_eof,
+                   auto_decompress)
+
+    cdef object _on_status_complete(self):
+        cdef int idx1, idx2
+        if not self._buf:
+            return
+        self._path = self._buf.decode('utf-8', 'surrogateescape')
+        try:
+            idx3 = len(self._path)
+            if self._cparser.method == cparser.HTTP_CONNECT:
+                # authority-form,
+                # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3
+                self._url = URL.build(authority=self._path, encoded=True)
+            elif idx3 > 1 and self._path[0] == '/':
+                # origin-form,
+                # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1
+                idx1 = self._path.find("?")
+                if idx1 == -1:
+                    query = ""
+                    idx2 = self._path.find("#")
+                    if idx2 == -1:
+                        path = self._path
+                        fragment = ""
+                    else:
+                        path = self._path[0: idx2]
+                        fragment = self._path[idx2+1:]
+
+                else:
+                    path = self._path[0:idx1]
+                    idx1 += 1
+                    idx2 = self._path.find("#", idx1+1)
+                    if idx2 == -1:
+                        query = self._path[idx1:]
+                        fragment = ""
+                    else:
+                        query = self._path[idx1: idx2]
+                        fragment = self._path[idx2+1:]
+
+                self._url = URL.build(
+                    path=path,
+                    query_string=query,
+                    fragment=fragment,
+                    encoded=True,
+                )
+            else:
+                # absolute-form for proxy maybe,
+                # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2
+                self._url = URL(self._path, encoded=True)
+        finally:
+            PyByteArray_Resize(self._buf, 0)
+
+
+cdef class HttpResponseParser(HttpParser):
+
+    def __init__(
+        self, protocol, loop, int limit, timer=None,
+            size_t max_line_size=8190, size_t max_headers=32768,
+            size_t max_field_size=8190, payload_exception=None,
+            bint response_with_body=True, bint read_until_eof=False,
+            bint auto_decompress=True
+    ):
+        self._init(cparser.HTTP_RESPONSE, protocol, loop, limit, timer,
+                   max_line_size, max_headers, max_field_size,
+                   payload_exception, response_with_body, read_until_eof,
+                   auto_decompress)
+        # Use strict parsing on dev mode, so users are warned about broken servers.
+        if not DEBUG:
+            cparser.llhttp_set_lenient_headers(self._cparser, 1)
+            cparser.llhttp_set_lenient_optional_cr_before_lf(self._cparser, 1)
+            cparser.llhttp_set_lenient_spaces_after_chunk_size(self._cparser, 1)
+
+    cdef object _on_status_complete(self):
+        if self._buf:
+            self._reason = self._buf.decode('utf-8', 'surrogateescape')
+            PyByteArray_Resize(self._buf, 0)
+        else:
+            self._reason = self._reason or ''
+
+cdef int cb_on_message_begin(cparser.llhttp_t* parser) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+
+    pyparser._started = True
+    pyparser._headers = CIMultiDict()
+    pyparser._raw_headers = []
+    PyByteArray_Resize(pyparser._buf, 0)
+    pyparser._path = None
+    pyparser._reason = None
+    return 0
+
+
+cdef int cb_on_url(cparser.llhttp_t* parser,
+                   const char *at, size_t length) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+    try:
+        if length > pyparser._max_line_size:
+            raise LineTooLong(
+                'Status line is too long', pyparser._max_line_size, length)
+        extend(pyparser._buf, at, length)
+    except BaseException as ex:
+        pyparser._last_error = ex
+        return -1
+    else:
+        return 0
+
+
+cdef int cb_on_status(cparser.llhttp_t* parser,
+                      const char *at, size_t length) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+    cdef str reason
+    try:
+        if length > pyparser._max_line_size:
+            raise LineTooLong(
+                'Status line is too long', pyparser._max_line_size, length)
+        extend(pyparser._buf, at, length)
+    except BaseException as ex:
+        pyparser._last_error = ex
+        return -1
+    else:
+        return 0
+
+
+cdef int cb_on_header_field(cparser.llhttp_t* parser,
+                            const char *at, size_t length) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+    cdef Py_ssize_t size
+    try:
+        pyparser._on_status_complete()
+        size = len(pyparser._raw_name) + length
+        if size > pyparser._max_field_size:
+            raise LineTooLong(
+                'Header name is too long', pyparser._max_field_size, size)
+        pyparser._on_header_field(at, length)
+    except BaseException as ex:
+        pyparser._last_error = ex
+        return -1
+    else:
+        return 0
+
+
+cdef int cb_on_header_value(cparser.llhttp_t* parser,
+                            const char *at, size_t length) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+    cdef Py_ssize_t size
+    try:
+        size = len(pyparser._raw_value) + length
+        if size > pyparser._max_field_size:
+            raise LineTooLong(
+                'Header value is too long', pyparser._max_field_size, size)
+        pyparser._on_header_value(at, length)
+    except BaseException as ex:
+        pyparser._last_error = ex
+        return -1
+    else:
+        return 0
+
+
+cdef int cb_on_headers_complete(cparser.llhttp_t* parser) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+    try:
+        pyparser._on_status_complete()
+        pyparser._on_headers_complete()
+    except BaseException as exc:
+        pyparser._last_error = exc
+        return -1
+    else:
+        if (
+            pyparser._cparser.upgrade or
+            pyparser._cparser.method == cparser.HTTP_CONNECT
+        ):
+            return 2
+        else:
+            return 0
+
+
+cdef int cb_on_body(cparser.llhttp_t* parser,
+                    const char *at, size_t length) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+    cdef bytes body = at[:length]
+    try:
+        pyparser._payload.feed_data(body, length)
+    except BaseException as exc:
+        if pyparser._payload_exception is not None:
+            pyparser._payload.set_exception(pyparser._payload_exception(str(exc)))
+        else:
+            pyparser._payload.set_exception(exc)
+        pyparser._payload_error = 1
+        return -1
+    else:
+        return 0
+
+
+cdef int cb_on_message_complete(cparser.llhttp_t* parser) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+    try:
+        pyparser._started = False
+        pyparser._on_message_complete()
+    except BaseException as exc:
+        pyparser._last_error = exc
+        return -1
+    else:
+        return 0
+
+
+cdef int cb_on_chunk_header(cparser.llhttp_t* parser) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+    try:
+        pyparser._on_chunk_header()
+    except BaseException as exc:
+        pyparser._last_error = exc
+        return -1
+    else:
+        return 0
+
+
+cdef int cb_on_chunk_complete(cparser.llhttp_t* parser) except -1:
+    cdef HttpParser pyparser = <HttpParser>parser.data
+    try:
+        pyparser._on_chunk_complete()
+    except BaseException as exc:
+        pyparser._last_error = exc
+        return -1
+    else:
+        return 0
+
+
+cdef parser_error_from_errno(cparser.llhttp_t* parser, data, pointer):
+    cdef cparser.llhttp_errno_t errno = cparser.llhttp_get_errno(parser)
+    cdef bytes desc = cparser.llhttp_get_error_reason(parser)
+
+    if errno in (cparser.HPE_CB_MESSAGE_BEGIN,
+                 cparser.HPE_CB_HEADERS_COMPLETE,
+                 cparser.HPE_CB_MESSAGE_COMPLETE,
+                 cparser.HPE_CB_CHUNK_HEADER,
+                 cparser.HPE_CB_CHUNK_COMPLETE,
+                 cparser.HPE_INVALID_CONSTANT,
+                 cparser.HPE_INVALID_HEADER_TOKEN,
+                 cparser.HPE_INVALID_CONTENT_LENGTH,
+                 cparser.HPE_INVALID_CHUNK_SIZE,
+                 cparser.HPE_INVALID_EOF_STATE,
+                 cparser.HPE_INVALID_TRANSFER_ENCODING):
+        cls = BadHttpMessage
+
+    elif errno == cparser.HPE_INVALID_STATUS:
+        cls = BadStatusLine
+
+    elif errno == cparser.HPE_INVALID_METHOD:
+        cls = BadStatusLine
+
+    elif errno == cparser.HPE_INVALID_VERSION:
+        cls = BadStatusLine
+
+    elif errno == cparser.HPE_INVALID_URL:
+        cls = InvalidURLError
+
+    else:
+        cls = BadHttpMessage
+
+    return cls("{}:\n\n  {!r}\n  {}".format(desc.decode("latin-1"), data, pointer))

aiohttp/_http_writer.pyx

@@ -0,0 +1,163 @@
+from cpython.bytes cimport PyBytes_FromStringAndSize
+from cpython.exc cimport PyErr_NoMemory
+from cpython.mem cimport PyMem_Free, PyMem_Malloc, PyMem_Realloc
+from cpython.object cimport PyObject_Str
+from libc.stdint cimport uint8_t, uint64_t
+from libc.string cimport memcpy
+
+from multidict import istr
+
+DEF BUF_SIZE = 16 * 1024  # 16KiB
+cdef char BUFFER[BUF_SIZE]
+
+cdef object _istr = istr
+
+
+# ----------------- writer ---------------------------
+
+cdef struct Writer:
+    char *buf
+    Py_ssize_t size
+    Py_ssize_t pos
+
+
+cdef inline void _init_writer(Writer* writer):
+    writer.buf = &BUFFER[0]
+    writer.size = BUF_SIZE
+    writer.pos = 0
+
+
+cdef inline void _release_writer(Writer* writer):
+    if writer.buf != BUFFER:
+        PyMem_Free(writer.buf)
+
+
+cdef inline int _write_byte(Writer* writer, uint8_t ch):
+    cdef char * buf
+    cdef Py_ssize_t size
+
+    if writer.pos == writer.size:
+        # reallocate
+        size = writer.size + BUF_SIZE
+        if writer.buf == BUFFER:
+            buf = <char*>PyMem_Malloc(size)
+            if buf == NULL:
+                PyErr_NoMemory()
+                return -1
+            memcpy(buf, writer.buf, writer.size)
+        else:
+            buf = <char*>PyMem_Realloc(writer.buf, size)
+            if buf == NULL:
+                PyErr_NoMemory()
+                return -1
+        writer.buf = buf
+        writer.size = size
+    writer.buf[writer.pos] = <char>ch
+    writer.pos += 1
+    return 0
+
+
+cdef inline int _write_utf8(Writer* writer, Py_UCS4 symbol):
+    cdef uint64_t utf = <uint64_t> symbol
+
+    if utf < 0x80:
+        return _write_byte(writer, <uint8_t>utf)
+    elif utf < 0x800:
+        if _write_byte(writer, <uint8_t>(0xc0 | (utf >> 6))) < 0:
+            return -1
+        return _write_byte(writer,  <uint8_t>(0x80 | (utf & 0x3f)))
+    elif 0xD800 <= utf <= 0xDFFF:
+        # surogate pair, ignored
+        return 0
+    elif utf < 0x10000:
+        if _write_byte(writer, <uint8_t>(0xe0 | (utf >> 12))) < 0:
+            return -1
+        if _write_byte(writer, <uint8_t>(0x80 | ((utf >> 6) & 0x3f))) < 0:
+            return -1
+        return _write_byte(writer, <uint8_t>(0x80 | (utf & 0x3f)))
+    elif utf > 0x10FFFF:
+        # symbol is too large
+        return 0
+    else:
+        if _write_byte(writer,  <uint8_t>(0xf0 | (utf >> 18))) < 0:
+            return -1
+        if _write_byte(writer,
+                       <uint8_t>(0x80 | ((utf >> 12) & 0x3f))) < 0:
+           return -1
+        if _write_byte(writer,
+                       <uint8_t>(0x80 | ((utf >> 6) & 0x3f))) < 0:
+            return -1
+        return _write_byte(writer, <uint8_t>(0x80 | (utf & 0x3f)))
+
+
+cdef inline int _write_str(Writer* writer, str s):
+    cdef Py_UCS4 ch
+    for ch in s:
+        if _write_utf8(writer, ch) < 0:
+            return -1
+
+
+# --------------- _serialize_headers ----------------------
+
+cdef str to_str(object s):
+    typ = type(s)
+    if typ is str:
+        return <str>s
+    elif typ is _istr:
+        return PyObject_Str(s)
+    elif not isinstance(s, str):
+        raise TypeError("Cannot serialize non-str key {!r}".format(s))
+    else:
+        return str(s)
+
+
+cdef void _safe_header(str string) except *:
+    if "\r" in string or "\n" in string:
+        raise ValueError(
+            "Newline or carriage return character detected in HTTP status message or "
+            "header. This is a potential security issue."
+        )
+
+
+def _serialize_headers(str status_line, headers):
+    cdef Writer writer
+    cdef object key
+    cdef object val
+    cdef bytes ret
+
+    _init_writer(&writer)
+
+    for key, val in headers.items():
+        _safe_header(to_str(key))
+        _safe_header(to_str(val))
+
+    try:
+        if _write_str(&writer, status_line) < 0:
+            raise
+        if _write_byte(&writer, b'\r') < 0:
+            raise
+        if _write_byte(&writer, b'\n') < 0:
+            raise
+
+        for key, val in headers.items():
+            if _write_str(&writer, to_str(key)) < 0:
+                raise
+            if _write_byte(&writer, b':') < 0:
+                raise
+            if _write_byte(&writer, b' ') < 0:
+                raise
+            if _write_str(&writer, to_str(val)) < 0:
+                raise
+            if _write_byte(&writer, b'\r') < 0:
+                raise
+            if _write_byte(&writer, b'\n') < 0:
+                raise
+
+        if _write_byte(&writer, b'\r') < 0:
+            raise
+        if _write_byte(&writer, b'\n') < 0:
+            raise
+
+        return PyBytes_FromStringAndSize(writer.buf, writer.pos)
+    finally:
+        _release_writer(&writer)

aiohttp/_websocket.pyx

@@ -0,0 +1,56 @@
+from cpython cimport PyBytes_AsString
+
+
+#from cpython cimport PyByteArray_AsString # cython still not exports that
+cdef extern from "Python.h":
+    char* PyByteArray_AsString(bytearray ba) except NULL
+
+from libc.stdint cimport uint32_t, uint64_t, uintmax_t
+
+
+def _websocket_mask_cython(object mask, object data):
+    """Note, this function mutates its `data` argument
+    """
+    cdef:
+        Py_ssize_t data_len, i
+        # bit operations on signed integers are implementation-specific
+        unsigned char * in_buf
+        const unsigned char * mask_buf
+        uint32_t uint32_msk
+        uint64_t uint64_msk
+
+    assert len(mask) == 4
+
+    if not isinstance(mask, bytes):
+        mask = bytes(mask)
+
+    if isinstance(data, bytearray):
+        data = <bytearray>data
+    else:
+        data = bytearray(data)
+
+    data_len = len(data)
+    in_buf = <unsigned char*>PyByteArray_AsString(data)
+    mask_buf = <const unsigned char*>PyBytes_AsString(mask)
+    uint32_msk = (<uint32_t*>mask_buf)[0]
+
+    # TODO: align in_data ptr to achieve even faster speeds
+    # does it need in python ?! malloc() always aligns to sizeof(long) bytes
+
+    if sizeof(size_t) >= 8:
+        uint64_msk = uint32_msk
+        uint64_msk = (uint64_msk << 32) | uint32_msk
+
+        while data_len >= 8:
+            (<uint64_t*>in_buf)[0] ^= uint64_msk
+            in_buf += 8
+            data_len -= 8
+
+
+    while data_len >= 4:
+        (<uint32_t*>in_buf)[0] ^= uint32_msk
+        in_buf += 4
+        data_len -= 4
+
+    for i in range(0, data_len):
+        in_buf[i] ^= mask_buf[i]

aiohttp/abc.py

@@ -0,0 +1,214 @@
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Sized
+from http.cookies import BaseCookie, Morsel
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+)
+
+from multidict import CIMultiDict
+from yarl import URL
+
+from .typedefs import LooseCookies
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .web_app import Application
+    from .web_exceptions import HTTPException
+    from .web_request import BaseRequest, Request
+    from .web_response import StreamResponse
+else:
+    BaseRequest = Request = Application = StreamResponse = None
+    HTTPException = None
+
+
+class AbstractRouter(ABC):
+    def __init__(self) -> None:
+        self._frozen = False
+
+    def post_init(self, app: Application) -> None:
+        """Post init stage.
+
+        Not an abstract method for sake of backward compatibility,
+        but if the router wants to be aware of the application
+        it can override this.
+        """
+
+    @property
+    def frozen(self) -> bool:
+        return self._frozen
+
+    def freeze(self) -> None:
+        """Freeze router."""
+        self._frozen = True
+
+    @abstractmethod
+    async def resolve(self, request: Request) -> "AbstractMatchInfo":
+        """Return MATCH_INFO for given request"""
+
+
+class AbstractMatchInfo(ABC):
+    @property  # pragma: no branch
+    @abstractmethod
+    def handler(self) -> Callable[[Request], Awaitable[StreamResponse]]:
+        """Execute matched request handler"""
+
+    @property
+    @abstractmethod
+    def expect_handler(
+        self,
+    ) -> Callable[[Request], Awaitable[Optional[StreamResponse]]]:
+        """Expect handler for 100-continue processing"""
+
+    @property  # pragma: no branch
+    @abstractmethod
+    def http_exception(self) -> Optional[HTTPException]:
+        """HTTPException instance raised on router's resolving, or None"""
+
+    @abstractmethod  # pragma: no branch
+    def get_info(self) -> Dict[str, Any]:
+        """Return a dict with additional info useful for introspection"""
+
+    @property  # pragma: no branch
+    @abstractmethod
+    def apps(self) -> Tuple[Application, ...]:
+        """Stack of nested applications.
+
+        Top level application is left-most element.
+
+        """
+
+    @abstractmethod
+    def add_app(self, app: Application) -> None:
+        """Add application to the nested apps stack."""
+
+    @abstractmethod
+    def freeze(self) -> None:
+        """Freeze the match info.
+
+        The method is called after route resolution.
+
+        After the call .add_app() is forbidden.
+
+        """
+
+
+class AbstractView(ABC):
+    """Abstract class based view."""
+
+    def __init__(self, request: Request) -> None:
+        self._request = request
+
+    @property
+    def request(self) -> Request:
+        """Request instance."""
+        return self._request
+
+    @abstractmethod
+    def __await__(self) -> Generator[Any, None, StreamResponse]:
+        """Execute the view handler."""
+
+
+class AbstractResolver(ABC):
+    """Abstract DNS resolver."""
+
+    @abstractmethod
+    async def resolve(self, host: str, port: int, family: int) -> List[Dict[str, Any]]:
+        """Return IP address for given hostname"""
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Release resolver"""
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    IterableBase = Iterable[Morsel[str]]
+else:
+    IterableBase = Iterable
+
+
+ClearCookiePredicate = Callable[["Morsel[str]"], bool]
+
+
+class AbstractCookieJar(Sized, IterableBase):
+    """Abstract Cookie Jar."""
+
+    @abstractmethod
+    def clear(self, predicate: Optional[ClearCookiePredicate] = None) -> None:
+        """Clear all cookies if no predicate is passed."""
+
+    @abstractmethod
+    def clear_domain(self, domain: str) -> None:
+        """Clear all cookies for domain and all subdomains."""
+
+    @abstractmethod
+    def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None:
+        """Update cookies."""
+
+    @abstractmethod
+    def filter_cookies(self, request_url: URL) -> "BaseCookie[str]":
+        """Return the jar's cookies filtered by their attributes."""
+
+
+class AbstractStreamWriter(ABC):
+    """Abstract stream writer."""
+
+    buffer_size = 0
+    output_size = 0
+    length: Optional[int] = 0
+
+    @abstractmethod
+    async def write(self, chunk: bytes) -> None:
+        """Write chunk into stream."""
+
+    @abstractmethod
+    async def write_eof(self, chunk: bytes = b"") -> None:
+        """Write last chunk."""
+
+    @abstractmethod
+    async def drain(self) -> None:
+        """Flush the write buffer."""
+
+    @abstractmethod
+    def enable_compression(self, encoding: str = "deflate") -> None:
+        """Enable HTTP body compression"""
+
+    @abstractmethod
+    def enable_chunking(self) -> None:
+        """Enable HTTP chunked mode"""
+
+    @abstractmethod
+    async def write_headers(
+        self, status_line: str, headers: "CIMultiDict[str]"
+    ) -> None:
+        """Write HTTP headers"""
+
+
+class AbstractAccessLogger(ABC):
+    """Abstract writer to access log."""
+
+    def __init__(self, logger: logging.Logger, log_format: str) -> None:
+        self.logger = logger
+        self.log_format = log_format
+
+    @abstractmethod
+    def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
+        """Emit log to logger."""
+
+
+class AbstractAsyncAccessLogger(ABC):
+    """Abstract asynchronous writer to access log."""
+
+    @abstractmethod
+    async def log(
+        self, request: BaseRequest, response: StreamResponse, request_start: float
+    ) -> None:
+        """Emit log to logger."""

aiohttp/base_protocol.py

@@ -0,0 +1,90 @@
+import asyncio
+from typing import Optional, cast
+
+from .tcp_helpers import tcp_nodelay
+
+
+class BaseProtocol(asyncio.Protocol):
+    __slots__ = (
+        "_loop",
+        "_paused",
+        "_drain_waiter",
+        "_connection_lost",
+        "_reading_paused",
+        "transport",
+    )
+
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        self._loop: asyncio.AbstractEventLoop = loop
+        self._paused = False
+        self._drain_waiter: Optional[asyncio.Future[None]] = None
+        self._reading_paused = False
+
+        self.transport: Optional[asyncio.Transport] = None
+
+    @property
+    def connected(self) -> bool:
+        """Return True if the connection is open."""
+        return self.transport is not None
+
+    def pause_writing(self) -> None:
+        assert not self._paused
+        self._paused = True
+
+    def resume_writing(self) -> None:
+        assert self._paused
+        self._paused = False
+
+        waiter = self._drain_waiter
+        if waiter is not None:
+            self._drain_waiter = None
+            if not waiter.done():
+                waiter.set_result(None)
+
+    def pause_reading(self) -> None:
+        if not self._reading_paused and self.transport is not None:
+            try:
+                self.transport.pause_reading()
+            except (AttributeError, NotImplementedError, RuntimeError):
+                pass
+            self._reading_paused = True
+
+    def resume_reading(self) -> None:
+        if self._reading_paused and self.transport is not None:
+            try:
+                self.transport.resume_reading()
+            except (AttributeError, NotImplementedError, RuntimeError):
+                pass
+            self._reading_paused = False
+
+    def connection_made(self, transport: asyncio.BaseTransport) -> None:
+        tr = cast(asyncio.Transport, transport)
+        tcp_nodelay(tr, True)
+        self.transport = tr
+
+    def connection_lost(self, exc: Optional[BaseException]) -> None:
+        # Wake up the writer if currently paused.
+        self.transport = None
+        if not self._paused:
+            return
+        waiter = self._drain_waiter
+        if waiter is None:
+            return
+        self._drain_waiter = None
+        if waiter.done():
+            return
+        if exc is None:
+            waiter.set_result(None)
+        else:
+            waiter.set_exception(exc)
+
+    async def _drain_helper(self) -> None:
+        if not self.connected:
+            raise ConnectionResetError("Connection lost")
+        if not self._paused:
+            return
+        waiter = self._drain_waiter
+        if waiter is None:
+            waiter = self._loop.create_future()
+            self._drain_waiter = waiter
+        await asyncio.shield(waiter)

aiohttp/client_exceptions.py

@@ -0,0 +1,313 @@
+"""HTTP related errors."""
+
+import asyncio
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
+
+from .http_parser import RawResponseMessage
+from .typedefs import LooseHeaders
+
+try:
+    import ssl
+
+    SSLContext = ssl.SSLContext
+except ImportError:  # pragma: no cover
+    ssl = SSLContext = None  # type: ignore[assignment]
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .client_reqrep import ClientResponse, ConnectionKey, Fingerprint, RequestInfo
+else:
+    RequestInfo = ClientResponse = ConnectionKey = None
+
+__all__ = (
+    "ClientError",
+    "ClientConnectionError",
+    "ClientOSError",
+    "ClientConnectorError",
+    "ClientProxyConnectionError",
+    "ClientSSLError",
+    "ClientConnectorSSLError",
+    "ClientConnectorCertificateError",
+    "ServerConnectionError",
+    "ServerTimeoutError",
+    "ServerDisconnectedError",
+    "ServerFingerprintMismatch",
+    "ClientResponseError",
+    "ClientHttpProxyError",
+    "WSServerHandshakeError",
+    "ContentTypeError",
+    "ClientPayloadError",
+    "InvalidURL",
+)
+
+
+class ClientError(Exception):
+    """Base class for client connection errors."""
+
+
+class ClientResponseError(ClientError):
+    """Base class for exceptions that occur after getting a response.
+
+    request_info: An instance of RequestInfo.
+    history: A sequence of responses, if redirects occurred.
+    status: HTTP status code.
+    message: Error message.
+    headers: Response headers.
+    """
+
+    def __init__(
+        self,
+        request_info: RequestInfo,
+        history: Tuple[ClientResponse, ...],
+        *,
+        status: Optional[int] = None,
+        message: str = "",
+        headers: Optional[LooseHeaders] = None,
+    ) -> None:
+        self.request_info = request_info
+        if status is not None:
+            self.status = status
+        else:
+            self.status = 0
+        self.message = message
+        self.headers = headers
+        self.history = history
+        self.args = (request_info, history)
+
+    def __str__(self) -> str:
+        return "{}, message={!r}, url={!r}".format(
+            self.status,
+            self.message,
+            self.request_info.real_url,
+        )
+
+    def __repr__(self) -> str:
+        args = f"{self.request_info!r}, {self.history!r}"
+        if self.status != 0:
+            args += f", status={self.status!r}"
+        if self.message != "":
+            args += f", message={self.message!r}"
+        if self.headers is not None:
+            args += f", headers={self.headers!r}"
+        return f"{type(self).__name__}({args})"
+
+
+class ContentTypeError(ClientResponseError):
+    """ContentType found is not valid."""
+
+
+class WSServerHandshakeError(ClientResponseError):
+    """websocket server handshake error."""
+
+
+class ClientHttpProxyError(ClientResponseError):
+    """HTTP proxy error.
+
+    Raised in :class:`aiohttp.connector.TCPConnector` if
+    proxy responds with status other than ``200 OK``
+    on ``CONNECT`` request.
+    """
+
+
+class TooManyRedirects(ClientResponseError):
+    """Client was redirected too many times."""
+
+
+class ClientConnectionError(ClientError):
+    """Base class for client socket errors."""
+
+
+class ClientOSError(ClientConnectionError, OSError):
+    """OSError error."""
+
+
+class ClientConnectorError(ClientOSError):
+    """Client connector error.
+
+    Raised in :class:`aiohttp.connector.TCPConnector` if
+        a connection can not be established.
+    """
+
+    def __init__(self, connection_key: ConnectionKey, os_error: OSError) -> None:
+        self._conn_key = connection_key
+        self._os_error = os_error
+        super().__init__(os_error.errno, os_error.strerror)
+        self.args = (connection_key, os_error)
+
+    @property
+    def os_error(self) -> OSError:
+        return self._os_error
+
+    @property
+    def host(self) -> str:
+        return self._conn_key.host
+
+    @property
+    def port(self) -> Optional[int]:
+        return self._conn_key.port
+
+    @property
+    def ssl(self) -> Union[SSLContext, None, bool, "Fingerprint"]:
+        return self._conn_key.ssl
+
+    def __str__(self) -> str:
+        return "Cannot connect to host {0.host}:{0.port} ssl:{1} [{2}]".format(
+            self, self.ssl if self.ssl is not None else "default", self.strerror
+        )
+
+    # OSError.__reduce__ does too much black magick
+    __reduce__ = BaseException.__reduce__
+
+
+class ClientProxyConnectionError(ClientConnectorError):
+    """Proxy connection error.
+
+    Raised in :class:`aiohttp.connector.TCPConnector` if
+        connection to proxy can not be established.
+    """
+
+
+class UnixClientConnectorError(ClientConnectorError):
+    """Unix connector error.
+
+    Raised in :py:class:`aiohttp.connector.UnixConnector`
+    if connection to unix socket can not be established.
+    """
+
+    def __init__(
+        self, path: str, connection_key: ConnectionKey, os_error: OSError
+    ) -> None:
+        self._path = path
+        super().__init__(connection_key, os_error)
+
+    @property
+    def path(self) -> str:
+        return self._path
+
+    def __str__(self) -> str:
+        return "Cannot connect to unix socket {0.path} ssl:{1} [{2}]".format(
+            self, self.ssl if self.ssl is not None else "default", self.strerror
+        )
+
+
+class ServerConnectionError(ClientConnectionError):
+    """Server connection errors."""
+
+
+class ServerDisconnectedError(ServerConnectionError):
+    """Server disconnected."""
+
+    def __init__(self, message: Union[RawResponseMessage, str, None] = None) -> None:
+        if message is None:
+            message = "Server disconnected"
+
+        self.args = (message,)
+        self.message = message
+
+
+class ServerTimeoutError(ServerConnectionError, asyncio.TimeoutError):
+    """Server timeout error."""
+
+
+class ServerFingerprintMismatch(ServerConnectionError):
+    """SSL certificate does not match expected fingerprint."""
+
+    def __init__(self, expected: bytes, got: bytes, host: str, port: int) -> None:
+        self.expected = expected
+        self.got = got
+        self.host = host
+        self.port = port
+        self.args = (expected, got, host, port)
+
+    def __repr__(self) -> str:
+        return "<{} expected={!r} got={!r} host={!r} port={!r}>".format(
+            self.__class__.__name__, self.expected, self.got, self.host, self.port
+        )
+
+
+class ClientPayloadError(ClientError):
+    """Response payload error."""
+
+
+class InvalidURL(ClientError, ValueError):
+    """Invalid URL.
+
+    URL used for fetching is malformed, e.g. it doesn't contains host
+    part.
+    """
+
+    # Derive from ValueError for backward compatibility
+
+    def __init__(self, url: Any) -> None:
+        # The type of url is not yarl.URL because the exception can be raised
+        # on URL(url) call
+        super().__init__(url)
+
+    @property
+    def url(self) -> Any:
+        return self.args[0]
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {self.url}>"
+
+
+class ClientSSLError(ClientConnectorError):
+    """Base error for ssl.*Errors."""
+
+
+if ssl is not None:
+    cert_errors = (ssl.CertificateError,)
+    cert_errors_bases = (
+        ClientSSLError,
+        ssl.CertificateError,
+    )
+
+    ssl_errors = (ssl.SSLError,)
+    ssl_error_bases = (ClientSSLError, ssl.SSLError)
+else:  # pragma: no cover
+    cert_errors = tuple()
+    cert_errors_bases = (
+        ClientSSLError,
+        ValueError,
+    )
+
+    ssl_errors = tuple()
+    ssl_error_bases = (ClientSSLError,)
+
+
+class ClientConnectorSSLError(*ssl_error_bases):  # type: ignore[misc]
+    """Response ssl error."""
+
+
+class ClientConnectorCertificateError(*cert_errors_bases):  # type: ignore[misc]
+    """Response certificate error."""
+
+    def __init__(
+        self, connection_key: ConnectionKey, certificate_error: Exception
+    ) -> None:
+        self._conn_key = connection_key
+        self._certificate_error = certificate_error
+        self.args = (connection_key, certificate_error)
+
+    @property
+    def certificate_error(self) -> Exception:
+        return self._certificate_error
+
+    @property
+    def host(self) -> str:
+        return self._conn_key.host
+
+    @property
+    def port(self) -> Optional[int]:
+        return self._conn_key.port
+
+    @property
+    def ssl(self) -> bool:
+        return self._conn_key.is_ssl
+
+    def __str__(self) -> str:
+        return (
+            "Cannot connect to host {0.host}:{0.port} ssl:{0.ssl} "
+            "[{0.certificate_error.__class__.__name__}: "
+            "{0.certificate_error.args}]".format(self)
+        )

aiohttp/client_proto.py

@@ -0,0 +1,269 @@
+import asyncio
+from contextlib import suppress
+from typing import Any, Optional, Tuple
+
+from .base_protocol import BaseProtocol
+from .client_exceptions import (
+    ClientOSError,
+    ClientPayloadError,
+    ServerDisconnectedError,
+    ServerTimeoutError,
+)
+from .helpers import BaseTimerContext, set_exception, set_result
+from .http import HttpResponseParser, RawResponseMessage, WebSocketReader
+from .streams import EMPTY_PAYLOAD, DataQueue, StreamReader
+
+
+class ResponseHandler(BaseProtocol, DataQueue[Tuple[RawResponseMessage, StreamReader]]):
+    """Helper class to adapt between Protocol and StreamReader."""
+
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        BaseProtocol.__init__(self, loop=loop)
+        DataQueue.__init__(self, loop)
+
+        self._should_close = False
+
+        self._payload: Optional[StreamReader] = None
+        self._skip_payload = False
+        self._payload_parser: Optional[WebSocketReader] = None
+
+        self._timer = None
+
+        self._tail = b""
+        self._upgraded = False
+        self._parser: Optional[HttpResponseParser] = None
+
+        self._read_timeout: Optional[float] = None
+        self._read_timeout_handle: Optional[asyncio.TimerHandle] = None
+
+        self._timeout_ceil_threshold: Optional[float] = 5
+
+        self.closed: asyncio.Future[None] = self._loop.create_future()
+
+    @property
+    def upgraded(self) -> bool:
+        return self._upgraded
+
+    @property
+    def should_close(self) -> bool:
+        if self._payload is not None and not self._payload.is_eof():
+            return True
+
+        return (
+            self._should_close
+            or self._upgraded
+            or self.exception() is not None
+            or self._payload_parser is not None
+            or len(self) > 0
+            or bool(self._tail)
+        )
+
+    def force_close(self) -> None:
+        self._should_close = True
+
+    def close(self) -> None:
+        transport = self.transport
+        if transport is not None:
+            transport.close()
+            self.transport = None
+            self._payload = None
+            self._drop_timeout()
+
+    def is_connected(self) -> bool:
+        return self.transport is not None and not self.transport.is_closing()
+
+    def connection_lost(self, exc: Optional[BaseException]) -> None:
+        self._drop_timeout()
+
+        if exc is not None:
+            set_exception(self.closed, exc)
+        else:
+            set_result(self.closed, None)
+
+        if self._payload_parser is not None:
+            with suppress(Exception):
+                self._payload_parser.feed_eof()
+
+        uncompleted = None
+        if self._parser is not None:
+            try:
+                uncompleted = self._parser.feed_eof()
+            except Exception:
+                if self._payload is not None:
+                    self._payload.set_exception(
+                        ClientPayloadError("Response payload is not completed")
+                    )
+
+        if not self.is_eof():
+            if isinstance(exc, OSError):
+                exc = ClientOSError(*exc.args)
+            if exc is None:
+                exc = ServerDisconnectedError(uncompleted)
+            # assigns self._should_close to True as side effect,
+            # we do it anyway below
+            self.set_exception(exc)
+
+        self._should_close = True
+        self._parser = None
+        self._payload = None
+        self._payload_parser = None
+        self._reading_paused = False
+
+        super().connection_lost(exc)
+
+    def eof_received(self) -> None:
+        # should call parser.feed_eof() most likely
+        self._drop_timeout()
+
+    def pause_reading(self) -> None:
+        super().pause_reading()
+        self._drop_timeout()
+
+    def resume_reading(self) -> None:
+        super().resume_reading()
+        self._reschedule_timeout()
+
+    def set_exception(self, exc: BaseException) -> None:
+        self._should_close = True
+        self._drop_timeout()
+        super().set_exception(exc)
+
+    def set_parser(self, parser: Any, payload: Any) -> None:
+        # TODO: actual types are:
+        #   parser: WebSocketReader
+        #   payload: FlowControlDataQueue
+        # but they are not generi enough
+        # Need an ABC for both types
+        self._payload = payload
+        self._payload_parser = parser
+
+        self._drop_timeout()
+
+        if self._tail:
+            data, self._tail = self._tail, b""
+            self.data_received(data)
+
+    def set_response_params(
+        self,
+        *,
+        timer: Optional[BaseTimerContext] = None,
+        skip_payload: bool = False,
+        read_until_eof: bool = False,
+        auto_decompress: bool = True,
+        read_timeout: Optional[float] = None,
+        read_bufsize: int = 2**16,
+        timeout_ceil_threshold: float = 5,
+        max_line_size: int = 8190,
+        max_field_size: int = 8190,
+    ) -> None:
+        self._skip_payload = skip_payload
+
+        self._read_timeout = read_timeout
+
+        self._timeout_ceil_threshold = timeout_ceil_threshold
+
+        self._parser = HttpResponseParser(
+            self,
+            self._loop,
+            read_bufsize,
+            timer=timer,
+            payload_exception=ClientPayloadError,
+            response_with_body=not skip_payload,
+            read_until_eof=read_until_eof,
+            auto_decompress=auto_decompress,
+            max_line_size=max_line_size,
+            max_field_size=max_field_size,
+        )
+
+        if self._tail:
+            data, self._tail = self._tail, b""
+            self.data_received(data)
+
+    def _drop_timeout(self) -> None:
+        if self._read_timeout_handle is not None:
+            self._read_timeout_handle.cancel()
+            self._read_timeout_handle = None
+
+    def _reschedule_timeout(self) -> None:
+        timeout = self._read_timeout
+        if self._read_timeout_handle is not None:
+            self._read_timeout_handle.cancel()
+
+        if timeout:
+            self._read_timeout_handle = self._loop.call_later(
+                timeout, self._on_read_timeout
+            )
+        else:
+            self._read_timeout_handle = None
+
+    def start_timeout(self) -> None:
+        self._reschedule_timeout()
+
+    def _on_read_timeout(self) -> None:
+        exc = ServerTimeoutError("Timeout on reading data from socket")
+        self.set_exception(exc)
+        if self._payload is not None:
+            self._payload.set_exception(exc)
+
+    def data_received(self, data: bytes) -> None:
+        self._reschedule_timeout()
+
+        if not data:
+            return
+
+        # custom payload parser
+        if self._payload_parser is not None:
+            eof, tail = self._payload_parser.feed_data(data)
+            if eof:
+                self._payload = None
+                self._payload_parser = None
+
+                if tail:
+                    self.data_received(tail)
+            return
+        else:
+            if self._upgraded or self._parser is None:
+                # i.e. websocket connection, websocket parser is not set yet
+                self._tail += data
+            else:
+                # parse http messages
+                try:
+                    messages, upgraded, tail = self._parser.feed_data(data)
+                except BaseException as exc:
+                    if self.transport is not None:
+                        # connection.release() could be called BEFORE
+                        # data_received(), the transport is already
+                        # closed in this case
+                        self.transport.close()
+                    # should_close is True after the call
+                    self.set_exception(exc)
+                    return
+
+                self._upgraded = upgraded
+
+                payload: Optional[StreamReader] = None
+                for message, payload in messages:
+                    if message.should_close:
+                        self._should_close = True
+
+                    self._payload = payload
+
+                    if self._skip_payload or message.code in (204, 304):
+                        self.feed_data((message, EMPTY_PAYLOAD), 0)
+                    else:
+                        self.feed_data((message, payload), 0)
+                if payload is not None:
+                    # new message(s) was processed
+                    # register timeout handler unsubscribing
+                    # either on end-of-stream or immediately for
+                    # EMPTY_PAYLOAD
+                    if payload is not EMPTY_PAYLOAD:
+                        payload.on_eof(self._drop_timeout)
+                    else:
+                        self._drop_timeout()
+
+                if tail:
+                    if upgraded:
+                        self.data_received(tail)
+                    else:
+                        self._tail = tail

aiohttp/client_ws.py

@@ -0,0 +1,327 @@
+"""WebSocket client for asyncio."""
+
+import asyncio
+import dataclasses
+import sys
+from typing import Any, Final, Optional, cast
+
+from .client_exceptions import ClientError
+from .client_reqrep import ClientResponse
+from .helpers import call_later, set_result
+from .http import (
+    WS_CLOSED_MESSAGE,
+    WS_CLOSING_MESSAGE,
+    WebSocketError,
+    WSCloseCode,
+    WSMessage,
+    WSMsgType,
+)
+from .http_websocket import WebSocketWriter  # WSMessage
+from .streams import EofStream, FlowControlDataQueue
+from .typedefs import (
+    DEFAULT_JSON_DECODER,
+    DEFAULT_JSON_ENCODER,
+    JSONDecoder,
+    JSONEncoder,
+)
+
+if sys.version_info >= (3, 11):
+    import asyncio as async_timeout
+else:
+    import async_timeout
+
+
+@dataclasses.dataclass(frozen=True)
+class ClientWSTimeout:
+    ws_receive: Optional[float] = None
+    ws_close: Optional[float] = None
+
+
+DEFAULT_WS_CLIENT_TIMEOUT: Final[ClientWSTimeout] = ClientWSTimeout(
+    ws_receive=None, ws_close=10.0
+)
+
+
+class ClientWebSocketResponse:
+    def __init__(
+        self,
+        reader: "FlowControlDataQueue[WSMessage]",
+        writer: WebSocketWriter,
+        protocol: Optional[str],
+        response: ClientResponse,
+        timeout: ClientWSTimeout,
+        autoclose: bool,
+        autoping: bool,
+        loop: asyncio.AbstractEventLoop,
+        *,
+        heartbeat: Optional[float] = None,
+        compress: int = 0,
+        client_notakeover: bool = False,
+    ) -> None:
+        self._response = response
+        self._conn = response.connection
+
+        self._writer = writer
+        self._reader = reader
+        self._protocol = protocol
+        self._closed = False
+        self._closing = False
+        self._close_code: Optional[int] = None
+        self._timeout: ClientWSTimeout = timeout
+        self._autoclose = autoclose
+        self._autoping = autoping
+        self._heartbeat = heartbeat
+        self._heartbeat_cb: Optional[asyncio.TimerHandle] = None
+        if heartbeat is not None:
+            self._pong_heartbeat = heartbeat / 2.0
+        self._pong_response_cb: Optional[asyncio.TimerHandle] = None
+        self._loop = loop
+        self._waiting: Optional[asyncio.Future[bool]] = None
+        self._exception: Optional[BaseException] = None
+        self._compress = compress
+        self._client_notakeover = client_notakeover
+
+        self._reset_heartbeat()
+
+    def _cancel_heartbeat(self) -> None:
+        if self._pong_response_cb is not None:
+            self._pong_response_cb.cancel()
+            self._pong_response_cb = None
+
+        if self._heartbeat_cb is not None:
+            self._heartbeat_cb.cancel()
+            self._heartbeat_cb = None
+
+    def _reset_heartbeat(self) -> None:
+        self._cancel_heartbeat()
+
+        if self._heartbeat is not None:
+            self._heartbeat_cb = call_later(
+                self._send_heartbeat,
+                self._heartbeat,
+                self._loop,
+                timeout_ceil_threshold=self._conn._connector._timeout_ceil_threshold
+                if self._conn is not None
+                else 5,
+            )
+
+    def _send_heartbeat(self) -> None:
+        if self._heartbeat is not None and not self._closed:
+            # fire-and-forget a task is not perfect but maybe ok for
+            # sending ping. Otherwise we need a long-living heartbeat
+            # task in the class.
+            self._loop.create_task(self._writer.ping())  # type: ignore[unused-awaitable]
+
+            if self._pong_response_cb is not None:
+                self._pong_response_cb.cancel()
+            self._pong_response_cb = call_later(
+                self._pong_not_received,
+                self._pong_heartbeat,
+                self._loop,
+                timeout_ceil_threshold=self._conn._connector._timeout_ceil_threshold
+                if self._conn is not None
+                else 5,
+            )
+
+    def _pong_not_received(self) -> None:
+        if not self._closed:
+            self._closed = True
+            self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+            self._exception = asyncio.TimeoutError()
+            self._response.close()
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    @property
+    def close_code(self) -> Optional[int]:
+        return self._close_code
+
+    @property
+    def protocol(self) -> Optional[str]:
+        return self._protocol
+
+    @property
+    def compress(self) -> int:
+        return self._compress
+
+    @property
+    def client_notakeover(self) -> bool:
+        return self._client_notakeover
+
+    def get_extra_info(self, name: str, default: Any = None) -> Any:
+        """extra info from connection transport"""
+        conn = self._response.connection
+        if conn is None:
+            return default
+        transport = conn.transport
+        if transport is None:
+            return default
+        return transport.get_extra_info(name, default)
+
+    def exception(self) -> Optional[BaseException]:
+        return self._exception
+
+    async def ping(self, message: bytes = b"") -> None:
+        await self._writer.ping(message)
+
+    async def pong(self, message: bytes = b"") -> None:
+        await self._writer.pong(message)
+
+    async def send_str(self, data: str, compress: Optional[int] = None) -> None:
+        if not isinstance(data, str):
+            raise TypeError("data argument must be str (%r)" % type(data))
+        await self._writer.send(data, binary=False, compress=compress)
+
+    async def send_bytes(self, data: bytes, compress: Optional[int] = None) -> None:
+        if not isinstance(data, (bytes, bytearray, memoryview)):
+            raise TypeError("data argument must be byte-ish (%r)" % type(data))
+        await self._writer.send(data, binary=True, compress=compress)
+
+    async def send_json(
+        self,
+        data: Any,
+        compress: Optional[int] = None,
+        *,
+        dumps: JSONEncoder = DEFAULT_JSON_ENCODER,
+    ) -> None:
+        await self.send_str(dumps(data), compress=compress)
+
+    async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bool:
+        # we need to break `receive()` cycle first,
+        # `close()` may be called from different task
+        if self._waiting is not None and not self._closed:
+            self._reader.feed_data(WS_CLOSING_MESSAGE, 0)
+            await self._waiting
+
+        if not self._closed:
+            self._cancel_heartbeat()
+            self._closed = True
+            try:
+                await self._writer.close(code, message)
+            except asyncio.CancelledError:
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                self._response.close()
+                raise
+            except Exception as exc:
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                self._exception = exc
+                self._response.close()
+                return True
+
+            if self._closing:
+                self._response.close()
+                return True
+
+            while True:
+                try:
+                    async with async_timeout.timeout(self._timeout.ws_close):
+                        msg = await self._reader.read()
+                except asyncio.CancelledError:
+                    self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                    self._response.close()
+                    raise
+                except Exception as exc:
+                    self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                    self._exception = exc
+                    self._response.close()
+                    return True
+
+                if msg.type == WSMsgType.CLOSE:
+                    self._close_code = msg.data
+                    self._response.close()
+                    return True
+        else:
+            return False
+
+    async def receive(self, timeout: Optional[float] = None) -> WSMessage:
+        while True:
+            if self._waiting is not None:
+                raise RuntimeError("Concurrent call to receive() is not allowed")
+
+            if self._closed:
+                return WS_CLOSED_MESSAGE
+            elif self._closing:
+                await self.close()
+                return WS_CLOSED_MESSAGE
+
+            try:
+                self._waiting = self._loop.create_future()
+                try:
+                    async with async_timeout.timeout(
+                        timeout or self._timeout.ws_receive
+                    ):
+                        msg = await self._reader.read()
+                    self._reset_heartbeat()
+                finally:
+                    waiter = self._waiting
+                    self._waiting = None
+                    set_result(waiter, True)
+            except (asyncio.CancelledError, asyncio.TimeoutError):
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                raise
+            except EofStream:
+                self._close_code = WSCloseCode.OK
+                await self.close()
+                return WSMessage(WSMsgType.CLOSED, None, None)
+            except ClientError:
+                self._closed = True
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                return WS_CLOSED_MESSAGE
+            except WebSocketError as exc:
+                self._close_code = exc.code
+                await self.close(code=exc.code)
+                return WSMessage(WSMsgType.ERROR, exc, None)
+            except Exception as exc:
+                self._exception = exc
+                self._closing = True
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                await self.close()
+                return WSMessage(WSMsgType.ERROR, exc, None)
+
+            if msg.type == WSMsgType.CLOSE:
+                self._closing = True
+                self._close_code = msg.data
+                # Could be closed elsewhere while awaiting reader
+                if not self._closed and self._autoclose:  # type: ignore[redundant-expr]
+                    await self.close()
+            elif msg.type == WSMsgType.CLOSING:
+                self._closing = True
+            elif msg.type == WSMsgType.PING and self._autoping:
+                await self.pong(msg.data)
+                continue
+            elif msg.type == WSMsgType.PONG and self._autoping:
+                continue
+
+            return msg
+
+    async def receive_str(self, *, timeout: Optional[float] = None) -> str:
+        msg = await self.receive(timeout)
+        if msg.type != WSMsgType.TEXT:
+            raise TypeError(f"Received message {msg.type}:{msg.data!r} is not str")
+        return cast(str, msg.data)
+
+    async def receive_bytes(self, *, timeout: Optional[float] = None) -> bytes:
+        msg = await self.receive(timeout)
+        if msg.type != WSMsgType.BINARY:
+            raise TypeError(f"Received message {msg.type}:{msg.data!r} is not bytes")
+        return cast(bytes, msg.data)
+
+    async def receive_json(
+        self,
+        *,
+        loads: JSONDecoder = DEFAULT_JSON_DECODER,
+        timeout: Optional[float] = None,
+    ) -> Any:
+        data = await self.receive_str(timeout=timeout)
+        return loads(data)
+
+    def __aiter__(self) -> "ClientWebSocketResponse":
+        return self
+
+    async def __anext__(self) -> WSMessage:
+        msg = await self.receive()
+        if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED):
+            raise StopAsyncIteration
+        return msg

aiohttp/compression_utils.py

@@ -0,0 +1,151 @@
+import asyncio
+import zlib
+from concurrent.futures import Executor
+from typing import Optional, cast
+
+try:
+    try:
+        import brotlicffi as brotli
+    except ImportError:
+        import brotli
+
+    HAS_BROTLI = True
+except ImportError:  # pragma: no cover
+    HAS_BROTLI = False
+
+MAX_SYNC_CHUNK_SIZE = 1024
+
+
+def encoding_to_mode(
+    encoding: Optional[str] = None,
+    suppress_deflate_header: bool = False,
+) -> int:
+    if encoding == "gzip":
+        return 16 + zlib.MAX_WBITS
+
+    return -zlib.MAX_WBITS if suppress_deflate_header else zlib.MAX_WBITS
+
+
+class ZlibBaseHandler:
+    def __init__(
+        self,
+        mode: int,
+        executor: Optional[Executor] = None,
+        max_sync_chunk_size: Optional[int] = MAX_SYNC_CHUNK_SIZE,
+    ):
+        self._mode = mode
+        self._executor = executor
+        self._max_sync_chunk_size = max_sync_chunk_size
+
+
+class ZLibCompressor(ZlibBaseHandler):
+    def __init__(
+        self,
+        encoding: Optional[str] = None,
+        suppress_deflate_header: bool = False,
+        level: Optional[int] = None,
+        wbits: Optional[int] = None,
+        strategy: int = zlib.Z_DEFAULT_STRATEGY,
+        executor: Optional[Executor] = None,
+        max_sync_chunk_size: Optional[int] = MAX_SYNC_CHUNK_SIZE,
+    ):
+        super().__init__(
+            mode=encoding_to_mode(encoding, suppress_deflate_header)
+            if wbits is None
+            else wbits,
+            executor=executor,
+            max_sync_chunk_size=max_sync_chunk_size,
+        )
+        if level is None:
+            self._compressor = zlib.compressobj(wbits=self._mode, strategy=strategy)
+        else:
+            self._compressor = zlib.compressobj(
+                wbits=self._mode, strategy=strategy, level=level
+            )
+
+    def compress_sync(self, data: bytes) -> bytes:
+        return self._compressor.compress(data)
+
+    async def compress(self, data: bytes) -> bytes:
+        if (
+            self._max_sync_chunk_size is not None
+            and len(data) > self._max_sync_chunk_size
+        ):
+            return await asyncio.get_event_loop().run_in_executor(
+                self._executor, self.compress_sync, data
+            )
+        return self.compress_sync(data)
+
+    def flush(self, mode: int = zlib.Z_FINISH) -> bytes:
+        return self._compressor.flush(mode)
+
+
+class ZLibDecompressor(ZlibBaseHandler):
+    def __init__(
+        self,
+        encoding: Optional[str] = None,
+        suppress_deflate_header: bool = False,
+        executor: Optional[Executor] = None,
+        max_sync_chunk_size: Optional[int] = MAX_SYNC_CHUNK_SIZE,
+    ):
+        super().__init__(
+            mode=encoding_to_mode(encoding, suppress_deflate_header),
+            executor=executor,
+            max_sync_chunk_size=max_sync_chunk_size,
+        )
+        self._decompressor = zlib.decompressobj(wbits=self._mode)
+
+    def decompress_sync(self, data: bytes, max_length: int = 0) -> bytes:
+        return self._decompressor.decompress(data, max_length)
+
+    async def decompress(self, data: bytes, max_length: int = 0) -> bytes:
+        if (
+            self._max_sync_chunk_size is not None
+            and len(data) > self._max_sync_chunk_size
+        ):
+            return await asyncio.get_event_loop().run_in_executor(
+                self._executor, self.decompress_sync, data, max_length
+            )
+        return self.decompress_sync(data, max_length)
+
+    def flush(self, length: int = 0) -> bytes:
+        return (
+            self._decompressor.flush(length)
+            if length > 0
+            else self._decompressor.flush()
+        )
+
+    @property
+    def eof(self) -> bool:
+        return self._decompressor.eof
+
+    @property
+    def unconsumed_tail(self) -> bytes:
+        return self._decompressor.unconsumed_tail
+
+    @property
+    def unused_data(self) -> bytes:
+        return self._decompressor.unused_data
+
+
+class BrotliDecompressor:
+    # Supports both 'brotlipy' and 'Brotli' packages
+    # since they share an import name. The top branches
+    # are for 'brotlipy' and bottom branches for 'Brotli'
+    def __init__(self) -> None:
+        if not HAS_BROTLI:
+            raise RuntimeError(
+                "The brotli decompression is not available. "
+                "Please install `Brotli` module"
+            )
+        self._obj = brotli.Decompressor()
+
+    def decompress_sync(self, data: bytes) -> bytes:
+        if hasattr(self._obj, "decompress"):
+            return cast(bytes, self._obj.decompress(data))
+        return cast(bytes, self._obj.process(data))
+
+    def flush(self) -> bytes:
+        if hasattr(self._obj, "flush"):
+            return cast(bytes, self._obj.flush())
+        return b""

aiohttp/cookiejar.py

@@ -0,0 +1,417 @@
+import contextlib
+import datetime
+import os  # noqa
+import pathlib
+import pickle
+import re
+import warnings
+from collections import defaultdict
+from http.cookies import BaseCookie, Morsel, SimpleCookie
+from typing import (  # noqa
+    DefaultDict,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+    cast,
+)
+
+from yarl import URL
+
+from .abc import AbstractCookieJar, ClearCookiePredicate
+from .helpers import is_ip_address, next_whole_second
+from .typedefs import LooseCookies, PathLike, StrOrURL
+
+__all__ = ("CookieJar", "DummyCookieJar")
+
+
+CookieItem = Union[str, "Morsel[str]"]
+
+
+class CookieJar(AbstractCookieJar):
+    """Implements cookie storage adhering to RFC 6265."""
+
+    DATE_TOKENS_RE = re.compile(
+        r"[\x09\x20-\x2F\x3B-\x40\x5B-\x60\x7B-\x7E]*"
+        r"(?P<token>[\x00-\x08\x0A-\x1F\d:a-zA-Z\x7F-\xFF]+)"
+    )
+
+    DATE_HMS_TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})")
+
+    DATE_DAY_OF_MONTH_RE = re.compile(r"(\d{1,2})")
+
+    DATE_MONTH_RE = re.compile(
+        "(jan)|(feb)|(mar)|(apr)|(may)|(jun)|(jul)|" "(aug)|(sep)|(oct)|(nov)|(dec)",
+        re.I,
+    )
+
+    DATE_YEAR_RE = re.compile(r"(\d{2,4})")
+
+    MAX_TIME = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
+
+    MAX_32BIT_TIME = datetime.datetime.fromtimestamp(2**31 - 1, datetime.timezone.utc)
+
+    def __init__(
+        self,
+        *,
+        unsafe: bool = False,
+        quote_cookie: bool = True,
+        treat_as_secure_origin: Union[StrOrURL, List[StrOrURL], None] = None
+    ) -> None:
+        self._cookies: DefaultDict[Tuple[str, str], SimpleCookie[str]] = defaultdict(
+            SimpleCookie
+        )
+        self._host_only_cookies: Set[Tuple[str, str]] = set()
+        self._unsafe = unsafe
+        self._quote_cookie = quote_cookie
+        if treat_as_secure_origin is None:
+            treat_as_secure_origin = []
+        elif isinstance(treat_as_secure_origin, URL):
+            treat_as_secure_origin = [treat_as_secure_origin.origin()]
+        elif isinstance(treat_as_secure_origin, str):
+            treat_as_secure_origin = [URL(treat_as_secure_origin).origin()]
+        else:
+            treat_as_secure_origin = [
+                URL(url).origin() if isinstance(url, str) else url.origin()
+                for url in treat_as_secure_origin
+            ]
+        self._treat_as_secure_origin = treat_as_secure_origin
+        self._next_expiration = next_whole_second()
+        self._expirations: Dict[Tuple[str, str, str], datetime.datetime] = {}
+        # #4515: datetime.max may not be representable on 32-bit platforms
+        self._max_time = self.MAX_TIME
+        try:
+            self._max_time.timestamp()
+        except OverflowError:
+            self._max_time = self.MAX_32BIT_TIME
+
+    def save(self, file_path: PathLike) -> None:
+        file_path = pathlib.Path(file_path)
+        with file_path.open(mode="wb") as f:
+            pickle.dump(self._cookies, f, pickle.HIGHEST_PROTOCOL)
+
+    def load(self, file_path: PathLike) -> None:
+        file_path = pathlib.Path(file_path)
+        with file_path.open(mode="rb") as f:
+            self._cookies = pickle.load(f)
+
+    def clear(self, predicate: Optional[ClearCookiePredicate] = None) -> None:
+        if predicate is None:
+            self._next_expiration = next_whole_second()
+            self._cookies.clear()
+            self._host_only_cookies.clear()
+            self._expirations.clear()
+            return
+
+        to_del = []
+        now = datetime.datetime.now(datetime.timezone.utc)
+        for (domain, path), cookie in self._cookies.items():
+            for name, morsel in cookie.items():
+                key = (domain, path, name)
+                if (
+                    key in self._expirations and self._expirations[key] <= now
+                ) or predicate(morsel):
+                    to_del.append(key)
+
+        for domain, path, name in to_del:
+            self._host_only_cookies.discard((domain, name))
+            key = (domain, path, name)
+            if key in self._expirations:
+                del self._expirations[(domain, path, name)]
+            self._cookies[(domain, path)].pop(name, None)
+
+        next_expiration = min(self._expirations.values(), default=self._max_time)
+        try:
+            self._next_expiration = next_expiration.replace(
+                microsecond=0
+            ) + datetime.timedelta(seconds=1)
+        except OverflowError:
+            self._next_expiration = self._max_time
+
+    def clear_domain(self, domain: str) -> None:
+        self.clear(lambda x: self._is_domain_match(domain, x["domain"]))
+
+    def __iter__(self) -> "Iterator[Morsel[str]]":
+        self._do_expiration()
+        for val in self._cookies.values():
+            yield from val.values()
+
+    def __len__(self) -> int:
+        return sum(1 for i in self)
+
+    def _do_expiration(self) -> None:
+        self.clear(lambda x: False)
+
+    def _expire_cookie(
+        self, when: datetime.datetime, domain: str, path: str, name: str
+    ) -> None:
+        self._next_expiration = min(self._next_expiration, when)
+        self._expirations[(domain, path, name)] = when
+
+    def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None:
+        """Update cookies."""
+        hostname = response_url.raw_host
+
+        if not self._unsafe and is_ip_address(hostname):
+            # Don't accept cookies from IPs
+            return
+
+        if isinstance(cookies, Mapping):
+            cookies = cookies.items()
+
+        for name, cookie in cookies:
+            if not isinstance(cookie, Morsel):
+                tmp: SimpleCookie[str] = SimpleCookie()
+                tmp[name] = cookie  # type: ignore[assignment]
+                cookie = tmp[name]
+
+            domain = cookie["domain"]
+
+            # ignore domains with trailing dots
+            if domain.endswith("."):
+                domain = ""
+                del cookie["domain"]
+
+            if not domain and hostname is not None:
+                # Set the cookie's domain to the response hostname
+                # and set its host-only-flag
+                self._host_only_cookies.add((hostname, name))
+                domain = cookie["domain"] = hostname
+
+            if domain.startswith("."):
+                # Remove leading dot
+                domain = domain[1:]
+                cookie["domain"] = domain
+
+            if hostname and not self._is_domain_match(domain, hostname):
+                # Setting cookies for different domains is not allowed
+                continue
+
+            path = cookie["path"]
+            if not path or not path.startswith("/"):
+                # Set the cookie's path to the response path
+                path = response_url.path
+                if not path.startswith("/"):
+                    path = "/"
+                else:
+                    # Cut everything from the last slash to the end
+                    path = "/" + path[1 : path.rfind("/")]
+                cookie["path"] = path
+
+            max_age = cookie["max-age"]
+            if max_age:
+                try:
+                    delta_seconds = int(max_age)
+                    try:
+                        max_age_expiration = datetime.datetime.now(
+                            datetime.timezone.utc
+                        ) + datetime.timedelta(seconds=delta_seconds)
+                    except OverflowError:
+                        max_age_expiration = self._max_time
+                    self._expire_cookie(max_age_expiration, domain, path, name)
+                except ValueError:
+                    cookie["max-age"] = ""
+
+            else:
+                expires = cookie["expires"]
+                if expires:
+                    expire_time = self._parse_date(expires)
+                    if expire_time:
+                        self._expire_cookie(expire_time, domain, path, name)
+                    else:
+                        cookie["expires"] = ""
+
+            self._cookies[(domain, path)][name] = cookie
+
+        self._do_expiration()
+
+    def filter_cookies(
+        self, request_url: URL = URL()
+    ) -> Union["BaseCookie[str]", "SimpleCookie[str]"]:
+        """Returns this jar's cookies filtered by their attributes."""
+        self._do_expiration()
+        if not isinstance(request_url, URL):
+            warnings.warn(
+                "The method accepts yarl.URL instances only, got {}".format(
+                    type(request_url)
+                ),
+                DeprecationWarning,
+            )
+            request_url = URL(request_url)
+        filtered: Union["SimpleCookie[str]", "BaseCookie[str]"] = (
+            SimpleCookie() if self._quote_cookie else BaseCookie()
+        )
+        hostname = request_url.raw_host or ""
+        request_origin = URL()
+        with contextlib.suppress(ValueError):
+            request_origin = request_url.origin()
+
+        is_not_secure = (
+            request_url.scheme not in ("https", "wss")
+            and request_origin not in self._treat_as_secure_origin
+        )
+
+        # Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4
+        for cookie in sorted(self, key=lambda c: len(c["path"])):
+            name = cookie.key
+            domain = cookie["domain"]
+
+            # Send shared cookies
+            if not domain:
+                filtered[name] = cookie.value
+                continue
+
+            if not self._unsafe and is_ip_address(hostname):
+                continue
+
+            if (domain, name) in self._host_only_cookies:
+                if domain != hostname:
+                    continue
+            elif not self._is_domain_match(domain, hostname):
+                continue
+
+            if not self._is_path_match(request_url.path, cookie["path"]):
+                continue
+
+            if is_not_secure and cookie["secure"]:
+                continue
+
+            # It's critical we use the Morsel so the coded_value
+            # (based on cookie version) is preserved
+            mrsl_val = cast("Morsel[str]", cookie.get(cookie.key, Morsel()))
+            mrsl_val.set(cookie.key, cookie.value, cookie.coded_value)
+            filtered[name] = mrsl_val
+
+        return filtered
+
+    @staticmethod
+    def _is_domain_match(domain: str, hostname: str) -> bool:
+        """Implements domain matching adhering to RFC 6265."""
+        if hostname == domain:
+            return True
+
+        if not hostname.endswith(domain):
+            return False
+
+        non_matching = hostname[: -len(domain)]
+
+        if not non_matching.endswith("."):
+            return False
+
+        return not is_ip_address(hostname)
+
+    @staticmethod
+    def _is_path_match(req_path: str, cookie_path: str) -> bool:
+        """Implements path matching adhering to RFC 6265."""
+        if not req_path.startswith("/"):
+            req_path = "/"
+
+        if req_path == cookie_path:
+            return True
+
+        if not req_path.startswith(cookie_path):
+            return False
+
+        if cookie_path.endswith("/"):
+            return True
+
+        non_matching = req_path[len(cookie_path) :]
+
+        return non_matching.startswith("/")
+
+    @classmethod
+    def _parse_date(cls, date_str: str) -> Optional[datetime.datetime]:
+        """Implements date string parsing adhering to RFC 6265."""
+        if not date_str:
+            return None
+
+        found_time = False
+        found_day = False
+        found_month = False
+        found_year = False
+
+        hour = minute = second = 0
+        day = 0
+        month = 0
+        year = 0
+
+        for token_match in cls.DATE_TOKENS_RE.finditer(date_str):
+            token = token_match.group("token")
+
+            if not found_time:
+                time_match = cls.DATE_HMS_TIME_RE.match(token)
+                if time_match:
+                    found_time = True
+                    hour, minute, second = (int(s) for s in time_match.groups())
+                    continue
+
+            if not found_day:
+                day_match = cls.DATE_DAY_OF_MONTH_RE.match(token)
+                if day_match:
+                    found_day = True
+                    day = int(day_match.group())
+                    continue
+
+            if not found_month:
+                month_match = cls.DATE_MONTH_RE.match(token)
+                if month_match:
+                    found_month = True
+                    assert month_match.lastindex is not None
+                    month = month_match.lastindex
+                    continue
+
+            if not found_year:
+                year_match = cls.DATE_YEAR_RE.match(token)
+                if year_match:
+                    found_year = True
+                    year = int(year_match.group())
+
+        if 70 <= year <= 99:
+            year += 1900
+        elif 0 <= year <= 69:
+            year += 2000
+
+        if False in (found_day, found_month, found_year, found_time):
+            return None
+
+        if not 1 <= day <= 31:
+            return None
+
+        if year < 1601 or hour > 23 or minute > 59 or second > 59:
+            return None
+
+        return datetime.datetime(
+            year, month, day, hour, minute, second, tzinfo=datetime.timezone.utc
+        )
+
+
+class DummyCookieJar(AbstractCookieJar):
+    """Implements a dummy cookie storage.
+
+    It can be used with the ClientSession when no cookie processing is needed.
+
+    """
+
+    def __iter__(self) -> "Iterator[Morsel[str]]":
+        while False:
+            yield None
+
+    def __len__(self) -> int:
+        return 0
+
+    def clear(self, predicate: Optional[ClearCookiePredicate] = None) -> None:
+        pass
+
+    def clear_domain(self, domain: str) -> None:
+        pass
+
+    def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None:
+        pass
+
+    def filter_cookies(self, request_url: URL) -> "BaseCookie[str]":
+        return SimpleCookie()

aiohttp/formdata.py

@@ -0,0 +1,173 @@
+import io
+from typing import Any, Iterable, List, Optional
+from urllib.parse import urlencode
+
+from multidict import MultiDict, MultiDictProxy
+
+from . import hdrs, multipart, payload
+from .helpers import guess_filename
+from .payload import Payload
+
+__all__ = ("FormData",)
+
+
+class FormData:
+    """Helper class for form body generation.
+
+    Supports multipart/form-data and application/x-www-form-urlencoded.
+    """
+
+    def __init__(
+        self,
+        fields: Iterable[Any] = (),
+        quote_fields: bool = True,
+        charset: Optional[str] = None,
+        boundary: Optional[str] = None,
+    ) -> None:
+        self._boundary = boundary
+        self._writer = multipart.MultipartWriter("form-data", boundary=self._boundary)
+        self._fields: List[Any] = []
+        self._is_multipart = False
+        self._is_processed = False
+        self._quote_fields = quote_fields
+        self._charset = charset
+
+        if isinstance(fields, dict):
+            fields = list(fields.items())
+        elif not isinstance(fields, (list, tuple)):
+            fields = (fields,)
+        self.add_fields(*fields)
+
+    @property
+    def is_multipart(self) -> bool:
+        return self._is_multipart
+
+    def add_field(
+        self,
+        name: str,
+        value: Any,
+        *,
+        content_type: Optional[str] = None,
+        filename: Optional[str] = None,
+        content_transfer_encoding: Optional[str] = None,
+    ) -> None:
+        if isinstance(value, io.IOBase):
+            self._is_multipart = True
+        elif isinstance(value, (bytes, bytearray, memoryview)):
+            if filename is None and content_transfer_encoding is None:
+                filename = name
+
+        type_options: MultiDict[str] = MultiDict({"name": name})
+        if filename is not None and not isinstance(filename, str):
+            raise TypeError(
+                "filename must be an instance of str. " "Got: %s" % filename
+            )
+        if filename is None and isinstance(value, io.IOBase):
+            filename = guess_filename(value, name)
+        if filename is not None:
+            type_options["filename"] = filename
+            self._is_multipart = True
+
+        headers = {}
+        if content_type is not None:
+            if not isinstance(content_type, str):
+                raise TypeError(
+                    "content_type must be an instance of str. " "Got: %s" % content_type
+                )
+            headers[hdrs.CONTENT_TYPE] = content_type
+            self._is_multipart = True
+        if content_transfer_encoding is not None:
+            if not isinstance(content_transfer_encoding, str):
+                raise TypeError(
+                    "content_transfer_encoding must be an instance"
+                    " of str. Got: %s" % content_transfer_encoding
+                )
+            headers[hdrs.CONTENT_TRANSFER_ENCODING] = content_transfer_encoding
+            self._is_multipart = True
+
+        self._fields.append((type_options, headers, value))
+
+    def add_fields(self, *fields: Any) -> None:
+        to_add = list(fields)
+
+        while to_add:
+            rec = to_add.pop(0)
+
+            if isinstance(rec, io.IOBase):
+                k = guess_filename(rec, "unknown")
+                self.add_field(k, rec)  # type: ignore[arg-type]
+
+            elif isinstance(rec, (MultiDictProxy, MultiDict)):
+                to_add.extend(rec.items())
+
+            elif isinstance(rec, (list, tuple)) and len(rec) == 2:
+                k, fp = rec
+                self.add_field(k, fp)  # type: ignore[arg-type]
+
+            else:
+                raise TypeError(
+                    "Only io.IOBase, multidict and (name, file) "
+                    "pairs allowed, use .add_field() for passing "
+                    "more complex parameters, got {!r}".format(rec)
+                )
+
+    def _gen_form_urlencoded(self) -> payload.BytesPayload:
+        # form data (x-www-form-urlencoded)
+        data = []
+        for type_options, _, value in self._fields:
+            data.append((type_options["name"], value))
+
+        charset = self._charset if self._charset is not None else "utf-8"
+
+        if charset == "utf-8":
+            content_type = "application/x-www-form-urlencoded"
+        else:
+            content_type = "application/x-www-form-urlencoded; " "charset=%s" % charset
+
+        return payload.BytesPayload(
+            urlencode(data, doseq=True, encoding=charset).encode(),
+            content_type=content_type,
+        )
+
+    def _gen_form_data(self) -> multipart.MultipartWriter:
+        """Encode a list of fields using the multipart/form-data MIME format"""
+        if self._is_processed:
+            raise RuntimeError("Form data has been processed already")
+        for dispparams, headers, value in self._fields:
+            try:
+                if hdrs.CONTENT_TYPE in headers:
+                    part = payload.get_payload(
+                        value,
+                        content_type=headers[hdrs.CONTENT_TYPE],
+                        headers=headers,
+                        encoding=self._charset,
+                    )
+                else:
+                    part = payload.get_payload(
+                        value, headers=headers, encoding=self._charset
+                    )
+            except Exception as exc:
+                raise TypeError(
+                    "Can not serialize value type: %r\n "
+                    "headers: %r\n value: %r" % (type(value), headers, value)
+                ) from exc
+
+            if dispparams:
+                part.set_content_disposition(
+                    "form-data", quote_fields=self._quote_fields, **dispparams
+                )
+                # FIXME cgi.FieldStorage doesn't likes body parts with
+                # Content-Length which were sent via chunked transfer encoding
+                assert part.headers is not None
+                part.headers.popall(hdrs.CONTENT_LENGTH, None)
+
+            self._writer.append_payload(part)
+
+        self._is_processed = True
+        return self._writer
+
+    def __call__(self) -> Payload:
+        if self._is_multipart:
+            return self._gen_form_data()
+        else:
+            return self._gen_form_urlencoded()

aiohttp/hdrs.py

@@ -0,0 +1,108 @@
+"""HTTP Headers constants."""
+
+# After changing the file content call ./tools/gen.py
+# to regenerate the headers parser
+from typing import Final, Set
+
+from multidict import istr
+
+METH_ANY: Final[str] = "*"
+METH_CONNECT: Final[str] = "CONNECT"
+METH_HEAD: Final[str] = "HEAD"
+METH_GET: Final[str] = "GET"
+METH_DELETE: Final[str] = "DELETE"
+METH_OPTIONS: Final[str] = "OPTIONS"
+METH_PATCH: Final[str] = "PATCH"
+METH_POST: Final[str] = "POST"
+METH_PUT: Final[str] = "PUT"
+METH_TRACE: Final[str] = "TRACE"
+
+METH_ALL: Final[Set[str]] = {
+    METH_CONNECT,
+    METH_HEAD,
+    METH_GET,
+    METH_DELETE,
+    METH_OPTIONS,
+    METH_PATCH,
+    METH_POST,
+    METH_PUT,
+    METH_TRACE,
+}
+
+ACCEPT: Final[istr] = istr("Accept")
+ACCEPT_CHARSET: Final[istr] = istr("Accept-Charset")
+ACCEPT_ENCODING: Final[istr] = istr("Accept-Encoding")
+ACCEPT_LANGUAGE: Final[istr] = istr("Accept-Language")
+ACCEPT_RANGES: Final[istr] = istr("Accept-Ranges")
+ACCESS_CONTROL_MAX_AGE: Final[istr] = istr("Access-Control-Max-Age")
+ACCESS_CONTROL_ALLOW_CREDENTIALS: Final[istr] = istr("Access-Control-Allow-Credentials")
+ACCESS_CONTROL_ALLOW_HEADERS: Final[istr] = istr("Access-Control-Allow-Headers")
+ACCESS_CONTROL_ALLOW_METHODS: Final[istr] = istr("Access-Control-Allow-Methods")
+ACCESS_CONTROL_ALLOW_ORIGIN: Final[istr] = istr("Access-Control-Allow-Origin")
+ACCESS_CONTROL_EXPOSE_HEADERS: Final[istr] = istr("Access-Control-Expose-Headers")
+ACCESS_CONTROL_REQUEST_HEADERS: Final[istr] = istr("Access-Control-Request-Headers")
+ACCESS_CONTROL_REQUEST_METHOD: Final[istr] = istr("Access-Control-Request-Method")
+AGE: Final[istr] = istr("Age")
+ALLOW: Final[istr] = istr("Allow")
+AUTHORIZATION: Final[istr] = istr("Authorization")
+CACHE_CONTROL: Final[istr] = istr("Cache-Control")
+CONNECTION: Final[istr] = istr("Connection")
+CONTENT_DISPOSITION: Final[istr] = istr("Content-Disposition")
+CONTENT_ENCODING: Final[istr] = istr("Content-Encoding")
+CONTENT_LANGUAGE: Final[istr] = istr("Content-Language")
+CONTENT_LENGTH: Final[istr] = istr("Content-Length")
+CONTENT_LOCATION: Final[istr] = istr("Content-Location")
+CONTENT_MD5: Final[istr] = istr("Content-MD5")
+CONTENT_RANGE: Final[istr] = istr("Content-Range")
+CONTENT_TRANSFER_ENCODING: Final[istr] = istr("Content-Transfer-Encoding")
+CONTENT_TYPE: Final[istr] = istr("Content-Type")
+COOKIE: Final[istr] = istr("Cookie")
+DATE: Final[istr] = istr("Date")
+DESTINATION: Final[istr] = istr("Destination")
+DIGEST: Final[istr] = istr("Digest")
+ETAG: Final[istr] = istr("Etag")
+EXPECT: Final[istr] = istr("Expect")
+EXPIRES: Final[istr] = istr("Expires")
+FORWARDED: Final[istr] = istr("Forwarded")
+FROM: Final[istr] = istr("From")
+HOST: Final[istr] = istr("Host")
+IF_MATCH: Final[istr] = istr("If-Match")
+IF_MODIFIED_SINCE: Final[istr] = istr("If-Modified-Since")
+IF_NONE_MATCH: Final[istr] = istr("If-None-Match")
+IF_RANGE: Final[istr] = istr("If-Range")
+IF_UNMODIFIED_SINCE: Final[istr] = istr("If-Unmodified-Since")
+KEEP_ALIVE: Final[istr] = istr("Keep-Alive")
+LAST_EVENT_ID: Final[istr] = istr("Last-Event-ID")
+LAST_MODIFIED: Final[istr] = istr("Last-Modified")
+LINK: Final[istr] = istr("Link")
+LOCATION: Final[istr] = istr("Location")
+MAX_FORWARDS: Final[istr] = istr("Max-Forwards")
+ORIGIN: Final[istr] = istr("Origin")
+PRAGMA: Final[istr] = istr("Pragma")
+PROXY_AUTHENTICATE: Final[istr] = istr("Proxy-Authenticate")
+PROXY_AUTHORIZATION: Final[istr] = istr("Proxy-Authorization")
+RANGE: Final[istr] = istr("Range")
+REFERER: Final[istr] = istr("Referer")
+RETRY_AFTER: Final[istr] = istr("Retry-After")
+SEC_WEBSOCKET_ACCEPT: Final[istr] = istr("Sec-WebSocket-Accept")
+SEC_WEBSOCKET_VERSION: Final[istr] = istr("Sec-WebSocket-Version")
+SEC_WEBSOCKET_PROTOCOL: Final[istr] = istr("Sec-WebSocket-Protocol")
+SEC_WEBSOCKET_EXTENSIONS: Final[istr] = istr("Sec-WebSocket-Extensions")
+SEC_WEBSOCKET_KEY: Final[istr] = istr("Sec-WebSocket-Key")
+SEC_WEBSOCKET_KEY1: Final[istr] = istr("Sec-WebSocket-Key1")
+SERVER: Final[istr] = istr("Server")
+SET_COOKIE: Final[istr] = istr("Set-Cookie")
+TE: Final[istr] = istr("TE")
+TRAILER: Final[istr] = istr("Trailer")
+TRANSFER_ENCODING: Final[istr] = istr("Transfer-Encoding")
+UPGRADE: Final[istr] = istr("Upgrade")
+URI: Final[istr] = istr("URI")
+USER_AGENT: Final[istr] = istr("User-Agent")
+VARY: Final[istr] = istr("Vary")
+VIA: Final[istr] = istr("Via")
+WANT_DIGEST: Final[istr] = istr("Want-Digest")
+WARNING: Final[istr] = istr("Warning")
+WWW_AUTHENTICATE: Final[istr] = istr("WWW-Authenticate")
+X_FORWARDED_FOR: Final[istr] = istr("X-Forwarded-For")
+X_FORWARDED_HOST: Final[istr] = istr("X-Forwarded-Host")
+X_FORWARDED_PROTO: Final[istr] = istr("X-Forwarded-Proto")

aiohttp/http.py

@@ -0,0 +1,60 @@
+import sys
+
+from . import __version__
+from .http_exceptions import HttpProcessingError
+from .http_parser import (
+    HeadersParser,
+    HttpParser,
+    HttpRequestParser,
+    HttpResponseParser,
+    RawRequestMessage,
+    RawResponseMessage,
+)
+from .http_websocket import (
+    WS_CLOSED_MESSAGE,
+    WS_CLOSING_MESSAGE,
+    WS_KEY,
+    WebSocketError,
+    WebSocketReader,
+    WebSocketWriter,
+    WSCloseCode,
+    WSMessage,
+    WSMsgType,
+    ws_ext_gen,
+    ws_ext_parse,
+)
+from .http_writer import HttpVersion, HttpVersion10, HttpVersion11, StreamWriter
+
+__all__ = (
+    "HttpProcessingError",
+    "SERVER_SOFTWARE",
+    # .http_writer
+    "StreamWriter",
+    "HttpVersion",
+    "HttpVersion10",
+    "HttpVersion11",
+    # .http_parser
+    "HeadersParser",
+    "HttpParser",
+    "HttpRequestParser",
+    "HttpResponseParser",
+    "RawRequestMessage",
+    "RawResponseMessage",
+    # .http_websocket
+    "WS_CLOSED_MESSAGE",
+    "WS_CLOSING_MESSAGE",
+    "WS_KEY",
+    "WebSocketReader",
+    "WebSocketWriter",
+    "ws_ext_gen",
+    "ws_ext_parse",
+    "WSMessage",
+    "WebSocketError",
+    "WSMsgType",
+    "WSCloseCode",
+)
+
+
+SERVER_SOFTWARE: str = "Python/{0[0]}.{0[1]} aiohttp/{1}".format(
+    sys.version_info, __version__
+)

aiohttp/http_exceptions.py

@@ -0,0 +1,105 @@
+"""Low-level http related exceptions."""
+
+
+from textwrap import indent
+from typing import Optional, Union
+
+from .typedefs import _CIMultiDict
+
+__all__ = ("HttpProcessingError",)
+
+
+class HttpProcessingError(Exception):
+    """HTTP error.
+
+    Shortcut for raising HTTP errors with custom code, message and headers.
+
+    code: HTTP Error code.
+    message: (optional) Error message.
+    headers: (optional) Headers to be sent in response, a list of pairs
+    """
+
+    code = 0
+    message = ""
+    headers = None
+
+    def __init__(
+        self,
+        *,
+        code: Optional[int] = None,
+        message: str = "",
+        headers: Optional[_CIMultiDict] = None,
+    ) -> None:
+        if code is not None:
+            self.code = code
+        self.headers = headers
+        self.message = message
+
+    def __str__(self) -> str:
+        msg = indent(self.message, "  ")
+        return f"{self.code}, message:\n{msg}"
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}: {self.code}, message={self.message!r}>"
+
+
+class BadHttpMessage(HttpProcessingError):
+    code = 400
+    message = "Bad Request"
+
+    def __init__(self, message: str, *, headers: Optional[_CIMultiDict] = None) -> None:
+        super().__init__(message=message, headers=headers)
+        self.args = (message,)
+
+
+class HttpBadRequest(BadHttpMessage):
+    code = 400
+    message = "Bad Request"
+
+
+class PayloadEncodingError(BadHttpMessage):
+    """Base class for payload errors"""
+
+
+class ContentEncodingError(PayloadEncodingError):
+    """Content encoding error."""
+
+
+class TransferEncodingError(PayloadEncodingError):
+    """transfer encoding error."""
+
+
+class ContentLengthError(PayloadEncodingError):
+    """Not enough data for satisfy content length header."""
+
+
+class LineTooLong(BadHttpMessage):
+    def __init__(
+        self, line: str, limit: str = "Unknown", actual_size: str = "Unknown"
+    ) -> None:
+        super().__init__(
+            f"Got more than {limit} bytes ({actual_size}) when reading {line}."
+        )
+        self.args = (line, limit, actual_size)
+
+
+class InvalidHeader(BadHttpMessage):
+    def __init__(self, hdr: Union[bytes, str]) -> None:
+        if isinstance(hdr, bytes):
+            hdr = hdr.decode("utf-8", "surrogateescape")
+        super().__init__(f"Invalid HTTP Header: {hdr}")
+        self.hdr = hdr
+        self.args = (hdr,)
+
+
+class BadStatusLine(BadHttpMessage):
+    def __init__(self, line: str = "") -> None:
+        if not isinstance(line, str):
+            line = repr(line)
+        super().__init__(f"Bad status line {line!r}")
+        self.args = (line,)
+        self.line = line
+
+
+class InvalidURLError(BadHttpMessage):
+    pass

aiohttp/http_parser.py

@@ -0,0 +1,930 @@
+import abc
+import asyncio
+import re
+import string
+from contextlib import suppress
+from enum import IntEnum
+from typing import (
+    Final,
+    Generic,
+    List,
+    NamedTuple,
+    Optional,
+    Pattern,
+    Set,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
+
+from multidict import CIMultiDict, CIMultiDictProxy, istr
+from yarl import URL
+
+from . import hdrs
+from .base_protocol import BaseProtocol
+from .compression_utils import HAS_BROTLI, BrotliDecompressor, ZLibDecompressor
+from .helpers import NO_EXTENSIONS, BaseTimerContext
+from .http_exceptions import (
+    BadHttpMessage,
+    BadStatusLine,
+    ContentEncodingError,
+    ContentLengthError,
+    InvalidHeader,
+    LineTooLong,
+    TransferEncodingError,
+)
+from .http_writer import HttpVersion, HttpVersion10
+from .log import internal_logger
+from .streams import EMPTY_PAYLOAD, StreamReader
+from .typedefs import RawHeaders
+
+__all__ = (
+    "HeadersParser",
+    "HttpParser",
+    "HttpRequestParser",
+    "HttpResponseParser",
+    "RawRequestMessage",
+    "RawResponseMessage",
+)
+
+ASCIISET: Final[Set[str]] = set(string.printable)
+
+# See https://tools.ietf.org/html/rfc7230#section-3.1.1
+# and https://tools.ietf.org/html/rfc7230#appendix-B
+#
+#     method = token
+#     tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." /
+#             "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA
+#     token = 1*tchar
+METHRE: Final[Pattern[str]] = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
+VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d+).(\d+)")
+HDRRE: Final[Pattern[bytes]] = re.compile(rb"[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\\\"]")
+
+
+class RawRequestMessage(NamedTuple):
+    method: str
+    path: str
+    version: HttpVersion
+    headers: CIMultiDictProxy[str]
+    raw_headers: RawHeaders
+    should_close: bool
+    compression: Optional[str]
+    upgrade: bool
+    chunked: bool
+    url: URL
+
+
+class RawResponseMessage(NamedTuple):
+    version: HttpVersion
+    code: int
+    reason: str
+    headers: CIMultiDictProxy[str]
+    raw_headers: RawHeaders
+    should_close: bool
+    compression: Optional[str]
+    upgrade: bool
+    chunked: bool
+
+
+_MsgT = TypeVar("_MsgT", RawRequestMessage, RawResponseMessage)
+
+
+class ParseState(IntEnum):
+    PARSE_NONE = 0
+    PARSE_LENGTH = 1
+    PARSE_CHUNKED = 2
+    PARSE_UNTIL_EOF = 3
+
+
+class ChunkState(IntEnum):
+    PARSE_CHUNKED_SIZE = 0
+    PARSE_CHUNKED_CHUNK = 1
+    PARSE_CHUNKED_CHUNK_EOF = 2
+    PARSE_MAYBE_TRAILERS = 3
+    PARSE_TRAILERS = 4
+
+
+class HeadersParser:
+    def __init__(
+        self,
+        max_line_size: int = 8190,
+        max_field_size: int = 8190,
+    ) -> None:
+        self.max_line_size = max_line_size
+        self.max_field_size = max_field_size
+
+    def parse_headers(
+        self, lines: List[bytes]
+    ) -> Tuple["CIMultiDictProxy[str]", RawHeaders]:
+        headers: CIMultiDict[str] = CIMultiDict()
+        raw_headers = []
+
+        lines_idx = 1
+        line = lines[1]
+        line_count = len(lines)
+
+        while line:
+            # Parse initial header name : value pair.
+            try:
+                bname, bvalue = line.split(b":", 1)
+            except ValueError:
+                raise InvalidHeader(line) from None
+
+            bname = bname.strip(b" \t")
+            bvalue = bvalue.lstrip()
+            if HDRRE.search(bname):
+                raise InvalidHeader(bname)
+            if len(bname) > self.max_field_size:
+                raise LineTooLong(
+                    "request header name {}".format(
+                        bname.decode("utf8", "backslashreplace")
+                    ),
+                    str(self.max_field_size),
+                    str(len(bname)),
+                )
+
+            header_length = len(bvalue)
+
+            # next line
+            lines_idx += 1
+            line = lines[lines_idx]
+
+            # consume continuation lines
+            continuation = line and line[0] in (32, 9)  # (' ', '\t')
+
+            if continuation:
+                bvalue_lst = [bvalue]
+                while continuation:
+                    header_length += len(line)
+                    if header_length > self.max_field_size:
+                        raise LineTooLong(
+                            "request header field {}".format(
+                                bname.decode("utf8", "backslashreplace")
+                            ),
+                            str(self.max_field_size),
+                            str(header_length),
+                        )
+                    bvalue_lst.append(line)
+
+                    # next line
+                    lines_idx += 1
+                    if lines_idx < line_count:
+                        line = lines[lines_idx]
+                        if line:
+                            continuation = line[0] in (32, 9)  # (' ', '\t')
+                    else:
+                        line = b""
+                        break
+                bvalue = b"".join(bvalue_lst)
+            else:
+                if header_length > self.max_field_size:
+                    raise LineTooLong(
+                        "request header field {}".format(
+                            bname.decode("utf8", "backslashreplace")
+                        ),
+                        str(self.max_field_size),
+                        str(header_length),
+                    )
+
+            bvalue = bvalue.strip()
+            name = bname.decode("utf-8", "surrogateescape")
+            value = bvalue.decode("utf-8", "surrogateescape")
+
+            headers.add(name, value)
+            raw_headers.append((bname, bvalue))
+
+        return (CIMultiDictProxy(headers), tuple(raw_headers))
+
+
+class HttpParser(abc.ABC, Generic[_MsgT]):
+    def __init__(
+        self,
+        protocol: BaseProtocol,
+        loop: asyncio.AbstractEventLoop,
+        limit: int,
+        max_line_size: int = 8190,
+        max_field_size: int = 8190,
+        timer: Optional[BaseTimerContext] = None,
+        code: Optional[int] = None,
+        method: Optional[str] = None,
+        readall: bool = False,
+        payload_exception: Optional[Type[BaseException]] = None,
+        response_with_body: bool = True,
+        read_until_eof: bool = False,
+        auto_decompress: bool = True,
+    ) -> None:
+        self.protocol = protocol
+        self.loop = loop
+        self.max_line_size = max_line_size
+        self.max_field_size = max_field_size
+        self.timer = timer
+        self.code = code
+        self.method = method
+        self.readall = readall
+        self.payload_exception = payload_exception
+        self.response_with_body = response_with_body
+        self.read_until_eof = read_until_eof
+
+        self._lines: List[bytes] = []
+        self._tail = b""
+        self._upgraded = False
+        self._payload = None
+        self._payload_parser: Optional[HttpPayloadParser] = None
+        self._auto_decompress = auto_decompress
+        self._limit = limit
+        self._headers_parser = HeadersParser(max_line_size, max_field_size)
+
+    @abc.abstractmethod
+    def parse_message(self, lines: List[bytes]) -> _MsgT:
+        pass
+
+    def feed_eof(self) -> Optional[_MsgT]:
+        if self._payload_parser is not None:
+            self._payload_parser.feed_eof()
+            self._payload_parser = None
+        else:
+            # try to extract partial message
+            if self._tail:
+                self._lines.append(self._tail)
+
+            if self._lines:
+                if self._lines[-1] != "\r\n":
+                    self._lines.append(b"")
+                with suppress(Exception):
+                    return self.parse_message(self._lines)
+        return None
+
+    def feed_data(
+        self,
+        data: bytes,
+        SEP: bytes = b"\r\n",
+        EMPTY: bytes = b"",
+        CONTENT_LENGTH: istr = hdrs.CONTENT_LENGTH,
+        METH_CONNECT: str = hdrs.METH_CONNECT,
+        SEC_WEBSOCKET_KEY1: istr = hdrs.SEC_WEBSOCKET_KEY1,
+    ) -> Tuple[List[Tuple[_MsgT, StreamReader]], bool, bytes]:
+        messages = []
+
+        if self._tail:
+            data, self._tail = self._tail + data, b""
+
+        data_len = len(data)
+        start_pos = 0
+        loop = self.loop
+
+        while start_pos < data_len:
+            # read HTTP message (request/response line + headers), \r\n\r\n
+            # and split by lines
+            if self._payload_parser is None and not self._upgraded:
+                pos = data.find(SEP, start_pos)
+                # consume \r\n
+                if pos == start_pos and not self._lines:
+                    start_pos = pos + 2
+                    continue
+
+                if pos >= start_pos:
+                    # line found
+                    self._lines.append(data[start_pos:pos])
+                    start_pos = pos + 2
+
+                    # \r\n\r\n found
+                    if self._lines[-1] == EMPTY:
+                        try:
+                            msg: _MsgT = self.parse_message(self._lines)
+                        finally:
+                            self._lines.clear()
+
+                        def get_content_length() -> Optional[int]:
+                            # payload length
+                            length_hdr = msg.headers.get(CONTENT_LENGTH)
+                            if length_hdr is None:
+                                return None
+
+                            try:
+                                length = int(length_hdr)
+                            except ValueError:
+                                raise InvalidHeader(CONTENT_LENGTH)
+
+                            if length < 0:
+                                raise InvalidHeader(CONTENT_LENGTH)
+
+                            return length
+
+                        length = get_content_length()
+                        # do not support old websocket spec
+                        if SEC_WEBSOCKET_KEY1 in msg.headers:
+                            raise InvalidHeader(SEC_WEBSOCKET_KEY1)
+
+                        self._upgraded = msg.upgrade
+
+                        method = getattr(msg, "method", self.method)
+
+                        assert self.protocol is not None
+                        # calculate payload
+                        if (
+                            (length is not None and length > 0)
+                            or msg.chunked
+                            and not msg.upgrade
+                        ):
+                            payload = StreamReader(
+                                self.protocol,
+                                timer=self.timer,
+                                loop=loop,
+                                limit=self._limit,
+                            )
+                            payload_parser = HttpPayloadParser(
+                                payload,
+                                length=length,
+                                chunked=msg.chunked,
+                                method=method,
+                                compression=msg.compression,
+                                code=self.code,
+                                readall=self.readall,
+                                response_with_body=self.response_with_body,
+                                auto_decompress=self._auto_decompress,
+                            )
+                            if not payload_parser.done:
+                                self._payload_parser = payload_parser
+                        elif method == METH_CONNECT:
+                            assert isinstance(msg, RawRequestMessage)
+                            payload = StreamReader(
+                                self.protocol,
+                                timer=self.timer,
+                                loop=loop,
+                                limit=self._limit,
+                            )
+                            self._upgraded = True
+                            self._payload_parser = HttpPayloadParser(
+                                payload,
+                                method=msg.method,
+                                compression=msg.compression,
+                                readall=True,
+                                auto_decompress=self._auto_decompress,
+                            )
+                        else:
+                            if (
+                                getattr(msg, "code", 100) >= 199
+                                and length is None
+                                and self.read_until_eof
+                            ):
+                                payload = StreamReader(
+                                    self.protocol,
+                                    timer=self.timer,
+                                    loop=loop,
+                                    limit=self._limit,
+                                )
+                                payload_parser = HttpPayloadParser(
+                                    payload,
+                                    length=length,
+                                    chunked=msg.chunked,
+                                    method=method,
+                                    compression=msg.compression,
+                                    code=self.code,
+                                    readall=True,
+                                    response_with_body=self.response_with_body,
+                                    auto_decompress=self._auto_decompress,
+                                )
+                                if not payload_parser.done:
+                                    self._payload_parser = payload_parser
+                            else:
+                                payload = EMPTY_PAYLOAD
+
+                        messages.append((msg, payload))
+                else:
+                    self._tail = data[start_pos:]
+                    data = EMPTY
+                    break
+
+            # no parser, just store
+            elif self._payload_parser is None and self._upgraded:
+                assert not self._lines
+                break
+
+            # feed payload
+            elif data and start_pos < data_len:
+                assert not self._lines
+                assert self._payload_parser is not None
+                try:
+                    eof, data = self._payload_parser.feed_data(data[start_pos:])
+                except BaseException as exc:
+                    if self.payload_exception is not None:
+                        self._payload_parser.payload.set_exception(
+                            self.payload_exception(str(exc))
+                        )
+                    else:
+                        self._payload_parser.payload.set_exception(exc)
+
+                    eof = True
+                    data = b""
+
+                if eof:
+                    start_pos = 0
+                    data_len = len(data)
+                    self._payload_parser = None
+                    continue
+            else:
+                break
+
+        if data and start_pos < data_len:
+            data = data[start_pos:]
+        else:
+            data = EMPTY
+
+        return messages, self._upgraded, data
+
+    def parse_headers(
+        self, lines: List[bytes]
+    ) -> Tuple[
+        "CIMultiDictProxy[str]", RawHeaders, Optional[bool], Optional[str], bool, bool
+    ]:
+        """Parses RFC 5322 headers from a stream.
+
+        Line continuations are supported. Returns list of header name
+        and value pairs. Header name is in upper case.
+        """
+        headers, raw_headers = self._headers_parser.parse_headers(lines)
+        close_conn = None
+        encoding = None
+        upgrade = False
+        chunked = False
+
+        # keep-alive
+        conn = headers.get(hdrs.CONNECTION)
+        if conn:
+            v = conn.lower()
+            if v == "close":
+                close_conn = True
+            elif v == "keep-alive":
+                close_conn = False
+            elif v == "upgrade":
+                upgrade = True
+
+        # encoding
+        enc = headers.get(hdrs.CONTENT_ENCODING)
+        if enc:
+            enc = enc.lower()
+            if enc in ("gzip", "deflate", "br"):
+                encoding = enc
+
+        # chunking
+        te = headers.get(hdrs.TRANSFER_ENCODING)
+        if te is not None:
+            if "chunked" == te.lower():
+                chunked = True
+            else:
+                raise BadHttpMessage("Request has invalid `Transfer-Encoding`")
+
+            if hdrs.CONTENT_LENGTH in headers:
+                raise BadHttpMessage(
+                    "Transfer-Encoding can't be present with Content-Length",
+                )
+
+        return (headers, raw_headers, close_conn, encoding, upgrade, chunked)
+
+    def set_upgraded(self, val: bool) -> None:
+        """Set connection upgraded (to websocket) mode.
+
+        :param bool val: new state.
+        """
+        self._upgraded = val
+
+
+class HttpRequestParser(HttpParser[RawRequestMessage]):
+    """Read request status line.
+
+    Exception .http_exceptions.BadStatusLine
+    could be raised in case of any errors in status line.
+    Returns RawRequestMessage.
+    """
+
+    def parse_message(self, lines: List[bytes]) -> RawRequestMessage:
+        # request line
+        line = lines[0].decode("utf-8", "surrogateescape")
+        try:
+            method, path, version = line.split(None, 2)
+        except ValueError:
+            raise BadStatusLine(line) from None
+
+        if len(path) > self.max_line_size:
+            raise LineTooLong(
+                "Status line is too long", str(self.max_line_size), str(len(path))
+            )
+
+        # method
+        if not METHRE.match(method):
+            raise BadStatusLine(method)
+
+        # version
+        try:
+            if version.startswith("HTTP/"):
+                n1, n2 = version[5:].split(".", 1)
+                version_o = HttpVersion(int(n1), int(n2))
+            else:
+                raise BadStatusLine(version)
+        except Exception:
+            raise BadStatusLine(version)
+
+        if method == "CONNECT":
+            # authority-form,
+            # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3
+            url = URL.build(authority=path, encoded=True)
+        elif path.startswith("/"):
+            # origin-form,
+            # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1
+            path_part, _hash_separator, url_fragment = path.partition("#")
+            path_part, _question_mark_separator, qs_part = path_part.partition("?")
+
+            # NOTE: `yarl.URL.build()` is used to mimic what the Cython-based
+            # NOTE: parser does, otherwise it results into the same
+            # NOTE: HTTP Request-Line input producing different
+            # NOTE: `yarl.URL()` objects
+            url = URL.build(
+                path=path_part,
+                query_string=qs_part,
+                fragment=url_fragment,
+                encoded=True,
+            )
+        else:
+            # absolute-form for proxy maybe,
+            # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2
+            url = URL(path, encoded=True)
+
+        # read headers
+        (
+            headers,
+            raw_headers,
+            close,
+            compression,
+            upgrade,
+            chunked,
+        ) = self.parse_headers(lines)
+
+        if close is None:  # then the headers weren't set in the request
+            if version_o <= HttpVersion10:  # HTTP 1.0 must asks to not close
+                close = True
+            else:  # HTTP 1.1 must ask to close.
+                close = False
+
+        return RawRequestMessage(
+            method,
+            path,
+            version_o,
+            headers,
+            raw_headers,
+            close,
+            compression,
+            upgrade,
+            chunked,
+            url,
+        )
+
+
+class HttpResponseParser(HttpParser[RawResponseMessage]):
+    """Read response status line and headers.
+
+    BadStatusLine could be raised in case of any errors in status line.
+    Returns RawResponseMessage.
+    """
+
+    def parse_message(self, lines: List[bytes]) -> RawResponseMessage:
+        line = lines[0].decode("utf-8", "surrogateescape")
+        try:
+            version, status = line.split(None, 1)
+        except ValueError:
+            raise BadStatusLine(line) from None
+
+        try:
+            status, reason = status.split(None, 1)
+        except ValueError:
+            reason = ""
+
+        if len(reason) > self.max_line_size:
+            raise LineTooLong(
+                "Status line is too long", str(self.max_line_size), str(len(reason))
+            )
+
+        # version
+        match = VERSRE.match(version)
+        if match is None:
+            raise BadStatusLine(line)
+        version_o = HttpVersion(int(match.group(1)), int(match.group(2)))
+
+        # The status code is a three-digit number
+        try:
+            status_i = int(status)
+        except ValueError:
+            raise BadStatusLine(line) from None
+
+        if status_i > 999:
+            raise BadStatusLine(line)
+
+        # read headers
+        (
+            headers,
+            raw_headers,
+            close,
+            compression,
+            upgrade,
+            chunked,
+        ) = self.parse_headers(lines)
+
+        if close is None:
+            close = version_o <= HttpVersion10
+
+        return RawResponseMessage(
+            version_o,
+            status_i,
+            reason.strip(),
+            headers,
+            raw_headers,
+            close,
+            compression,
+            upgrade,
+            chunked,
+        )
+
+
+class HttpPayloadParser:
+    def __init__(
+        self,
+        payload: StreamReader,
+        length: Optional[int] = None,
+        chunked: bool = False,
+        compression: Optional[str] = None,
+        code: Optional[int] = None,
+        method: Optional[str] = None,
+        readall: bool = False,
+        response_with_body: bool = True,
+        auto_decompress: bool = True,
+    ) -> None:
+        self._length = 0
+        self._type = ParseState.PARSE_NONE
+        self._chunk = ChunkState.PARSE_CHUNKED_SIZE
+        self._chunk_size = 0
+        self._chunk_tail = b""
+        self._auto_decompress = auto_decompress
+        self.done = False
+
+        # payload decompression wrapper
+        if response_with_body and compression and self._auto_decompress:
+            real_payload: Union[StreamReader, DeflateBuffer] = DeflateBuffer(
+                payload, compression
+            )
+        else:
+            real_payload = payload
+
+        # payload parser
+        if not response_with_body:
+            # don't parse payload if it's not expected to be received
+            self._type = ParseState.PARSE_NONE
+            real_payload.feed_eof()
+            self.done = True
+
+        elif chunked:
+            self._type = ParseState.PARSE_CHUNKED
+        elif length is not None:
+            self._type = ParseState.PARSE_LENGTH
+            self._length = length
+            if self._length == 0:
+                real_payload.feed_eof()
+                self.done = True
+        else:
+            if readall and code != 204:
+                self._type = ParseState.PARSE_UNTIL_EOF
+            elif method in ("PUT", "POST"):
+                internal_logger.warning(  # pragma: no cover
+                    "Content-Length or Transfer-Encoding header is required"
+                )
+                self._type = ParseState.PARSE_NONE
+                real_payload.feed_eof()
+                self.done = True
+
+        self.payload = real_payload
+
+    def feed_eof(self) -> None:
+        if self._type == ParseState.PARSE_UNTIL_EOF:
+            self.payload.feed_eof()
+        elif self._type == ParseState.PARSE_LENGTH:
+            raise ContentLengthError(
+                "Not enough data for satisfy content length header."
+            )
+        elif self._type == ParseState.PARSE_CHUNKED:
+            raise TransferEncodingError(
+                "Not enough data for satisfy transfer length header."
+            )
+
+    def feed_data(
+        self, chunk: bytes, SEP: bytes = b"\r\n", CHUNK_EXT: bytes = b";"
+    ) -> Tuple[bool, bytes]:
+        # Read specified amount of bytes
+        if self._type == ParseState.PARSE_LENGTH:
+            required = self._length
+            chunk_len = len(chunk)
+
+            if required >= chunk_len:
+                self._length = required - chunk_len
+                self.payload.feed_data(chunk, chunk_len)
+                if self._length == 0:
+                    self.payload.feed_eof()
+                    return True, b""
+            else:
+                self._length = 0
+                self.payload.feed_data(chunk[:required], required)
+                self.payload.feed_eof()
+                return True, chunk[required:]
+
+        # Chunked transfer encoding parser
+        elif self._type == ParseState.PARSE_CHUNKED:
+            if self._chunk_tail:
+                chunk = self._chunk_tail + chunk
+                self._chunk_tail = b""
+
+            while chunk:
+                # read next chunk size
+                if self._chunk == ChunkState.PARSE_CHUNKED_SIZE:
+                    pos = chunk.find(SEP)
+                    if pos >= 0:
+                        i = chunk.find(CHUNK_EXT, 0, pos)
+                        if i >= 0:
+                            size_b = chunk[:i]  # strip chunk-extensions
+                        else:
+                            size_b = chunk[:pos]
+
+                        try:
+                            size = int(bytes(size_b), 16)
+                        except ValueError:
+                            exc = TransferEncodingError(
+                                chunk[:pos].decode("ascii", "surrogateescape")
+                            )
+                            self.payload.set_exception(exc)
+                            raise exc from None
+
+                        chunk = chunk[pos + 2 :]
+                        if size == 0:  # eof marker
+                            self._chunk = ChunkState.PARSE_MAYBE_TRAILERS
+                        else:
+                            self._chunk = ChunkState.PARSE_CHUNKED_CHUNK
+                            self._chunk_size = size
+                            self.payload.begin_http_chunk_receiving()
+                    else:
+                        self._chunk_tail = chunk
+                        return False, b""
+
+                # read chunk and feed buffer
+                if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK:
+                    required = self._chunk_size
+                    chunk_len = len(chunk)
+
+                    if required > chunk_len:
+                        self._chunk_size = required - chunk_len
+                        self.payload.feed_data(chunk, chunk_len)
+                        return False, b""
+                    else:
+                        self._chunk_size = 0
+                        self.payload.feed_data(chunk[:required], required)
+                        chunk = chunk[required:]
+                        self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF
+                        self.payload.end_http_chunk_receiving()
+
+                # toss the CRLF at the end of the chunk
+                if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK_EOF:
+                    if chunk[:2] == SEP:
+                        chunk = chunk[2:]
+                        self._chunk = ChunkState.PARSE_CHUNKED_SIZE
+                    else:
+                        self._chunk_tail = chunk
+                        return False, b""
+
+                # if stream does not contain trailer, after 0\r\n
+                # we should get another \r\n otherwise
+                # trailers needs to be skiped until \r\n\r\n
+                if self._chunk == ChunkState.PARSE_MAYBE_TRAILERS:
+                    head = chunk[:2]
+                    if head == SEP:
+                        # end of stream
+                        self.payload.feed_eof()
+                        return True, chunk[2:]
+                    # Both CR and LF, or only LF may not be received yet. It is
+                    # expected that CRLF or LF will be shown at the very first
+                    # byte next time, otherwise trailers should come. The last
+                    # CRLF which marks the end of response might not be
+                    # contained in the same TCP segment which delivered the
+                    # size indicator.
+                    if not head:
+                        return False, b""
+                    if head == SEP[:1]:
+                        self._chunk_tail = head
+                        return False, b""
+                    self._chunk = ChunkState.PARSE_TRAILERS
+
+                # read and discard trailer up to the CRLF terminator
+                if self._chunk == ChunkState.PARSE_TRAILERS:
+                    pos = chunk.find(SEP)
+                    if pos >= 0:
+                        chunk = chunk[pos + 2 :]
+                        self._chunk = ChunkState.PARSE_MAYBE_TRAILERS
+                    else:
+                        self._chunk_tail = chunk
+                        return False, b""
+
+        # Read all bytes until eof
+        elif self._type == ParseState.PARSE_UNTIL_EOF:
+            self.payload.feed_data(chunk, len(chunk))
+
+        return False, b""
+
+
+class DeflateBuffer:
+    """DeflateStream decompress stream and feed data into specified stream."""
+
+    def __init__(self, out: StreamReader, encoding: Optional[str]) -> None:
+        self.out = out
+        self.size = 0
+        self.encoding = encoding
+        self._started_decoding = False
+
+        self.decompressor: Union[BrotliDecompressor, ZLibDecompressor]
+        if encoding == "br":
+            if not HAS_BROTLI:  # pragma: no cover
+                raise ContentEncodingError(
+                    "Can not decode content-encoding: brotli (br). "
+                    "Please install `Brotli`"
+                )
+            self.decompressor = BrotliDecompressor()
+        else:
+            self.decompressor = ZLibDecompressor(encoding=encoding)
+
+    def set_exception(self, exc: BaseException) -> None:
+        self.out.set_exception(exc)
+
+    def feed_data(self, chunk: bytes, size: int) -> None:
+        if not size:
+            return
+
+        self.size += size
+
+        # RFC1950
+        # bits 0..3 = CM = 0b1000 = 8 = "deflate"
+        # bits 4..7 = CINFO = 1..7 = windows size.
+        if (
+            not self._started_decoding
+            and self.encoding == "deflate"
+            and chunk[0] & 0xF != 8
+        ):
+            # Change the decoder to decompress incorrectly compressed data
+            # Actually we should issue a warning about non-RFC-compliant data.
+            self.decompressor = ZLibDecompressor(
+                encoding=self.encoding, suppress_deflate_header=True
+            )
+
+        try:
+            chunk = self.decompressor.decompress_sync(chunk)
+        except Exception:
+            raise ContentEncodingError(
+                "Can not decode content-encoding: %s" % self.encoding
+            )
+
+        self._started_decoding = True
+
+        if chunk:
+            self.out.feed_data(chunk, len(chunk))
+
+    def feed_eof(self) -> None:
+        chunk = self.decompressor.flush()
+
+        if chunk or self.size > 0:
+            self.out.feed_data(chunk, len(chunk))
+            # decompressor is not brotli unless encoding is "br"
+            if self.encoding == "deflate" and not self.decompressor.eof:  # type: ignore[union-attr]
+                raise ContentEncodingError("deflate")
+
+        self.out.feed_eof()
+
+    def begin_http_chunk_receiving(self) -> None:
+        self.out.begin_http_chunk_receiving()
+
+    def end_http_chunk_receiving(self) -> None:
+        self.out.end_http_chunk_receiving()
+
+
+HttpRequestParserPy = HttpRequestParser
+HttpResponseParserPy = HttpResponseParser
+RawRequestMessagePy = RawRequestMessage
+RawResponseMessagePy = RawResponseMessage
+
+try:
+    if not NO_EXTENSIONS:
+        from ._http_parser import (  # type: ignore[import,no-redef]
+            HttpRequestParser,
+            HttpResponseParser,
+            RawRequestMessage,
+            RawResponseMessage,
+        )
+
+        HttpRequestParserC = HttpRequestParser
+        HttpResponseParserC = HttpResponseParser
+        RawRequestMessageC = RawRequestMessage
+        RawResponseMessageC = RawResponseMessage
+except ImportError:  # pragma: no cover
+    pass

aiohttp/http_websocket.py

@@ -0,0 +1,710 @@
+"""WebSocket protocol versions 13 and 8."""
+
+import asyncio
+import functools
+import json
+import random
+import re
+import sys
+import zlib
+from enum import IntEnum
+from struct import Struct
+from typing import (
+    Any,
+    Callable,
+    Final,
+    List,
+    NamedTuple,
+    Optional,
+    Pattern,
+    Set,
+    Tuple,
+    Union,
+    cast,
+)
+
+from .base_protocol import BaseProtocol
+from .compression_utils import ZLibCompressor, ZLibDecompressor
+from .helpers import NO_EXTENSIONS
+from .streams import DataQueue
+
+__all__ = (
+    "WS_CLOSED_MESSAGE",
+    "WS_CLOSING_MESSAGE",
+    "WS_KEY",
+    "WebSocketReader",
+    "WebSocketWriter",
+    "WSMessage",
+    "WebSocketError",
+    "WSMsgType",
+    "WSCloseCode",
+)
+
+
+class WSCloseCode(IntEnum):
+    OK = 1000
+    GOING_AWAY = 1001
+    PROTOCOL_ERROR = 1002
+    UNSUPPORTED_DATA = 1003
+    ABNORMAL_CLOSURE = 1006
+    INVALID_TEXT = 1007
+    POLICY_VIOLATION = 1008
+    MESSAGE_TOO_BIG = 1009
+    MANDATORY_EXTENSION = 1010
+    INTERNAL_ERROR = 1011
+    SERVICE_RESTART = 1012
+    TRY_AGAIN_LATER = 1013
+    BAD_GATEWAY = 1014
+
+
+ALLOWED_CLOSE_CODES: Final[Set[int]] = {int(i) for i in WSCloseCode}
+
+
+class WSMsgType(IntEnum):
+    # websocket spec types
+    CONTINUATION = 0x0
+    TEXT = 0x1
+    BINARY = 0x2
+    PING = 0x9
+    PONG = 0xA
+    CLOSE = 0x8
+
+    # aiohttp specific types
+    CLOSING = 0x100
+    CLOSED = 0x101
+    ERROR = 0x102
+
+
+WS_KEY: Final[bytes] = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+
+
+UNPACK_LEN2 = Struct("!H").unpack_from
+UNPACK_LEN3 = Struct("!Q").unpack_from
+UNPACK_CLOSE_CODE = Struct("!H").unpack
+PACK_LEN1 = Struct("!BB").pack
+PACK_LEN2 = Struct("!BBH").pack
+PACK_LEN3 = Struct("!BBQ").pack
+PACK_CLOSE_CODE = Struct("!H").pack
+MSG_SIZE: Final[int] = 2**14
+DEFAULT_LIMIT: Final[int] = 2**16
+
+
+class WSMessage(NamedTuple):
+    type: WSMsgType
+    # To type correctly, this would need some kind of tagged union for each type.
+    data: Any
+    extra: Optional[str]
+
+    def json(self, *, loads: Callable[[Any], Any] = json.loads) -> Any:
+        """Return parsed JSON data.
+
+        .. versionadded:: 0.22
+        """
+        return loads(self.data)
+
+
+WS_CLOSED_MESSAGE = WSMessage(WSMsgType.CLOSED, None, None)
+WS_CLOSING_MESSAGE = WSMessage(WSMsgType.CLOSING, None, None)
+
+
+class WebSocketError(Exception):
+    """WebSocket protocol parser error."""
+
+    def __init__(self, code: int, message: str) -> None:
+        self.code = code
+        super().__init__(code, message)
+
+    def __str__(self) -> str:
+        return cast(str, self.args[1])
+
+
+class WSHandshakeError(Exception):
+    """WebSocket protocol handshake error."""
+
+
+native_byteorder: Final[str] = sys.byteorder
+
+
+# Used by _websocket_mask_python
+@functools.lru_cache()
+def _xor_table() -> List[bytes]:
+    return [bytes(a ^ b for a in range(256)) for b in range(256)]
+
+
+def _websocket_mask_python(mask: bytes, data: bytearray) -> None:
+    """Websocket masking function.
+
+    `mask` is a `bytes` object of length 4; `data` is a `bytearray`
+    object of any length. The contents of `data` are masked with `mask`,
+    as specified in section 5.3 of RFC 6455.
+
+    Note that this function mutates the `data` argument.
+
+    This pure-python implementation may be replaced by an optimized
+    version when available.
+
+    """
+    assert isinstance(data, bytearray), data
+    assert len(mask) == 4, mask
+
+    if data:
+        _XOR_TABLE = _xor_table()
+        a, b, c, d = (_XOR_TABLE[n] for n in mask)
+        data[::4] = data[::4].translate(a)
+        data[1::4] = data[1::4].translate(b)
+        data[2::4] = data[2::4].translate(c)
+        data[3::4] = data[3::4].translate(d)
+
+
+if NO_EXTENSIONS:  # pragma: no cover
+    _websocket_mask = _websocket_mask_python
+else:
+    try:
+        from ._websocket import _websocket_mask_cython  # type: ignore[import]
+
+        _websocket_mask = _websocket_mask_cython
+    except ImportError:  # pragma: no cover
+        _websocket_mask = _websocket_mask_python
+
+_WS_DEFLATE_TRAILING: Final[bytes] = bytes([0x00, 0x00, 0xFF, 0xFF])
+
+
+_WS_EXT_RE: Final[Pattern[str]] = re.compile(
+    r"^(?:;\s*(?:"
+    r"(server_no_context_takeover)|"
+    r"(client_no_context_takeover)|"
+    r"(server_max_window_bits(?:=(\d+))?)|"
+    r"(client_max_window_bits(?:=(\d+))?)))*$"
+)
+
+_WS_EXT_RE_SPLIT: Final[Pattern[str]] = re.compile(r"permessage-deflate([^,]+)?")
+
+
+def ws_ext_parse(extstr: Optional[str], isserver: bool = False) -> Tuple[int, bool]:
+    if not extstr:
+        return 0, False
+
+    compress = 0
+    notakeover = False
+    for ext in _WS_EXT_RE_SPLIT.finditer(extstr):
+        defext = ext.group(1)
+        # Return compress = 15 when get `permessage-deflate`
+        if not defext:
+            compress = 15
+            break
+        match = _WS_EXT_RE.match(defext)
+        if match:
+            compress = 15
+            if isserver:
+                # Server never fail to detect compress handshake.
+                # Server does not need to send max wbit to client
+                if match.group(4):
+                    compress = int(match.group(4))
+                    # Group3 must match if group4 matches
+                    # Compress wbit 8 does not support in zlib
+                    # If compress level not support,
+                    # CONTINUE to next extension
+                    if compress > 15 or compress < 9:
+                        compress = 0
+                        continue
+                if match.group(1):
+                    notakeover = True
+                # Ignore regex group 5 & 6 for client_max_window_bits
+                break
+            else:
+                if match.group(6):
+                    compress = int(match.group(6))
+                    # Group5 must match if group6 matches
+                    # Compress wbit 8 does not support in zlib
+                    # If compress level not support,
+                    # FAIL the parse progress
+                    if compress > 15 or compress < 9:
+                        raise WSHandshakeError("Invalid window size")
+                if match.group(2):
+                    notakeover = True
+                # Ignore regex group 5 & 6 for client_max_window_bits
+                break
+        # Return Fail if client side and not match
+        elif not isserver:
+            raise WSHandshakeError("Extension for deflate not supported" + ext.group(1))
+
+    return compress, notakeover
+
+
+def ws_ext_gen(
+    compress: int = 15, isserver: bool = False, server_notakeover: bool = False
+) -> str:
+    # client_notakeover=False not used for server
+    # compress wbit 8 does not support in zlib
+    if compress < 9 or compress > 15:
+        raise ValueError(
+            "Compress wbits must between 9 and 15, " "zlib does not support wbits=8"
+        )
+    enabledext = ["permessage-deflate"]
+    if not isserver:
+        enabledext.append("client_max_window_bits")
+
+    if compress < 15:
+        enabledext.append("server_max_window_bits=" + str(compress))
+    if server_notakeover:
+        enabledext.append("server_no_context_takeover")
+    # if client_notakeover:
+    #     enabledext.append('client_no_context_takeover')
+    return "; ".join(enabledext)
+
+
+class WSParserState(IntEnum):
+    READ_HEADER = 1
+    READ_PAYLOAD_LENGTH = 2
+    READ_PAYLOAD_MASK = 3
+    READ_PAYLOAD = 4
+
+
+class WebSocketReader:
+    def __init__(
+        self, queue: DataQueue[WSMessage], max_msg_size: int, compress: bool = True
+    ) -> None:
+        self.queue = queue
+        self._max_msg_size = max_msg_size
+
+        self._exc: Optional[BaseException] = None
+        self._partial = bytearray()
+        self._state = WSParserState.READ_HEADER
+
+        self._opcode: Optional[int] = None
+        self._frame_fin = False
+        self._frame_opcode: Optional[int] = None
+        self._frame_payload = bytearray()
+
+        self._tail = b""
+        self._has_mask = False
+        self._frame_mask: Optional[bytes] = None
+        self._payload_length = 0
+        self._payload_length_flag = 0
+        self._compressed: Optional[bool] = None
+        self._decompressobj: Optional[ZLibDecompressor] = None
+        self._compress = compress
+
+    def feed_eof(self) -> None:
+        self.queue.feed_eof()
+
+    def feed_data(self, data: bytes) -> Tuple[bool, bytes]:
+        if self._exc:
+            return True, data
+
+        try:
+            return self._feed_data(data)
+        except Exception as exc:
+            self._exc = exc
+            self.queue.set_exception(exc)
+            return True, b""
+
+    def _feed_data(self, data: bytes) -> Tuple[bool, bytes]:
+        for fin, opcode, payload, compressed in self.parse_frame(data):
+            if compressed and not self._decompressobj:
+                self._decompressobj = ZLibDecompressor(suppress_deflate_header=True)
+            if opcode == WSMsgType.CLOSE:
+                if len(payload) >= 2:
+                    close_code = UNPACK_CLOSE_CODE(payload[:2])[0]
+                    if close_code < 3000 and close_code not in ALLOWED_CLOSE_CODES:
+                        raise WebSocketError(
+                            WSCloseCode.PROTOCOL_ERROR,
+                            f"Invalid close code: {close_code}",
+                        )
+                    try:
+                        close_message = payload[2:].decode("utf-8")
+                    except UnicodeDecodeError as exc:
+                        raise WebSocketError(
+                            WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message"
+                        ) from exc
+                    msg = WSMessage(WSMsgType.CLOSE, close_code, close_message)
+                elif payload:
+                    raise WebSocketError(
+                        WSCloseCode.PROTOCOL_ERROR,
+                        f"Invalid close frame: {fin} {opcode} {payload!r}",
+                    )
+                else:
+                    msg = WSMessage(WSMsgType.CLOSE, 0, "")
+
+                self.queue.feed_data(msg, 0)
+
+            elif opcode == WSMsgType.PING:
+                self.queue.feed_data(
+                    WSMessage(WSMsgType.PING, payload, ""), len(payload)
+                )
+
+            elif opcode == WSMsgType.PONG:
+                self.queue.feed_data(
+                    WSMessage(WSMsgType.PONG, payload, ""), len(payload)
+                )
+
+            elif (
+                opcode not in (WSMsgType.TEXT, WSMsgType.BINARY)
+                and self._opcode is None
+            ):
+                raise WebSocketError(
+                    WSCloseCode.PROTOCOL_ERROR, f"Unexpected opcode={opcode!r}"
+                )
+            else:
+                # load text/binary
+                if not fin:
+                    # got partial frame payload
+                    if opcode != WSMsgType.CONTINUATION:
+                        self._opcode = opcode
+                    self._partial.extend(payload)
+                    if self._max_msg_size and len(self._partial) >= self._max_msg_size:
+                        raise WebSocketError(
+                            WSCloseCode.MESSAGE_TOO_BIG,
+                            "Message size {} exceeds limit {}".format(
+                                len(self._partial), self._max_msg_size
+                            ),
+                        )
+                else:
+                    # previous frame was non finished
+                    # we should get continuation opcode
+                    if self._partial:
+                        if opcode != WSMsgType.CONTINUATION:
+                            raise WebSocketError(
+                                WSCloseCode.PROTOCOL_ERROR,
+                                "The opcode in non-fin frame is expected "
+                                "to be zero, got {!r}".format(opcode),
+                            )
+
+                    if opcode == WSMsgType.CONTINUATION:
+                        assert self._opcode is not None
+                        opcode = self._opcode
+                        self._opcode = None
+
+                    self._partial.extend(payload)
+                    if self._max_msg_size and len(self._partial) >= self._max_msg_size:
+                        raise WebSocketError(
+                            WSCloseCode.MESSAGE_TOO_BIG,
+                            "Message size {} exceeds limit {}".format(
+                                len(self._partial), self._max_msg_size
+                            ),
+                        )
+
+                    # Decompress process must to be done after all packets
+                    # received.
+                    if compressed:
+                        assert self._decompressobj is not None
+                        self._partial.extend(_WS_DEFLATE_TRAILING)
+                        payload_merged = self._decompressobj.decompress_sync(
+                            self._partial, self._max_msg_size
+                        )
+                        if self._decompressobj.unconsumed_tail:
+                            left = len(self._decompressobj.unconsumed_tail)
+                            raise WebSocketError(
+                                WSCloseCode.MESSAGE_TOO_BIG,
+                                "Decompressed message size {} exceeds limit {}".format(
+                                    self._max_msg_size + left, self._max_msg_size
+                                ),
+                            )
+                    else:
+                        payload_merged = bytes(self._partial)
+
+                    self._partial.clear()
+
+                    if opcode == WSMsgType.TEXT:
+                        try:
+                            text = payload_merged.decode("utf-8")
+                            self.queue.feed_data(
+                                WSMessage(WSMsgType.TEXT, text, ""), len(text)
+                            )
+                        except UnicodeDecodeError as exc:
+                            raise WebSocketError(
+                                WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message"
+                            ) from exc
+                    else:
+                        self.queue.feed_data(
+                            WSMessage(WSMsgType.BINARY, payload_merged, ""),
+                            len(payload_merged),
+                        )
+
+        return False, b""
+
+    def parse_frame(
+        self, buf: bytes
+    ) -> List[Tuple[bool, Optional[int], bytearray, Optional[bool]]]:
+        """Return the next frame from the socket."""
+        frames = []
+        if self._tail:
+            buf, self._tail = self._tail + buf, b""
+
+        start_pos = 0
+        buf_length = len(buf)
+
+        while True:
+            # read header
+            if self._state == WSParserState.READ_HEADER:
+                if buf_length - start_pos >= 2:
+                    data = buf[start_pos : start_pos + 2]
+                    start_pos += 2
+                    first_byte, second_byte = data
+
+                    fin = (first_byte >> 7) & 1
+                    rsv1 = (first_byte >> 6) & 1
+                    rsv2 = (first_byte >> 5) & 1
+                    rsv3 = (first_byte >> 4) & 1
+                    opcode = first_byte & 0xF
+
+                    # frame-fin = %x0 ; more frames of this message follow
+                    #           / %x1 ; final frame of this message
+                    # frame-rsv1 = %x0 ;
+                    #    1 bit, MUST be 0 unless negotiated otherwise
+                    # frame-rsv2 = %x0 ;
+                    #    1 bit, MUST be 0 unless negotiated otherwise
+                    # frame-rsv3 = %x0 ;
+                    #    1 bit, MUST be 0 unless negotiated otherwise
+                    #
+                    # Remove rsv1 from this test for deflate development
+                    if rsv2 or rsv3 or (rsv1 and not self._compress):
+                        raise WebSocketError(
+                            WSCloseCode.PROTOCOL_ERROR,
+                            "Received frame with non-zero reserved bits",
+                        )
+
+                    if opcode > 0x7 and fin == 0:
+                        raise WebSocketError(
+                            WSCloseCode.PROTOCOL_ERROR,
+                            "Received fragmented control frame",
+                        )
+
+                    has_mask = (second_byte >> 7) & 1
+                    length = second_byte & 0x7F
+
+                    # Control frames MUST have a payload
+                    # length of 125 bytes or less
+                    if opcode > 0x7 and length > 125:
+                        raise WebSocketError(
+                            WSCloseCode.PROTOCOL_ERROR,
+                            "Control frame payload cannot be " "larger than 125 bytes",
+                        )
+
+                    # Set compress status if last package is FIN
+                    # OR set compress status if this is first fragment
+                    # Raise error if not first fragment with rsv1 = 0x1
+                    if self._frame_fin or self._compressed is None:
+                        self._compressed = True if rsv1 else False
+                    elif rsv1:
+                        raise WebSocketError(
+                            WSCloseCode.PROTOCOL_ERROR,
+                            "Received frame with non-zero reserved bits",
+                        )
+
+                    self._frame_fin = bool(fin)
+                    self._frame_opcode = opcode
+                    self._has_mask = bool(has_mask)
+                    self._payload_length_flag = length
+                    self._state = WSParserState.READ_PAYLOAD_LENGTH
+                else:
+                    break
+
+            # read payload length
+            if self._state == WSParserState.READ_PAYLOAD_LENGTH:
+                length = self._payload_length_flag
+                if length == 126:
+                    if buf_length - start_pos >= 2:
+                        data = buf[start_pos : start_pos + 2]
+                        start_pos += 2
+                        length = UNPACK_LEN2(data)[0]
+                        self._payload_length = length
+                        self._state = (
+                            WSParserState.READ_PAYLOAD_MASK
+                            if self._has_mask
+                            else WSParserState.READ_PAYLOAD
+                        )
+                    else:
+                        break
+                elif length > 126:
+                    if buf_length - start_pos >= 8:
+                        data = buf[start_pos : start_pos + 8]
+                        start_pos += 8
+                        length = UNPACK_LEN3(data)[0]
+                        self._payload_length = length
+                        self._state = (
+                            WSParserState.READ_PAYLOAD_MASK
+                            if self._has_mask
+                            else WSParserState.READ_PAYLOAD
+                        )
+                    else:
+                        break
+                else:
+                    self._payload_length = length
+                    self._state = (
+                        WSParserState.READ_PAYLOAD_MASK
+                        if self._has_mask
+                        else WSParserState.READ_PAYLOAD
+                    )
+
+            # read payload mask
+            if self._state == WSParserState.READ_PAYLOAD_MASK:
+                if buf_length - start_pos >= 4:
+                    self._frame_mask = buf[start_pos : start_pos + 4]
+                    start_pos += 4
+                    self._state = WSParserState.READ_PAYLOAD
+                else:
+                    break
+
+            if self._state == WSParserState.READ_PAYLOAD:
+                length = self._payload_length
+                payload = self._frame_payload
+
+                chunk_len = buf_length - start_pos
+                if length >= chunk_len:
+                    self._payload_length = length - chunk_len
+                    payload.extend(buf[start_pos:])
+                    start_pos = buf_length
+                else:
+                    self._payload_length = 0
+                    payload.extend(buf[start_pos : start_pos + length])
+                    start_pos = start_pos + length
+
+                if self._payload_length == 0:
+                    if self._has_mask:
+                        assert self._frame_mask is not None
+                        _websocket_mask(self._frame_mask, payload)
+
+                    frames.append(
+                        (self._frame_fin, self._frame_opcode, payload, self._compressed)
+                    )
+
+                    self._frame_payload = bytearray()
+                    self._state = WSParserState.READ_HEADER
+                else:
+                    break
+
+        self._tail = buf[start_pos:]
+
+        return frames
+
+
+class WebSocketWriter:
+    def __init__(
+        self,
+        protocol: BaseProtocol,
+        transport: asyncio.Transport,
+        *,
+        use_mask: bool = False,
+        limit: int = DEFAULT_LIMIT,
+        random: Any = random.Random(),
+        compress: int = 0,
+        notakeover: bool = False,
+    ) -> None:
+        self.protocol = protocol
+        self.transport = transport
+        self.use_mask = use_mask
+        self.randrange = random.randrange
+        self.compress = compress
+        self.notakeover = notakeover
+        self._closing = False
+        self._limit = limit
+        self._output_size = 0
+        self._compressobj: Any = None  # actually compressobj
+
+    async def _send_frame(
+        self, message: bytes, opcode: int, compress: Optional[int] = None
+    ) -> None:
+        """Send a frame over the websocket with message as its payload."""
+        if self._closing and not (opcode & WSMsgType.CLOSE):
+            raise ConnectionResetError("Cannot write to closing transport")
+
+        rsv = 0
+
+        # Only compress larger packets (disabled)
+        # Does small packet needs to be compressed?
+        # if self.compress and opcode < 8 and len(message) > 124:
+        if (compress or self.compress) and opcode < 8:
+            if compress:
+                # Do not set self._compress if compressing is for this frame
+                compressobj = ZLibCompressor(level=zlib.Z_BEST_SPEED, wbits=-compress)
+            else:  # self.compress
+                if not self._compressobj:
+                    self._compressobj = ZLibCompressor(
+                        level=zlib.Z_BEST_SPEED, wbits=-self.compress
+                    )
+                compressobj = self._compressobj
+
+            message = await compressobj.compress(message)
+            message += compressobj.flush(
+                zlib.Z_FULL_FLUSH if self.notakeover else zlib.Z_SYNC_FLUSH
+            )
+            if message.endswith(_WS_DEFLATE_TRAILING):
+                message = message[:-4]
+            rsv = rsv | 0x40
+
+        msg_length = len(message)
+
+        use_mask = self.use_mask
+        if use_mask:
+            mask_bit = 0x80
+        else:
+            mask_bit = 0
+
+        if msg_length < 126:
+            header = PACK_LEN1(0x80 | rsv | opcode, msg_length | mask_bit)
+        elif msg_length < (1 << 16):
+            header = PACK_LEN2(0x80 | rsv | opcode, 126 | mask_bit, msg_length)
+        else:
+            header = PACK_LEN3(0x80 | rsv | opcode, 127 | mask_bit, msg_length)
+        if use_mask:
+            mask = self.randrange(0, 0xFFFFFFFF)
+            mask = mask.to_bytes(4, "big")
+            message = bytearray(message)
+            _websocket_mask(mask, message)
+            self._write(header + mask + message)
+            self._output_size += len(header) + len(mask) + len(message)
+        else:
+            if len(message) > MSG_SIZE:
+                self._write(header)
+                self._write(message)
+            else:
+                self._write(header + message)
+
+            self._output_size += len(header) + len(message)
+
+        if self._output_size > self._limit:
+            self._output_size = 0
+            await self.protocol._drain_helper()
+
+    def _write(self, data: bytes) -> None:
+        if self.transport.is_closing():
+            raise ConnectionResetError("Cannot write to closing transport")
+        self.transport.write(data)
+
+    async def pong(self, message: Union[bytes, str] = b"") -> None:
+        """Send pong message."""
+        if isinstance(message, str):
+            message = message.encode("utf-8")
+        await self._send_frame(message, WSMsgType.PONG)
+
+    async def ping(self, message: Union[bytes, str] = b"") -> None:
+        """Send ping message."""
+        if isinstance(message, str):
+            message = message.encode("utf-8")
+        await self._send_frame(message, WSMsgType.PING)
+
+    async def send(
+        self,
+        message: Union[str, bytes],
+        binary: bool = False,
+        compress: Optional[int] = None,
+    ) -> None:
+        """Send a frame over the websocket with message as its payload."""
+        if isinstance(message, str):
+            message = message.encode("utf-8")
+        if binary:
+            await self._send_frame(message, WSMsgType.BINARY, compress)
+        else:
+            await self._send_frame(message, WSMsgType.TEXT, compress)
+
+    async def close(self, code: int = 1000, message: Union[bytes, str] = b"") -> None:
+        """Close the websocket, sending the specified code and message."""
+        if isinstance(message, str):
+            message = message.encode("utf-8")
+        try:
+            await self._send_frame(
+                PACK_CLOSE_CODE(code) + message, opcode=WSMsgType.CLOSE
+            )
+        finally:
+            self._closing = True

aiohttp/http_writer.py

@@ -0,0 +1,198 @@
+"""Http related parsers and protocol."""
+
+import asyncio
+import zlib
+from typing import Any, Awaitable, Callable, NamedTuple, Optional, Union  # noqa
+
+from multidict import CIMultiDict
+
+from .abc import AbstractStreamWriter
+from .base_protocol import BaseProtocol
+from .compression_utils import ZLibCompressor
+from .helpers import NO_EXTENSIONS
+
+__all__ = ("StreamWriter", "HttpVersion", "HttpVersion10", "HttpVersion11")
+
+
+class HttpVersion(NamedTuple):
+    major: int
+    minor: int
+
+
+HttpVersion10 = HttpVersion(1, 0)
+HttpVersion11 = HttpVersion(1, 1)
+
+
+_T_OnChunkSent = Optional[Callable[[bytes], Awaitable[None]]]
+_T_OnHeadersSent = Optional[Callable[["CIMultiDict[str]"], Awaitable[None]]]
+
+
+class StreamWriter(AbstractStreamWriter):
+    def __init__(
+        self,
+        protocol: BaseProtocol,
+        loop: asyncio.AbstractEventLoop,
+        on_chunk_sent: _T_OnChunkSent = None,
+        on_headers_sent: _T_OnHeadersSent = None,
+    ) -> None:
+        self._protocol = protocol
+
+        self.loop = loop
+        self.length = None
+        self.chunked = False
+        self.buffer_size = 0
+        self.output_size = 0
+
+        self._eof = False
+        self._compress: Optional[ZLibCompressor] = None
+        self._drain_waiter = None
+
+        self._on_chunk_sent: _T_OnChunkSent = on_chunk_sent
+        self._on_headers_sent: _T_OnHeadersSent = on_headers_sent
+
+    @property
+    def transport(self) -> Optional[asyncio.Transport]:
+        return self._protocol.transport
+
+    @property
+    def protocol(self) -> BaseProtocol:
+        return self._protocol
+
+    def enable_chunking(self) -> None:
+        self.chunked = True
+
+    def enable_compression(
+        self, encoding: str = "deflate", strategy: int = zlib.Z_DEFAULT_STRATEGY
+    ) -> None:
+        self._compress = ZLibCompressor(encoding=encoding, strategy=strategy)
+
+    def _write(self, chunk: bytes) -> None:
+        size = len(chunk)
+        self.buffer_size += size
+        self.output_size += size
+        transport = self.transport
+        if not self._protocol.connected or transport is None or transport.is_closing():
+            raise ConnectionResetError("Cannot write to closing transport")
+        transport.write(chunk)
+
+    async def write(
+        self, chunk: bytes, *, drain: bool = True, LIMIT: int = 0x10000
+    ) -> None:
+        """Writes chunk of data to a stream.
+
+        write_eof() indicates end of stream.
+        writer can't be used after write_eof() method being called.
+        write() return drain future.
+        """
+        if self._on_chunk_sent is not None:
+            await self._on_chunk_sent(chunk)
+
+        if isinstance(chunk, memoryview):
+            if chunk.nbytes != len(chunk):
+                # just reshape it
+                chunk = chunk.cast("c")
+
+        if self._compress is not None:
+            chunk = await self._compress.compress(chunk)
+            if not chunk:
+                return
+
+        if self.length is not None:
+            chunk_len = len(chunk)
+            if self.length >= chunk_len:
+                self.length = self.length - chunk_len
+            else:
+                chunk = chunk[: self.length]
+                self.length = 0
+                if not chunk:
+                    return
+
+        if chunk:
+            if self.chunked:
+                chunk_len_pre = ("%x\r\n" % len(chunk)).encode("ascii")
+                chunk = chunk_len_pre + chunk + b"\r\n"
+
+            self._write(chunk)
+
+            if self.buffer_size > LIMIT and drain:
+                self.buffer_size = 0
+                await self.drain()
+
+    async def write_headers(
+        self, status_line: str, headers: "CIMultiDict[str]"
+    ) -> None:
+        """Write request/response status and headers."""
+        if self._on_headers_sent is not None:
+            await self._on_headers_sent(headers)
+
+        # status + headers
+        buf = _serialize_headers(status_line, headers)
+        self._write(buf)
+
+    async def write_eof(self, chunk: bytes = b"") -> None:
+        if self._eof:
+            return
+
+        if chunk and self._on_chunk_sent is not None:
+            await self._on_chunk_sent(chunk)
+
+        if self._compress:
+            if chunk:
+                chunk = await self._compress.compress(chunk)
+
+            chunk += self._compress.flush()
+            if chunk and self.chunked:
+                chunk_len = ("%x\r\n" % len(chunk)).encode("ascii")
+                chunk = chunk_len + chunk + b"\r\n0\r\n\r\n"
+        else:
+            if self.chunked:
+                if chunk:
+                    chunk_len = ("%x\r\n" % len(chunk)).encode("ascii")
+                    chunk = chunk_len + chunk + b"\r\n0\r\n\r\n"
+                else:
+                    chunk = b"0\r\n\r\n"
+
+        if chunk:
+            self._write(chunk)
+
+        await self.drain()
+
+        self._eof = True
+
+    async def drain(self) -> None:
+        """Flush the write buffer.
+
+        The intended use is to write
+
+          await w.write(data)
+          await w.drain()
+        """
+        if self._protocol.transport is not None:
+            await self._protocol._drain_helper()
+
+
+def _safe_header(string: str) -> str:
+    if "\r" in string or "\n" in string:
+        raise ValueError(
+            "Newline or carriage return detected in headers. "
+            "Potential header injection attack."
+        )
+    return string
+
+
+def _py_serialize_headers(status_line: str, headers: "CIMultiDict[str]") -> bytes:
+    headers_gen = (_safe_header(k) + ": " + _safe_header(v) for k, v in headers.items())
+    line = status_line + "\r\n" + "\r\n".join(headers_gen) + "\r\n\r\n"
+    return line.encode("utf-8")
+
+
+_serialize_headers = _py_serialize_headers
+
+try:
+    import aiohttp._http_writer as _http_writer  # type: ignore[import]
+
+    _c_serialize_headers = _http_writer._serialize_headers
+    if not NO_EXTENSIONS:
+        _serialize_headers = _c_serialize_headers
+except ImportError:
+    pass

aiohttp/locks.py

@@ -0,0 +1,41 @@
+import asyncio
+import collections
+from typing import Any, Deque, Optional
+
+
+class EventResultOrError:
+    """Event asyncio lock helper class.
+
+    Wraps the Event asyncio lock allowing either to awake the
+    locked Tasks without any error or raising an exception.
+
+    thanks to @vorpalsmith for the simple design.
+    """
+
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        self._loop = loop
+        self._exc: Optional[BaseException] = None
+        self._event = asyncio.Event()
+        self._waiters: Deque[asyncio.Future[Any]] = collections.deque()
+
+    def set(self, exc: Optional[BaseException] = None) -> None:
+        self._exc = exc
+        self._event.set()
+
+    async def wait(self) -> Any:
+        waiter = self._loop.create_task(self._event.wait())
+        self._waiters.append(waiter)
+        try:
+            val = await waiter
+        finally:
+            self._waiters.remove(waiter)
+
+        if self._exc is not None:
+            raise self._exc
+
+        return val
+
+    def cancel(self) -> None:
+        """Cancel all waiters"""
+        for waiter in self._waiters:
+            waiter.cancel()

aiohttp/log.py

@@ -0,0 +1,8 @@
+import logging
+
+access_logger = logging.getLogger("aiohttp.access")
+client_logger = logging.getLogger("aiohttp.client")
+internal_logger = logging.getLogger("aiohttp.internal")
+server_logger = logging.getLogger("aiohttp.server")
+web_logger = logging.getLogger("aiohttp.web")
+ws_logger = logging.getLogger("aiohttp.websocket")

aiohttp/payload.py

@@ -0,0 +1,458 @@
+import asyncio
+import enum
+import io
+import json
+import mimetypes
+import os
+import warnings
+from abc import ABC, abstractmethod
+from itertools import chain
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    ByteString,
+    Dict,
+    Final,
+    Iterable,
+    Optional,
+    TextIO,
+    Tuple,
+    Type,
+    Union,
+)
+
+from multidict import CIMultiDict
+
+from . import hdrs
+from .abc import AbstractStreamWriter
+from .helpers import (
+    _SENTINEL,
+    content_disposition_header,
+    guess_filename,
+    parse_mimetype,
+    sentinel,
+)
+from .streams import StreamReader
+from .typedefs import JSONEncoder, _CIMultiDict
+
+__all__ = (
+    "PAYLOAD_REGISTRY",
+    "get_payload",
+    "payload_type",
+    "Payload",
+    "BytesPayload",
+    "StringPayload",
+    "IOBasePayload",
+    "BytesIOPayload",
+    "BufferedReaderPayload",
+    "TextIOPayload",
+    "StringIOPayload",
+    "JsonPayload",
+    "AsyncIterablePayload",
+)
+
+TOO_LARGE_BYTES_BODY: Final[int] = 2**20  # 1 MB
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import List
+
+
+class LookupError(Exception):
+    pass
+
+
+class Order(str, enum.Enum):
+    normal = "normal"
+    try_first = "try_first"
+    try_last = "try_last"
+
+
+def get_payload(data: Any, *args: Any, **kwargs: Any) -> "Payload":
+    return PAYLOAD_REGISTRY.get(data, *args, **kwargs)
+
+
+def register_payload(
+    factory: Type["Payload"], type: Any, *, order: Order = Order.normal
+) -> None:
+    PAYLOAD_REGISTRY.register(factory, type, order=order)
+
+
+class payload_type:
+    def __init__(self, type: Any, *, order: Order = Order.normal) -> None:
+        self.type = type
+        self.order = order
+
+    def __call__(self, factory: Type["Payload"]) -> Type["Payload"]:
+        register_payload(factory, self.type, order=self.order)
+        return factory
+
+
+PayloadType = Type["Payload"]
+_PayloadRegistryItem = Tuple[PayloadType, Any]
+
+
+class PayloadRegistry:
+    """Payload registry.
+
+    note: we need zope.interface for more efficient adapter search
+    """
+
+    def __init__(self) -> None:
+        self._first: List[_PayloadRegistryItem] = []
+        self._normal: List[_PayloadRegistryItem] = []
+        self._last: List[_PayloadRegistryItem] = []
+
+    def get(
+        self,
+        data: Any,
+        *args: Any,
+        _CHAIN: "Type[chain[_PayloadRegistryItem]]" = chain,
+        **kwargs: Any,
+    ) -> "Payload":
+        if isinstance(data, Payload):
+            return data
+        for factory, type in _CHAIN(self._first, self._normal, self._last):
+            if isinstance(data, type):
+                return factory(data, *args, **kwargs)
+
+        raise LookupError()
+
+    def register(
+        self, factory: PayloadType, type: Any, *, order: Order = Order.normal
+    ) -> None:
+        if order is Order.try_first:
+            self._first.append((factory, type))
+        elif order is Order.normal:
+            self._normal.append((factory, type))
+        elif order is Order.try_last:
+            self._last.append((factory, type))
+        else:
+            raise ValueError(f"Unsupported order {order!r}")
+
+
+class Payload(ABC):
+    _default_content_type: str = "application/octet-stream"
+    _size: Optional[int] = None
+
+    def __init__(
+        self,
+        value: Any,
+        headers: Optional[
+            Union[_CIMultiDict, Dict[str, str], Iterable[Tuple[str, str]]]
+        ] = None,
+        content_type: Union[None, str, _SENTINEL] = sentinel,
+        filename: Optional[str] = None,
+        encoding: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        self._encoding = encoding
+        self._filename = filename
+        self._headers: _CIMultiDict = CIMultiDict()
+        self._value = value
+        if content_type is not sentinel and content_type is not None:
+            assert isinstance(content_type, str)
+            self._headers[hdrs.CONTENT_TYPE] = content_type
+        elif self._filename is not None:
+            content_type = mimetypes.guess_type(self._filename)[0]
+            if content_type is None:
+                content_type = self._default_content_type
+            self._headers[hdrs.CONTENT_TYPE] = content_type
+        else:
+            self._headers[hdrs.CONTENT_TYPE] = self._default_content_type
+        self._headers.update(headers or {})
+
+    @property
+    def size(self) -> Optional[int]:
+        """Size of the payload."""
+        return self._size
+
+    @property
+    def filename(self) -> Optional[str]:
+        """Filename of the payload."""
+        return self._filename
+
+    @property
+    def headers(self) -> _CIMultiDict:
+        """Custom item headers"""
+        return self._headers
+
+    @property
+    def _binary_headers(self) -> bytes:
+        return (
+            "".join([k + ": " + v + "\r\n" for k, v in self.headers.items()]).encode(
+                "utf-8"
+            )
+            + b"\r\n"
+        )
+
+    @property
+    def encoding(self) -> Optional[str]:
+        """Payload encoding"""
+        return self._encoding
+
+    @property
+    def content_type(self) -> str:
+        """Content type"""
+        return self._headers[hdrs.CONTENT_TYPE]
+
+    def set_content_disposition(
+        self,
+        disptype: str,
+        quote_fields: bool = True,
+        _charset: str = "utf-8",
+        **params: Any,
+    ) -> None:
+        """Sets ``Content-Disposition`` header."""
+        self._headers[hdrs.CONTENT_DISPOSITION] = content_disposition_header(
+            disptype, quote_fields=quote_fields, _charset=_charset, **params
+        )
+
+    @abstractmethod
+    async def write(self, writer: AbstractStreamWriter) -> None:
+        """Write payload.
+
+        writer is an AbstractStreamWriter instance:
+        """
+
+
+class BytesPayload(Payload):
+    def __init__(self, value: ByteString, *args: Any, **kwargs: Any) -> None:
+        if not isinstance(value, (bytes, bytearray, memoryview)):
+            raise TypeError(f"value argument must be byte-ish, not {type(value)!r}")
+
+        if "content_type" not in kwargs:
+            kwargs["content_type"] = "application/octet-stream"
+
+        super().__init__(value, *args, **kwargs)
+
+        if isinstance(value, memoryview):
+            self._size = value.nbytes
+        else:
+            self._size = len(value)
+
+        if self._size > TOO_LARGE_BYTES_BODY:
+            warnings.warn(
+                "Sending a large body directly with raw bytes might"
+                " lock the event loop. You should probably pass an "
+                "io.BytesIO object instead",
+                ResourceWarning,
+                source=self,
+            )
+
+    async def write(self, writer: AbstractStreamWriter) -> None:
+        await writer.write(self._value)
+
+
+class StringPayload(BytesPayload):
+    def __init__(
+        self,
+        value: str,
+        *args: Any,
+        encoding: Optional[str] = None,
+        content_type: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        if encoding is None:
+            if content_type is None:
+                real_encoding = "utf-8"
+                content_type = "text/plain; charset=utf-8"
+            else:
+                mimetype = parse_mimetype(content_type)
+                real_encoding = mimetype.parameters.get("charset", "utf-8")
+        else:
+            if content_type is None:
+                content_type = "text/plain; charset=%s" % encoding
+            real_encoding = encoding
+
+        super().__init__(
+            value.encode(real_encoding),
+            encoding=real_encoding,
+            content_type=content_type,
+            *args,
+            **kwargs,
+        )
+
+
+class StringIOPayload(StringPayload):
+    def __init__(self, value: IO[str], *args: Any, **kwargs: Any) -> None:
+        super().__init__(value.read(), *args, **kwargs)
+
+
+class IOBasePayload(Payload):
+    _value: IO[Any]
+
+    def __init__(
+        self, value: IO[Any], disposition: str = "attachment", *args: Any, **kwargs: Any
+    ) -> None:
+        if "filename" not in kwargs:
+            kwargs["filename"] = guess_filename(value)
+
+        super().__init__(value, *args, **kwargs)
+
+        if self._filename is not None and disposition is not None:
+            if hdrs.CONTENT_DISPOSITION not in self.headers:
+                self.set_content_disposition(disposition, filename=self._filename)
+
+    async def write(self, writer: AbstractStreamWriter) -> None:
+        loop = asyncio.get_event_loop()
+        try:
+            chunk = await loop.run_in_executor(None, self._value.read, 2**16)
+            while chunk:
+                await writer.write(chunk)
+                chunk = await loop.run_in_executor(None, self._value.read, 2**16)
+        finally:
+            await loop.run_in_executor(None, self._value.close)
+
+
+class TextIOPayload(IOBasePayload):
+    _value: TextIO
+
+    def __init__(
+        self,
+        value: TextIO,
+        *args: Any,
+        encoding: Optional[str] = None,
+        content_type: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        if encoding is None:
+            if content_type is None:
+                encoding = "utf-8"
+                content_type = "text/plain; charset=utf-8"
+            else:
+                mimetype = parse_mimetype(content_type)
+                encoding = mimetype.parameters.get("charset", "utf-8")
+        else:
+            if content_type is None:
+                content_type = "text/plain; charset=%s" % encoding
+
+        super().__init__(
+            value,
+            content_type=content_type,
+            encoding=encoding,
+            *args,
+            **kwargs,
+        )
+
+    @property
+    def size(self) -> Optional[int]:
+        try:
+            return os.fstat(self._value.fileno()).st_size - self._value.tell()
+        except OSError:
+            return None
+
+    async def write(self, writer: AbstractStreamWriter) -> None:
+        loop = asyncio.get_event_loop()
+        try:
+            chunk = await loop.run_in_executor(None, self._value.read, 2**16)
+            while chunk:
+                data = (
+                    chunk.encode(encoding=self._encoding)
+                    if self._encoding
+                    else chunk.encode()
+                )
+                await writer.write(data)
+                chunk = await loop.run_in_executor(None, self._value.read, 2**16)
+        finally:
+            await loop.run_in_executor(None, self._value.close)
+
+
+class BytesIOPayload(IOBasePayload):
+    @property
+    def size(self) -> int:
+        position = self._value.tell()
+        end = self._value.seek(0, os.SEEK_END)
+        self._value.seek(position)
+        return end - position
+
+
+class BufferedReaderPayload(IOBasePayload):
+    @property
+    def size(self) -> Optional[int]:
+        try:
+            return os.fstat(self._value.fileno()).st_size - self._value.tell()
+        except OSError:
+            # data.fileno() is not supported, e.g.
+            # io.BufferedReader(io.BytesIO(b'data'))
+            return None
+
+
+class JsonPayload(BytesPayload):
+    def __init__(
+        self,
+        value: Any,
+        encoding: str = "utf-8",
+        content_type: str = "application/json",
+        dumps: JSONEncoder = json.dumps,
+        *args: Any,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(
+            dumps(value).encode(encoding),
+            content_type=content_type,
+            encoding=encoding,
+            *args,
+            **kwargs,
+        )
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import AsyncIterable, AsyncIterator
+
+    _AsyncIterator = AsyncIterator[bytes]
+    _AsyncIterable = AsyncIterable[bytes]
+else:
+    from collections.abc import AsyncIterable, AsyncIterator
+
+    _AsyncIterator = AsyncIterator
+    _AsyncIterable = AsyncIterable
+
+
+class AsyncIterablePayload(Payload):
+    _iter: Optional[_AsyncIterator] = None
+
+    def __init__(self, value: _AsyncIterable, *args: Any, **kwargs: Any) -> None:
+        if not isinstance(value, AsyncIterable):
+            raise TypeError(
+                "value argument must support "
+                "collections.abc.AsyncIterable interface, "
+                "got {!r}".format(type(value))
+            )
+
+        if "content_type" not in kwargs:
+            kwargs["content_type"] = "application/octet-stream"
+
+        super().__init__(value, *args, **kwargs)
+
+        self._iter = value.__aiter__()
+
+    async def write(self, writer: AbstractStreamWriter) -> None:
+        if self._iter:
+            try:
+                # iter is not None check prevents rare cases
+                # when the case iterable is used twice
+                while True:
+                    chunk = await self._iter.__anext__()
+                    await writer.write(chunk)
+            except StopAsyncIteration:
+                self._iter = None
+
+
+class StreamReaderPayload(AsyncIterablePayload):
+    def __init__(self, value: StreamReader, *args: Any, **kwargs: Any) -> None:
+        super().__init__(value.iter_any(), *args, **kwargs)
+
+
+PAYLOAD_REGISTRY = PayloadRegistry()
+PAYLOAD_REGISTRY.register(BytesPayload, (bytes, bytearray, memoryview))
+PAYLOAD_REGISTRY.register(StringPayload, str)
+PAYLOAD_REGISTRY.register(StringIOPayload, io.StringIO)
+PAYLOAD_REGISTRY.register(TextIOPayload, io.TextIOBase)
+PAYLOAD_REGISTRY.register(BytesIOPayload, io.BytesIO)
+PAYLOAD_REGISTRY.register(BufferedReaderPayload, (io.BufferedReader, io.BufferedRandom))
+PAYLOAD_REGISTRY.register(IOBasePayload, io.IOBase)
+PAYLOAD_REGISTRY.register(StreamReaderPayload, StreamReader)
+# try_last for giving a chance to more specialized async interables like
+# multidict.BodyPartReaderPayload override the default
+PAYLOAD_REGISTRY.register(AsyncIterablePayload, AsyncIterable, order=Order.try_last)

aiohttp/py.typed

@@ -0,0 +1 @@
+Marker

aiohttp/pytest_plugin.py

@@ -0,0 +1,363 @@
+import asyncio
+import contextlib
+import inspect
+import warnings
+from typing import Any, Awaitable, Callable, Dict, Iterator, Optional, Type, Union
+
+import pytest
+
+from aiohttp.web import Application
+
+from .test_utils import (
+    BaseTestServer,
+    RawTestServer,
+    TestClient,
+    TestServer,
+    loop_context,
+    setup_test_loop,
+    teardown_test_loop,
+    unused_port as _unused_port,
+)
+
+try:
+    import uvloop
+except ImportError:  # pragma: no cover
+    uvloop = None  # type: ignore[assignment]
+
+AiohttpClient = Callable[[Union[Application, BaseTestServer]], Awaitable[TestClient]]
+AiohttpRawServer = Callable[[Application], Awaitable[RawTestServer]]
+AiohttpServer = Callable[[Application], Awaitable[TestServer]]
+
+
+def pytest_addoption(parser):  # type: ignore[no-untyped-def]
+    parser.addoption(
+        "--aiohttp-fast",
+        action="store_true",
+        default=False,
+        help="run tests faster by disabling extra checks",
+    )
+    parser.addoption(
+        "--aiohttp-loop",
+        action="store",
+        default="pyloop",
+        help="run tests with specific loop: pyloop, uvloop or all",
+    )
+    parser.addoption(
+        "--aiohttp-enable-loop-debug",
+        action="store_true",
+        default=False,
+        help="enable event loop debug mode",
+    )
+
+
+def pytest_fixture_setup(fixturedef):  # type: ignore[no-untyped-def]
+    """Set up pytest fixture.
+
+    Allow fixtures to be coroutines. Run coroutine fixtures in an event loop.
+    """
+    func = fixturedef.func
+
+    if inspect.isasyncgenfunction(func):
+        # async generator fixture
+        is_async_gen = True
+    elif asyncio.iscoroutinefunction(func):
+        # regular async fixture
+        is_async_gen = False
+    else:
+        # not an async fixture, nothing to do
+        return
+
+    strip_request = False
+    if "request" not in fixturedef.argnames:
+        fixturedef.argnames += ("request",)
+        strip_request = True
+
+    def wrapper(*args, **kwargs):  # type: ignore[no-untyped-def]
+        request = kwargs["request"]
+        if strip_request:
+            del kwargs["request"]
+
+        # if neither the fixture nor the test use the 'loop' fixture,
+        # 'getfixturevalue' will fail because the test is not parameterized
+        # (this can be removed someday if 'loop' is no longer parameterized)
+        if "loop" not in request.fixturenames:
+            raise Exception(
+                "Asynchronous fixtures must depend on the 'loop' fixture or "
+                "be used in tests depending from it."
+            )
+
+        _loop = request.getfixturevalue("loop")
+
+        if is_async_gen:
+            # for async generators, we need to advance the generator once,
+            # then advance it again in a finalizer
+            gen = func(*args, **kwargs)
+
+            def finalizer():  # type: ignore[no-untyped-def]
+                try:
+                    return _loop.run_until_complete(gen.__anext__())
+                except StopAsyncIteration:
+                    pass
+
+            request.addfinalizer(finalizer)
+            return _loop.run_until_complete(gen.__anext__())
+        else:
+            return _loop.run_until_complete(func(*args, **kwargs))
+
+    fixturedef.func = wrapper
+
+
+@pytest.fixture
+def fast(request):  # type: ignore[no-untyped-def]
+    """--fast config option"""
+    return request.config.getoption("--aiohttp-fast")
+
+
+@pytest.fixture
+def loop_debug(request):  # type: ignore[no-untyped-def]
+    """--enable-loop-debug config option"""
+    return request.config.getoption("--aiohttp-enable-loop-debug")
+
+
+@contextlib.contextmanager
+def _runtime_warning_context():  # type: ignore[no-untyped-def]
+    """Context manager which checks for RuntimeWarnings.
+
+    This exists specifically to
+    avoid "coroutine 'X' was never awaited" warnings being missed.
+
+    If RuntimeWarnings occur in the context a RuntimeError is raised.
+    """
+    with warnings.catch_warnings(record=True) as _warnings:
+        yield
+        rw = [
+            "{w.filename}:{w.lineno}:{w.message}".format(w=w)
+            for w in _warnings
+            if w.category == RuntimeWarning
+        ]
+        if rw:
+            raise RuntimeError(
+                "{} Runtime Warning{},\n{}".format(
+                    len(rw), "" if len(rw) == 1 else "s", "\n".join(rw)
+                )
+            )
+
+    # Propagate warnings to pytest
+    for msg in _warnings:
+        warnings.showwarning(
+            msg.message, msg.category, msg.filename, msg.lineno, msg.file, msg.line
+        )
+
+
+@contextlib.contextmanager
+def _passthrough_loop_context(loop, fast=False):  # type: ignore[no-untyped-def]
+    """Passthrough loop context.
+
+    Sets up and tears down a loop unless one is passed in via the loop
+    argument when it's passed straight through.
+    """
+    if loop:
+        # loop already exists, pass it straight through
+        yield loop
+    else:
+        # this shadows loop_context's standard behavior
+        loop = setup_test_loop()
+        yield loop
+        teardown_test_loop(loop, fast=fast)
+
+
+def pytest_pycollect_makeitem(collector, name, obj):  # type: ignore[no-untyped-def]
+    """Fix pytest collecting for coroutines."""
+    if collector.funcnamefilter(name) and asyncio.iscoroutinefunction(obj):
+        return list(collector._genfunctions(name, obj))
+
+
+def pytest_pyfunc_call(pyfuncitem):  # type: ignore[no-untyped-def]
+    """Run coroutines in an event loop instead of a normal function call."""
+    fast = pyfuncitem.config.getoption("--aiohttp-fast")
+    if asyncio.iscoroutinefunction(pyfuncitem.function):
+        existing_loop = pyfuncitem.funcargs.get(
+            "proactor_loop"
+        ) or pyfuncitem.funcargs.get("loop", None)
+        with _runtime_warning_context():
+            with _passthrough_loop_context(existing_loop, fast=fast) as _loop:
+                testargs = {
+                    arg: pyfuncitem.funcargs[arg]
+                    for arg in pyfuncitem._fixtureinfo.argnames
+                }
+                _loop.run_until_complete(pyfuncitem.obj(**testargs))
+
+        return True
+
+
+def pytest_generate_tests(metafunc):  # type: ignore[no-untyped-def]
+    if "loop_factory" not in metafunc.fixturenames:
+        return
+
+    loops = metafunc.config.option.aiohttp_loop
+    avail_factories: Dict[str, Type[asyncio.AbstractEventLoopPolicy]]
+    avail_factories = {"pyloop": asyncio.DefaultEventLoopPolicy}
+
+    if uvloop is not None:  # pragma: no cover
+        avail_factories["uvloop"] = uvloop.EventLoopPolicy
+
+    if loops == "all":
+        loops = "pyloop,uvloop?"
+
+    factories = {}  # type: ignore[var-annotated]
+    for name in loops.split(","):
+        required = not name.endswith("?")
+        name = name.strip(" ?")
+        if name not in avail_factories:  # pragma: no cover
+            if required:
+                raise ValueError(
+                    "Unknown loop '%s', available loops: %s"
+                    % (name, list(factories.keys()))
+                )
+            else:
+                continue
+        factories[name] = avail_factories[name]
+    metafunc.parametrize(
+        "loop_factory", list(factories.values()), ids=list(factories.keys())
+    )
+
+
+@pytest.fixture
+def loop(loop_factory, fast, loop_debug):  # type: ignore[no-untyped-def]
+    """Return an instance of the event loop."""
+    policy = loop_factory()
+    asyncio.set_event_loop_policy(policy)
+    with loop_context(fast=fast) as _loop:
+        if loop_debug:
+            _loop.set_debug(True)  # pragma: no cover
+        asyncio.set_event_loop(_loop)
+        yield _loop
+
+
+@pytest.fixture
+def proactor_loop():  # type: ignore[no-untyped-def]
+    policy = asyncio.WindowsProactorEventLoopPolicy()  # type: ignore[attr-defined]
+    asyncio.set_event_loop_policy(policy)
+
+    with loop_context(policy.new_event_loop) as _loop:
+        asyncio.set_event_loop(_loop)
+        yield _loop
+
+
+@pytest.fixture
+def aiohttp_unused_port() -> Callable[[], int]:
+    """Return a port that is unused on the current host."""
+    return _unused_port
+
+
+@pytest.fixture
+def aiohttp_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpServer]:
+    """Factory to create a TestServer instance, given an app.
+
+    aiohttp_server(app, **kwargs)
+    """
+    servers = []
+
+    async def go(app, *, port=None, **kwargs):  # type: ignore[no-untyped-def]
+        server = TestServer(app, port=port)
+        await server.start_server(**kwargs)
+        servers.append(server)
+        return server
+
+    yield go
+
+    async def finalize() -> None:
+        while servers:
+            await servers.pop().close()
+
+    loop.run_until_complete(finalize())
+
+
+@pytest.fixture
+def aiohttp_raw_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpRawServer]:
+    """Factory to create a RawTestServer instance, given a web handler.
+
+    aiohttp_raw_server(handler, **kwargs)
+    """
+    servers = []
+
+    async def go(handler, *, port=None, **kwargs):  # type: ignore[no-untyped-def]
+        server = RawTestServer(handler, port=port)
+        await server.start_server(**kwargs)
+        servers.append(server)
+        return server
+
+    yield go
+
+    async def finalize() -> None:
+        while servers:
+            await servers.pop().close()
+
+    loop.run_until_complete(finalize())
+
+
+@pytest.fixture
+def aiohttp_client_cls() -> Type[TestClient]:
+    """
+    Client class to use in ``aiohttp_client`` factory.
+
+    Use it for passing custom ``TestClient`` implementations.
+
+    Example::
+
+       class MyClient(TestClient):
+           async def login(self, *, user, pw):
+               payload = {"username": user, "password": pw}
+               return await self.post("/login", json=payload)
+
+       @pytest.fixture
+       def aiohttp_client_cls():
+           return MyClient
+
+       def test_login(aiohttp_client):
+           app = web.Application()
+           client = await aiohttp_client(app)
+           await client.login(user="admin", pw="s3cr3t")
+
+    """
+    return TestClient
+
+
+@pytest.fixture
+def aiohttp_client(
+    loop: asyncio.AbstractEventLoop, aiohttp_client_cls: Type[TestClient]
+) -> Iterator[AiohttpClient]:
+    """Factory to create a TestClient instance.
+
+    aiohttp_client(app, **kwargs)
+    aiohttp_client(server, **kwargs)
+    aiohttp_client(raw_server, **kwargs)
+    """
+    clients = []
+
+    async def go(
+        __param: Union[Application, BaseTestServer],
+        *,
+        server_kwargs: Optional[Dict[str, Any]] = None,
+        **kwargs: Any
+    ) -> TestClient:
+        if isinstance(__param, Application):
+            server_kwargs = server_kwargs or {}
+            server = TestServer(__param, **server_kwargs)
+            client = aiohttp_client_cls(server, **kwargs)
+        elif isinstance(__param, BaseTestServer):
+            client = aiohttp_client_cls(__param, **kwargs)
+        else:
+            raise ValueError("Unknown argument type: %r" % type(__param))
+
+        await client.start_server()
+        clients.append(client)
+        return client
+
+    yield go
+
+    async def finalize() -> None:
+        while clients:
+            await clients.pop().close()
+
+    loop.run_until_complete(finalize())

aiohttp/resolver.py

@@ -0,0 +1,118 @@
+import asyncio
+import socket
+from typing import Any, Dict, List, Type, Union
+
+from .abc import AbstractResolver
+
+__all__ = ("ThreadedResolver", "AsyncResolver", "DefaultResolver")
+
+try:
+    import aiodns
+
+    # aiodns_default = hasattr(aiodns.DNSResolver, 'gethostbyname')
+except ImportError:  # pragma: no cover
+    aiodns = None
+
+aiodns_default = False
+
+
+class ThreadedResolver(AbstractResolver):
+    """Threaded resolver.
+
+    Uses an Executor for synchronous getaddrinfo() calls.
+    concurrent.futures.ThreadPoolExecutor is used by default.
+    """
+
+    def __init__(self) -> None:
+        self._loop = asyncio.get_running_loop()
+
+    async def resolve(
+        self, hostname: str, port: int = 0, family: int = socket.AF_INET
+    ) -> List[Dict[str, Any]]:
+        infos = await self._loop.getaddrinfo(
+            hostname,
+            port,
+            type=socket.SOCK_STREAM,
+            family=family,
+            flags=socket.AI_ADDRCONFIG,
+        )
+
+        hosts = []
+        for family, _, proto, _, address in infos:
+            if family == socket.AF_INET6:
+                if len(address) < 3:
+                    # IPv6 is not supported by Python build,
+                    # or IPv6 is not enabled in the host
+                    continue
+                if address[3]:  # type: ignore[misc]
+                    # This is essential for link-local IPv6 addresses.
+                    # LL IPv6 is a VERY rare case. Strictly speaking, we should use
+                    # getnameinfo() unconditionally, but performance makes sense.
+                    host, _port = socket.getnameinfo(
+                        address, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV
+                    )
+                    port = int(_port)
+                else:
+                    host, port = address[:2]
+            else:  # IPv4
+                assert family == socket.AF_INET
+                host, port = address  # type: ignore[misc]
+            hosts.append(
+                {
+                    "hostname": hostname,
+                    "host": host,
+                    "port": port,
+                    "family": family,
+                    "proto": proto,
+                    "flags": socket.AI_NUMERICHOST | socket.AI_NUMERICSERV,
+                }
+            )
+
+        return hosts
+
+    async def close(self) -> None:
+        pass
+
+
+class AsyncResolver(AbstractResolver):
+    """Use the `aiodns` package to make asynchronous DNS lookups"""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        if aiodns is None:
+            raise RuntimeError("Resolver requires aiodns library")
+
+        self._loop = asyncio.get_running_loop()
+        self._resolver = aiodns.DNSResolver(*args, loop=self._loop, **kwargs)
+
+    async def resolve(
+        self, host: str, port: int = 0, family: int = socket.AF_INET
+    ) -> List[Dict[str, Any]]:
+        try:
+            resp = await self._resolver.gethostbyname(host, family)
+        except aiodns.error.DNSError as exc:
+            msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed"
+            raise OSError(msg) from exc
+        hosts = []
+        for address in resp.addresses:
+            hosts.append(
+                {
+                    "hostname": host,
+                    "host": address,
+                    "port": port,
+                    "family": family,
+                    "proto": 0,
+                    "flags": socket.AI_NUMERICHOST | socket.AI_NUMERICSERV,
+                }
+            )
+
+        if not hosts:
+            raise OSError("DNS lookup failed")
+
+        return hosts
+
+    async def close(self) -> None:
+        self._resolver.cancel()
+
+
+_DefaultType = Type[Union[AsyncResolver, ThreadedResolver]]
+DefaultResolver: _DefaultType = AsyncResolver if aiodns_default else ThreadedResolver

aiohttp/streams.py

@@ -0,0 +1,652 @@
+import asyncio
+import collections
+import warnings
+from typing import (
+    Awaitable,
+    Callable,
+    Deque,
+    Final,
+    Generic,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+)
+
+from .base_protocol import BaseProtocol
+from .helpers import BaseTimerContext, TimerNoop, set_exception, set_result
+from .log import internal_logger
+
+__all__ = (
+    "EMPTY_PAYLOAD",
+    "EofStream",
+    "StreamReader",
+    "DataQueue",
+    "FlowControlDataQueue",
+)
+
+_T = TypeVar("_T")
+
+
+class EofStream(Exception):
+    """eof stream indication."""
+
+
+class AsyncStreamIterator(Generic[_T]):
+    def __init__(self, read_func: Callable[[], Awaitable[_T]]) -> None:
+        self.read_func = read_func
+
+    def __aiter__(self) -> "AsyncStreamIterator[_T]":
+        return self
+
+    async def __anext__(self) -> _T:
+        try:
+            rv = await self.read_func()
+        except EofStream:
+            raise StopAsyncIteration
+        if rv == b"":
+            raise StopAsyncIteration
+        return rv
+
+
+class ChunkTupleAsyncStreamIterator:
+    def __init__(self, stream: "StreamReader") -> None:
+        self._stream = stream
+
+    def __aiter__(self) -> "ChunkTupleAsyncStreamIterator":
+        return self
+
+    async def __anext__(self) -> Tuple[bytes, bool]:
+        rv = await self._stream.readchunk()
+        if rv == (b"", False):
+            raise StopAsyncIteration
+        return rv
+
+
+class AsyncStreamReaderMixin:
+    def __aiter__(self) -> AsyncStreamIterator[bytes]:
+        return AsyncStreamIterator(self.readline)  # type: ignore[attr-defined]
+
+    def iter_chunked(self, n: int) -> AsyncStreamIterator[bytes]:
+        """Returns an asynchronous iterator that yields chunks of size n."""
+        return AsyncStreamIterator(lambda: self.read(n))  # type: ignore[attr-defined]
+
+    def iter_any(self) -> AsyncStreamIterator[bytes]:
+        """Yield all available data as soon as it is received."""
+        return AsyncStreamIterator(self.readany)  # type: ignore[attr-defined]
+
+    def iter_chunks(self) -> ChunkTupleAsyncStreamIterator:
+        """Yield chunks of data as they are received by the server.
+
+        The yielded objects are tuples
+        of (bytes, bool) as returned by the StreamReader.readchunk method.
+        """
+        return ChunkTupleAsyncStreamIterator(self)  # type: ignore[arg-type]
+
+
+class StreamReader(AsyncStreamReaderMixin):
+    """An enhancement of asyncio.StreamReader.
+
+    Supports asynchronous iteration by line, chunk or as available::
+
+        async for line in reader:
+            ...
+        async for chunk in reader.iter_chunked(1024):
+            ...
+        async for slice in reader.iter_any():
+            ...
+
+    """
+
+    total_bytes = 0
+
+    def __init__(
+        self,
+        protocol: BaseProtocol,
+        limit: int,
+        *,
+        timer: Optional[BaseTimerContext] = None,
+        loop: asyncio.AbstractEventLoop,
+    ) -> None:
+        self._protocol = protocol
+        self._low_water = limit
+        self._high_water = limit * 2
+        if loop is None:
+            loop = asyncio.get_event_loop()
+        self._loop = loop
+        self._size = 0
+        self._cursor = 0
+        self._http_chunk_splits: Optional[List[int]] = None
+        self._buffer: Deque[bytes] = collections.deque()
+        self._buffer_offset = 0
+        self._eof = False
+        self._waiter: Optional[asyncio.Future[None]] = None
+        self._eof_waiter: Optional[asyncio.Future[None]] = None
+        self._exception: Optional[BaseException] = None
+        self._timer = TimerNoop() if timer is None else timer
+        self._eof_callbacks: List[Callable[[], None]] = []
+
+    def __repr__(self) -> str:
+        info = [self.__class__.__name__]
+        if self._size:
+            info.append("%d bytes" % self._size)
+        if self._eof:
+            info.append("eof")
+        if self._low_water != 2**16:  # default limit
+            info.append("low=%d high=%d" % (self._low_water, self._high_water))
+        if self._waiter:
+            info.append("w=%r" % self._waiter)
+        if self._exception:
+            info.append("e=%r" % self._exception)
+        return "<%s>" % " ".join(info)
+
+    def get_read_buffer_limits(self) -> Tuple[int, int]:
+        return (self._low_water, self._high_water)
+
+    def exception(self) -> Optional[BaseException]:
+        return self._exception
+
+    def set_exception(self, exc: BaseException) -> None:
+        self._exception = exc
+        self._eof_callbacks.clear()
+
+        waiter = self._waiter
+        if waiter is not None:
+            self._waiter = None
+            set_exception(waiter, exc)
+
+        waiter = self._eof_waiter
+        if waiter is not None:
+            self._eof_waiter = None
+            set_exception(waiter, exc)
+
+    def on_eof(self, callback: Callable[[], None]) -> None:
+        if self._eof:
+            try:
+                callback()
+            except Exception:
+                internal_logger.exception("Exception in eof callback")
+        else:
+            self._eof_callbacks.append(callback)
+
+    def feed_eof(self) -> None:
+        self._eof = True
+
+        waiter = self._waiter
+        if waiter is not None:
+            self._waiter = None
+            set_result(waiter, None)
+
+        waiter = self._eof_waiter
+        if waiter is not None:
+            self._eof_waiter = None
+            set_result(waiter, None)
+
+        for cb in self._eof_callbacks:
+            try:
+                cb()
+            except Exception:
+                internal_logger.exception("Exception in eof callback")
+
+        self._eof_callbacks.clear()
+
+    def is_eof(self) -> bool:
+        """Return True if  'feed_eof' was called."""
+        return self._eof
+
+    def at_eof(self) -> bool:
+        """Return True if the buffer is empty and 'feed_eof' was called."""
+        return self._eof and not self._buffer
+
+    async def wait_eof(self) -> None:
+        if self._eof:
+            return
+
+        assert self._eof_waiter is None
+        self._eof_waiter = self._loop.create_future()
+        try:
+            await self._eof_waiter
+        finally:
+            self._eof_waiter = None
+
+    def unread_data(self, data: bytes) -> None:
+        """rollback reading some data from stream, inserting it to buffer head."""
+        warnings.warn(
+            "unread_data() is deprecated "
+            "and will be removed in future releases (#3260)",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        if not data:
+            return
+
+        if self._buffer_offset:
+            self._buffer[0] = self._buffer[0][self._buffer_offset :]
+            self._buffer_offset = 0
+        self._size += len(data)
+        self._cursor -= len(data)
+        self._buffer.appendleft(data)
+        self._eof_counter = 0
+
+    # TODO: size is ignored, remove the param later
+    def feed_data(self, data: bytes, size: int = 0) -> None:
+        assert not self._eof, "feed_data after feed_eof"
+
+        if not data:
+            return
+
+        self._size += len(data)
+        self._buffer.append(data)
+        self.total_bytes += len(data)
+
+        waiter = self._waiter
+        if waiter is not None:
+            self._waiter = None
+            set_result(waiter, None)
+
+        if self._size > self._high_water and not self._protocol._reading_paused:
+            self._protocol.pause_reading()
+
+    def begin_http_chunk_receiving(self) -> None:
+        if self._http_chunk_splits is None:
+            if self.total_bytes:
+                raise RuntimeError(
+                    "Called begin_http_chunk_receiving when" "some data was already fed"
+                )
+            self._http_chunk_splits = []
+
+    def end_http_chunk_receiving(self) -> None:
+        if self._http_chunk_splits is None:
+            raise RuntimeError(
+                "Called end_chunk_receiving without calling "
+                "begin_chunk_receiving first"
+            )
+
+        # self._http_chunk_splits contains logical byte offsets from start of
+        # the body transfer. Each offset is the offset of the end of a chunk.
+        # "Logical" means bytes, accessible for a user.
+        # If no chunks containing logical data were received, current position
+        # is difinitely zero.
+        pos = self._http_chunk_splits[-1] if self._http_chunk_splits else 0
+
+        if self.total_bytes == pos:
+            # We should not add empty chunks here. So we check for that.
+            # Note, when chunked + gzip is used, we can receive a chunk
+            # of compressed data, but that data may not be enough for gzip FSM
+            # to yield any uncompressed data. That's why current position may
+            # not change after receiving a chunk.
+            return
+
+        self._http_chunk_splits.append(self.total_bytes)
+
+        # wake up readchunk when end of http chunk received
+        waiter = self._waiter
+        if waiter is not None:
+            self._waiter = None
+            set_result(waiter, None)
+
+    async def _wait(self, func_name: str) -> None:
+        # StreamReader uses a future to link the protocol feed_data() method
+        # to a read coroutine. Running two read coroutines at the same time
+        # would have an unexpected behaviour. It would not possible to know
+        # which coroutine would get the next data.
+        if self._waiter is not None:
+            raise RuntimeError(
+                "%s() called while another coroutine is "
+                "already waiting for incoming data" % func_name
+            )
+
+        waiter = self._waiter = self._loop.create_future()
+        try:
+            with self._timer:
+                await waiter
+        finally:
+            self._waiter = None
+
+    async def readline(self) -> bytes:
+        return await self.readuntil()
+
+    async def readuntil(self, separator: bytes = b"\n") -> bytes:
+        seplen = len(separator)
+        if seplen == 0:
+            raise ValueError("Separator should be at least one-byte string")
+
+        if self._exception is not None:
+            raise self._exception
+
+        chunk = b""
+        chunk_size = 0
+        not_enough = True
+
+        while not_enough:
+            while self._buffer and not_enough:
+                offset = self._buffer_offset
+                ichar = self._buffer[0].find(separator, offset) + 1
+                # Read from current offset to found separator or to the end.
+                data = self._read_nowait_chunk(
+                    ichar - offset + seplen - 1 if ichar else -1
+                )
+                chunk += data
+                chunk_size += len(data)
+                if ichar:
+                    not_enough = False
+
+                if chunk_size > self._high_water:
+                    raise ValueError("Chunk too big")
+
+            if self._eof:
+                break
+
+            if not_enough:
+                await self._wait("readuntil")
+
+        return chunk
+
+    async def read(self, n: int = -1) -> bytes:
+        if self._exception is not None:
+            raise self._exception
+
+        if not n:
+            return b""
+
+        if n < 0:
+            # This used to just loop creating a new waiter hoping to
+            # collect everything in self._buffer, but that would
+            # deadlock if the subprocess sends more than self.limit
+            # bytes.  So just call self.readany() until EOF.
+            blocks = []
+            while True:
+                block = await self.readany()
+                if not block:
+                    break
+                blocks.append(block)
+            return b"".join(blocks)
+
+        # TODO: should be `if` instead of `while`
+        # because waiter maybe triggered on chunk end,
+        # without feeding any data
+        while not self._buffer and not self._eof:
+            await self._wait("read")
+
+        return self._read_nowait(n)
+
+    async def readany(self) -> bytes:
+        if self._exception is not None:
+            raise self._exception
+
+        # TODO: should be `if` instead of `while`
+        # because waiter maybe triggered on chunk end,
+        # without feeding any data
+        while not self._buffer and not self._eof:
+            await self._wait("readany")
+
+        return self._read_nowait(-1)
+
+    async def readchunk(self) -> Tuple[bytes, bool]:
+        """Returns a tuple of (data, end_of_http_chunk).
+
+        When chunked transfer
+        encoding is used, end_of_http_chunk is a boolean indicating if the end
+        of the data corresponds to the end of a HTTP chunk , otherwise it is
+        always False.
+        """
+        while True:
+            if self._exception is not None:
+                raise self._exception
+
+            while self._http_chunk_splits:
+                pos = self._http_chunk_splits.pop(0)
+                if pos == self._cursor:
+                    return (b"", True)
+                if pos > self._cursor:
+                    return (self._read_nowait(pos - self._cursor), True)
+                internal_logger.warning(
+                    "Skipping HTTP chunk end due to data "
+                    "consumption beyond chunk boundary"
+                )
+
+            if self._buffer:
+                return (self._read_nowait_chunk(-1), False)
+                # return (self._read_nowait(-1), False)
+
+            if self._eof:
+                # Special case for signifying EOF.
+                # (b'', True) is not a final return value actually.
+                return (b"", False)
+
+            await self._wait("readchunk")
+
+    async def readexactly(self, n: int) -> bytes:
+        if self._exception is not None:
+            raise self._exception
+
+        blocks: List[bytes] = []
+        while n > 0:
+            block = await self.read(n)
+            if not block:
+                partial = b"".join(blocks)
+                raise asyncio.IncompleteReadError(partial, len(partial) + n)
+            blocks.append(block)
+            n -= len(block)
+
+        return b"".join(blocks)
+
+    def read_nowait(self, n: int = -1) -> bytes:
+        # default was changed to be consistent with .read(-1)
+        #
+        # I believe the most users don't know about the method and
+        # they are not affected.
+        if self._exception is not None:
+            raise self._exception
+
+        if self._waiter and not self._waiter.done():
+            raise RuntimeError(
+                "Called while some coroutine is waiting for incoming data."
+            )
+
+        return self._read_nowait(n)
+
+    def _read_nowait_chunk(self, n: int) -> bytes:
+        first_buffer = self._buffer[0]
+        offset = self._buffer_offset
+        if n != -1 and len(first_buffer) - offset > n:
+            data = first_buffer[offset : offset + n]
+            self._buffer_offset += n
+
+        elif offset:
+            self._buffer.popleft()
+            data = first_buffer[offset:]
+            self._buffer_offset = 0
+
+        else:
+            data = self._buffer.popleft()
+
+        self._size -= len(data)
+        self._cursor += len(data)
+
+        chunk_splits = self._http_chunk_splits
+        # Prevent memory leak: drop useless chunk splits
+        while chunk_splits and chunk_splits[0] < self._cursor:
+            chunk_splits.pop(0)
+
+        if self._size < self._low_water and self._protocol._reading_paused:
+            self._protocol.resume_reading()
+        return data
+
+    def _read_nowait(self, n: int) -> bytes:
+        """Read not more than n bytes, or whole buffer if n == -1"""
+        self._timer.assert_timeout()
+
+        chunks = []
+        while self._buffer:
+            chunk = self._read_nowait_chunk(n)
+            chunks.append(chunk)
+            if n != -1:
+                n -= len(chunk)
+                if n == 0:
+                    break
+
+        return b"".join(chunks) if chunks else b""
+
+
+class EmptyStreamReader(StreamReader):  # lgtm [py/missing-call-to-init]
+    def __init__(self) -> None:
+        self._read_eof_chunk = False
+
+    def __repr__(self) -> str:
+        return "<%s>" % self.__class__.__name__
+
+    def exception(self) -> Optional[BaseException]:
+        return None
+
+    def set_exception(self, exc: BaseException) -> None:
+        pass
+
+    def on_eof(self, callback: Callable[[], None]) -> None:
+        try:
+            callback()
+        except Exception:
+            internal_logger.exception("Exception in eof callback")
+
+    def feed_eof(self) -> None:
+        pass
+
+    def is_eof(self) -> bool:
+        return True
+
+    def at_eof(self) -> bool:
+        return True
+
+    async def wait_eof(self) -> None:
+        return
+
+    def feed_data(self, data: bytes, n: int = 0) -> None:
+        pass
+
+    async def readline(self) -> bytes:
+        return b""
+
+    async def read(self, n: int = -1) -> bytes:
+        return b""
+
+    # TODO add async def readuntil
+
+    async def readany(self) -> bytes:
+        return b""
+
+    async def readchunk(self) -> Tuple[bytes, bool]:
+        if not self._read_eof_chunk:
+            self._read_eof_chunk = True
+            return (b"", False)
+
+        return (b"", True)
+
+    async def readexactly(self, n: int) -> bytes:
+        raise asyncio.IncompleteReadError(b"", n)
+
+    def read_nowait(self, n: int = -1) -> bytes:
+        return b""
+
+
+EMPTY_PAYLOAD: Final[StreamReader] = EmptyStreamReader()
+
+
+class DataQueue(Generic[_T]):
+    """DataQueue is a general-purpose blocking queue with one reader."""
+
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        self._loop = loop
+        self._eof = False
+        self._waiter: Optional[asyncio.Future[None]] = None
+        self._exception: Optional[BaseException] = None
+        self._size = 0
+        self._buffer: Deque[Tuple[_T, int]] = collections.deque()
+
+    def __len__(self) -> int:
+        return len(self._buffer)
+
+    def is_eof(self) -> bool:
+        return self._eof
+
+    def at_eof(self) -> bool:
+        return self._eof and not self._buffer
+
+    def exception(self) -> Optional[BaseException]:
+        return self._exception
+
+    def set_exception(self, exc: BaseException) -> None:
+        self._eof = True
+        self._exception = exc
+
+        waiter = self._waiter
+        if waiter is not None:
+            self._waiter = None
+            set_exception(waiter, exc)
+
+    def feed_data(self, data: _T, size: int = 0) -> None:
+        self._size += size
+        self._buffer.append((data, size))
+
+        waiter = self._waiter
+        if waiter is not None:
+            self._waiter = None
+            set_result(waiter, None)
+
+    def feed_eof(self) -> None:
+        self._eof = True
+
+        waiter = self._waiter
+        if waiter is not None:
+            self._waiter = None
+            set_result(waiter, None)
+
+    async def read(self) -> _T:
+        if not self._buffer and not self._eof:
+            assert not self._waiter
+            self._waiter = self._loop.create_future()
+            try:
+                await self._waiter
+            except (asyncio.CancelledError, asyncio.TimeoutError):
+                self._waiter = None
+                raise
+
+        if self._buffer:
+            data, size = self._buffer.popleft()
+            self._size -= size
+            return data
+        else:
+            if self._exception is not None:
+                raise self._exception
+            else:
+                raise EofStream
+
+    def __aiter__(self) -> AsyncStreamIterator[_T]:
+        return AsyncStreamIterator(self.read)
+
+
+class FlowControlDataQueue(DataQueue[_T]):
+    """FlowControlDataQueue resumes and pauses an underlying stream.
+
+    It is a destination for parsed data.
+    """
+
+    def __init__(
+        self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop
+    ) -> None:
+        super().__init__(loop=loop)
+
+        self._protocol = protocol
+        self._limit = limit * 2
+
+    def feed_data(self, data: _T, size: int = 0) -> None:
+        super().feed_data(data, size)
+
+        if self._size > self._limit and not self._protocol._reading_paused:
+            self._protocol.pause_reading()
+
+    async def read(self) -> _T:
+        try:
+            return await super().read()
+        finally:
+            if self._size < self._limit and self._protocol._reading_paused:
+                self._protocol.resume_reading()

aiohttp/tcp_helpers.py

@@ -0,0 +1,37 @@
+"""Helper methods to tune a TCP connection"""
+
+import asyncio
+import socket
+from contextlib import suppress
+from typing import Optional  # noqa
+
+__all__ = ("tcp_keepalive", "tcp_nodelay")
+
+
+if hasattr(socket, "SO_KEEPALIVE"):
+
+    def tcp_keepalive(transport: asyncio.Transport) -> None:
+        sock = transport.get_extra_info("socket")
+        if sock is not None:
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+
+else:
+
+    def tcp_keepalive(transport: asyncio.Transport) -> None:  # pragma: no cover
+        pass
+
+
+def tcp_nodelay(transport: asyncio.Transport, value: bool) -> None:
+    sock = transport.get_extra_info("socket")
+
+    if sock is None:
+        return
+
+    if sock.family not in (socket.AF_INET, socket.AF_INET6):
+        return
+
+    value = bool(value)
+
+    # socket may be closed already, on windows OSError get raised
+    with suppress(OSError):
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, value)

aiohttp/test_utils.py

@@ -0,0 +1,618 @@
+"""Utilities shared by tests."""
+
+import asyncio
+import contextlib
+import gc
+import inspect
+import ipaddress
+import os
+import socket
+import sys
+from abc import ABC, abstractmethod
+from types import TracebackType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Iterator,
+    List,
+    Optional,
+    Type,
+    Union,
+    cast,
+)
+from unittest import IsolatedAsyncioTestCase, mock
+
+from aiosignal import Signal
+from multidict import CIMultiDict, CIMultiDictProxy
+from yarl import URL
+
+import aiohttp
+from aiohttp.client import _RequestContextManager, _WSRequestContextManager
+
+from . import ClientSession, hdrs
+from .abc import AbstractCookieJar
+from .client_reqrep import ClientResponse
+from .client_ws import ClientWebSocketResponse
+from .helpers import _SENTINEL, sentinel
+from .http import HttpVersion, RawRequestMessage
+from .typedefs import StrOrURL
+from .web import (
+    Application,
+    AppRunner,
+    BaseRunner,
+    Request,
+    Server,
+    ServerRunner,
+    SockSite,
+    UrlMappingMatchInfo,
+)
+from .web_protocol import _RequestHandler
+
+if TYPE_CHECKING:  # pragma: no cover
+    from ssl import SSLContext
+else:
+    SSLContext = None
+
+REUSE_ADDRESS = os.name == "posix" and sys.platform != "cygwin"
+
+
+def get_unused_port_socket(
+    host: str, family: socket.AddressFamily = socket.AF_INET
+) -> socket.socket:
+    return get_port_socket(host, 0, family)
+
+
+def get_port_socket(
+    host: str, port: int, family: socket.AddressFamily = socket.AF_INET
+) -> socket.socket:
+    s = socket.socket(family, socket.SOCK_STREAM)
+    if REUSE_ADDRESS:
+        # Windows has different semantics for SO_REUSEADDR,
+        # so don't set it. Ref:
+        # https://docs.microsoft.com/en-us/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    s.bind((host, port))
+    return s
+
+
+def unused_port() -> int:
+    """Return a port that is unused on the current host."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return cast(int, s.getsockname()[1])
+
+
+class BaseTestServer(ABC):
+    __test__ = False
+
+    def __init__(
+        self,
+        *,
+        scheme: Union[str, _SENTINEL] = sentinel,
+        host: str = "127.0.0.1",
+        port: Optional[int] = None,
+        skip_url_asserts: bool = False,
+        socket_factory: Callable[
+            [str, int, socket.AddressFamily], socket.socket
+        ] = get_port_socket,
+        **kwargs: Any,
+    ) -> None:
+        self.runner: Optional[BaseRunner] = None
+        self._root: Optional[URL] = None
+        self.host = host
+        self.port = port
+        self._closed = False
+        self.scheme = scheme
+        self.skip_url_asserts = skip_url_asserts
+        self.socket_factory = socket_factory
+
+    async def start_server(self, **kwargs: Any) -> None:
+        if self.runner:
+            return
+        self._ssl = kwargs.pop("ssl", None)
+        self.runner = await self._make_runner(handler_cancellation=True, **kwargs)
+        await self.runner.setup()
+        if not self.port:
+            self.port = 0
+        absolute_host = self.host
+        try:
+            version = ipaddress.ip_address(self.host).version
+        except ValueError:
+            version = 4
+        if version == 6:
+            absolute_host = f"[{self.host}]"
+        family = socket.AF_INET6 if version == 6 else socket.AF_INET
+        _sock = self.socket_factory(self.host, self.port, family)
+        self.host, self.port = _sock.getsockname()[:2]
+        site = SockSite(self.runner, sock=_sock, ssl_context=self._ssl)
+        await site.start()
+        server = site._server
+        assert server is not None
+        sockets = server.sockets  # type: ignore[attr-defined]
+        assert sockets is not None
+        self.port = sockets[0].getsockname()[1]
+        if self.scheme is sentinel:
+            if self._ssl:
+                scheme = "https"
+            else:
+                scheme = "http"
+            self.scheme = scheme
+        self._root = URL(f"{self.scheme}://{absolute_host}:{self.port}")
+
+    @abstractmethod  # pragma: no cover
+    async def _make_runner(self, **kwargs: Any) -> BaseRunner:
+        pass
+
+    def make_url(self, path: StrOrURL) -> URL:
+        assert self._root is not None
+        url = URL(path)
+        if not self.skip_url_asserts:
+            assert not url.is_absolute()
+            return self._root.join(url)
+        else:
+            return URL(str(self._root) + str(path))
+
+    @property
+    def started(self) -> bool:
+        return self.runner is not None
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    @property
+    def handler(self) -> Server:
+        # for backward compatibility
+        # web.Server instance
+        runner = self.runner
+        assert runner is not None
+        assert runner.server is not None
+        return runner.server
+
+    async def close(self) -> None:
+        """Close all fixtures created by the test client.
+
+        After that point, the TestClient is no longer usable.
+
+        This is an idempotent function: running close multiple times
+        will not have any additional effects.
+
+        close is also run when the object is garbage collected, and on
+        exit when used as a context manager.
+
+        """
+        if self.started and not self.closed:
+            assert self.runner is not None
+            await self.runner.cleanup()
+            self._root = None
+            self.port = None
+            self._closed = True
+
+    async def __aenter__(self) -> "BaseTestServer":
+        await self.start_server()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None:
+        await self.close()
+
+
+class TestServer(BaseTestServer):
+    def __init__(
+        self,
+        app: Application,
+        *,
+        scheme: Union[str, _SENTINEL] = sentinel,
+        host: str = "127.0.0.1",
+        port: Optional[int] = None,
+        **kwargs: Any,
+    ):
+        self.app = app
+        super().__init__(scheme=scheme, host=host, port=port, **kwargs)
+
+    async def _make_runner(self, **kwargs: Any) -> BaseRunner:
+        return AppRunner(self.app, **kwargs)
+
+
+class RawTestServer(BaseTestServer):
+    def __init__(
+        self,
+        handler: _RequestHandler,
+        *,
+        scheme: Union[str, _SENTINEL] = sentinel,
+        host: str = "127.0.0.1",
+        port: Optional[int] = None,
+        **kwargs: Any,
+    ) -> None:
+        self._handler = handler
+        super().__init__(scheme=scheme, host=host, port=port, **kwargs)
+
+    async def _make_runner(self, **kwargs: Any) -> ServerRunner:
+        srv = Server(self._handler, **kwargs)
+        return ServerRunner(srv, **kwargs)
+
+
+class TestClient:
+    """
+    A test client implementation.
+
+    To write functional tests for aiohttp based servers.
+
+    """
+
+    __test__ = False
+
+    def __init__(
+        self,
+        server: BaseTestServer,
+        *,
+        cookie_jar: Optional[AbstractCookieJar] = None,
+        **kwargs: Any,
+    ) -> None:
+        if not isinstance(server, BaseTestServer):
+            raise TypeError(
+                "server must be TestServer " "instance, found type: %r" % type(server)
+            )
+        self._server = server
+        if cookie_jar is None:
+            cookie_jar = aiohttp.CookieJar(unsafe=True)
+        self._session = ClientSession(cookie_jar=cookie_jar, **kwargs)
+        self._closed = False
+        self._responses: List[ClientResponse] = []
+        self._websockets: List[ClientWebSocketResponse] = []
+
+    async def start_server(self) -> None:
+        await self._server.start_server()
+
+    @property
+    def scheme(self) -> Union[str, object]:
+        return self._server.scheme
+
+    @property
+    def host(self) -> str:
+        return self._server.host
+
+    @property
+    def port(self) -> Optional[int]:
+        return self._server.port
+
+    @property
+    def server(self) -> BaseTestServer:
+        return self._server
+
+    @property
+    def app(self) -> Optional[Application]:
+        return cast(Optional[Application], getattr(self._server, "app", None))
+
+    @property
+    def session(self) -> ClientSession:
+        """An internal aiohttp.ClientSession.
+
+        Unlike the methods on the TestClient, client session requests
+        do not automatically include the host in the url queried, and
+        will require an absolute path to the resource.
+
+        """
+        return self._session
+
+    def make_url(self, path: StrOrURL) -> URL:
+        return self._server.make_url(path)
+
+    async def _request(
+        self, method: str, path: StrOrURL, **kwargs: Any
+    ) -> ClientResponse:
+        resp = await self._session.request(method, self.make_url(path), **kwargs)
+        # save it to close later
+        self._responses.append(resp)
+        return resp
+
+    def request(
+        self, method: str, path: StrOrURL, **kwargs: Any
+    ) -> _RequestContextManager:
+        """Routes a request to tested http server.
+
+        The interface is identical to aiohttp.ClientSession.request,
+        except the loop kwarg is overridden by the instance used by the
+        test server.
+
+        """
+        return _RequestContextManager(self._request(method, path, **kwargs))
+
+    def get(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
+        """Perform an HTTP GET request."""
+        return _RequestContextManager(self._request(hdrs.METH_GET, path, **kwargs))
+
+    def post(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
+        """Perform an HTTP POST request."""
+        return _RequestContextManager(self._request(hdrs.METH_POST, path, **kwargs))
+
+    def options(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
+        """Perform an HTTP OPTIONS request."""
+        return _RequestContextManager(self._request(hdrs.METH_OPTIONS, path, **kwargs))
+
+    def head(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
+        """Perform an HTTP HEAD request."""
+        return _RequestContextManager(self._request(hdrs.METH_HEAD, path, **kwargs))
+
+    def put(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
+        """Perform an HTTP PUT request."""
+        return _RequestContextManager(self._request(hdrs.METH_PUT, path, **kwargs))
+
+    def patch(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
+        """Perform an HTTP PATCH request."""
+        return _RequestContextManager(self._request(hdrs.METH_PATCH, path, **kwargs))
+
+    def delete(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
+        """Perform an HTTP PATCH request."""
+        return _RequestContextManager(self._request(hdrs.METH_DELETE, path, **kwargs))
+
+    def ws_connect(self, path: StrOrURL, **kwargs: Any) -> _WSRequestContextManager:
+        """Initiate websocket connection.
+
+        The api corresponds to aiohttp.ClientSession.ws_connect.
+
+        """
+        return _WSRequestContextManager(self._ws_connect(path, **kwargs))
+
+    async def _ws_connect(
+        self, path: StrOrURL, **kwargs: Any
+    ) -> ClientWebSocketResponse:
+        ws = await self._session.ws_connect(self.make_url(path), **kwargs)
+        self._websockets.append(ws)
+        return ws
+
+    async def close(self) -> None:
+        """Close all fixtures created by the test client.
+
+        After that point, the TestClient is no longer usable.
+
+        This is an idempotent function: running close multiple times
+        will not have any additional effects.
+
+        close is also run on exit when used as a(n) (asynchronous)
+        context manager.
+
+        """
+        if not self._closed:
+            for resp in self._responses:
+                resp.close()
+            for ws in self._websockets:
+                await ws.close()
+            await self._session.close()
+            await self._server.close()
+            self._closed = True
+
+    async def __aenter__(self) -> "TestClient":
+        await self.start_server()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc: Optional[BaseException],
+        tb: Optional[TracebackType],
+    ) -> None:
+        await self.close()
+
+
+class AioHTTPTestCase(IsolatedAsyncioTestCase, ABC):
+    """A base class to allow for unittest web applications using aiohttp.
+
+    Provides the following:
+
+    * self.client (aiohttp.test_utils.TestClient): an aiohttp test client.
+    * self.app (aiohttp.web.Application): the application returned by
+        self.get_application()
+
+    Note that the TestClient's methods are asynchronous: you have to
+    execute function on the test client using asynchronous methods.
+    """
+
+    @abstractmethod
+    async def get_application(self) -> Application:
+        """Get application.
+
+        This method should be overridden to return the aiohttp.web.Application
+        object to test.
+        """
+
+    async def asyncSetUp(self) -> None:
+        self.app = await self.get_application()
+        self.server = await self.get_server(self.app)
+        self.client = await self.get_client(self.server)
+
+        await self.client.start_server()
+
+    async def asyncTearDown(self) -> None:
+        await self.client.close()
+
+    async def get_server(self, app: Application) -> TestServer:
+        """Return a TestServer instance."""
+        return TestServer(app)
+
+    async def get_client(self, server: TestServer) -> TestClient:
+        """Return a TestClient instance."""
+        return TestClient(server)
+
+
+_LOOP_FACTORY = Callable[[], asyncio.AbstractEventLoop]
+
+
+@contextlib.contextmanager
+def loop_context(
+    loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, fast: bool = False
+) -> Iterator[asyncio.AbstractEventLoop]:
+    """A contextmanager that creates an event_loop, for test purposes.
+
+    Handles the creation and cleanup of a test loop.
+    """
+    loop = setup_test_loop(loop_factory)
+    yield loop
+    teardown_test_loop(loop, fast=fast)
+
+
+def setup_test_loop(
+    loop_factory: _LOOP_FACTORY = asyncio.new_event_loop,
+) -> asyncio.AbstractEventLoop:
+    """Create and return an asyncio.BaseEventLoop instance.
+
+    The caller should also call teardown_test_loop,
+    once they are done with the loop.
+    """
+    loop = loop_factory()
+    asyncio.set_event_loop(loop)
+    return loop
+
+
+def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None:
+    """Teardown and cleanup an event_loop created by setup_test_loop."""
+    closed = loop.is_closed()
+    if not closed:
+        loop.call_soon(loop.stop)
+        loop.run_forever()
+        loop.close()
+
+    if not fast:
+        gc.collect()
+
+    asyncio.set_event_loop(None)
+
+
+def _create_app_mock() -> mock.MagicMock:
+    def get_dict(app: Any, key: str) -> Any:
+        return app.__app_dict[key]
+
+    def set_dict(app: Any, key: str, value: Any) -> None:
+        app.__app_dict[key] = value
+
+    app = mock.MagicMock(spec=Application)
+    app.__app_dict = {}
+    app.__getitem__ = get_dict
+    app.__setitem__ = set_dict
+
+    app.on_response_prepare = Signal(app)
+    app.on_response_prepare.freeze()
+    return app
+
+
+def _create_transport(sslcontext: Optional[SSLContext] = None) -> mock.Mock:
+    transport = mock.Mock()
+
+    def get_extra_info(key: str) -> Optional[SSLContext]:
+        if key == "sslcontext":
+            return sslcontext
+        else:
+            return None
+
+    transport.get_extra_info.side_effect = get_extra_info
+    return transport
+
+
+def make_mocked_request(
+    method: str,
+    path: str,
+    headers: Any = None,
+    *,
+    match_info: Any = sentinel,
+    version: HttpVersion = HttpVersion(1, 1),
+    closing: bool = False,
+    app: Any = None,
+    writer: Any = sentinel,
+    protocol: Any = sentinel,
+    transport: Any = sentinel,
+    payload: Any = sentinel,
+    sslcontext: Optional[SSLContext] = None,
+    client_max_size: int = 1024**2,
+    loop: Any = ...,
+) -> Request:
+    """Creates mocked web.Request testing purposes.
+
+    Useful in unit tests, when spinning full web server is overkill or
+    specific conditions and errors are hard to trigger.
+    """
+    task = mock.Mock()
+    if loop is ...:
+        loop = mock.Mock()
+        loop.create_future.return_value = ()
+
+    if version < HttpVersion(1, 1):
+        closing = True
+
+    if headers:
+        headers = CIMultiDictProxy(CIMultiDict(headers))
+        raw_hdrs = tuple(
+            (k.encode("utf-8"), v.encode("utf-8")) for k, v in headers.items()
+        )
+    else:
+        headers = CIMultiDictProxy(CIMultiDict())
+        raw_hdrs = ()
+
+    chunked = "chunked" in headers.get(hdrs.TRANSFER_ENCODING, "").lower()
+
+    message = RawRequestMessage(
+        method,
+        path,
+        version,
+        headers,
+        raw_hdrs,
+        closing,
+        None,
+        False,
+        chunked,
+        URL(path),
+    )
+    if app is None:
+        app = _create_app_mock()
+
+    if transport is sentinel:
+        transport = _create_transport(sslcontext)
+
+    if protocol is sentinel:
+        protocol = mock.Mock()
+        protocol.transport = transport
+
+    if writer is sentinel:
+        writer = mock.Mock()
+        writer.write_headers = make_mocked_coro(None)
+        writer.write = make_mocked_coro(None)
+        writer.write_eof = make_mocked_coro(None)
+        writer.drain = make_mocked_coro(None)
+        writer.transport = transport
+
+    protocol.transport = transport
+    protocol.writer = writer
+
+    if payload is sentinel:
+        payload = mock.Mock()
+
+    req = Request(
+        message, payload, protocol, writer, task, loop, client_max_size=client_max_size
+    )
+
+    match_info = UrlMappingMatchInfo(
+        {} if match_info is sentinel else match_info, mock.Mock()
+    )
+    match_info.add_app(app)
+    req._match_info = match_info
+
+    return req
+
+
+def make_mocked_coro(
+    return_value: Any = sentinel, raise_exception: Any = sentinel
+) -> Any:
+    """Creates a coroutine mock."""
+
+    async def mock_coro(*args: Any, **kwargs: Any) -> Any:
+        if raise_exception is not sentinel:
+            raise raise_exception
+        if not inspect.isawaitable(return_value):
+            return return_value
+        await return_value
+
+    return mock.Mock(wraps=mock_coro)

aiohttp/tracing.py

@@ -0,0 +1,471 @@
+import dataclasses
+from types import SimpleNamespace
+from typing import TYPE_CHECKING, Awaitable, Optional, Protocol, Type, TypeVar
+
+from aiosignal import Signal
+from multidict import CIMultiDict
+from yarl import URL
+
+from .client_reqrep import ClientResponse
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .client import ClientSession
+
+    _ParamT_contra = TypeVar("_ParamT_contra", contravariant=True)
+
+    class _SignalCallback(Protocol[_ParamT_contra]):
+        def __call__(
+            self,
+            __client_session: ClientSession,
+            __trace_config_ctx: SimpleNamespace,
+            __params: _ParamT_contra,
+        ) -> Awaitable[None]:
+            ...
+
+
+__all__ = (
+    "TraceConfig",
+    "TraceRequestStartParams",
+    "TraceRequestEndParams",
+    "TraceRequestExceptionParams",
+    "TraceConnectionQueuedStartParams",
+    "TraceConnectionQueuedEndParams",
+    "TraceConnectionCreateStartParams",
+    "TraceConnectionCreateEndParams",
+    "TraceConnectionReuseconnParams",
+    "TraceDnsResolveHostStartParams",
+    "TraceDnsResolveHostEndParams",
+    "TraceDnsCacheHitParams",
+    "TraceDnsCacheMissParams",
+    "TraceRequestRedirectParams",
+    "TraceRequestChunkSentParams",
+    "TraceResponseChunkReceivedParams",
+    "TraceRequestHeadersSentParams",
+)
+
+
+class TraceConfig:
+    """First-class used to trace requests launched via ClientSession objects."""
+
+    def __init__(
+        self, trace_config_ctx_factory: Type[SimpleNamespace] = SimpleNamespace
+    ) -> None:
+        self._on_request_start: Signal[
+            _SignalCallback[TraceRequestStartParams]
+        ] = Signal(self)
+        self._on_request_chunk_sent: Signal[
+            _SignalCallback[TraceRequestChunkSentParams]
+        ] = Signal(self)
+        self._on_response_chunk_received: Signal[
+            _SignalCallback[TraceResponseChunkReceivedParams]
+        ] = Signal(self)
+        self._on_request_end: Signal[_SignalCallback[TraceRequestEndParams]] = Signal(
+            self
+        )
+        self._on_request_exception: Signal[
+            _SignalCallback[TraceRequestExceptionParams]
+        ] = Signal(self)
+        self._on_request_redirect: Signal[
+            _SignalCallback[TraceRequestRedirectParams]
+        ] = Signal(self)
+        self._on_connection_queued_start: Signal[
+            _SignalCallback[TraceConnectionQueuedStartParams]
+        ] = Signal(self)
+        self._on_connection_queued_end: Signal[
+            _SignalCallback[TraceConnectionQueuedEndParams]
+        ] = Signal(self)
+        self._on_connection_create_start: Signal[
+            _SignalCallback[TraceConnectionCreateStartParams]
+        ] = Signal(self)
+        self._on_connection_create_end: Signal[
+            _SignalCallback[TraceConnectionCreateEndParams]
+        ] = Signal(self)
+        self._on_connection_reuseconn: Signal[
+            _SignalCallback[TraceConnectionReuseconnParams]
+        ] = Signal(self)
+        self._on_dns_resolvehost_start: Signal[
+            _SignalCallback[TraceDnsResolveHostStartParams]
+        ] = Signal(self)
+        self._on_dns_resolvehost_end: Signal[
+            _SignalCallback[TraceDnsResolveHostEndParams]
+        ] = Signal(self)
+        self._on_dns_cache_hit: Signal[
+            _SignalCallback[TraceDnsCacheHitParams]
+        ] = Signal(self)
+        self._on_dns_cache_miss: Signal[
+            _SignalCallback[TraceDnsCacheMissParams]
+        ] = Signal(self)
+        self._on_request_headers_sent: Signal[
+            _SignalCallback[TraceRequestHeadersSentParams]
+        ] = Signal(self)
+
+        self._trace_config_ctx_factory = trace_config_ctx_factory
+
+    def trace_config_ctx(
+        self, trace_request_ctx: Optional[SimpleNamespace] = None
+    ) -> SimpleNamespace:
+        """Return a new trace_config_ctx instance"""
+        return self._trace_config_ctx_factory(trace_request_ctx=trace_request_ctx)
+
+    def freeze(self) -> None:
+        self._on_request_start.freeze()
+        self._on_request_chunk_sent.freeze()
+        self._on_response_chunk_received.freeze()
+        self._on_request_end.freeze()
+        self._on_request_exception.freeze()
+        self._on_request_redirect.freeze()
+        self._on_connection_queued_start.freeze()
+        self._on_connection_queued_end.freeze()
+        self._on_connection_create_start.freeze()
+        self._on_connection_create_end.freeze()
+        self._on_connection_reuseconn.freeze()
+        self._on_dns_resolvehost_start.freeze()
+        self._on_dns_resolvehost_end.freeze()
+        self._on_dns_cache_hit.freeze()
+        self._on_dns_cache_miss.freeze()
+        self._on_request_headers_sent.freeze()
+
+    @property
+    def on_request_start(self) -> "Signal[_SignalCallback[TraceRequestStartParams]]":
+        return self._on_request_start
+
+    @property
+    def on_request_chunk_sent(
+        self,
+    ) -> "Signal[_SignalCallback[TraceRequestChunkSentParams]]":
+        return self._on_request_chunk_sent
+
+    @property
+    def on_response_chunk_received(
+        self,
+    ) -> "Signal[_SignalCallback[TraceResponseChunkReceivedParams]]":
+        return self._on_response_chunk_received
+
+    @property
+    def on_request_end(self) -> "Signal[_SignalCallback[TraceRequestEndParams]]":
+        return self._on_request_end
+
+    @property
+    def on_request_exception(
+        self,
+    ) -> "Signal[_SignalCallback[TraceRequestExceptionParams]]":
+        return self._on_request_exception
+
+    @property
+    def on_request_redirect(
+        self,
+    ) -> "Signal[_SignalCallback[TraceRequestRedirectParams]]":
+        return self._on_request_redirect
+
+    @property
+    def on_connection_queued_start(
+        self,
+    ) -> "Signal[_SignalCallback[TraceConnectionQueuedStartParams]]":
+        return self._on_connection_queued_start
+
+    @property
+    def on_connection_queued_end(
+        self,
+    ) -> "Signal[_SignalCallback[TraceConnectionQueuedEndParams]]":
+        return self._on_connection_queued_end
+
+    @property
+    def on_connection_create_start(
+        self,
+    ) -> "Signal[_SignalCallback[TraceConnectionCreateStartParams]]":
+        return self._on_connection_create_start
+
+    @property
+    def on_connection_create_end(
+        self,
+    ) -> "Signal[_SignalCallback[TraceConnectionCreateEndParams]]":
+        return self._on_connection_create_end
+
+    @property
+    def on_connection_reuseconn(
+        self,
+    ) -> "Signal[_SignalCallback[TraceConnectionReuseconnParams]]":
+        return self._on_connection_reuseconn
+
+    @property
+    def on_dns_resolvehost_start(
+        self,
+    ) -> "Signal[_SignalCallback[TraceDnsResolveHostStartParams]]":
+        return self._on_dns_resolvehost_start
+
+    @property
+    def on_dns_resolvehost_end(
+        self,
+    ) -> "Signal[_SignalCallback[TraceDnsResolveHostEndParams]]":
+        return self._on_dns_resolvehost_end
+
+    @property
+    def on_dns_cache_hit(self) -> "Signal[_SignalCallback[TraceDnsCacheHitParams]]":
+        return self._on_dns_cache_hit
+
+    @property
+    def on_dns_cache_miss(self) -> "Signal[_SignalCallback[TraceDnsCacheMissParams]]":
+        return self._on_dns_cache_miss
+
+    @property
+    def on_request_headers_sent(
+        self,
+    ) -> "Signal[_SignalCallback[TraceRequestHeadersSentParams]]":
+        return self._on_request_headers_sent
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceRequestStartParams:
+    """Parameters sent by the `on_request_start` signal"""
+
+    method: str
+    url: URL
+    headers: "CIMultiDict[str]"
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceRequestChunkSentParams:
+    """Parameters sent by the `on_request_chunk_sent` signal"""
+
+    method: str
+    url: URL
+    chunk: bytes
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceResponseChunkReceivedParams:
+    """Parameters sent by the `on_response_chunk_received` signal"""
+
+    method: str
+    url: URL
+    chunk: bytes
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceRequestEndParams:
+    """Parameters sent by the `on_request_end` signal"""
+
+    method: str
+    url: URL
+    headers: "CIMultiDict[str]"
+    response: ClientResponse
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceRequestExceptionParams:
+    """Parameters sent by the `on_request_exception` signal"""
+
+    method: str
+    url: URL
+    headers: "CIMultiDict[str]"
+    exception: BaseException
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceRequestRedirectParams:
+    """Parameters sent by the `on_request_redirect` signal"""
+
+    method: str
+    url: URL
+    headers: "CIMultiDict[str]"
+    response: ClientResponse
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceConnectionQueuedStartParams:
+    """Parameters sent by the `on_connection_queued_start` signal"""
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceConnectionQueuedEndParams:
+    """Parameters sent by the `on_connection_queued_end` signal"""
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceConnectionCreateStartParams:
+    """Parameters sent by the `on_connection_create_start` signal"""
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceConnectionCreateEndParams:
+    """Parameters sent by the `on_connection_create_end` signal"""
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceConnectionReuseconnParams:
+    """Parameters sent by the `on_connection_reuseconn` signal"""
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceDnsResolveHostStartParams:
+    """Parameters sent by the `on_dns_resolvehost_start` signal"""
+
+    host: str
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceDnsResolveHostEndParams:
+    """Parameters sent by the `on_dns_resolvehost_end` signal"""
+
+    host: str
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceDnsCacheHitParams:
+    """Parameters sent by the `on_dns_cache_hit` signal"""
+
+    host: str
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceDnsCacheMissParams:
+    """Parameters sent by the `on_dns_cache_miss` signal"""
+
+    host: str
+
+
+@dataclasses.dataclass(frozen=True)
+class TraceRequestHeadersSentParams:
+    """Parameters sent by the `on_request_headers_sent` signal"""
+
+    method: str
+    url: URL
+    headers: "CIMultiDict[str]"
+
+
+class Trace:
+    """Internal dependency holder class.
+
+    Used to keep together the main dependencies used
+    at the moment of send a signal.
+    """
+
+    def __init__(
+        self,
+        session: "ClientSession",
+        trace_config: TraceConfig,
+        trace_config_ctx: SimpleNamespace,
+    ) -> None:
+        self._trace_config = trace_config
+        self._trace_config_ctx = trace_config_ctx
+        self._session = session
+
+    async def send_request_start(
+        self, method: str, url: URL, headers: "CIMultiDict[str]"
+    ) -> None:
+        return await self._trace_config.on_request_start.send(
+            self._session,
+            self._trace_config_ctx,
+            TraceRequestStartParams(method, url, headers),
+        )
+
+    async def send_request_chunk_sent(
+        self, method: str, url: URL, chunk: bytes
+    ) -> None:
+        return await self._trace_config.on_request_chunk_sent.send(
+            self._session,
+            self._trace_config_ctx,
+            TraceRequestChunkSentParams(method, url, chunk),
+        )
+
+    async def send_response_chunk_received(
+        self, method: str, url: URL, chunk: bytes
+    ) -> None:
+        return await self._trace_config.on_response_chunk_received.send(
+            self._session,
+            self._trace_config_ctx,
+            TraceResponseChunkReceivedParams(method, url, chunk),
+        )
+
+    async def send_request_end(
+        self,
+        method: str,
+        url: URL,
+        headers: "CIMultiDict[str]",
+        response: ClientResponse,
+    ) -> None:
+        return await self._trace_config.on_request_end.send(
+            self._session,
+            self._trace_config_ctx,
+            TraceRequestEndParams(method, url, headers, response),
+        )
+
+    async def send_request_exception(
+        self,
+        method: str,
+        url: URL,
+        headers: "CIMultiDict[str]",
+        exception: BaseException,
+    ) -> None:
+        return await self._trace_config.on_request_exception.send(
+            self._session,
+            self._trace_config_ctx,
+            TraceRequestExceptionParams(method, url, headers, exception),
+        )
+
+    async def send_request_redirect(
+        self,
+        method: str,
+        url: URL,
+        headers: "CIMultiDict[str]",
+        response: ClientResponse,
+    ) -> None:
+        return await self._trace_config._on_request_redirect.send(
+            self._session,
+            self._trace_config_ctx,
+            TraceRequestRedirectParams(method, url, headers, response),
+        )
+
+    async def send_connection_queued_start(self) -> None:
+        return await self._trace_config.on_connection_queued_start.send(
+            self._session, self._trace_config_ctx, TraceConnectionQueuedStartParams()
+        )
+
+    async def send_connection_queued_end(self) -> None:
+        return await self._trace_config.on_connection_queued_end.send(
+            self._session, self._trace_config_ctx, TraceConnectionQueuedEndParams()
+        )
+
+    async def send_connection_create_start(self) -> None:
+        return await self._trace_config.on_connection_create_start.send(
+            self._session, self._trace_config_ctx, TraceConnectionCreateStartParams()
+        )
+
+    async def send_connection_create_end(self) -> None:
+        return await self._trace_config.on_connection_create_end.send(
+            self._session, self._trace_config_ctx, TraceConnectionCreateEndParams()
+        )
+
+    async def send_connection_reuseconn(self) -> None:
+        return await self._trace_config.on_connection_reuseconn.send(
+            self._session, self._trace_config_ctx, TraceConnectionReuseconnParams()
+        )
+
+    async def send_dns_resolvehost_start(self, host: str) -> None:
+        return await self._trace_config.on_dns_resolvehost_start.send(
+            self._session, self._trace_config_ctx, TraceDnsResolveHostStartParams(host)
+        )
+
+    async def send_dns_resolvehost_end(self, host: str) -> None:
+        return await self._trace_config.on_dns_resolvehost_end.send(
+            self._session, self._trace_config_ctx, TraceDnsResolveHostEndParams(host)
+        )
+
+    async def send_dns_cache_hit(self, host: str) -> None:
+        return await self._trace_config.on_dns_cache_hit.send(
+            self._session, self._trace_config_ctx, TraceDnsCacheHitParams(host)
+        )
+
+    async def send_dns_cache_miss(self, host: str) -> None:
+        return await self._trace_config.on_dns_cache_miss.send(
+            self._session, self._trace_config_ctx, TraceDnsCacheMissParams(host)
+        )
+
+    async def send_request_headers(
+        self, method: str, url: URL, headers: "CIMultiDict[str]"
+    ) -> None:
+        return await self._trace_config._on_request_headers_sent.send(
+            self._session,
+            self._trace_config_ctx,
+            TraceRequestHeadersSentParams(method, url, headers),
+        )

aiohttp/typedefs.py

@@ -0,0 +1,54 @@
+import json
+import os
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Iterable,
+    Mapping,
+    Tuple,
+    Union,
+)
+
+from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy, istr
+from yarl import URL
+
+DEFAULT_JSON_ENCODER = json.dumps
+DEFAULT_JSON_DECODER = json.loads
+
+if TYPE_CHECKING:  # pragma: no cover
+    _CIMultiDict = CIMultiDict[str]
+    _CIMultiDictProxy = CIMultiDictProxy[str]
+    _MultiDict = MultiDict[str]
+    _MultiDictProxy = MultiDictProxy[str]
+    from http.cookies import BaseCookie, Morsel
+
+    from .web import Request, StreamResponse
+else:
+    _CIMultiDict = CIMultiDict
+    _CIMultiDictProxy = CIMultiDictProxy
+    _MultiDict = MultiDict
+    _MultiDictProxy = MultiDictProxy
+
+Byteish = Union[bytes, bytearray, memoryview]
+JSONEncoder = Callable[[Any], str]
+JSONDecoder = Callable[[str], Any]
+LooseHeaders = Union[Mapping[Union[str, istr], str], _CIMultiDict, _CIMultiDictProxy]
+RawHeaders = Tuple[Tuple[bytes, bytes], ...]
+StrOrURL = Union[str, URL]
+
+LooseCookiesMappings = Mapping[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]]
+LooseCookiesIterables = Iterable[
+    Tuple[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]]
+]
+LooseCookies = Union[
+    LooseCookiesMappings,
+    LooseCookiesIterables,
+    "BaseCookie[str]",
+]
+
+Handler = Callable[["Request"], Awaitable["StreamResponse"]]
+Middleware = Callable[["Request", Handler], Awaitable["StreamResponse"]]
+
+PathLike = Union[str, "os.PathLike[str]"]

aiohttp/web.py

@@ -0,0 +1,573 @@
+import asyncio
+import logging
+import os
+import socket
+import sys
+from argparse import ArgumentParser
+from collections.abc import Iterable
+from importlib import import_module
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Iterable as TypingIterable,
+    List,
+    Optional,
+    Set,
+    Type,
+    Union,
+    cast,
+)
+
+from .abc import AbstractAccessLogger
+from .helpers import AppKey
+from .log import access_logger
+from .typedefs import PathLike
+from .web_app import Application, CleanupError
+from .web_exceptions import (
+    HTTPAccepted,
+    HTTPBadGateway,
+    HTTPBadRequest,
+    HTTPClientError,
+    HTTPConflict,
+    HTTPCreated,
+    HTTPError,
+    HTTPException,
+    HTTPExpectationFailed,
+    HTTPFailedDependency,
+    HTTPForbidden,
+    HTTPFound,
+    HTTPGatewayTimeout,
+    HTTPGone,
+    HTTPInsufficientStorage,
+    HTTPInternalServerError,
+    HTTPLengthRequired,
+    HTTPMethodNotAllowed,
+    HTTPMisdirectedRequest,
+    HTTPMove,
+    HTTPMovedPermanently,
+    HTTPMultipleChoices,
+    HTTPNetworkAuthenticationRequired,
+    HTTPNoContent,
+    HTTPNonAuthoritativeInformation,
+    HTTPNotAcceptable,
+    HTTPNotExtended,
+    HTTPNotFound,
+    HTTPNotImplemented,
+    HTTPNotModified,
+    HTTPOk,
+    HTTPPartialContent,
+    HTTPPaymentRequired,
+    HTTPPermanentRedirect,
+    HTTPPreconditionFailed,
+    HTTPPreconditionRequired,
+    HTTPProxyAuthenticationRequired,
+    HTTPRedirection,
+    HTTPRequestEntityTooLarge,
+    HTTPRequestHeaderFieldsTooLarge,
+    HTTPRequestRangeNotSatisfiable,
+    HTTPRequestTimeout,
+    HTTPRequestURITooLong,
+    HTTPResetContent,
+    HTTPSeeOther,
+    HTTPServerError,
+    HTTPServiceUnavailable,
+    HTTPSuccessful,
+    HTTPTemporaryRedirect,
+    HTTPTooManyRequests,
+    HTTPUnauthorized,
+    HTTPUnavailableForLegalReasons,
+    HTTPUnprocessableEntity,
+    HTTPUnsupportedMediaType,
+    HTTPUpgradeRequired,
+    HTTPUseProxy,
+    HTTPVariantAlsoNegotiates,
+    HTTPVersionNotSupported,
+)
+from .web_fileresponse import FileResponse
+from .web_log import AccessLogger
+from .web_middlewares import middleware, normalize_path_middleware
+from .web_protocol import PayloadAccessError, RequestHandler, RequestPayloadError
+from .web_request import BaseRequest, FileField, Request
+from .web_response import ContentCoding, Response, StreamResponse, json_response
+from .web_routedef import (
+    AbstractRouteDef,
+    RouteDef,
+    RouteTableDef,
+    StaticDef,
+    delete,
+    get,
+    head,
+    options,
+    patch,
+    post,
+    put,
+    route,
+    static,
+    view,
+)
+from .web_runner import (
+    AppRunner,
+    BaseRunner,
+    BaseSite,
+    GracefulExit,
+    NamedPipeSite,
+    ServerRunner,
+    SockSite,
+    TCPSite,
+    UnixSite,
+)
+from .web_server import Server
+from .web_urldispatcher import (
+    AbstractResource,
+    AbstractRoute,
+    DynamicResource,
+    PlainResource,
+    PrefixedSubAppResource,
+    Resource,
+    ResourceRoute,
+    StaticResource,
+    UrlDispatcher,
+    UrlMappingMatchInfo,
+    View,
+)
+from .web_ws import WebSocketReady, WebSocketResponse, WSMsgType
+
+__all__ = (
+    # web_app
+    "AppKey",
+    "Application",
+    "CleanupError",
+    # web_exceptions
+    "HTTPAccepted",
+    "HTTPBadGateway",
+    "HTTPBadRequest",
+    "HTTPClientError",
+    "HTTPConflict",
+    "HTTPCreated",
+    "HTTPError",
+    "HTTPException",
+    "HTTPExpectationFailed",
+    "HTTPFailedDependency",
+    "HTTPForbidden",
+    "HTTPFound",
+    "HTTPGatewayTimeout",
+    "HTTPGone",
+    "HTTPInsufficientStorage",
+    "HTTPInternalServerError",
+    "HTTPLengthRequired",
+    "HTTPMethodNotAllowed",
+    "HTTPMisdirectedRequest",
+    "HTTPMove",
+    "HTTPMovedPermanently",
+    "HTTPMultipleChoices",
+    "HTTPNetworkAuthenticationRequired",
+    "HTTPNoContent",
+    "HTTPNonAuthoritativeInformation",
+    "HTTPNotAcceptable",
+    "HTTPNotExtended",
+    "HTTPNotFound",
+    "HTTPNotImplemented",
+    "HTTPNotModified",
+    "HTTPOk",
+    "HTTPPartialContent",
+    "HTTPPaymentRequired",
+    "HTTPPermanentRedirect",
+    "HTTPPreconditionFailed",
+    "HTTPPreconditionRequired",
+    "HTTPProxyAuthenticationRequired",
+    "HTTPRedirection",
+    "HTTPRequestEntityTooLarge",
+    "HTTPRequestHeaderFieldsTooLarge",
+    "HTTPRequestRangeNotSatisfiable",
+    "HTTPRequestTimeout",
+    "HTTPRequestURITooLong",
+    "HTTPResetContent",
+    "HTTPSeeOther",
+    "HTTPServerError",
+    "HTTPServiceUnavailable",
+    "HTTPSuccessful",
+    "HTTPTemporaryRedirect",
+    "HTTPTooManyRequests",
+    "HTTPUnauthorized",
+    "HTTPUnavailableForLegalReasons",
+    "HTTPUnprocessableEntity",
+    "HTTPUnsupportedMediaType",
+    "HTTPUpgradeRequired",
+    "HTTPUseProxy",
+    "HTTPVariantAlsoNegotiates",
+    "HTTPVersionNotSupported",
+    # web_fileresponse
+    "FileResponse",
+    # web_middlewares
+    "middleware",
+    "normalize_path_middleware",
+    # web_protocol
+    "PayloadAccessError",
+    "RequestHandler",
+    "RequestPayloadError",
+    # web_request
+    "BaseRequest",
+    "FileField",
+    "Request",
+    # web_response
+    "ContentCoding",
+    "Response",
+    "StreamResponse",
+    "json_response",
+    # web_routedef
+    "AbstractRouteDef",
+    "RouteDef",
+    "RouteTableDef",
+    "StaticDef",
+    "delete",
+    "get",
+    "head",
+    "options",
+    "patch",
+    "post",
+    "put",
+    "route",
+    "static",
+    "view",
+    # web_runner
+    "AppRunner",
+    "BaseRunner",
+    "BaseSite",
+    "GracefulExit",
+    "ServerRunner",
+    "SockSite",
+    "TCPSite",
+    "UnixSite",
+    "NamedPipeSite",
+    # web_server
+    "Server",
+    # web_urldispatcher
+    "AbstractResource",
+    "AbstractRoute",
+    "DynamicResource",
+    "PlainResource",
+    "PrefixedSubAppResource",
+    "Resource",
+    "ResourceRoute",
+    "StaticResource",
+    "UrlDispatcher",
+    "UrlMappingMatchInfo",
+    "View",
+    # web_ws
+    "WebSocketReady",
+    "WebSocketResponse",
+    "WSMsgType",
+    # web
+    "run_app",
+)
+
+
+try:
+    from ssl import SSLContext
+except ImportError:  # pragma: no cover
+    SSLContext = Any  # type: ignore[misc,assignment]
+
+HostSequence = TypingIterable[str]
+
+
+async def _run_app(
+    app: Union[Application, Awaitable[Application]],
+    *,
+    host: Optional[Union[str, HostSequence]] = None,
+    port: Optional[int] = None,
+    path: Union[PathLike, TypingIterable[PathLike], None] = None,
+    sock: Optional[Union[socket.socket, TypingIterable[socket.socket]]] = None,
+    shutdown_timeout: float = 60.0,
+    keepalive_timeout: float = 75.0,
+    ssl_context: Optional[SSLContext] = None,
+    print: Optional[Callable[..., None]] = print,
+    backlog: int = 128,
+    access_log_class: Type[AbstractAccessLogger] = AccessLogger,
+    access_log_format: str = AccessLogger.LOG_FORMAT,
+    access_log: Optional[logging.Logger] = access_logger,
+    handle_signals: bool = True,
+    reuse_address: Optional[bool] = None,
+    reuse_port: Optional[bool] = None,
+    handler_cancellation: bool = False,
+) -> None:
+    # An internal function to actually do all dirty job for application running
+    if asyncio.iscoroutine(app):
+        app = await app
+
+    app = cast(Application, app)
+
+    runner = AppRunner(
+        app,
+        handle_signals=handle_signals,
+        access_log_class=access_log_class,
+        access_log_format=access_log_format,
+        access_log=access_log,
+        keepalive_timeout=keepalive_timeout,
+        handler_cancellation=handler_cancellation,
+    )
+
+    await runner.setup()
+
+    sites: List[BaseSite] = []
+
+    try:
+        if host is not None:
+            if isinstance(host, (str, bytes, bytearray, memoryview)):
+                sites.append(
+                    TCPSite(
+                        runner,
+                        host,
+                        port,
+                        shutdown_timeout=shutdown_timeout,
+                        ssl_context=ssl_context,
+                        backlog=backlog,
+                        reuse_address=reuse_address,
+                        reuse_port=reuse_port,
+                    )
+                )
+            else:
+                for h in host:
+                    sites.append(
+                        TCPSite(
+                            runner,
+                            h,
+                            port,
+                            shutdown_timeout=shutdown_timeout,
+                            ssl_context=ssl_context,
+                            backlog=backlog,
+                            reuse_address=reuse_address,
+                            reuse_port=reuse_port,
+                        )
+                    )
+        elif path is None and sock is None or port is not None:
+            sites.append(
+                TCPSite(
+                    runner,
+                    port=port,
+                    shutdown_timeout=shutdown_timeout,
+                    ssl_context=ssl_context,
+                    backlog=backlog,
+                    reuse_address=reuse_address,
+                    reuse_port=reuse_port,
+                )
+            )
+
+        if path is not None:
+            if isinstance(path, (str, os.PathLike)):
+                sites.append(
+                    UnixSite(
+                        runner,
+                        path,
+                        shutdown_timeout=shutdown_timeout,
+                        ssl_context=ssl_context,
+                        backlog=backlog,
+                    )
+                )
+            else:
+                for p in path:
+                    sites.append(
+                        UnixSite(
+                            runner,
+                            p,
+                            shutdown_timeout=shutdown_timeout,
+                            ssl_context=ssl_context,
+                            backlog=backlog,
+                        )
+                    )
+
+        if sock is not None:
+            if not isinstance(sock, Iterable):
+                sites.append(
+                    SockSite(
+                        runner,
+                        sock,
+                        shutdown_timeout=shutdown_timeout,
+                        ssl_context=ssl_context,
+                        backlog=backlog,
+                    )
+                )
+            else:
+                for s in sock:
+                    sites.append(
+                        SockSite(
+                            runner,
+                            s,
+                            shutdown_timeout=shutdown_timeout,
+                            ssl_context=ssl_context,
+                            backlog=backlog,
+                        )
+                    )
+        for site in sites:
+            await site.start()
+
+        if print:  # pragma: no branch
+            names = sorted(str(s.name) for s in runner.sites)
+            print(
+                "======== Running on {} ========\n"
+                "(Press CTRL+C to quit)".format(", ".join(names))
+            )
+
+        # sleep forever by 1 hour intervals,
+        while True:
+            await asyncio.sleep(3600)
+    finally:
+        await runner.cleanup()
+
+
+def _cancel_tasks(
+    to_cancel: Set["asyncio.Task[Any]"], loop: asyncio.AbstractEventLoop
+) -> None:
+    if not to_cancel:
+        return
+
+    for task in to_cancel:
+        task.cancel()
+
+    loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))
+
+    for task in to_cancel:
+        if task.cancelled():
+            continue
+        if task.exception() is not None:
+            loop.call_exception_handler(
+                {
+                    "message": "unhandled exception during asyncio.run() shutdown",
+                    "exception": task.exception(),
+                    "task": task,
+                }
+            )
+
+
+def run_app(
+    app: Union[Application, Awaitable[Application]],
+    *,
+    debug: bool = False,
+    host: Optional[Union[str, HostSequence]] = None,
+    port: Optional[int] = None,
+    path: Union[PathLike, TypingIterable[PathLike], None] = None,
+    sock: Optional[Union[socket.socket, TypingIterable[socket.socket]]] = None,
+    shutdown_timeout: float = 60.0,
+    keepalive_timeout: float = 75.0,
+    ssl_context: Optional[SSLContext] = None,
+    print: Optional[Callable[..., None]] = print,
+    backlog: int = 128,
+    access_log_class: Type[AbstractAccessLogger] = AccessLogger,
+    access_log_format: str = AccessLogger.LOG_FORMAT,
+    access_log: Optional[logging.Logger] = access_logger,
+    handle_signals: bool = True,
+    reuse_address: Optional[bool] = None,
+    reuse_port: Optional[bool] = None,
+    handler_cancellation: bool = False,
+    loop: Optional[asyncio.AbstractEventLoop] = None,
+) -> None:
+    """Run an app locally"""
+    if loop is None:
+        loop = asyncio.new_event_loop()
+    loop.set_debug(debug)
+
+    # Configure if and only if in debugging mode and using the default logger
+    if loop.get_debug() and access_log and access_log.name == "aiohttp.access":
+        if access_log.level == logging.NOTSET:
+            access_log.setLevel(logging.DEBUG)
+        if not access_log.hasHandlers():
+            access_log.addHandler(logging.StreamHandler())
+
+    main_task = loop.create_task(
+        _run_app(
+            app,
+            host=host,
+            port=port,
+            path=path,
+            sock=sock,
+            shutdown_timeout=shutdown_timeout,
+            keepalive_timeout=keepalive_timeout,
+            ssl_context=ssl_context,
+            print=print,
+            backlog=backlog,
+            access_log_class=access_log_class,
+            access_log_format=access_log_format,
+            access_log=access_log,
+            handle_signals=handle_signals,
+            reuse_address=reuse_address,
+            reuse_port=reuse_port,
+            handler_cancellation=handler_cancellation,
+        )
+    )
+
+    try:
+        asyncio.set_event_loop(loop)
+        loop.run_until_complete(main_task)
+    except (GracefulExit, KeyboardInterrupt):  # pragma: no cover
+        pass
+    finally:
+        _cancel_tasks({main_task}, loop)
+        _cancel_tasks(asyncio.all_tasks(loop), loop)
+        loop.run_until_complete(loop.shutdown_asyncgens())
+        loop.close()
+        asyncio.set_event_loop(None)
+
+
+def main(argv: List[str]) -> None:
+    arg_parser = ArgumentParser(
+        description="aiohttp.web Application server", prog="aiohttp.web"
+    )
+    arg_parser.add_argument(
+        "entry_func",
+        help=(
+            "Callable returning the `aiohttp.web.Application` instance to "
+            "run. Should be specified in the 'module:function' syntax."
+        ),
+        metavar="entry-func",
+    )
+    arg_parser.add_argument(
+        "-H",
+        "--hostname",
+        help="TCP/IP hostname to serve on (default: %(default)r)",
+        default="localhost",
+    )
+    arg_parser.add_argument(
+        "-P",
+        "--port",
+        help="TCP/IP port to serve on (default: %(default)r)",
+        type=int,
+        default="8080",
+    )
+    arg_parser.add_argument(
+        "-U",
+        "--path",
+        help="Unix file system path to serve on. Specifying a path will cause "
+        "hostname and port arguments to be ignored.",
+    )
+    args, extra_argv = arg_parser.parse_known_args(argv)
+
+    # Import logic
+    mod_str, _, func_str = args.entry_func.partition(":")
+    if not func_str or not mod_str:
+        arg_parser.error("'entry-func' not in 'module:function' syntax")
+    if mod_str.startswith("."):
+        arg_parser.error("relative module names not supported")
+    try:
+        module = import_module(mod_str)
+    except ImportError as ex:
+        arg_parser.error(f"unable to import {mod_str}: {ex}")
+    try:
+        func = getattr(module, func_str)
+    except AttributeError:
+        arg_parser.error(f"module {mod_str!r} has no attribute {func_str!r}")
+
+    # Compatibility logic
+    if args.path is not None and not hasattr(socket, "AF_UNIX"):
+        arg_parser.error(
+            "file system paths not supported by your operating" " environment"
+        )
+
+    logging.basicConfig(level=logging.DEBUG)
+
+    app = func(extra_argv)
+    run_app(app, host=args.hostname, port=args.port, path=args.path)
+    arg_parser.exit(message="Stopped\n")
+
+
+if __name__ == "__main__":  # pragma: no branch
+    main(sys.argv[1:])  # pragma: no cover

aiohttp/web_app.py

@@ -0,0 +1,445 @@
+import asyncio
+import logging
+import warnings
+from functools import partial, update_wrapper
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncIterator,
+    Awaitable,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Sequence,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+    final,
+    overload,
+)
+
+from aiosignal import Signal
+from frozenlist import FrozenList
+
+from . import hdrs
+from .helpers import AppKey
+from .log import web_logger
+from .typedefs import Middleware
+from .web_middlewares import _fix_request_current_app
+from .web_request import Request
+from .web_response import StreamResponse
+from .web_routedef import AbstractRouteDef
+from .web_urldispatcher import (
+    AbstractResource,
+    AbstractRoute,
+    Domain,
+    MaskDomain,
+    MatchedSubAppResource,
+    PrefixedSubAppResource,
+    UrlDispatcher,
+)
+
+__all__ = ("Application", "CleanupError")
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    _AppSignal = Signal[Callable[["Application"], Awaitable[None]]]
+    _RespPrepareSignal = Signal[Callable[[Request, StreamResponse], Awaitable[None]]]
+    _Middlewares = FrozenList[Middleware]
+    _MiddlewaresHandlers = Sequence[Middleware]
+    _Subapps = List["Application"]
+else:
+    # No type checker mode, skip types
+    _AppSignal = Signal
+    _RespPrepareSignal = Signal
+    _Handler = Callable
+    _Middlewares = FrozenList
+    _MiddlewaresHandlers = Sequence
+    _Subapps = List
+
+_T = TypeVar("_T")
+_U = TypeVar("_U")
+
+
+@final
+class Application(MutableMapping[Union[str, AppKey[Any]], Any]):
+    __slots__ = (
+        "logger",
+        "_debug",
+        "_router",
+        "_loop",
+        "_handler_args",
+        "_middlewares",
+        "_middlewares_handlers",
+        "_run_middlewares",
+        "_state",
+        "_frozen",
+        "_pre_frozen",
+        "_subapps",
+        "_on_response_prepare",
+        "_on_startup",
+        "_on_shutdown",
+        "_on_cleanup",
+        "_client_max_size",
+        "_cleanup_ctx",
+    )
+
+    def __init__(
+        self,
+        *,
+        logger: logging.Logger = web_logger,
+        middlewares: Iterable[Middleware] = (),
+        handler_args: Optional[Mapping[str, Any]] = None,
+        client_max_size: int = 1024**2,
+        debug: Any = ...,  # mypy doesn't support ellipsis
+    ) -> None:
+        if debug is not ...:
+            warnings.warn(
+                "debug argument is no-op since 4.0 " "and scheduled for removal in 5.0",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        self._router = UrlDispatcher()
+        self._handler_args = handler_args
+        self.logger = logger
+
+        self._middlewares: _Middlewares = FrozenList(middlewares)
+
+        # initialized on freezing
+        self._middlewares_handlers: _MiddlewaresHandlers = tuple()
+        # initialized on freezing
+        self._run_middlewares: Optional[bool] = None
+
+        self._state: Dict[Union[AppKey[Any], str], object] = {}
+        self._frozen = False
+        self._pre_frozen = False
+        self._subapps: _Subapps = []
+
+        self._on_response_prepare: _RespPrepareSignal = Signal(self)
+        self._on_startup: _AppSignal = Signal(self)
+        self._on_shutdown: _AppSignal = Signal(self)
+        self._on_cleanup: _AppSignal = Signal(self)
+        self._cleanup_ctx = CleanupContext()
+        self._on_startup.append(self._cleanup_ctx._on_startup)
+        self._on_cleanup.append(self._cleanup_ctx._on_cleanup)
+        self._client_max_size = client_max_size
+
+    def __init_subclass__(cls: Type["Application"]) -> None:
+        raise TypeError(
+            "Inheritance class {} from web.Application "
+            "is forbidden".format(cls.__name__)
+        )
+
+    # MutableMapping API
+
+    def __eq__(self, other: object) -> bool:
+        return self is other
+
+    @overload  # type: ignore[override]
+    def __getitem__(self, key: AppKey[_T]) -> _T:
+        ...
+
+    @overload
+    def __getitem__(self, key: str) -> Any:
+        ...
+
+    def __getitem__(self, key: Union[str, AppKey[_T]]) -> Any:
+        return self._state[key]
+
+    def _check_frozen(self) -> None:
+        if self._frozen:
+            raise RuntimeError(
+                "Changing state of started or joined " "application is forbidden"
+            )
+
+    @overload  # type: ignore[override]
+    def __setitem__(self, key: AppKey[_T], value: _T) -> None:
+        ...
+
+    @overload
+    def __setitem__(self, key: str, value: Any) -> None:
+        ...
+
+    def __setitem__(self, key: Union[str, AppKey[_T]], value: Any) -> None:
+        self._check_frozen()
+        if not isinstance(key, AppKey):
+            warnings.warn(
+                "It is recommended to use web.AppKey instances for keys.\n"
+                + "https://docs.aiohttp.org/en/stable/web_advanced.html"
+                + "#application-s-config",
+                stacklevel=2,
+            )
+        self._state[key] = value
+
+    def __delitem__(self, key: Union[str, AppKey[_T]]) -> None:
+        self._check_frozen()
+        del self._state[key]
+
+    def __len__(self) -> int:
+        return len(self._state)
+
+    def __iter__(self) -> Iterator[Union[str, AppKey[Any]]]:
+        return iter(self._state)
+
+    @overload  # type: ignore[override]
+    def get(self, key: AppKey[_T], default: None = ...) -> Optional[_T]:
+        ...
+
+    @overload
+    def get(self, key: AppKey[_T], default: _U) -> Union[_T, _U]:
+        ...
+
+    @overload
+    def get(self, key: str, default: Any = ...) -> Any:
+        ...
+
+    def get(self, key: Union[str, AppKey[_T]], default: Any = None) -> Any:
+        return self._state.get(key, default)
+
+    ########
+    def _set_loop(self, loop: Optional[asyncio.AbstractEventLoop]) -> None:
+        warnings.warn(
+            "_set_loop() is no-op since 4.0 " "and scheduled for removal in 5.0",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+    @property
+    def pre_frozen(self) -> bool:
+        return self._pre_frozen
+
+    def pre_freeze(self) -> None:
+        if self._pre_frozen:
+            return
+
+        self._pre_frozen = True
+        self._middlewares.freeze()
+        self._router.freeze()
+        self._on_response_prepare.freeze()
+        self._cleanup_ctx.freeze()
+        self._on_startup.freeze()
+        self._on_shutdown.freeze()
+        self._on_cleanup.freeze()
+        self._middlewares_handlers = tuple(self._prepare_middleware())
+
+        # If current app and any subapp do not have middlewares avoid run all
+        # of the code footprint that it implies, which have a middleware
+        # hardcoded per app that sets up the current_app attribute. If no
+        # middlewares are configured the handler will receive the proper
+        # current_app without needing all of this code.
+        self._run_middlewares = True if self.middlewares else False
+
+        for subapp in self._subapps:
+            subapp.pre_freeze()
+            self._run_middlewares = self._run_middlewares or subapp._run_middlewares
+
+    @property
+    def frozen(self) -> bool:
+        return self._frozen
+
+    def freeze(self) -> None:
+        if self._frozen:
+            return
+
+        self.pre_freeze()
+        self._frozen = True
+        for subapp in self._subapps:
+            subapp.freeze()
+
+    @property
+    def debug(self) -> bool:
+        warnings.warn(
+            "debug property is deprecated since 4.0" "and scheduled for removal in 5.0",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return asyncio.get_event_loop().get_debug()
+
+    def _reg_subapp_signals(self, subapp: "Application") -> None:
+        def reg_handler(signame: str) -> None:
+            subsig = getattr(subapp, signame)
+
+            async def handler(app: "Application") -> None:
+                await subsig.send(subapp)
+
+            appsig = getattr(self, signame)
+            appsig.append(handler)
+
+        reg_handler("on_startup")
+        reg_handler("on_shutdown")
+        reg_handler("on_cleanup")
+
+    def add_subapp(self, prefix: str, subapp: "Application") -> AbstractResource:
+        if not isinstance(prefix, str):
+            raise TypeError("Prefix must be str")
+        prefix = prefix.rstrip("/")
+        if not prefix:
+            raise ValueError("Prefix cannot be empty")
+        factory = partial(PrefixedSubAppResource, prefix, subapp)
+        return self._add_subapp(factory, subapp)
+
+    def _add_subapp(
+        self, resource_factory: Callable[[], AbstractResource], subapp: "Application"
+    ) -> AbstractResource:
+        if self.frozen:
+            raise RuntimeError("Cannot add sub application to frozen application")
+        if subapp.frozen:
+            raise RuntimeError("Cannot add frozen application")
+        resource = resource_factory()
+        self.router.register_resource(resource)
+        self._reg_subapp_signals(subapp)
+        self._subapps.append(subapp)
+        subapp.pre_freeze()
+        return resource
+
+    def add_domain(self, domain: str, subapp: "Application") -> AbstractResource:
+        if not isinstance(domain, str):
+            raise TypeError("Domain must be str")
+        elif "*" in domain:
+            rule: Domain = MaskDomain(domain)
+        else:
+            rule = Domain(domain)
+        factory = partial(MatchedSubAppResource, rule, subapp)
+        return self._add_subapp(factory, subapp)
+
+    def add_routes(self, routes: Iterable[AbstractRouteDef]) -> List[AbstractRoute]:
+        return self.router.add_routes(routes)
+
+    @property
+    def on_response_prepare(self) -> _RespPrepareSignal:
+        return self._on_response_prepare
+
+    @property
+    def on_startup(self) -> _AppSignal:
+        return self._on_startup
+
+    @property
+    def on_shutdown(self) -> _AppSignal:
+        return self._on_shutdown
+
+    @property
+    def on_cleanup(self) -> _AppSignal:
+        return self._on_cleanup
+
+    @property
+    def cleanup_ctx(self) -> "CleanupContext":
+        return self._cleanup_ctx
+
+    @property
+    def router(self) -> UrlDispatcher:
+        return self._router
+
+    @property
+    def middlewares(self) -> _Middlewares:
+        return self._middlewares
+
+    async def startup(self) -> None:
+        """Causes on_startup signal
+
+        Should be called in the event loop along with the request handler.
+        """
+        await self.on_startup.send(self)
+
+    async def shutdown(self) -> None:
+        """Causes on_shutdown signal
+
+        Should be called before cleanup()
+        """
+        await self.on_shutdown.send(self)
+
+    async def cleanup(self) -> None:
+        """Causes on_cleanup signal
+
+        Should be called after shutdown()
+        """
+        if self.on_cleanup.frozen:
+            await self.on_cleanup.send(self)
+        else:
+            # If an exception occurs in startup, ensure cleanup contexts are completed.
+            await self._cleanup_ctx._on_cleanup(self)
+
+    def _prepare_middleware(self) -> Iterator[Middleware]:
+        yield from reversed(self._middlewares)
+        yield _fix_request_current_app(self)
+
+    async def _handle(self, request: Request) -> StreamResponse:
+        match_info = await self._router.resolve(request)
+        match_info.add_app(self)
+        match_info.freeze()
+
+        resp = None
+        request._match_info = match_info
+        expect = request.headers.get(hdrs.EXPECT)
+        if expect:
+            resp = await match_info.expect_handler(request)
+            await request.writer.drain()
+
+        if resp is None:
+            handler = match_info.handler
+
+            if self._run_middlewares:
+                for app in match_info.apps[::-1]:
+                    assert app.pre_frozen, "middleware handlers are not ready"
+                    for m in app._middlewares_handlers:
+                        handler = update_wrapper(partial(m, handler=handler), handler)
+
+            resp = await handler(request)
+
+        return resp
+
+    def __call__(self) -> "Application":
+        """gunicorn compatibility"""
+        return self
+
+    def __repr__(self) -> str:
+        return f"<Application 0x{id(self):x}>"
+
+    def __bool__(self) -> bool:
+        return True
+
+
+class CleanupError(RuntimeError):
+    @property
+    def exceptions(self) -> List[BaseException]:
+        return cast(List[BaseException], self.args[1])
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    _CleanupContextBase = FrozenList[Callable[[Application], AsyncIterator[None]]]
+else:
+    _CleanupContextBase = FrozenList
+
+
+class CleanupContext(_CleanupContextBase):
+    def __init__(self) -> None:
+        super().__init__()
+        self._exits: List[AsyncIterator[None]] = []
+
+    async def _on_startup(self, app: Application) -> None:
+        for cb in self:
+            it = cb(app).__aiter__()
+            await it.__anext__()
+            self._exits.append(it)
+
+    async def _on_cleanup(self, app: Application) -> None:
+        errors = []
+        for it in reversed(self._exits):
+            try:
+                await it.__anext__()
+            except StopAsyncIteration:
+                pass
+            except Exception as exc:
+                errors.append(exc)
+            else:
+                errors.append(RuntimeError(f"{it!r} has more than one 'yield'"))
+        if errors:
+            if len(errors) == 1:
+                raise errors[0]
+            else:
+                raise CleanupError("Multiple errors on cleanup stage", errors)

aiohttp/web_exceptions.py

@@ -0,0 +1,509 @@
+import warnings
+from http import HTTPStatus
+from typing import Any, Iterable, Optional, Set, Tuple
+
+from multidict import CIMultiDict
+from yarl import URL
+
+from . import hdrs
+from .helpers import CookieMixin
+from .typedefs import LooseHeaders, StrOrURL
+
+__all__ = (
+    "HTTPException",
+    "HTTPError",
+    "HTTPRedirection",
+    "HTTPSuccessful",
+    "HTTPOk",
+    "HTTPCreated",
+    "HTTPAccepted",
+    "HTTPNonAuthoritativeInformation",
+    "HTTPNoContent",
+    "HTTPResetContent",
+    "HTTPPartialContent",
+    "HTTPMove",
+    "HTTPMultipleChoices",
+    "HTTPMovedPermanently",
+    "HTTPFound",
+    "HTTPSeeOther",
+    "HTTPNotModified",
+    "HTTPUseProxy",
+    "HTTPTemporaryRedirect",
+    "HTTPPermanentRedirect",
+    "HTTPClientError",
+    "HTTPBadRequest",
+    "HTTPUnauthorized",
+    "HTTPPaymentRequired",
+    "HTTPForbidden",
+    "HTTPNotFound",
+    "HTTPMethodNotAllowed",
+    "HTTPNotAcceptable",
+    "HTTPProxyAuthenticationRequired",
+    "HTTPRequestTimeout",
+    "HTTPConflict",
+    "HTTPGone",
+    "HTTPLengthRequired",
+    "HTTPPreconditionFailed",
+    "HTTPRequestEntityTooLarge",
+    "HTTPRequestURITooLong",
+    "HTTPUnsupportedMediaType",
+    "HTTPRequestRangeNotSatisfiable",
+    "HTTPExpectationFailed",
+    "HTTPMisdirectedRequest",
+    "HTTPUnprocessableEntity",
+    "HTTPFailedDependency",
+    "HTTPUpgradeRequired",
+    "HTTPPreconditionRequired",
+    "HTTPTooManyRequests",
+    "HTTPRequestHeaderFieldsTooLarge",
+    "HTTPUnavailableForLegalReasons",
+    "HTTPServerError",
+    "HTTPInternalServerError",
+    "HTTPNotImplemented",
+    "HTTPBadGateway",
+    "HTTPServiceUnavailable",
+    "HTTPGatewayTimeout",
+    "HTTPVersionNotSupported",
+    "HTTPVariantAlsoNegotiates",
+    "HTTPInsufficientStorage",
+    "HTTPNotExtended",
+    "HTTPNetworkAuthenticationRequired",
+)
+
+
+############################################################
+# HTTP Exceptions
+############################################################
+
+
+class HTTPException(CookieMixin, Exception):
+    # You should set in subclasses:
+    # status = 200
+
+    status_code = -1
+    empty_body = False
+    default_reason = ""  # Initialized at the end of the module
+
+    def __init__(
+        self,
+        *,
+        headers: Optional[LooseHeaders] = None,
+        reason: Optional[str] = None,
+        text: Optional[str] = None,
+        content_type: Optional[str] = None,
+    ) -> None:
+        super().__init__()
+        if reason is None:
+            reason = self.default_reason
+
+        if text is None:
+            if not self.empty_body:
+                text = f"{self.status_code}: {reason}"
+        else:
+            if self.empty_body:
+                warnings.warn(
+                    "text argument is deprecated for HTTP status {} "
+                    "since 4.0 and scheduled for removal in 5.0 (#3462),"
+                    "the response should be provided without a body".format(
+                        self.status_code
+                    ),
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
+
+        if headers is not None:
+            real_headers = CIMultiDict(headers)
+        else:
+            real_headers = CIMultiDict()
+
+        if content_type is not None:
+            if not text:
+                warnings.warn(
+                    "content_type without text is deprecated "
+                    "since 4.0 and scheduled for removal in 5.0 "
+                    "(#3462)",
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
+            real_headers[hdrs.CONTENT_TYPE] = content_type
+        elif hdrs.CONTENT_TYPE not in real_headers and text:
+            real_headers[hdrs.CONTENT_TYPE] = "text/plain"
+
+        self._reason = reason
+        self._text = text
+        self._headers = real_headers
+        self.args = ()
+
+    def __bool__(self) -> bool:
+        return True
+
+    @property
+    def status(self) -> int:
+        return self.status_code
+
+    @property
+    def reason(self) -> str:
+        return self._reason
+
+    @property
+    def text(self) -> Optional[str]:
+        return self._text
+
+    @property
+    def headers(self) -> "CIMultiDict[str]":
+        return self._headers
+
+    def __str__(self) -> str:
+        return self.reason
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}: {self.reason}>"
+
+    __reduce__ = object.__reduce__
+
+    def __getnewargs__(self) -> Tuple[Any, ...]:
+        return self.args
+
+
+class HTTPError(HTTPException):
+    """Base class for exceptions with status codes in the 400s and 500s."""
+
+
+class HTTPRedirection(HTTPException):
+    """Base class for exceptions with status codes in the 300s."""
+
+
+class HTTPSuccessful(HTTPException):
+    """Base class for exceptions with status codes in the 200s."""
+
+
+class HTTPOk(HTTPSuccessful):
+    status_code = 200
+
+
+class HTTPCreated(HTTPSuccessful):
+    status_code = 201
+
+
+class HTTPAccepted(HTTPSuccessful):
+    status_code = 202
+
+
+class HTTPNonAuthoritativeInformation(HTTPSuccessful):
+    status_code = 203
+
+
+class HTTPNoContent(HTTPSuccessful):
+    status_code = 204
+    empty_body = True
+
+
+class HTTPResetContent(HTTPSuccessful):
+    status_code = 205
+    empty_body = True
+
+
+class HTTPPartialContent(HTTPSuccessful):
+    status_code = 206
+
+
+############################################################
+# 3xx redirection
+############################################################
+
+
+class HTTPMove(HTTPRedirection):
+    def __init__(
+        self,
+        location: StrOrURL,
+        *,
+        headers: Optional[LooseHeaders] = None,
+        reason: Optional[str] = None,
+        text: Optional[str] = None,
+        content_type: Optional[str] = None,
+    ) -> None:
+        if not location:
+            raise ValueError("HTTP redirects need a location to redirect to.")
+        super().__init__(
+            headers=headers, reason=reason, text=text, content_type=content_type
+        )
+        self._location = URL(location)
+        self.headers["Location"] = str(self.location)
+
+    @property
+    def location(self) -> URL:
+        return self._location
+
+
+class HTTPMultipleChoices(HTTPMove):
+    status_code = 300
+
+
+class HTTPMovedPermanently(HTTPMove):
+    status_code = 301
+
+
+class HTTPFound(HTTPMove):
+    status_code = 302
+
+
+# This one is safe after a POST (the redirected location will be
+# retrieved with GET):
+class HTTPSeeOther(HTTPMove):
+    status_code = 303
+
+
+class HTTPNotModified(HTTPRedirection):
+    # FIXME: this should include a date or etag header
+    status_code = 304
+    empty_body = True
+
+
+class HTTPUseProxy(HTTPMove):
+    # Not a move, but looks a little like one
+    status_code = 305
+
+
+class HTTPTemporaryRedirect(HTTPMove):
+    status_code = 307
+
+
+class HTTPPermanentRedirect(HTTPMove):
+    status_code = 308
+
+
+############################################################
+# 4xx client error
+############################################################
+
+
+class HTTPClientError(HTTPError):
+    pass
+
+
+class HTTPBadRequest(HTTPClientError):
+    status_code = 400
+
+
+class HTTPUnauthorized(HTTPClientError):
+    status_code = 401
+
+
+class HTTPPaymentRequired(HTTPClientError):
+    status_code = 402
+
+
+class HTTPForbidden(HTTPClientError):
+    status_code = 403
+
+
+class HTTPNotFound(HTTPClientError):
+    status_code = 404
+
+
+class HTTPMethodNotAllowed(HTTPClientError):
+    status_code = 405
+
+    def __init__(
+        self,
+        method: str,
+        allowed_methods: Iterable[str],
+        *,
+        headers: Optional[LooseHeaders] = None,
+        reason: Optional[str] = None,
+        text: Optional[str] = None,
+        content_type: Optional[str] = None,
+    ) -> None:
+        allow = ",".join(sorted(allowed_methods))
+        super().__init__(
+            headers=headers, reason=reason, text=text, content_type=content_type
+        )
+        self.headers["Allow"] = allow
+        self._allowed: Set[str] = set(allowed_methods)
+        self._method = method
+
+    @property
+    def allowed_methods(self) -> Set[str]:
+        return self._allowed
+
+    @property
+    def method(self) -> str:
+        return self._method
+
+
+class HTTPNotAcceptable(HTTPClientError):
+    status_code = 406
+
+
+class HTTPProxyAuthenticationRequired(HTTPClientError):
+    status_code = 407
+
+
+class HTTPRequestTimeout(HTTPClientError):
+    status_code = 408
+
+
+class HTTPConflict(HTTPClientError):
+    status_code = 409
+
+
+class HTTPGone(HTTPClientError):
+    status_code = 410
+
+
+class HTTPLengthRequired(HTTPClientError):
+    status_code = 411
+
+
+class HTTPPreconditionFailed(HTTPClientError):
+    status_code = 412
+
+
+class HTTPRequestEntityTooLarge(HTTPClientError):
+    status_code = 413
+
+    def __init__(self, max_size: int, actual_size: int, **kwargs: Any) -> None:
+        kwargs.setdefault(
+            "text",
+            "Maximum request body size {} exceeded, "
+            "actual body size {}".format(max_size, actual_size),
+        )
+        super().__init__(**kwargs)
+
+
+class HTTPRequestURITooLong(HTTPClientError):
+    status_code = 414
+
+
+class HTTPUnsupportedMediaType(HTTPClientError):
+    status_code = 415
+
+
+class HTTPRequestRangeNotSatisfiable(HTTPClientError):
+    status_code = 416
+
+
+class HTTPExpectationFailed(HTTPClientError):
+    status_code = 417
+
+
+class HTTPMisdirectedRequest(HTTPClientError):
+    status_code = 421
+
+
+class HTTPUnprocessableEntity(HTTPClientError):
+    status_code = 422
+
+
+class HTTPFailedDependency(HTTPClientError):
+    status_code = 424
+
+
+class HTTPUpgradeRequired(HTTPClientError):
+    status_code = 426
+
+
+class HTTPPreconditionRequired(HTTPClientError):
+    status_code = 428
+
+
+class HTTPTooManyRequests(HTTPClientError):
+    status_code = 429
+
+
+class HTTPRequestHeaderFieldsTooLarge(HTTPClientError):
+    status_code = 431
+
+
+class HTTPUnavailableForLegalReasons(HTTPClientError):
+    status_code = 451
+
+    def __init__(
+        self,
+        link: StrOrURL,
+        *,
+        headers: Optional[LooseHeaders] = None,
+        reason: Optional[str] = None,
+        text: Optional[str] = None,
+        content_type: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            headers=headers, reason=reason, text=text, content_type=content_type
+        )
+        self.headers["Link"] = f'<{str(link)}>; rel="blocked-by"'
+        self._link = URL(link)
+
+    @property
+    def link(self) -> URL:
+        return self._link
+
+
+############################################################
+# 5xx Server Error
+############################################################
+#  Response status codes beginning with the digit "5" indicate cases in
+#  which the server is aware that it has erred or is incapable of
+#  performing the request. Except when responding to a HEAD request, the
+#  server SHOULD include an entity containing an explanation of the error
+#  situation, and whether it is a temporary or permanent condition. User
+#  agents SHOULD display any included entity to the user. These response
+#  codes are applicable to any request method.
+
+
+class HTTPServerError(HTTPError):
+    pass
+
+
+class HTTPInternalServerError(HTTPServerError):
+    status_code = 500
+
+
+class HTTPNotImplemented(HTTPServerError):
+    status_code = 501
+
+
+class HTTPBadGateway(HTTPServerError):
+    status_code = 502
+
+
+class HTTPServiceUnavailable(HTTPServerError):
+    status_code = 503
+
+
+class HTTPGatewayTimeout(HTTPServerError):
+    status_code = 504
+
+
+class HTTPVersionNotSupported(HTTPServerError):
+    status_code = 505
+
+
+class HTTPVariantAlsoNegotiates(HTTPServerError):
+    status_code = 506
+
+
+class HTTPInsufficientStorage(HTTPServerError):
+    status_code = 507
+
+
+class HTTPNotExtended(HTTPServerError):
+    status_code = 510
+
+
+class HTTPNetworkAuthenticationRequired(HTTPServerError):
+    status_code = 511
+
+
+def _initialize_default_reason() -> None:
+    for obj in globals().values():
+        if isinstance(obj, type) and issubclass(obj, HTTPException):
+            if obj.status_code >= 0:
+                try:
+                    status = HTTPStatus(obj.status_code)
+                    obj.default_reason = status.phrase
+                except ValueError:
+                    pass
+
+
+_initialize_default_reason()
+del _initialize_default_reason

aiohttp/web_fileresponse.py

@@ -0,0 +1,281 @@
+import asyncio
+import mimetypes
+import os
+import pathlib
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Final,
+    Optional,
+    Tuple,
+    cast,
+)
+
+from . import hdrs
+from .abc import AbstractStreamWriter
+from .helpers import ETAG_ANY, ETag
+from .typedefs import LooseHeaders, PathLike
+from .web_exceptions import (
+    HTTPNotModified,
+    HTTPPartialContent,
+    HTTPPreconditionFailed,
+    HTTPRequestRangeNotSatisfiable,
+)
+from .web_response import StreamResponse
+
+__all__ = ("FileResponse",)
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .web_request import BaseRequest
+
+
+_T_OnChunkSent = Optional[Callable[[bytes], Awaitable[None]]]
+
+
+NOSENDFILE: Final[bool] = bool(os.environ.get("AIOHTTP_NOSENDFILE"))
+
+
+class FileResponse(StreamResponse):
+    """A response object can be used to send files."""
+
+    def __init__(
+        self,
+        path: PathLike,
+        chunk_size: int = 256 * 1024,
+        status: int = 200,
+        reason: Optional[str] = None,
+        headers: Optional[LooseHeaders] = None,
+    ) -> None:
+        super().__init__(status=status, reason=reason, headers=headers)
+
+        self._path = pathlib.Path(path)
+        self._chunk_size = chunk_size
+
+    async def _sendfile_fallback(
+        self, writer: AbstractStreamWriter, fobj: IO[Any], offset: int, count: int
+    ) -> AbstractStreamWriter:
+        # To keep memory usage low,fobj is transferred in chunks
+        # controlled by the constructor's chunk_size argument.
+
+        chunk_size = self._chunk_size
+        loop = asyncio.get_event_loop()
+
+        await loop.run_in_executor(None, fobj.seek, offset)
+
+        chunk = await loop.run_in_executor(None, fobj.read, chunk_size)
+        while chunk:
+            await writer.write(chunk)
+            count = count - chunk_size
+            if count <= 0:
+                break
+            chunk = await loop.run_in_executor(None, fobj.read, min(chunk_size, count))
+
+        await writer.drain()
+        return writer
+
+    async def _sendfile(
+        self, request: "BaseRequest", fobj: IO[Any], offset: int, count: int
+    ) -> AbstractStreamWriter:
+        writer = await super().prepare(request)
+        assert writer is not None
+
+        if NOSENDFILE or self.compression:
+            return await self._sendfile_fallback(writer, fobj, offset, count)
+
+        loop = request._loop
+        transport = request.transport
+        assert transport is not None
+
+        try:
+            await loop.sendfile(transport, fobj, offset, count)
+        except NotImplementedError:
+            return await self._sendfile_fallback(writer, fobj, offset, count)
+
+        await super().write_eof()
+        return writer
+
+    @staticmethod
+    def _strong_etag_match(etag_value: str, etags: Tuple[ETag, ...]) -> bool:
+        if len(etags) == 1 and etags[0].value == ETAG_ANY:
+            return True
+        return any(etag.value == etag_value for etag in etags if not etag.is_weak)
+
+    async def _not_modified(
+        self, request: "BaseRequest", etag_value: str, last_modified: float
+    ) -> Optional[AbstractStreamWriter]:
+        self.set_status(HTTPNotModified.status_code)
+        self._length_check = False
+        self.etag = etag_value  # type: ignore[assignment]
+        self.last_modified = last_modified  # type: ignore[assignment]
+        # Delete any Content-Length headers provided by user. HTTP 304
+        # should always have empty response body
+        return await super().prepare(request)
+
+    async def _precondition_failed(
+        self, request: "BaseRequest"
+    ) -> Optional[AbstractStreamWriter]:
+        self.set_status(HTTPPreconditionFailed.status_code)
+        self.content_length = 0
+        return await super().prepare(request)
+
+    async def prepare(self, request: "BaseRequest") -> Optional[AbstractStreamWriter]:
+        filepath = self._path
+
+        gzip = False
+        if "gzip" in request.headers.get(hdrs.ACCEPT_ENCODING, ""):
+            gzip_path = filepath.with_name(filepath.name + ".gz")
+
+            if gzip_path.is_file():
+                filepath = gzip_path
+                gzip = True
+
+        loop = asyncio.get_event_loop()
+        st: os.stat_result = await loop.run_in_executor(None, filepath.stat)
+
+        etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}"
+        last_modified = st.st_mtime
+
+        # https://tools.ietf.org/html/rfc7232#section-6
+        ifmatch = request.if_match
+        if ifmatch is not None and not self._strong_etag_match(etag_value, ifmatch):
+            return await self._precondition_failed(request)
+
+        unmodsince = request.if_unmodified_since
+        if (
+            unmodsince is not None
+            and ifmatch is None
+            and st.st_mtime > unmodsince.timestamp()
+        ):
+            return await self._precondition_failed(request)
+
+        ifnonematch = request.if_none_match
+        if ifnonematch is not None and self._strong_etag_match(etag_value, ifnonematch):
+            return await self._not_modified(request, etag_value, last_modified)
+
+        modsince = request.if_modified_since
+        if (
+            modsince is not None
+            and ifnonematch is None
+            and st.st_mtime <= modsince.timestamp()
+        ):
+            return await self._not_modified(request, etag_value, last_modified)
+
+        ct = None
+        if hdrs.CONTENT_TYPE not in self.headers:
+            ct, encoding = mimetypes.guess_type(str(filepath))
+            if not ct:
+                ct = "application/octet-stream"
+        else:
+            encoding = "gzip" if gzip else None
+
+        status = self._status
+        file_size = st.st_size
+        count = file_size
+
+        start = None
+
+        ifrange = request.if_range
+        if ifrange is None or st.st_mtime <= ifrange.timestamp():
+            # If-Range header check:
+            # condition = cached date >= last modification date
+            # return 206 if True else 200.
+            # if False:
+            #   Range header would not be processed, return 200
+            # if True but Range header missing
+            #   return 200
+            try:
+                rng = request.http_range
+                start = rng.start
+                end = rng.stop
+            except ValueError:
+                # https://tools.ietf.org/html/rfc7233:
+                # A server generating a 416 (Range Not Satisfiable) response to
+                # a byte-range request SHOULD send a Content-Range header field
+                # with an unsatisfied-range value.
+                # The complete-length in a 416 response indicates the current
+                # length of the selected representation.
+                #
+                # Will do the same below. Many servers ignore this and do not
+                # send a Content-Range header with HTTP 416
+                self.headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}"
+                self.set_status(HTTPRequestRangeNotSatisfiable.status_code)
+                return await super().prepare(request)
+
+            # If a range request has been made, convert start, end slice
+            # notation into file pointer offset and count
+            if start is not None or end is not None:
+                if start < 0 and end is None:  # return tail of file
+                    start += file_size
+                    if start < 0:
+                        # if Range:bytes=-1000 in request header but file size
+                        # is only 200, there would be trouble without this
+                        start = 0
+                    count = file_size - start
+                else:
+                    # rfc7233:If the last-byte-pos value is
+                    # absent, or if the value is greater than or equal to
+                    # the current length of the representation data,
+                    # the byte range is interpreted as the remainder
+                    # of the representation (i.e., the server replaces the
+                    # value of last-byte-pos with a value that is one less than
+                    # the current length of the selected representation).
+                    count = (
+                        min(end if end is not None else file_size, file_size) - start
+                    )
+
+                if start >= file_size:
+                    # HTTP 416 should be returned in this case.
+                    #
+                    # According to https://tools.ietf.org/html/rfc7233:
+                    # If a valid byte-range-set includes at least one
+                    # byte-range-spec with a first-byte-pos that is less than
+                    # the current length of the representation, or at least one
+                    # suffix-byte-range-spec with a non-zero suffix-length,
+                    # then the byte-range-set is satisfiable. Otherwise, the
+                    # byte-range-set is unsatisfiable.
+                    self.headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}"
+                    self.set_status(HTTPRequestRangeNotSatisfiable.status_code)
+                    return await super().prepare(request)
+
+                status = HTTPPartialContent.status_code
+                # Even though you are sending the whole file, you should still
+                # return a HTTP 206 for a Range request.
+                self.set_status(status)
+
+        if ct:
+            self.content_type = ct
+        if encoding:
+            self.headers[hdrs.CONTENT_ENCODING] = encoding
+        if gzip:
+            self.headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING
+
+        self.etag = etag_value  # type: ignore[assignment]
+        self.last_modified = st.st_mtime  # type: ignore[assignment]
+        self.content_length = count
+
+        self.headers[hdrs.ACCEPT_RANGES] = "bytes"
+
+        real_start = cast(int, start)
+
+        if status == HTTPPartialContent.status_code:
+            self.headers[hdrs.CONTENT_RANGE] = "bytes {}-{}/{}".format(
+                real_start, real_start + count - 1, file_size
+            )
+
+        # If we are sending 0 bytes calling sendfile() will throw a ValueError
+        if count == 0 or request.method == hdrs.METH_HEAD or self.status in [204, 304]:
+            return await super().prepare(request)
+
+        fobj = await loop.run_in_executor(None, filepath.open, "rb")
+        if start:  # be aware that start could be None or int=0 here.
+            offset = start
+        else:
+            offset = 0
+
+        try:
+            return await self._sendfile(request, fobj, offset, count)
+        finally:
+            await asyncio.shield(loop.run_in_executor(None, fobj.close))

aiohttp/web_log.py

@@ -0,0 +1,213 @@
+import datetime
+import functools
+import logging
+import os
+import re
+import time as time_mod
+from collections import namedtuple
+from typing import Any, Callable, Dict, Iterable, List, Tuple  # noqa
+
+from .abc import AbstractAccessLogger
+from .web_request import BaseRequest
+from .web_response import StreamResponse
+
+KeyMethod = namedtuple("KeyMethod", "key method")
+
+
+class AccessLogger(AbstractAccessLogger):
+    """Helper object to log access.
+
+    Usage:
+        log = logging.getLogger("spam")
+        log_format = "%a %{User-Agent}i"
+        access_logger = AccessLogger(log, log_format)
+        access_logger.log(request, response, time)
+
+    Format:
+        %%  The percent sign
+        %a  Remote IP-address (IP-address of proxy if using reverse proxy)
+        %t  Time when the request was started to process
+        %P  The process ID of the child that serviced the request
+        %r  First line of request
+        %s  Response status code
+        %b  Size of response in bytes, including HTTP headers
+        %T  Time taken to serve the request, in seconds
+        %Tf Time taken to serve the request, in seconds with floating fraction
+            in .06f format
+        %D  Time taken to serve the request, in microseconds
+        %{FOO}i  request.headers['FOO']
+        %{FOO}o  response.headers['FOO']
+        %{FOO}e  os.environ['FOO']
+
+    """
+
+    LOG_FORMAT_MAP = {
+        "a": "remote_address",
+        "t": "request_start_time",
+        "P": "process_id",
+        "r": "first_request_line",
+        "s": "response_status",
+        "b": "response_size",
+        "T": "request_time",
+        "Tf": "request_time_frac",
+        "D": "request_time_micro",
+        "i": "request_header",
+        "o": "response_header",
+    }
+
+    LOG_FORMAT = '%a %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"'
+    FORMAT_RE = re.compile(r"%(\{([A-Za-z0-9\-_]+)\}([ioe])|[atPrsbOD]|Tf?)")
+    CLEANUP_RE = re.compile(r"(%[^s])")
+    _FORMAT_CACHE: Dict[str, Tuple[str, List[KeyMethod]]] = {}
+
+    def __init__(self, logger: logging.Logger, log_format: str = LOG_FORMAT) -> None:
+        """Initialise the logger.
+
+        logger is a logger object to be used for logging.
+        log_format is a string with apache compatible log format description.
+
+        """
+        super().__init__(logger, log_format=log_format)
+
+        _compiled_format = AccessLogger._FORMAT_CACHE.get(log_format)
+        if not _compiled_format:
+            _compiled_format = self.compile_format(log_format)
+            AccessLogger._FORMAT_CACHE[log_format] = _compiled_format
+
+        self._log_format, self._methods = _compiled_format
+
+    def compile_format(self, log_format: str) -> Tuple[str, List[KeyMethod]]:
+        """Translate log_format into form usable by modulo formatting
+
+        All known atoms will be replaced with %s
+        Also methods for formatting of those atoms will be added to
+        _methods in appropriate order
+
+        For example we have log_format = "%a %t"
+        This format will be translated to "%s %s"
+        Also contents of _methods will be
+        [self._format_a, self._format_t]
+        These method will be called and results will be passed
+        to translated string format.
+
+        Each _format_* method receive 'args' which is list of arguments
+        given to self.log
+
+        Exceptions are _format_e, _format_i and _format_o methods which
+        also receive key name (by functools.partial)
+
+        """
+        # list of (key, method) tuples, we don't use an OrderedDict as users
+        # can repeat the same key more than once
+        methods = list()
+
+        for atom in self.FORMAT_RE.findall(log_format):
+            if atom[1] == "":
+                format_key1 = self.LOG_FORMAT_MAP[atom[0]]
+                m = getattr(AccessLogger, "_format_%s" % atom[0])
+                key_method = KeyMethod(format_key1, m)
+            else:
+                format_key2 = (self.LOG_FORMAT_MAP[atom[2]], atom[1])
+                m = getattr(AccessLogger, "_format_%s" % atom[2])
+                key_method = KeyMethod(format_key2, functools.partial(m, atom[1]))
+
+            methods.append(key_method)
+
+        log_format = self.FORMAT_RE.sub(r"%s", log_format)
+        log_format = self.CLEANUP_RE.sub(r"%\1", log_format)
+        return log_format, methods
+
+    @staticmethod
+    def _format_i(
+        key: str, request: BaseRequest, response: StreamResponse, time: float
+    ) -> str:
+        if request is None:
+            return "(no headers)"
+
+        # suboptimal, make istr(key) once
+        return request.headers.get(key, "-")
+
+    @staticmethod
+    def _format_o(
+        key: str, request: BaseRequest, response: StreamResponse, time: float
+    ) -> str:
+        # suboptimal, make istr(key) once
+        return response.headers.get(key, "-")
+
+    @staticmethod
+    def _format_a(request: BaseRequest, response: StreamResponse, time: float) -> str:
+        if request is None:
+            return "-"
+        ip = request.remote
+        return ip if ip is not None else "-"
+
+    @staticmethod
+    def _format_t(request: BaseRequest, response: StreamResponse, time: float) -> str:
+        tz = datetime.timezone(datetime.timedelta(seconds=-time_mod.timezone))
+        now = datetime.datetime.now(tz)
+        start_time = now - datetime.timedelta(seconds=time)
+        return start_time.strftime("[%d/%b/%Y:%H:%M:%S %z]")
+
+    @staticmethod
+    def _format_P(request: BaseRequest, response: StreamResponse, time: float) -> str:
+        return "<%s>" % os.getpid()
+
+    @staticmethod
+    def _format_r(request: BaseRequest, response: StreamResponse, time: float) -> str:
+        if request is None:
+            return "-"
+        return "{} {} HTTP/{}.{}".format(
+            request.method,
+            request.path_qs,
+            request.version.major,
+            request.version.minor,
+        )
+
+    @staticmethod
+    def _format_s(request: BaseRequest, response: StreamResponse, time: float) -> int:
+        return response.status
+
+    @staticmethod
+    def _format_b(request: BaseRequest, response: StreamResponse, time: float) -> int:
+        return response.body_length
+
+    @staticmethod
+    def _format_T(request: BaseRequest, response: StreamResponse, time: float) -> str:
+        return str(round(time))
+
+    @staticmethod
+    def _format_Tf(request: BaseRequest, response: StreamResponse, time: float) -> str:
+        return "%06f" % time
+
+    @staticmethod
+    def _format_D(request: BaseRequest, response: StreamResponse, time: float) -> str:
+        return str(round(time * 1000000))
+
+    def _format_line(
+        self, request: BaseRequest, response: StreamResponse, time: float
+    ) -> Iterable[Tuple[str, Callable[[BaseRequest, StreamResponse, float], str]]]:
+        return [(key, method(request, response, time)) for key, method in self._methods]
+
+    def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
+        if not self.logger.isEnabledFor(logging.INFO):
+            # Avoid formatting the log line if it will not be emitted.
+            return
+        try:
+            fmt_info = self._format_line(request, response, time)
+
+            values = list()
+            extra = dict()
+            for key, value in fmt_info:
+                values.append(value)
+
+                if key.__class__ is str:
+                    extra[key] = value
+                else:
+                    k1, k2 = key  # type: ignore[misc]
+                    dct = extra.get(k1, {})  # type: ignore[var-annotated,has-type]
+                    dct[k2] = value  # type: ignore[index,has-type]
+                    extra[k1] = dct  # type: ignore[has-type,assignment]
+
+            self.logger.info(self._log_format % tuple(values), extra=extra)
+        except Exception:
+            self.logger.exception("Error in logging")

aiohttp/web_middlewares.py

@@ -0,0 +1,121 @@
+import re
+import warnings
+from typing import TYPE_CHECKING, Tuple, Type, TypeVar
+
+from .typedefs import Handler, Middleware
+from .web_exceptions import HTTPMove, HTTPPermanentRedirect
+from .web_request import Request
+from .web_response import StreamResponse
+from .web_urldispatcher import SystemRoute
+
+__all__ = (
+    "middleware",
+    "normalize_path_middleware",
+)
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .web_app import Application
+
+_Func = TypeVar("_Func")
+
+
+async def _check_request_resolves(request: Request, path: str) -> Tuple[bool, Request]:
+    alt_request = request.clone(rel_url=path)
+
+    match_info = await request.app.router.resolve(alt_request)
+    alt_request._match_info = match_info
+
+    if match_info.http_exception is None:
+        return True, alt_request
+
+    return False, request
+
+
+def middleware(f: _Func) -> _Func:
+    warnings.warn(
+        "Middleware decorator is deprecated since 4.0 "
+        "and its behaviour is default, "
+        "you can simply remove this decorator.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return f
+
+
+def normalize_path_middleware(
+    *,
+    append_slash: bool = True,
+    remove_slash: bool = False,
+    merge_slashes: bool = True,
+    redirect_class: Type[HTTPMove] = HTTPPermanentRedirect,
+) -> Middleware:
+    """Factory for producing a middleware that normalizes the path of a request.
+
+    Normalizing means:
+        - Add or remove a trailing slash to the path.
+        - Double slashes are replaced by one.
+
+    The middleware returns as soon as it finds a path that resolves
+    correctly. The order if both merge and append/remove are enabled is
+        1) merge slashes
+        2) append/remove slash
+        3) both merge slashes and append/remove slash.
+    If the path resolves with at least one of those conditions, it will
+    redirect to the new path.
+
+    Only one of `append_slash` and `remove_slash` can be enabled. If both
+    are `True` the factory will raise an assertion error
+
+    If `append_slash` is `True` the middleware will append a slash when
+    needed. If a resource is defined with trailing slash and the request
+    comes without it, it will append it automatically.
+
+    If `remove_slash` is `True`, `append_slash` must be `False`. When enabled
+    the middleware will remove trailing slashes and redirect if the resource
+    is defined
+
+    If merge_slashes is True, merge multiple consecutive slashes in the
+    path into one.
+    """
+    correct_configuration = not (append_slash and remove_slash)
+    assert correct_configuration, "Cannot both remove and append slash"
+
+    async def impl(request: Request, handler: Handler) -> StreamResponse:
+        if isinstance(request.match_info.route, SystemRoute):
+            paths_to_check = []
+            if "?" in request.raw_path:
+                path, query = request.raw_path.split("?", 1)
+                query = "?" + query
+            else:
+                query = ""
+                path = request.raw_path
+
+            if merge_slashes:
+                paths_to_check.append(re.sub("//+", "/", path))
+            if append_slash and not request.path.endswith("/"):
+                paths_to_check.append(path + "/")
+            if remove_slash and request.path.endswith("/"):
+                paths_to_check.append(path[:-1])
+            if merge_slashes and append_slash:
+                paths_to_check.append(re.sub("//+", "/", path + "/"))
+            if merge_slashes and remove_slash and path.endswith("/"):
+                merged_slashes = re.sub("//+", "/", path)
+                paths_to_check.append(merged_slashes[:-1])
+
+            for path in paths_to_check:
+                path = re.sub("^//+", "/", path)  # SECURITY: GHSA-v6wp-4m6f-gcjg
+                resolves, request = await _check_request_resolves(request, path)
+                if resolves:
+                    raise redirect_class(request.raw_path + query)
+
+        return await handler(request)
+
+    return impl
+
+
+def _fix_request_current_app(app: "Application") -> Middleware:
+    async def impl(request: Request, handler: Handler) -> StreamResponse:
+        with request.match_info.set_current_app(app):
+            return await handler(request)
+
+    return impl

aiohttp/web_protocol.py

@@ -0,0 +1,711 @@
+import asyncio
+import asyncio.streams
+import dataclasses
+import traceback
+from collections import deque
+from contextlib import suppress
+from html import escape as html_escape
+from http import HTTPStatus
+from logging import Logger
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Deque,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
+
+import yarl
+
+from .abc import AbstractAccessLogger, AbstractAsyncAccessLogger, AbstractStreamWriter
+from .base_protocol import BaseProtocol
+from .helpers import ceil_timeout
+from .http import (
+    HttpProcessingError,
+    HttpRequestParser,
+    HttpVersion10,
+    RawRequestMessage,
+    StreamWriter,
+)
+from .log import access_logger, server_logger
+from .streams import EMPTY_PAYLOAD, StreamReader
+from .tcp_helpers import tcp_keepalive
+from .web_exceptions import HTTPException
+from .web_log import AccessLogger
+from .web_request import BaseRequest
+from .web_response import Response, StreamResponse
+
+__all__ = ("RequestHandler", "RequestPayloadError", "PayloadAccessError")
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .web_server import Server
+
+
+_RequestFactory = Callable[
+    [
+        RawRequestMessage,
+        StreamReader,
+        "RequestHandler",
+        AbstractStreamWriter,
+        "asyncio.Task[None]",
+    ],
+    BaseRequest,
+]
+
+_RequestHandler = Callable[[BaseRequest], Awaitable[StreamResponse]]
+_AnyAbstractAccessLogger = Union[
+    Type[AbstractAsyncAccessLogger],
+    Type[AbstractAccessLogger],
+]
+
+ERROR = RawRequestMessage(
+    "UNKNOWN",
+    "/",
+    HttpVersion10,
+    {},  # type: ignore[arg-type]
+    {},  # type: ignore[arg-type]
+    True,
+    None,
+    False,
+    False,
+    yarl.URL("/"),
+)
+
+
+class RequestPayloadError(Exception):
+    """Payload parsing error."""
+
+
+class PayloadAccessError(Exception):
+    """Payload was accessed after response was sent."""
+
+
+class AccessLoggerWrapper(AbstractAsyncAccessLogger):
+    """Wrap an AbstractAccessLogger so it behaves like an AbstractAsyncAccessLogger."""
+
+    def __init__(
+        self, access_logger: AbstractAccessLogger, loop: asyncio.AbstractEventLoop
+    ) -> None:
+        self.access_logger = access_logger
+        self._loop = loop
+        super().__init__()
+
+    async def log(
+        self, request: BaseRequest, response: StreamResponse, request_start: float
+    ) -> None:
+        self.access_logger.log(request, response, self._loop.time() - request_start)
+
+
+@dataclasses.dataclass(frozen=True)
+class _ErrInfo:
+    status: int
+    exc: BaseException
+    message: str
+
+
+_MsgType = Tuple[Union[RawRequestMessage, _ErrInfo], StreamReader]
+
+
+class RequestHandler(BaseProtocol):
+    """HTTP protocol implementation.
+
+    RequestHandler handles incoming HTTP request. It reads request line,
+    request headers and request payload and calls handle_request() method.
+    By default it always returns with 404 response.
+
+    RequestHandler handles errors in incoming request, like bad
+    status line, bad headers or incomplete payload. If any error occurs,
+    connection gets closed.
+
+    keepalive_timeout -- number of seconds before closing
+                         keep-alive connection
+
+    tcp_keepalive -- TCP keep-alive is on, default is on
+
+    logger -- custom logger object
+
+    access_log_class -- custom class for access_logger
+
+    access_log -- custom logging object
+
+    access_log_format -- access log format string
+
+    loop -- Optional event loop
+
+    max_line_size -- Optional maximum header line size
+
+    max_field_size -- Optional maximum header field size
+
+    timeout_ceil_threshold -- Optional value to specify
+                              threshold to ceil() timeout
+                              values
+
+    """
+
+    KEEPALIVE_RESCHEDULE_DELAY = 1
+
+    __slots__ = (
+        "_request_count",
+        "_keepalive",
+        "_manager",
+        "_request_handler",
+        "_request_factory",
+        "_tcp_keepalive",
+        "_keepalive_time",
+        "_keepalive_handle",
+        "_keepalive_timeout",
+        "_lingering_time",
+        "_messages",
+        "_message_tail",
+        "_waiter",
+        "_task_handler",
+        "_upgrade",
+        "_payload_parser",
+        "_request_parser",
+        "logger",
+        "access_log",
+        "access_logger",
+        "_close",
+        "_force_close",
+        "_current_request",
+        "_timeout_ceil_threshold",
+    )
+
+    def __init__(
+        self,
+        manager: "Server",
+        *,
+        loop: asyncio.AbstractEventLoop,
+        keepalive_timeout: float = 75.0,  # NGINX default is 75 secs
+        tcp_keepalive: bool = True,
+        logger: Logger = server_logger,
+        access_log_class: _AnyAbstractAccessLogger = AccessLogger,
+        access_log: Optional[Logger] = access_logger,
+        access_log_format: str = AccessLogger.LOG_FORMAT,
+        max_line_size: int = 8190,
+        max_field_size: int = 8190,
+        lingering_time: float = 10.0,
+        read_bufsize: int = 2**16,
+        auto_decompress: bool = True,
+        timeout_ceil_threshold: float = 5,
+    ):
+        super().__init__(loop)
+
+        self._request_count = 0
+        self._keepalive = False
+        self._current_request: Optional[BaseRequest] = None
+        self._manager: Optional[Server] = manager
+        self._request_handler: Optional[_RequestHandler] = manager.request_handler
+        self._request_factory: Optional[_RequestFactory] = manager.request_factory
+
+        self._tcp_keepalive = tcp_keepalive
+        # placeholder to be replaced on keepalive timeout setup
+        self._keepalive_time = 0.0
+        self._keepalive_handle: Optional[asyncio.Handle] = None
+        self._keepalive_timeout = keepalive_timeout
+        self._lingering_time = float(lingering_time)
+
+        self._messages: Deque[_MsgType] = deque()
+        self._message_tail = b""
+
+        self._waiter: Optional[asyncio.Future[None]] = None
+        self._task_handler: Optional[asyncio.Task[None]] = None
+
+        self._upgrade = False
+        self._payload_parser: Any = None
+        self._request_parser: Optional[HttpRequestParser] = HttpRequestParser(
+            self,
+            loop,
+            read_bufsize,
+            max_line_size=max_line_size,
+            max_field_size=max_field_size,
+            payload_exception=RequestPayloadError,
+            auto_decompress=auto_decompress,
+        )
+
+        self._timeout_ceil_threshold: float = 5
+        try:
+            self._timeout_ceil_threshold = float(timeout_ceil_threshold)
+        except (TypeError, ValueError):
+            pass
+
+        self.logger = logger
+        self.access_log = access_log
+        if access_log:
+            if issubclass(access_log_class, AbstractAsyncAccessLogger):
+                self.access_logger: Optional[
+                    AbstractAsyncAccessLogger
+                ] = access_log_class()
+            else:
+                access_logger = access_log_class(access_log, access_log_format)
+                self.access_logger = AccessLoggerWrapper(
+                    access_logger,
+                    self._loop,
+                )
+        else:
+            self.access_logger = None
+
+        self._close = False
+        self._force_close = False
+
+    def __repr__(self) -> str:
+        return "<{} {}>".format(
+            self.__class__.__name__,
+            "connected" if self.transport is not None else "disconnected",
+        )
+
+    @property
+    def keepalive_timeout(self) -> float:
+        return self._keepalive_timeout
+
+    async def shutdown(self, timeout: Optional[float] = 15.0) -> None:
+        """Do worker process exit preparations.
+
+        We need to clean up everything and stop accepting requests.
+        It is especially important for keep-alive connections.
+        """
+        self._force_close = True
+
+        if self._keepalive_handle is not None:
+            self._keepalive_handle.cancel()
+
+        if self._waiter:
+            self._waiter.cancel()
+
+        # wait for handlers
+        with suppress(asyncio.CancelledError, asyncio.TimeoutError):
+            async with ceil_timeout(timeout):
+                if self._current_request is not None:
+                    self._current_request._cancel(asyncio.CancelledError())
+
+                if self._task_handler is not None and not self._task_handler.done():
+                    await self._task_handler
+
+        # force-close non-idle handler
+        if self._task_handler is not None:
+            self._task_handler.cancel()
+
+        if self.transport is not None:
+            self.transport.close()
+            self.transport = None
+
+    def connection_made(self, transport: asyncio.BaseTransport) -> None:
+        super().connection_made(transport)
+
+        real_transport = cast(asyncio.Transport, transport)
+        if self._tcp_keepalive:
+            tcp_keepalive(real_transport)
+
+        self._task_handler = self._loop.create_task(self.start())
+        assert self._manager is not None
+        self._manager.connection_made(self, real_transport)
+
+    def connection_lost(self, exc: Optional[BaseException]) -> None:
+        if self._manager is None:
+            return
+        self._manager.connection_lost(self, exc)
+
+        super().connection_lost(exc)
+
+        # Grab value before setting _manager to None.
+        handler_cancellation = self._manager.handler_cancellation
+
+        self._manager = None
+        self._force_close = True
+        self._request_factory = None
+        self._request_handler = None
+        self._request_parser = None
+
+        if self._keepalive_handle is not None:
+            self._keepalive_handle.cancel()
+
+        if self._current_request is not None:
+            if exc is None:
+                exc = ConnectionResetError("Connection lost")
+            self._current_request._cancel(exc)
+
+        if self._waiter is not None:
+            self._waiter.cancel()
+
+        if handler_cancellation and self._task_handler is not None:
+            self._task_handler.cancel()
+
+        self._task_handler = None
+
+        if self._payload_parser is not None:
+            self._payload_parser.feed_eof()
+            self._payload_parser = None
+
+    def set_parser(self, parser: Any) -> None:
+        # Actual type is WebReader
+        assert self._payload_parser is None
+
+        self._payload_parser = parser
+
+        if self._message_tail:
+            self._payload_parser.feed_data(self._message_tail)
+            self._message_tail = b""
+
+    def eof_received(self) -> None:
+        pass
+
+    def data_received(self, data: bytes) -> None:
+        if self._force_close or self._close:
+            return
+        # parse http messages
+        messages: Sequence[_MsgType]
+        if self._payload_parser is None and not self._upgrade:
+            assert self._request_parser is not None
+            try:
+                messages, upgraded, tail = self._request_parser.feed_data(data)
+            except HttpProcessingError as exc:
+                messages = [
+                    (_ErrInfo(status=400, exc=exc, message=exc.message), EMPTY_PAYLOAD)
+                ]
+                upgraded = False
+                tail = b""
+
+            for msg, payload in messages or ():
+                self._request_count += 1
+                self._messages.append((msg, payload))
+
+            waiter = self._waiter
+            if messages and waiter is not None and not waiter.done():
+                # don't set result twice
+                waiter.set_result(None)
+
+            self._upgrade = upgraded
+            if upgraded and tail:
+                self._message_tail = tail
+
+        # no parser, just store
+        elif self._payload_parser is None and self._upgrade and data:
+            self._message_tail += data
+
+        # feed payload
+        elif data:
+            eof, tail = self._payload_parser.feed_data(data)
+            if eof:
+                self.close()
+
+    def keep_alive(self, val: bool) -> None:
+        """Set keep-alive connection mode.
+
+        :param bool val: new state.
+        """
+        self._keepalive = val
+        if self._keepalive_handle:
+            self._keepalive_handle.cancel()
+            self._keepalive_handle = None
+
+    def close(self) -> None:
+        """Close connection.
+
+        Stop accepting new pipelining messages and close
+        connection when handlers done processing messages.
+        """
+        self._close = True
+        if self._waiter:
+            self._waiter.cancel()
+
+    def force_close(self) -> None:
+        """Forcefully close connection."""
+        self._force_close = True
+        if self._waiter:
+            self._waiter.cancel()
+        if self.transport is not None:
+            self.transport.close()
+            self.transport = None
+
+    async def log_access(
+        self, request: BaseRequest, response: StreamResponse, request_start: float
+    ) -> None:
+        if self.access_logger is not None:
+            await self.access_logger.log(request, response, request_start)
+
+    def log_debug(self, *args: Any, **kw: Any) -> None:
+        if self._loop.get_debug():
+            self.logger.debug(*args, **kw)
+
+    def log_exception(self, *args: Any, **kw: Any) -> None:
+        self.logger.exception(*args, **kw)
+
+    def _process_keepalive(self) -> None:
+        if self._force_close or not self._keepalive:
+            return
+
+        next = self._keepalive_time + self._keepalive_timeout
+
+        # handler in idle state
+        if self._waiter:
+            if self._loop.time() > next:
+                self.force_close()
+                return
+
+        # not all request handlers are done,
+        # reschedule itself to next second
+        self._keepalive_handle = self._loop.call_later(
+            self.KEEPALIVE_RESCHEDULE_DELAY,
+            self._process_keepalive,
+        )
+
+    async def _handle_request(
+        self,
+        request: BaseRequest,
+        start_time: float,
+        request_handler: Callable[[BaseRequest], Awaitable[StreamResponse]],
+    ) -> Tuple[StreamResponse, bool]:
+        assert self._request_handler is not None
+        try:
+            try:
+                self._current_request = request
+                resp = await request_handler(request)
+            finally:
+                self._current_request = None
+        except HTTPException as exc:
+            resp = Response(
+                status=exc.status, reason=exc.reason, text=exc.text, headers=exc.headers
+            )
+            resp._cookies = exc._cookies
+            reset = await self.finish_response(request, resp, start_time)
+        except asyncio.CancelledError:
+            raise
+        except asyncio.TimeoutError as exc:
+            self.log_debug("Request handler timed out.", exc_info=exc)
+            resp = self.handle_error(request, 504)
+            reset = await self.finish_response(request, resp, start_time)
+        except Exception as exc:
+            resp = self.handle_error(request, 500, exc)
+            reset = await self.finish_response(request, resp, start_time)
+        else:
+            reset = await self.finish_response(request, resp, start_time)
+
+        return resp, reset
+
+    async def start(self) -> None:
+        """Process incoming request.
+
+        It reads request line, request headers and request payload, then
+        calls handle_request() method. Subclass has to override
+        handle_request(). start() handles various exceptions in request
+        or response handling. Connection is being closed always unless
+        keep_alive(True) specified.
+        """
+        loop = self._loop
+        handler = self._task_handler
+        assert handler is not None
+        manager = self._manager
+        assert manager is not None
+        keepalive_timeout = self._keepalive_timeout
+        resp = None
+        assert self._request_factory is not None
+        assert self._request_handler is not None
+
+        while not self._force_close:
+            if not self._messages:
+                try:
+                    # wait for next request
+                    self._waiter = loop.create_future()
+                    await self._waiter
+                except asyncio.CancelledError:
+                    break
+                finally:
+                    self._waiter = None
+
+            message, payload = self._messages.popleft()
+
+            start = loop.time()
+
+            manager.requests_count += 1
+            writer = StreamWriter(self, loop)
+            if isinstance(message, _ErrInfo):
+                # make request_factory work
+                request_handler = self._make_error_handler(message)
+                message = ERROR
+            else:
+                request_handler = self._request_handler
+
+            request = self._request_factory(message, payload, self, writer, handler)
+            try:
+                # a new task is used for copy context vars (#3406)
+                task = self._loop.create_task(
+                    self._handle_request(request, start, request_handler)
+                )
+                try:
+                    resp, reset = await task
+                except (asyncio.CancelledError, ConnectionError):
+                    self.log_debug("Ignored premature client disconnection")
+                    break
+
+                # Drop the processed task from asyncio.Task.all_tasks() early
+                del task
+                # https://github.com/python/mypy/issues/14309
+                if reset:  # type: ignore[possibly-undefined]
+                    self.log_debug("Ignored premature client disconnection 2")
+                    break
+
+                # notify server about keep-alive
+                self._keepalive = bool(resp.keep_alive)
+
+                # check payload
+                if not payload.is_eof():
+                    lingering_time = self._lingering_time
+                    # Could be force closed while awaiting above tasks.
+                    if not self._force_close and lingering_time:  # type: ignore[redundant-expr]
+                        self.log_debug(
+                            "Start lingering close timer for %s sec.", lingering_time
+                        )
+
+                        now = loop.time()
+                        end_t = now + lingering_time
+
+                        with suppress(asyncio.TimeoutError, asyncio.CancelledError):
+                            while not payload.is_eof() and now < end_t:
+                                async with ceil_timeout(end_t - now):
+                                    # read and ignore
+                                    await payload.readany()
+                                now = loop.time()
+
+                    # if payload still uncompleted
+                    if not payload.is_eof() and not self._force_close:
+                        self.log_debug("Uncompleted request.")
+                        self.close()
+
+                payload.set_exception(PayloadAccessError())
+
+            except asyncio.CancelledError:
+                self.log_debug("Ignored premature client disconnection ")
+                break
+            except RuntimeError as exc:
+                if self._loop.get_debug():
+                    self.log_exception("Unhandled runtime exception", exc_info=exc)
+                self.force_close()
+            except Exception as exc:
+                self.log_exception("Unhandled exception", exc_info=exc)
+                self.force_close()
+            finally:
+                if self.transport is None and resp is not None:
+                    self.log_debug("Ignored premature client disconnection.")
+                elif not self._force_close:
+                    if self._keepalive and not self._close:
+                        # start keep-alive timer
+                        if keepalive_timeout is not None:
+                            now = self._loop.time()
+                            self._keepalive_time = now
+                            if self._keepalive_handle is None:
+                                self._keepalive_handle = loop.call_at(
+                                    now + keepalive_timeout, self._process_keepalive
+                                )
+                    else:
+                        break
+
+        # remove handler, close transport if no handlers left
+        if not self._force_close:
+            self._task_handler = None
+            if self.transport is not None:
+                self.transport.close()
+
+    async def finish_response(
+        self, request: BaseRequest, resp: StreamResponse, start_time: float
+    ) -> bool:
+        """Prepare the response and write_eof, then log access.
+
+        This has to
+        be called within the context of any exception so the access logger
+        can get exception information. Returns True if the client disconnects
+        prematurely.
+        """
+        request._finish()
+        if self._request_parser is not None:
+            self._request_parser.set_upgraded(False)
+            self._upgrade = False
+            if self._message_tail:
+                self._request_parser.feed_data(self._message_tail)
+                self._message_tail = b""
+        try:
+            prepare_meth = resp.prepare
+        except AttributeError:
+            if resp is None:
+                raise RuntimeError("Missing return " "statement on request handler")
+            else:
+                raise RuntimeError(
+                    "Web-handler should return "
+                    "a response instance, "
+                    "got {!r}".format(resp)
+                )
+        try:
+            await prepare_meth(request)
+            await resp.write_eof()
+        except ConnectionError:
+            await self.log_access(request, resp, start_time)
+            return True
+        else:
+            await self.log_access(request, resp, start_time)
+            return False
+
+    def handle_error(
+        self,
+        request: BaseRequest,
+        status: int = 500,
+        exc: Optional[BaseException] = None,
+        message: Optional[str] = None,
+    ) -> StreamResponse:
+        """Handle errors.
+
+        Returns HTTP response with specific status code. Logs additional
+        information. It always closes current connection.
+        """
+        self.log_exception("Error handling request", exc_info=exc)
+
+        # some data already got sent, connection is broken
+        if request.writer.output_size > 0:
+            raise ConnectionError(
+                "Response is sent already, cannot send another response "
+                "with the error message"
+            )
+
+        ct = "text/plain"
+        if status == HTTPStatus.INTERNAL_SERVER_ERROR:
+            title = "{0.value} {0.phrase}".format(HTTPStatus.INTERNAL_SERVER_ERROR)
+            msg = HTTPStatus.INTERNAL_SERVER_ERROR.description
+            tb = None
+            if self._loop.get_debug():
+                with suppress(Exception):
+                    tb = traceback.format_exc()
+
+            if "text/html" in request.headers.get("Accept", ""):
+                if tb:
+                    tb = html_escape(tb)
+                    msg = f"<h2>Traceback:</h2>\n<pre>{tb}</pre>"
+                message = (
+                    "<html><head>"
+                    "<title>{title}</title>"
+                    "</head><body>\n<h1>{title}</h1>"
+                    "\n{msg}\n</body></html>\n"
+                ).format(title=title, msg=msg)
+                ct = "text/html"
+            else:
+                if tb:
+                    msg = tb
+                message = title + "\n\n" + msg
+
+        resp = Response(status=status, text=message, content_type=ct)
+        resp.force_close()
+
+        return resp
+
+    def _make_error_handler(
+        self, err_info: _ErrInfo
+    ) -> Callable[[BaseRequest], Awaitable[StreamResponse]]:
+        async def handler(request: BaseRequest) -> StreamResponse:
+            return self.handle_error(
+                request, err_info.status, err_info.exc, err_info.message
+            )
+
+        return handler

aiohttp/web_request.py

@@ -0,0 +1,912 @@
+import asyncio
+import dataclasses
+import datetime
+import io
+import re
+import socket
+import string
+import tempfile
+import types
+from http.cookies import SimpleCookie
+from types import MappingProxyType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Final,
+    Iterator,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Pattern,
+    Set,
+    Tuple,
+    Union,
+    cast,
+)
+from urllib.parse import parse_qsl
+
+from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy
+from yarl import URL
+
+from . import hdrs
+from .abc import AbstractStreamWriter
+from .helpers import (
+    _SENTINEL,
+    ETAG_ANY,
+    LIST_QUOTED_ETAG_RE,
+    ChainMapProxy,
+    ETag,
+    HeadersMixin,
+    is_expected_content_type,
+    parse_http_date,
+    reify,
+    sentinel,
+    set_result,
+)
+from .http_parser import RawRequestMessage
+from .http_writer import HttpVersion
+from .multipart import BodyPartReader, MultipartReader
+from .streams import EmptyStreamReader, StreamReader
+from .typedefs import (
+    DEFAULT_JSON_DECODER,
+    JSONDecoder,
+    LooseHeaders,
+    RawHeaders,
+    StrOrURL,
+)
+from .web_exceptions import (
+    HTTPBadRequest,
+    HTTPRequestEntityTooLarge,
+    HTTPUnsupportedMediaType,
+)
+from .web_response import StreamResponse
+
+__all__ = ("BaseRequest", "FileField", "Request")
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .web_app import Application
+    from .web_protocol import RequestHandler
+    from .web_urldispatcher import UrlMappingMatchInfo
+
+
+@dataclasses.dataclass(frozen=True)
+class FileField:
+    name: str
+    filename: str
+    file: io.BufferedReader
+    content_type: str
+    headers: "CIMultiDictProxy[str]"
+
+
+_TCHAR: Final[str] = string.digits + string.ascii_letters + r"!#$%&'*+.^_`|~-"
+# '-' at the end to prevent interpretation as range in a char class
+
+_TOKEN: Final[str] = rf"[{_TCHAR}]+"
+
+_QDTEXT: Final[str] = r"[{}]".format(
+    r"".join(chr(c) for c in (0x09, 0x20, 0x21) + tuple(range(0x23, 0x7F)))
+)
+# qdtext includes 0x5C to escape 0x5D ('\]')
+# qdtext excludes obs-text (because obsoleted, and encoding not specified)
+
+_QUOTED_PAIR: Final[str] = r"\\[\t !-~]"
+
+_QUOTED_STRING: Final[str] = r'"(?:{quoted_pair}|{qdtext})*"'.format(
+    qdtext=_QDTEXT, quoted_pair=_QUOTED_PAIR
+)
+
+_FORWARDED_PAIR: Final[
+    str
+] = r"({token})=({token}|{quoted_string})(:\d{{1,4}})?".format(
+    token=_TOKEN, quoted_string=_QUOTED_STRING
+)
+
+_QUOTED_PAIR_REPLACE_RE: Final[Pattern[str]] = re.compile(r"\\([\t !-~])")
+# same pattern as _QUOTED_PAIR but contains a capture group
+
+_FORWARDED_PAIR_RE: Final[Pattern[str]] = re.compile(_FORWARDED_PAIR)
+
+############################################################
+# HTTP Request
+############################################################
+
+
+class BaseRequest(MutableMapping[str, Any], HeadersMixin):
+    POST_METHODS = {
+        hdrs.METH_PATCH,
+        hdrs.METH_POST,
+        hdrs.METH_PUT,
+        hdrs.METH_TRACE,
+        hdrs.METH_DELETE,
+    }
+
+    __slots__ = (
+        "_message",
+        "_protocol",
+        "_payload_writer",
+        "_payload",
+        "_headers",
+        "_method",
+        "_version",
+        "_rel_url",
+        "_post",
+        "_read_bytes",
+        "_state",
+        "_cache",
+        "_task",
+        "_client_max_size",
+        "_loop",
+        "_transport_sslcontext",
+        "_transport_peername",
+        "_disconnection_waiters",
+        "__weakref__",
+    )
+
+    def __init__(
+        self,
+        message: RawRequestMessage,
+        payload: StreamReader,
+        protocol: "RequestHandler",
+        payload_writer: AbstractStreamWriter,
+        task: "asyncio.Task[None]",
+        loop: asyncio.AbstractEventLoop,
+        *,
+        client_max_size: int = 1024**2,
+        state: Optional[Dict[str, Any]] = None,
+        scheme: Optional[str] = None,
+        host: Optional[str] = None,
+        remote: Optional[str] = None,
+    ) -> None:
+        super().__init__()
+        if state is None:
+            state = {}
+        self._message = message
+        self._protocol = protocol
+        self._payload_writer = payload_writer
+
+        self._payload = payload
+        self._headers = message.headers
+        self._method = message.method
+        self._version = message.version
+        self._cache: Dict[str, Any] = {}
+        url = message.url
+        if url.is_absolute():
+            # absolute URL is given,
+            # override auto-calculating url, host, and scheme
+            # all other properties should be good
+            self._cache["url"] = url
+            self._cache["host"] = url.host
+            self._cache["scheme"] = url.scheme
+            self._rel_url = url.relative()
+        else:
+            self._rel_url = message.url
+        self._post: Optional[MultiDictProxy[Union[str, bytes, FileField]]] = None
+        self._read_bytes: Optional[bytes] = None
+
+        self._state = state
+        self._task = task
+        self._client_max_size = client_max_size
+        self._loop = loop
+        self._disconnection_waiters: Set[asyncio.Future[None]] = set()
+
+        transport = self._protocol.transport
+        assert transport is not None
+        self._transport_sslcontext = transport.get_extra_info("sslcontext")
+        self._transport_peername = transport.get_extra_info("peername")
+
+        if scheme is not None:
+            self._cache["scheme"] = scheme
+        if host is not None:
+            self._cache["host"] = host
+        if remote is not None:
+            self._cache["remote"] = remote
+
+    def clone(
+        self,
+        *,
+        method: Union[str, _SENTINEL] = sentinel,
+        rel_url: Union[StrOrURL, _SENTINEL] = sentinel,
+        headers: Union[LooseHeaders, _SENTINEL] = sentinel,
+        scheme: Union[str, _SENTINEL] = sentinel,
+        host: Union[str, _SENTINEL] = sentinel,
+        remote: Union[str, _SENTINEL] = sentinel,
+        client_max_size: Union[int, _SENTINEL] = sentinel,
+    ) -> "BaseRequest":
+        """Clone itself with replacement some attributes.
+
+        Creates and returns a new instance of Request object. If no parameters
+        are given, an exact copy is returned. If a parameter is not passed, it
+        will reuse the one from the current request object.
+        """
+        if self._read_bytes:
+            raise RuntimeError("Cannot clone request " "after reading its content")
+
+        dct: Dict[str, Any] = {}
+        if method is not sentinel:
+            dct["method"] = method
+        if rel_url is not sentinel:
+            new_url: URL = URL(rel_url)
+            dct["url"] = new_url
+            dct["path"] = str(new_url)
+        if headers is not sentinel:
+            # a copy semantic
+            new_headers = CIMultiDictProxy(CIMultiDict(headers))
+            dct["headers"] = new_headers
+            dct["raw_headers"] = tuple(
+                (k.encode("utf-8"), v.encode("utf-8")) for k, v in new_headers.items()
+            )
+
+        message = self._message._replace(**dct)
+
+        kwargs: Dict[str, str] = {}
+        if scheme is not sentinel:
+            kwargs["scheme"] = scheme
+        if host is not sentinel:
+            kwargs["host"] = host
+        if remote is not sentinel:
+            kwargs["remote"] = remote
+        if client_max_size is sentinel:
+            client_max_size = self._client_max_size
+
+        return self.__class__(
+            message,
+            self._payload,
+            self._protocol,
+            self._payload_writer,
+            self._task,
+            self._loop,
+            client_max_size=client_max_size,
+            state=self._state.copy(),
+            **kwargs,
+        )
+
+    @property
+    def task(self) -> "asyncio.Task[None]":
+        return self._task
+
+    @property
+    def protocol(self) -> "RequestHandler":
+        return self._protocol
+
+    @property
+    def transport(self) -> Optional[asyncio.Transport]:
+        if self._protocol is None:
+            return None
+        return self._protocol.transport
+
+    @property
+    def writer(self) -> AbstractStreamWriter:
+        return self._payload_writer
+
+    @property
+    def client_max_size(self) -> int:
+        return self._client_max_size
+
+    @reify
+    def rel_url(self) -> URL:
+        return self._rel_url
+
+    # MutableMapping API
+
+    def __getitem__(self, key: str) -> Any:
+        return self._state[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self._state[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self._state[key]
+
+    def __len__(self) -> int:
+        return len(self._state)
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._state)
+
+    ########
+
+    @reify
+    def secure(self) -> bool:
+        """A bool indicating if the request is handled with SSL."""
+        return self.scheme == "https"
+
+    @reify
+    def forwarded(self) -> Tuple[Mapping[str, str], ...]:
+        """A tuple containing all parsed Forwarded header(s).
+
+        Makes an effort to parse Forwarded headers as specified by RFC 7239:
+
+        - It adds one (immutable) dictionary per Forwarded 'field-value', ie
+          per proxy. The element corresponds to the data in the Forwarded
+          field-value added by the first proxy encountered by the client. Each
+          subsequent item corresponds to those added by later proxies.
+        - It checks that every value has valid syntax in general as specified
+          in section 4: either a 'token' or a 'quoted-string'.
+        - It un-escapes found escape sequences.
+        - It does NOT validate 'by' and 'for' contents as specified in section
+          6.
+        - It does NOT validate 'host' contents (Host ABNF).
+        - It does NOT validate 'proto' contents for valid URI scheme names.
+
+        Returns a tuple containing one or more immutable dicts
+        """
+        elems = []
+        for field_value in self._message.headers.getall(hdrs.FORWARDED, ()):
+            length = len(field_value)
+            pos = 0
+            need_separator = False
+            elem: Dict[str, str] = {}
+            elems.append(types.MappingProxyType(elem))
+            while 0 <= pos < length:
+                match = _FORWARDED_PAIR_RE.match(field_value, pos)
+                if match is not None:  # got a valid forwarded-pair
+                    if need_separator:
+                        # bad syntax here, skip to next comma
+                        pos = field_value.find(",", pos)
+                    else:
+                        name, value, port = match.groups()
+                        if value[0] == '"':
+                            # quoted string: remove quotes and unescape
+                            value = _QUOTED_PAIR_REPLACE_RE.sub(r"\1", value[1:-1])
+                        if port:
+                            value += port
+                        elem[name.lower()] = value
+                        pos += len(match.group(0))
+                        need_separator = True
+                elif field_value[pos] == ",":  # next forwarded-element
+                    need_separator = False
+                    elem = {}
+                    elems.append(types.MappingProxyType(elem))
+                    pos += 1
+                elif field_value[pos] == ";":  # next forwarded-pair
+                    need_separator = False
+                    pos += 1
+                elif field_value[pos] in " \t":
+                    # Allow whitespace even between forwarded-pairs, though
+                    # RFC 7239 doesn't. This simplifies code and is in line
+                    # with Postel's law.
+                    pos += 1
+                else:
+                    # bad syntax here, skip to next comma
+                    pos = field_value.find(",", pos)
+        return tuple(elems)
+
+    @reify
+    def scheme(self) -> str:
+        """A string representing the scheme of the request.
+
+        Hostname is resolved in this order:
+
+        - overridden value by .clone(scheme=new_scheme) call.
+        - type of connection to peer: HTTPS if socket is SSL, HTTP otherwise.
+
+        'http' or 'https'.
+        """
+        if self._transport_sslcontext:
+            return "https"
+        else:
+            return "http"
+
+    @reify
+    def method(self) -> str:
+        """Read only property for getting HTTP method.
+
+        The value is upper-cased str like 'GET', 'POST', 'PUT' etc.
+        """
+        return self._method
+
+    @reify
+    def version(self) -> HttpVersion:
+        """Read only property for getting HTTP version of request.
+
+        Returns aiohttp.protocol.HttpVersion instance.
+        """
+        return self._version
+
+    @reify
+    def host(self) -> str:
+        """Hostname of the request.
+
+        Hostname is resolved in this order:
+
+        - overridden value by .clone(host=new_host) call.
+        - HOST HTTP header
+        - socket.getfqdn() value
+        """
+        host = self._message.headers.get(hdrs.HOST)
+        if host is not None:
+            return host
+        return socket.getfqdn()
+
+    @reify
+    def remote(self) -> Optional[str]:
+        """Remote IP of client initiated HTTP request.
+
+        The IP is resolved in this order:
+
+        - overridden value by .clone(remote=new_remote) call.
+        - peername of opened socket
+        """
+        if self._transport_peername is None:
+            return None
+        if isinstance(self._transport_peername, (list, tuple)):
+            return str(self._transport_peername[0])
+        return str(self._transport_peername)
+
+    @reify
+    def url(self) -> URL:
+        url = URL.build(scheme=self.scheme, host=self.host)
+        return url.join(self._rel_url)
+
+    @reify
+    def path(self) -> str:
+        """The URL including *PATH INFO* without the host or scheme.
+
+        E.g., ``/app/blog``
+        """
+        return self._rel_url.path
+
+    @reify
+    def path_qs(self) -> str:
+        """The URL including PATH_INFO and the query string.
+
+        E.g, /app/blog?id=10
+        """
+        return str(self._rel_url)
+
+    @reify
+    def raw_path(self) -> str:
+        """The URL including raw *PATH INFO* without the host or scheme.
+
+        Warning, the path is unquoted and may contains non valid URL characters
+
+        E.g., ``/my%2Fpath%7Cwith%21some%25strange%24characters``
+        """
+        return self._message.path
+
+    @reify
+    def query(self) -> MultiDictProxy[str]:
+        """A multidict with all the variables in the query string."""
+        return MultiDictProxy(self._rel_url.query)
+
+    @reify
+    def query_string(self) -> str:
+        """The query string in the URL.
+
+        E.g., id=10
+        """
+        return self._rel_url.query_string
+
+    @reify
+    def headers(self) -> "CIMultiDictProxy[str]":
+        """A case-insensitive multidict proxy with all headers."""
+        return self._headers
+
+    @reify
+    def raw_headers(self) -> RawHeaders:
+        """A sequence of pairs for all headers."""
+        return self._message.raw_headers
+
+    @reify
+    def if_modified_since(self) -> Optional[datetime.datetime]:
+        """The value of If-Modified-Since HTTP header, or None.
+
+        This header is represented as a `datetime` object.
+        """
+        return parse_http_date(self.headers.get(hdrs.IF_MODIFIED_SINCE))
+
+    @reify
+    def if_unmodified_since(self) -> Optional[datetime.datetime]:
+        """The value of If-Unmodified-Since HTTP header, or None.
+
+        This header is represented as a `datetime` object.
+        """
+        return parse_http_date(self.headers.get(hdrs.IF_UNMODIFIED_SINCE))
+
+    @staticmethod
+    def _etag_values(etag_header: str) -> Iterator[ETag]:
+        """Extract `ETag` objects from raw header."""
+        if etag_header == ETAG_ANY:
+            yield ETag(
+                is_weak=False,
+                value=ETAG_ANY,
+            )
+        else:
+            for match in LIST_QUOTED_ETAG_RE.finditer(etag_header):
+                is_weak, value, garbage = match.group(2, 3, 4)
+                # Any symbol captured by 4th group means
+                # that the following sequence is invalid.
+                if garbage:
+                    break
+
+                yield ETag(
+                    is_weak=bool(is_weak),
+                    value=value,
+                )
+
+    @classmethod
+    def _if_match_or_none_impl(
+        cls, header_value: Optional[str]
+    ) -> Optional[Tuple[ETag, ...]]:
+        if not header_value:
+            return None
+
+        return tuple(cls._etag_values(header_value))
+
+    @reify
+    def if_match(self) -> Optional[Tuple[ETag, ...]]:
+        """The value of If-Match HTTP header, or None.
+
+        This header is represented as a `tuple` of `ETag` objects.
+        """
+        return self._if_match_or_none_impl(self.headers.get(hdrs.IF_MATCH))
+
+    @reify
+    def if_none_match(self) -> Optional[Tuple[ETag, ...]]:
+        """The value of If-None-Match HTTP header, or None.
+
+        This header is represented as a `tuple` of `ETag` objects.
+        """
+        return self._if_match_or_none_impl(self.headers.get(hdrs.IF_NONE_MATCH))
+
+    @reify
+    def if_range(self) -> Optional[datetime.datetime]:
+        """The value of If-Range HTTP header, or None.
+
+        This header is represented as a `datetime` object.
+        """
+        return parse_http_date(self.headers.get(hdrs.IF_RANGE))
+
+    @reify
+    def keep_alive(self) -> bool:
+        """Is keepalive enabled by client?"""
+        return not self._message.should_close
+
+    @reify
+    def cookies(self) -> Mapping[str, str]:
+        """Return request cookies.
+
+        A read-only dictionary-like object.
+        """
+        raw = self.headers.get(hdrs.COOKIE, "")
+        parsed: SimpleCookie[str] = SimpleCookie(raw)
+        return MappingProxyType({key: val.value for key, val in parsed.items()})
+
+    @reify
+    def http_range(self) -> slice:
+        """The content of Range HTTP header.
+
+        Return a slice instance.
+
+        """
+        rng = self._headers.get(hdrs.RANGE)
+        start, end = None, None
+        if rng is not None:
+            try:
+                pattern = r"^bytes=(\d*)-(\d*)$"
+                start, end = re.findall(pattern, rng)[0]
+            except IndexError:  # pattern was not found in header
+                raise ValueError("range not in acceptable format")
+
+            end = int(end) if end else None
+            start = int(start) if start else None
+
+            if start is None and end is not None:
+                # end with no start is to return tail of content
+                start = -end
+                end = None
+
+            if start is not None and end is not None:
+                # end is inclusive in range header, exclusive for slice
+                end += 1
+
+                if start >= end:
+                    raise ValueError("start cannot be after end")
+
+            if start is end is None:  # No valid range supplied
+                raise ValueError("No start or end of range specified")
+
+        return slice(start, end, 1)
+
+    @reify
+    def content(self) -> StreamReader:
+        """Return raw payload stream."""
+        return self._payload
+
+    @property
+    def can_read_body(self) -> bool:
+        """Return True if request's HTTP BODY can be read, False otherwise."""
+        return not self._payload.at_eof()
+
+    @reify
+    def body_exists(self) -> bool:
+        """Return True if request has HTTP BODY, False otherwise."""
+        return type(self._payload) is not EmptyStreamReader
+
+    async def release(self) -> None:
+        """Release request.
+
+        Eat unread part of HTTP BODY if present.
+        """
+        while not self._payload.at_eof():
+            await self._payload.readany()
+
+    async def read(self) -> bytes:
+        """Read request body if present.
+
+        Returns bytes object with full request content.
+        """
+        if self._read_bytes is None:
+            body = bytearray()
+            while True:
+                chunk = await self._payload.readany()
+                body.extend(chunk)
+                if self._client_max_size:
+                    body_size = len(body)
+                    if body_size > self._client_max_size:
+                        raise HTTPRequestEntityTooLarge(
+                            max_size=self._client_max_size, actual_size=body_size
+                        )
+                if not chunk:
+                    break
+            self._read_bytes = bytes(body)
+        return self._read_bytes
+
+    async def text(self) -> str:
+        """Return BODY as text using encoding from .charset."""
+        bytes_body = await self.read()
+        encoding = self.charset or "utf-8"
+        try:
+            return bytes_body.decode(encoding)
+        except LookupError:
+            raise HTTPUnsupportedMediaType()
+
+    async def json(
+        self,
+        *,
+        loads: JSONDecoder = DEFAULT_JSON_DECODER,
+        content_type: Optional[str] = "application/json",
+    ) -> Any:
+        """Return BODY as JSON."""
+        body = await self.text()
+        if content_type:
+            if not is_expected_content_type(self.content_type, content_type):
+                raise HTTPBadRequest(
+                    text=(
+                        "Attempt to decode JSON with "
+                        "unexpected mimetype: %s" % self.content_type
+                    )
+                )
+
+        return loads(body)
+
+    async def multipart(self) -> MultipartReader:
+        """Return async iterator to process BODY as multipart."""
+        return MultipartReader(self._headers, self._payload)
+
+    async def post(self) -> "MultiDictProxy[Union[str, bytes, FileField]]":
+        """Return POST parameters."""
+        if self._post is not None:
+            return self._post
+        if self._method not in self.POST_METHODS:
+            self._post = MultiDictProxy(MultiDict())
+            return self._post
+
+        content_type = self.content_type
+        if content_type not in (
+            "",
+            "application/x-www-form-urlencoded",
+            "multipart/form-data",
+        ):
+            self._post = MultiDictProxy(MultiDict())
+            return self._post
+
+        out: MultiDict[Union[str, bytes, FileField]] = MultiDict()
+
+        if content_type == "multipart/form-data":
+            multipart = await self.multipart()
+            max_size = self._client_max_size
+
+            field = await multipart.next()
+            while field is not None:
+                size = 0
+                field_ct = field.headers.get(hdrs.CONTENT_TYPE)
+
+                if isinstance(field, BodyPartReader):
+                    assert field.name is not None
+
+                    # Note that according to RFC 7578, the Content-Type header
+                    # is optional, even for files, so we can't assume it's
+                    # present.
+                    # https://tools.ietf.org/html/rfc7578#section-4.4
+                    if field.filename:
+                        # store file in temp file
+                        tmp = tempfile.TemporaryFile()
+                        chunk = await field.read_chunk(size=2**16)
+                        while chunk:
+                            chunk = field.decode(chunk)
+                            tmp.write(chunk)
+                            size += len(chunk)
+                            if 0 < max_size < size:
+                                tmp.close()
+                                raise HTTPRequestEntityTooLarge(
+                                    max_size=max_size, actual_size=size
+                                )
+                            chunk = await field.read_chunk(size=2**16)
+                        tmp.seek(0)
+
+                        if field_ct is None:
+                            field_ct = "application/octet-stream"
+
+                        ff = FileField(
+                            field.name,
+                            field.filename,
+                            cast(io.BufferedReader, tmp),
+                            field_ct,
+                            field.headers,
+                        )
+                        out.add(field.name, ff)
+                    else:
+                        # deal with ordinary data
+                        value = await field.read(decode=True)
+                        if field_ct is None or field_ct.startswith("text/"):
+                            charset = field.get_charset(default="utf-8")
+                            out.add(field.name, value.decode(charset))
+                        else:
+                            out.add(field.name, value)
+                        size += len(value)
+                        if 0 < max_size < size:
+                            raise HTTPRequestEntityTooLarge(
+                                max_size=max_size, actual_size=size
+                            )
+                else:
+                    raise ValueError(
+                        "To decode nested multipart you need " "to use custom reader",
+                    )
+
+                field = await multipart.next()
+        else:
+            data = await self.read()
+            if data:
+                charset = self.charset or "utf-8"
+                bytes_query = data.rstrip()
+                try:
+                    query = bytes_query.decode(charset)
+                except LookupError:
+                    raise HTTPUnsupportedMediaType()
+                out.extend(
+                    parse_qsl(qs=query, keep_blank_values=True, encoding=charset)
+                )
+
+        self._post = MultiDictProxy(out)
+        return self._post
+
+    def get_extra_info(self, name: str, default: Any = None) -> Any:
+        """Extra info from protocol transport"""
+        protocol = self._protocol
+        if protocol is None:
+            return default
+
+        transport = protocol.transport
+        if transport is None:
+            return default
+
+        return transport.get_extra_info(name, default)
+
+    def __repr__(self) -> str:
+        ascii_encodable_path = self.path.encode("ascii", "backslashreplace").decode(
+            "ascii"
+        )
+        return "<{} {} {} >".format(
+            self.__class__.__name__, self._method, ascii_encodable_path
+        )
+
+    def __eq__(self, other: object) -> bool:
+        return id(self) == id(other)
+
+    def __bool__(self) -> bool:
+        return True
+
+    async def _prepare_hook(self, response: StreamResponse) -> None:
+        return
+
+    def _cancel(self, exc: BaseException) -> None:
+        self._payload.set_exception(exc)
+        for fut in self._disconnection_waiters:
+            set_result(fut, None)
+
+    def _finish(self) -> None:
+        for fut in self._disconnection_waiters:
+            fut.cancel()
+
+        if self._post is None or self.content_type != "multipart/form-data":
+            return
+
+        # NOTE: Release file descriptors for the
+        # NOTE: `tempfile.Temporaryfile`-created `_io.BufferedRandom`
+        # NOTE: instances of files sent within multipart request body
+        # NOTE: via HTTP POST request.
+        for file_name, file_field_object in self._post.items():
+            if not isinstance(file_field_object, FileField):
+                continue
+
+            file_field_object.file.close()
+
+    async def wait_for_disconnection(self) -> None:
+        loop = asyncio.get_event_loop()
+        fut: asyncio.Future[None] = loop.create_future()
+        self._disconnection_waiters.add(fut)
+        try:
+            await fut
+        finally:
+            self._disconnection_waiters.remove(fut)
+
+
+class Request(BaseRequest):
+    __slots__ = ("_match_info",)
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+
+        # matchdict, route_name, handler
+        # or information about traversal lookup
+
+        # initialized after route resolving
+        self._match_info: Optional[UrlMappingMatchInfo] = None
+
+    def clone(
+        self,
+        *,
+        method: Union[str, _SENTINEL] = sentinel,
+        rel_url: Union[StrOrURL, _SENTINEL] = sentinel,
+        headers: Union[LooseHeaders, _SENTINEL] = sentinel,
+        scheme: Union[str, _SENTINEL] = sentinel,
+        host: Union[str, _SENTINEL] = sentinel,
+        remote: Union[str, _SENTINEL] = sentinel,
+        client_max_size: Union[int, _SENTINEL] = sentinel,
+    ) -> "Request":
+        ret = super().clone(
+            method=method,
+            rel_url=rel_url,
+            headers=headers,
+            scheme=scheme,
+            host=host,
+            remote=remote,
+            client_max_size=client_max_size,
+        )
+        new_ret = cast(Request, ret)
+        new_ret._match_info = self._match_info
+        return new_ret
+
+    @reify
+    def match_info(self) -> "UrlMappingMatchInfo":
+        """Result of route resolving."""
+        match_info = self._match_info
+        assert match_info is not None
+        return match_info
+
+    @property
+    def app(self) -> "Application":
+        """Application instance."""
+        match_info = self._match_info
+        assert match_info is not None
+        return match_info.current_app
+
+    @property
+    def config_dict(self) -> ChainMapProxy:
+        match_info = self._match_info
+        assert match_info is not None
+        lst = match_info.apps
+        app = self.app
+        idx = lst.index(app)
+        sublist = list(reversed(lst[: idx + 1]))
+        return ChainMapProxy(sublist)
+
+    async def _prepare_hook(self, response: StreamResponse) -> None:
+        match_info = self._match_info
+        if match_info is None:
+            return
+        for app in match_info._apps:
+            await app.on_response_prepare.send(self, response)

aiohttp/web_response.py

@@ -0,0 +1,728 @@
+import asyncio
+import collections.abc
+import datetime
+import enum
+import json
+import math
+import time
+import warnings
+from concurrent.futures import Executor
+from http import HTTPStatus
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterator,
+    MutableMapping,
+    Optional,
+    Union,
+    cast,
+)
+
+from multidict import CIMultiDict, istr
+
+from . import hdrs, payload
+from .abc import AbstractStreamWriter
+from .compression_utils import ZLibCompressor
+from .helpers import (
+    ETAG_ANY,
+    QUOTED_ETAG_RE,
+    CookieMixin,
+    ETag,
+    HeadersMixin,
+    parse_http_date,
+    populate_with_cookies,
+    rfc822_formatted_time,
+    sentinel,
+    validate_etag_value,
+)
+from .http import SERVER_SOFTWARE, HttpVersion10, HttpVersion11
+from .payload import Payload
+from .typedefs import JSONEncoder, LooseHeaders
+
+__all__ = ("ContentCoding", "StreamResponse", "Response", "json_response")
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .web_request import BaseRequest
+
+    BaseClass = MutableMapping[str, Any]
+else:
+    BaseClass = collections.abc.MutableMapping
+
+
+class ContentCoding(enum.Enum):
+    # The content codings that we have support for.
+    #
+    # Additional registered codings are listed at:
+    # https://www.iana.org/assignments/http-parameters/http-parameters.xhtml#content-coding
+    deflate = "deflate"
+    gzip = "gzip"
+    identity = "identity"
+
+
+############################################################
+# HTTP Response classes
+############################################################
+
+
+class StreamResponse(BaseClass, HeadersMixin, CookieMixin):
+    __slots__ = (
+        "_length_check",
+        "_body",
+        "_keep_alive",
+        "_chunked",
+        "_compression",
+        "_compression_force",
+        "_req",
+        "_payload_writer",
+        "_eof_sent",
+        "_body_length",
+        "_state",
+        "_headers",
+        "_status",
+        "_reason",
+        "_cookies",
+        "__weakref__",
+    )
+
+    def __init__(
+        self,
+        *,
+        status: int = 200,
+        reason: Optional[str] = None,
+        headers: Optional[LooseHeaders] = None,
+    ) -> None:
+        super().__init__()
+        self._length_check = True
+        self._body = None
+        self._keep_alive: Optional[bool] = None
+        self._chunked = False
+        self._compression = False
+        self._compression_force: Optional[ContentCoding] = None
+
+        self._req: Optional[BaseRequest] = None
+        self._payload_writer: Optional[AbstractStreamWriter] = None
+        self._eof_sent = False
+        self._body_length = 0
+        self._state: Dict[str, Any] = {}
+
+        if headers is not None:
+            self._headers: CIMultiDict[str] = CIMultiDict(headers)
+        else:
+            self._headers = CIMultiDict()
+
+        self.set_status(status, reason)
+
+    @property
+    def prepared(self) -> bool:
+        return self._payload_writer is not None
+
+    @property
+    def task(self) -> "Optional[asyncio.Task[None]]":
+        if self._req:
+            return self._req.task
+        else:
+            return None
+
+    @property
+    def status(self) -> int:
+        return self._status
+
+    @property
+    def chunked(self) -> bool:
+        return self._chunked
+
+    @property
+    def compression(self) -> bool:
+        return self._compression
+
+    @property
+    def reason(self) -> str:
+        return self._reason
+
+    def set_status(
+        self,
+        status: int,
+        reason: Optional[str] = None,
+    ) -> None:
+        assert not self.prepared, (
+            "Cannot change the response status code after " "the headers have been sent"
+        )
+        self._status = int(status)
+        if reason is None:
+            try:
+                reason = HTTPStatus(self._status).phrase
+            except ValueError:
+                reason = ""
+        self._reason = reason
+
+    @property
+    def keep_alive(self) -> Optional[bool]:
+        return self._keep_alive
+
+    def force_close(self) -> None:
+        self._keep_alive = False
+
+    @property
+    def body_length(self) -> int:
+        return self._body_length
+
+    def enable_chunked_encoding(self) -> None:
+        """Enables automatic chunked transfer encoding."""
+        self._chunked = True
+
+        if hdrs.CONTENT_LENGTH in self._headers:
+            raise RuntimeError(
+                "You can't enable chunked encoding when " "a content length is set"
+            )
+
+    def enable_compression(self, force: Optional[ContentCoding] = None) -> None:
+        """Enables response compression encoding."""
+        # Backwards compatibility for when force was a bool <0.17.
+        self._compression = True
+        self._compression_force = force
+
+    @property
+    def headers(self) -> "CIMultiDict[str]":
+        return self._headers
+
+    @property
+    def content_length(self) -> Optional[int]:
+        # Just a placeholder for adding setter
+        return super().content_length
+
+    @content_length.setter
+    def content_length(self, value: Optional[int]) -> None:
+        if value is not None:
+            value = int(value)
+            if self._chunked:
+                raise RuntimeError(
+                    "You can't set content length when " "chunked encoding is enable"
+                )
+            self._headers[hdrs.CONTENT_LENGTH] = str(value)
+        else:
+            self._headers.pop(hdrs.CONTENT_LENGTH, None)
+
+    @property
+    def content_type(self) -> str:
+        # Just a placeholder for adding setter
+        return super().content_type
+
+    @content_type.setter
+    def content_type(self, value: str) -> None:
+        self.content_type  # read header values if needed
+        self._content_type = str(value)
+        self._generate_content_type_header()
+
+    @property
+    def charset(self) -> Optional[str]:
+        # Just a placeholder for adding setter
+        return super().charset
+
+    @charset.setter
+    def charset(self, value: Optional[str]) -> None:
+        ctype = self.content_type  # read header values if needed
+        if ctype == "application/octet-stream":
+            raise RuntimeError(
+                "Setting charset for application/octet-stream "
+                "doesn't make sense, setup content_type first"
+            )
+        assert self._content_dict is not None
+        if value is None:
+            self._content_dict.pop("charset", None)
+        else:
+            self._content_dict["charset"] = str(value).lower()
+        self._generate_content_type_header()
+
+    @property
+    def last_modified(self) -> Optional[datetime.datetime]:
+        """The value of Last-Modified HTTP header, or None.
+
+        This header is represented as a `datetime` object.
+        """
+        return parse_http_date(self._headers.get(hdrs.LAST_MODIFIED))
+
+    @last_modified.setter
+    def last_modified(
+        self, value: Optional[Union[int, float, datetime.datetime, str]]
+    ) -> None:
+        if value is None:
+            self._headers.pop(hdrs.LAST_MODIFIED, None)
+        elif isinstance(value, (int, float)):
+            self._headers[hdrs.LAST_MODIFIED] = time.strftime(
+                "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value))
+            )
+        elif isinstance(value, datetime.datetime):
+            self._headers[hdrs.LAST_MODIFIED] = time.strftime(
+                "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple()
+            )
+        elif isinstance(value, str):
+            self._headers[hdrs.LAST_MODIFIED] = value
+
+    @property
+    def etag(self) -> Optional[ETag]:
+        quoted_value = self._headers.get(hdrs.ETAG)
+        if not quoted_value:
+            return None
+        elif quoted_value == ETAG_ANY:
+            return ETag(value=ETAG_ANY)
+        match = QUOTED_ETAG_RE.fullmatch(quoted_value)
+        if not match:
+            return None
+        is_weak, value = match.group(1, 2)
+        return ETag(
+            is_weak=bool(is_weak),
+            value=value,
+        )
+
+    @etag.setter
+    def etag(self, value: Optional[Union[ETag, str]]) -> None:
+        if value is None:
+            self._headers.pop(hdrs.ETAG, None)
+        elif (isinstance(value, str) and value == ETAG_ANY) or (
+            isinstance(value, ETag) and value.value == ETAG_ANY
+        ):
+            self._headers[hdrs.ETAG] = ETAG_ANY
+        elif isinstance(value, str):
+            validate_etag_value(value)
+            self._headers[hdrs.ETAG] = f'"{value}"'
+        elif isinstance(value, ETag) and isinstance(value.value, str):  # type: ignore[redundant-expr]
+            validate_etag_value(value.value)
+            hdr_value = f'W/"{value.value}"' if value.is_weak else f'"{value.value}"'
+            self._headers[hdrs.ETAG] = hdr_value
+        else:
+            raise ValueError(
+                f"Unsupported etag type: {type(value)}. "
+                f"etag must be str, ETag or None"
+            )
+
+    def _generate_content_type_header(
+        self, CONTENT_TYPE: istr = hdrs.CONTENT_TYPE
+    ) -> None:
+        assert self._content_dict is not None
+        assert self._content_type is not None
+        params = "; ".join(f"{k}={v}" for k, v in self._content_dict.items())
+        if params:
+            ctype = self._content_type + "; " + params
+        else:
+            ctype = self._content_type
+        self._headers[CONTENT_TYPE] = ctype
+
+    async def _do_start_compression(self, coding: ContentCoding) -> None:
+        if coding != ContentCoding.identity:
+            assert self._payload_writer is not None
+            self._headers[hdrs.CONTENT_ENCODING] = coding.value
+            self._payload_writer.enable_compression(coding.value)
+            # Compressed payload may have different content length,
+            # remove the header
+            self._headers.popall(hdrs.CONTENT_LENGTH, None)
+
+    async def _start_compression(self, request: "BaseRequest") -> None:
+        if self._compression_force:
+            await self._do_start_compression(self._compression_force)
+        else:
+            accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower()
+            for coding in ContentCoding:
+                if coding.value in accept_encoding:
+                    await self._do_start_compression(coding)
+                    return
+
+    async def prepare(self, request: "BaseRequest") -> Optional[AbstractStreamWriter]:
+        if self._eof_sent:
+            return None
+        if self._payload_writer is not None:
+            return self._payload_writer
+
+        return await self._start(request)
+
+    async def _start(self, request: "BaseRequest") -> AbstractStreamWriter:
+        self._req = request
+        writer = self._payload_writer = request._payload_writer
+
+        await self._prepare_headers()
+        await request._prepare_hook(self)
+        await self._write_headers()
+
+        return writer
+
+    async def _prepare_headers(self) -> None:
+        request = self._req
+        assert request is not None
+        writer = self._payload_writer
+        assert writer is not None
+        keep_alive = self._keep_alive
+        if keep_alive is None:
+            keep_alive = request.keep_alive
+        self._keep_alive = keep_alive
+
+        version = request.version
+
+        headers = self._headers
+        populate_with_cookies(headers, self.cookies)
+
+        if self._compression:
+            await self._start_compression(request)
+
+        if self._chunked:
+            if version != HttpVersion11:
+                raise RuntimeError(
+                    "Using chunked encoding is forbidden "
+                    "for HTTP/{0.major}.{0.minor}".format(request.version)
+                )
+            writer.enable_chunking()
+            headers[hdrs.TRANSFER_ENCODING] = "chunked"
+            if hdrs.CONTENT_LENGTH in headers:
+                del headers[hdrs.CONTENT_LENGTH]
+        elif self._length_check:
+            writer.length = self.content_length
+            if writer.length is None:
+                if version >= HttpVersion11 and self.status != 204:
+                    writer.enable_chunking()
+                    headers[hdrs.TRANSFER_ENCODING] = "chunked"
+                    if hdrs.CONTENT_LENGTH in headers:
+                        del headers[hdrs.CONTENT_LENGTH]
+                else:
+                    keep_alive = False
+            # HTTP 1.1: https://tools.ietf.org/html/rfc7230#section-3.3.2
+            # HTTP 1.0: https://tools.ietf.org/html/rfc1945#section-10.4
+            elif version >= HttpVersion11 and self.status in (100, 101, 102, 103, 204):
+                del headers[hdrs.CONTENT_LENGTH]
+
+        if self.status not in (204, 304):
+            headers.setdefault(hdrs.CONTENT_TYPE, "application/octet-stream")
+        headers.setdefault(hdrs.DATE, rfc822_formatted_time())
+        headers.setdefault(hdrs.SERVER, SERVER_SOFTWARE)
+
+        # connection header
+        if hdrs.CONNECTION not in headers:
+            if keep_alive:
+                if version == HttpVersion10:
+                    headers[hdrs.CONNECTION] = "keep-alive"
+            else:
+                if version == HttpVersion11:
+                    headers[hdrs.CONNECTION] = "close"
+
+    async def _write_headers(self) -> None:
+        request = self._req
+        assert request is not None
+        writer = self._payload_writer
+        assert writer is not None
+        # status line
+        version = request.version
+        status_line = "HTTP/{}.{} {} {}".format(
+            version[0], version[1], self._status, self._reason
+        )
+        await writer.write_headers(status_line, self._headers)
+
+    async def write(self, data: bytes) -> None:
+        assert isinstance(
+            data, (bytes, bytearray, memoryview)
+        ), "data argument must be byte-ish (%r)" % type(data)
+
+        if self._eof_sent:
+            raise RuntimeError("Cannot call write() after write_eof()")
+        if self._payload_writer is None:
+            raise RuntimeError("Cannot call write() before prepare()")
+
+        await self._payload_writer.write(data)
+
+    async def drain(self) -> None:
+        assert not self._eof_sent, "EOF has already been sent"
+        assert self._payload_writer is not None, "Response has not been started"
+        warnings.warn(
+            "drain method is deprecated, use await resp.write()",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        await self._payload_writer.drain()
+
+    async def write_eof(self, data: bytes = b"") -> None:
+        assert isinstance(
+            data, (bytes, bytearray, memoryview)
+        ), "data argument must be byte-ish (%r)" % type(data)
+
+        if self._eof_sent:
+            return
+
+        assert self._payload_writer is not None, "Response has not been started"
+
+        await self._payload_writer.write_eof(data)
+        self._eof_sent = True
+        self._req = None
+        self._body_length = self._payload_writer.output_size
+        self._payload_writer = None
+
+    def __repr__(self) -> str:
+        if self._eof_sent:
+            info = "eof"
+        elif self.prepared:
+            assert self._req is not None
+            info = f"{self._req.method} {self._req.path} "
+        else:
+            info = "not prepared"
+        return f"<{self.__class__.__name__} {self.reason} {info}>"
+
+    def __getitem__(self, key: str) -> Any:
+        return self._state[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self._state[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self._state[key]
+
+    def __len__(self) -> int:
+        return len(self._state)
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._state)
+
+    def __hash__(self) -> int:
+        return hash(id(self))
+
+    def __eq__(self, other: object) -> bool:
+        return self is other
+
+
+class Response(StreamResponse):
+    __slots__ = (
+        "_body_payload",
+        "_compressed_body",
+        "_zlib_executor_size",
+        "_zlib_executor",
+    )
+
+    def __init__(
+        self,
+        *,
+        body: Any = None,
+        status: int = 200,
+        reason: Optional[str] = None,
+        text: Optional[str] = None,
+        headers: Optional[LooseHeaders] = None,
+        content_type: Optional[str] = None,
+        charset: Optional[str] = None,
+        zlib_executor_size: Optional[int] = None,
+        zlib_executor: Optional[Executor] = None,
+    ) -> None:
+        if body is not None and text is not None:
+            raise ValueError("body and text are not allowed together")
+
+        if headers is None:
+            real_headers: CIMultiDict[str] = CIMultiDict()
+        elif not isinstance(headers, CIMultiDict):
+            real_headers = CIMultiDict(headers)
+        else:
+            real_headers = headers  # = cast('CIMultiDict[str]', headers)
+
+        if content_type is not None and "charset" in content_type:
+            raise ValueError("charset must not be in content_type " "argument")
+
+        if text is not None:
+            if hdrs.CONTENT_TYPE in real_headers:
+                if content_type or charset:
+                    raise ValueError(
+                        "passing both Content-Type header and "
+                        "content_type or charset params "
+                        "is forbidden"
+                    )
+            else:
+                # fast path for filling headers
+                if not isinstance(text, str):
+                    raise TypeError("text argument must be str (%r)" % type(text))
+                if content_type is None:
+                    content_type = "text/plain"
+                if charset is None:
+                    charset = "utf-8"
+                real_headers[hdrs.CONTENT_TYPE] = content_type + "; charset=" + charset
+                body = text.encode(charset)
+                text = None
+        else:
+            if hdrs.CONTENT_TYPE in real_headers:
+                if content_type is not None or charset is not None:
+                    raise ValueError(
+                        "passing both Content-Type header and "
+                        "content_type or charset params "
+                        "is forbidden"
+                    )
+            else:
+                if content_type is not None:
+                    if charset is not None:
+                        content_type += "; charset=" + charset
+                    real_headers[hdrs.CONTENT_TYPE] = content_type
+
+        super().__init__(status=status, reason=reason, headers=real_headers)
+
+        if text is not None:
+            self.text = text
+        else:
+            self.body = body
+
+        self._compressed_body: Optional[bytes] = None
+        self._zlib_executor_size = zlib_executor_size
+        self._zlib_executor = zlib_executor
+
+    @property
+    def body(self) -> Optional[Union[bytes, Payload]]:
+        return self._body
+
+    @body.setter
+    def body(self, body: bytes) -> None:
+        if body is None:
+            self._body: Optional[bytes] = None
+            self._body_payload: bool = False
+        elif isinstance(body, (bytes, bytearray)):
+            self._body = body
+            self._body_payload = False
+        else:
+            try:
+                self._body = body = payload.PAYLOAD_REGISTRY.get(body)
+            except payload.LookupError:
+                raise ValueError("Unsupported body type %r" % type(body))
+
+            self._body_payload = True
+
+            headers = self._headers
+
+            # set content-type
+            if hdrs.CONTENT_TYPE not in headers:
+                headers[hdrs.CONTENT_TYPE] = body.content_type
+
+            # copy payload headers
+            if body.headers:
+                for key, value in body.headers.items():
+                    if key not in headers:
+                        headers[key] = value
+
+        self._compressed_body = None
+
+    @property
+    def text(self) -> Optional[str]:
+        if self._body is None:
+            return None
+        return self._body.decode(self.charset or "utf-8")
+
+    @text.setter
+    def text(self, text: str) -> None:
+        assert isinstance(text, str), "text argument must be str (%r)" % type(text)
+
+        if self.content_type == "application/octet-stream":
+            self.content_type = "text/plain"
+        if self.charset is None:
+            self.charset = "utf-8"
+
+        self._body = text.encode(self.charset)
+        self._body_payload = False
+        self._compressed_body = None
+
+    @property
+    def content_length(self) -> Optional[int]:
+        if self._chunked:
+            return None
+
+        if hdrs.CONTENT_LENGTH in self._headers:
+            return super().content_length
+
+        if self._compressed_body is not None:
+            # Return length of the compressed body
+            return len(self._compressed_body)
+        elif self._body_payload:
+            # A payload without content length, or a compressed payload
+            return None
+        elif self._body is not None:
+            return len(self._body)
+        else:
+            return 0
+
+    @content_length.setter
+    def content_length(self, value: Optional[int]) -> None:
+        raise RuntimeError("Content length is set automatically")
+
+    async def write_eof(self, data: bytes = b"") -> None:
+        if self._eof_sent:
+            return
+        if self._compressed_body is None:
+            body: Optional[Union[bytes, Payload]] = self._body
+        else:
+            body = self._compressed_body
+        assert not data, f"data arg is not supported, got {data!r}"
+        assert self._req is not None
+        assert self._payload_writer is not None
+        if body is not None:
+            if self._req._method == hdrs.METH_HEAD or self._status in [204, 304]:
+                await super().write_eof()
+            elif self._body_payload:
+                payload = cast(Payload, body)
+                await payload.write(self._payload_writer)
+                await super().write_eof()
+            else:
+                await super().write_eof(cast(bytes, body))
+        else:
+            await super().write_eof()
+
+    async def _start(self, request: "BaseRequest") -> AbstractStreamWriter:
+        if not self._chunked and hdrs.CONTENT_LENGTH not in self._headers:
+            if self._body_payload:
+                size = cast(Payload, self._body).size
+                if size is not None:
+                    self._headers[hdrs.CONTENT_LENGTH] = str(size)
+            else:
+                body_len = len(self._body) if self._body else "0"
+                self._headers[hdrs.CONTENT_LENGTH] = str(body_len)
+
+        return await super()._start(request)
+
+    async def _do_start_compression(self, coding: ContentCoding) -> None:
+        if self._body_payload or self._chunked:
+            return await super()._do_start_compression(coding)
+
+        if coding != ContentCoding.identity:
+            # Instead of using _payload_writer.enable_compression,
+            # compress the whole body
+            compressor = ZLibCompressor(
+                encoding=str(coding.value),
+                max_sync_chunk_size=self._zlib_executor_size,
+                executor=self._zlib_executor,
+            )
+            assert self._body is not None
+            if self._zlib_executor_size is None and len(self._body) > 1024 * 1024:
+                warnings.warn(
+                    "Synchronous compression of large response bodies "
+                    f"({len(self._body)} bytes) might block the async event loop. "
+                    "Consider providing a custom value to zlib_executor_size/"
+                    "zlib_executor response properties or disabling compression on it."
+                )
+            self._compressed_body = (
+                await compressor.compress(self._body) + compressor.flush()
+            )
+            assert self._compressed_body is not None
+
+            self._headers[hdrs.CONTENT_ENCODING] = coding.value
+            self._headers[hdrs.CONTENT_LENGTH] = str(len(self._compressed_body))
+
+
+def json_response(
+    data: Any = sentinel,
+    *,
+    text: Optional[str] = None,
+    body: Optional[bytes] = None,
+    status: int = 200,
+    reason: Optional[str] = None,
+    headers: Optional[LooseHeaders] = None,
+    content_type: str = "application/json",
+    dumps: JSONEncoder = json.dumps,
+) -> Response:
+    if data is not sentinel:
+        if text or body:
+            raise ValueError("only one of data, text, or body should be specified")
+        else:
+            text = dumps(data)
+    return Response(
+        text=text,
+        body=body,
+        status=status,
+        reason=reason,
+        headers=headers,
+        content_type=content_type,
+    )

aiohttp/web_routedef.py

@@ -0,0 +1,215 @@
+import abc
+import dataclasses
+import os  # noqa
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Sequence,
+    Type,
+    Union,
+    overload,
+)
+
+from . import hdrs
+from .abc import AbstractView
+from .typedefs import Handler, PathLike
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .web_request import Request
+    from .web_response import StreamResponse
+    from .web_urldispatcher import AbstractRoute, UrlDispatcher
+else:
+    Request = StreamResponse = UrlDispatcher = AbstractRoute = None
+
+
+__all__ = (
+    "AbstractRouteDef",
+    "RouteDef",
+    "StaticDef",
+    "RouteTableDef",
+    "head",
+    "options",
+    "get",
+    "post",
+    "patch",
+    "put",
+    "delete",
+    "route",
+    "view",
+    "static",
+)
+
+
+class AbstractRouteDef(abc.ABC):
+    @abc.abstractmethod
+    def register(self, router: UrlDispatcher) -> List[AbstractRoute]:
+        pass  # pragma: no cover
+
+
+_HandlerType = Union[Type[AbstractView], Handler]
+
+
+@dataclasses.dataclass(frozen=True, repr=False)
+class RouteDef(AbstractRouteDef):
+    method: str
+    path: str
+    handler: _HandlerType
+    kwargs: Dict[str, Any]
+
+    def __repr__(self) -> str:
+        info = []
+        for name, value in sorted(self.kwargs.items()):
+            info.append(f", {name}={value!r}")
+        return "<RouteDef {method} {path} -> {handler.__name__!r}" "{info}>".format(
+            method=self.method, path=self.path, handler=self.handler, info="".join(info)
+        )
+
+    def register(self, router: UrlDispatcher) -> List[AbstractRoute]:
+        if self.method in hdrs.METH_ALL:
+            reg = getattr(router, "add_" + self.method.lower())
+            return [reg(self.path, self.handler, **self.kwargs)]
+        else:
+            return [
+                router.add_route(self.method, self.path, self.handler, **self.kwargs)
+            ]
+
+
+@dataclasses.dataclass(frozen=True, repr=False)
+class StaticDef(AbstractRouteDef):
+    prefix: str
+    path: PathLike
+    kwargs: Dict[str, Any]
+
+    def __repr__(self) -> str:
+        info = []
+        for name, value in sorted(self.kwargs.items()):
+            info.append(f", {name}={value!r}")
+        return "<StaticDef {prefix} -> {path}" "{info}>".format(
+            prefix=self.prefix, path=self.path, info="".join(info)
+        )
+
+    def register(self, router: UrlDispatcher) -> List[AbstractRoute]:
+        resource = router.add_static(self.prefix, self.path, **self.kwargs)
+        routes = resource.get_info().get("routes", {})
+        return list(routes.values())
+
+
+def route(method: str, path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
+    return RouteDef(method, path, handler, kwargs)
+
+
+def head(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
+    return route(hdrs.METH_HEAD, path, handler, **kwargs)
+
+
+def options(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
+    return route(hdrs.METH_OPTIONS, path, handler, **kwargs)
+
+
+def get(
+    path: str,
+    handler: _HandlerType,
+    *,
+    name: Optional[str] = None,
+    allow_head: bool = True,
+    **kwargs: Any,
+) -> RouteDef:
+    return route(
+        hdrs.METH_GET, path, handler, name=name, allow_head=allow_head, **kwargs
+    )
+
+
+def post(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
+    return route(hdrs.METH_POST, path, handler, **kwargs)
+
+
+def put(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
+    return route(hdrs.METH_PUT, path, handler, **kwargs)
+
+
+def patch(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
+    return route(hdrs.METH_PATCH, path, handler, **kwargs)
+
+
+def delete(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
+    return route(hdrs.METH_DELETE, path, handler, **kwargs)
+
+
+def view(path: str, handler: Type[AbstractView], **kwargs: Any) -> RouteDef:
+    return route(hdrs.METH_ANY, path, handler, **kwargs)
+
+
+def static(prefix: str, path: PathLike, **kwargs: Any) -> StaticDef:
+    return StaticDef(prefix, path, kwargs)
+
+
+_Deco = Callable[[_HandlerType], _HandlerType]
+
+
+class RouteTableDef(Sequence[AbstractRouteDef]):
+    """Route definition table"""
+
+    def __init__(self) -> None:
+        self._items: List[AbstractRouteDef] = []
+
+    def __repr__(self) -> str:
+        return f"<RouteTableDef count={len(self._items)}>"
+
+    @overload
+    def __getitem__(self, index: int) -> AbstractRouteDef:
+        ...
+
+    @overload
+    def __getitem__(self, index: slice) -> List[AbstractRouteDef]:
+        ...
+
+    def __getitem__(self, index):  # type: ignore[no-untyped-def]
+        return self._items[index]
+
+    def __iter__(self) -> Iterator[AbstractRouteDef]:
+        return iter(self._items)
+
+    def __len__(self) -> int:
+        return len(self._items)
+
+    def __contains__(self, item: object) -> bool:
+        return item in self._items
+
+    def route(self, method: str, path: str, **kwargs: Any) -> _Deco:
+        def inner(handler: _HandlerType) -> _HandlerType:
+            self._items.append(RouteDef(method, path, handler, kwargs))
+            return handler
+
+        return inner
+
+    def head(self, path: str, **kwargs: Any) -> _Deco:
+        return self.route(hdrs.METH_HEAD, path, **kwargs)
+
+    def get(self, path: str, **kwargs: Any) -> _Deco:
+        return self.route(hdrs.METH_GET, path, **kwargs)
+
+    def post(self, path: str, **kwargs: Any) -> _Deco:
+        return self.route(hdrs.METH_POST, path, **kwargs)
+
+    def put(self, path: str, **kwargs: Any) -> _Deco:
+        return self.route(hdrs.METH_PUT, path, **kwargs)
+
+    def patch(self, path: str, **kwargs: Any) -> _Deco:
+        return self.route(hdrs.METH_PATCH, path, **kwargs)
+
+    def delete(self, path: str, **kwargs: Any) -> _Deco:
+        return self.route(hdrs.METH_DELETE, path, **kwargs)
+
+    def options(self, path: str, **kwargs: Any) -> _Deco:
+        return self.route(hdrs.METH_OPTIONS, path, **kwargs)
+
+    def view(self, path: str, **kwargs: Any) -> _Deco:
+        return self.route(hdrs.METH_ANY, path, **kwargs)
+
+    def static(self, prefix: str, path: PathLike, **kwargs: Any) -> None:
+        self._items.append(StaticDef(prefix, path, kwargs))

aiohttp/web_runner.py

@@ -0,0 +1,450 @@
+import asyncio
+import signal
+import socket
+from abc import ABC, abstractmethod
+from contextlib import suppress
+from typing import Any, List, Optional, Set, Type
+
+from yarl import URL
+
+from .abc import AbstractAccessLogger, AbstractStreamWriter
+from .http_parser import RawRequestMessage
+from .streams import StreamReader
+from .typedefs import PathLike
+from .web_app import Application
+from .web_log import AccessLogger
+from .web_protocol import RequestHandler
+from .web_request import Request
+from .web_server import Server
+
+try:
+    from ssl import SSLContext
+except ImportError:
+    SSLContext = object  # type: ignore[misc,assignment]
+
+
+__all__ = (
+    "BaseSite",
+    "TCPSite",
+    "UnixSite",
+    "NamedPipeSite",
+    "SockSite",
+    "BaseRunner",
+    "AppRunner",
+    "ServerRunner",
+    "GracefulExit",
+)
+
+
+class GracefulExit(SystemExit):
+    code = 1
+
+
+def _raise_graceful_exit() -> None:
+    raise GracefulExit()
+
+
+class BaseSite(ABC):
+    __slots__ = ("_runner", "_shutdown_timeout", "_ssl_context", "_backlog", "_server")
+
+    def __init__(
+        self,
+        runner: "BaseRunner",
+        *,
+        shutdown_timeout: float = 60.0,
+        ssl_context: Optional[SSLContext] = None,
+        backlog: int = 128,
+    ) -> None:
+        if runner.server is None:
+            raise RuntimeError("Call runner.setup() before making a site")
+        self._runner = runner
+        self._shutdown_timeout = shutdown_timeout
+        self._ssl_context = ssl_context
+        self._backlog = backlog
+        self._server: Optional[asyncio.AbstractServer] = None
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        pass  # pragma: no cover
+
+    @abstractmethod
+    async def start(self) -> None:
+        self._runner._reg_site(self)
+
+    async def stop(self) -> None:
+        self._runner._check_site(self)
+        if self._server is None:
+            self._runner._unreg_site(self)
+            return  # not started yet
+        self._server.close()
+        # named pipes do not have wait_closed property
+        if hasattr(self._server, "wait_closed"):
+            await self._server.wait_closed()
+
+        # Wait for pending tasks for a given time limit.
+        with suppress(asyncio.TimeoutError):
+            await asyncio.wait_for(
+                self._wait(asyncio.current_task()), timeout=self._shutdown_timeout
+            )
+
+        await self._runner.shutdown()
+        assert self._runner.server
+        await self._runner.server.shutdown(self._shutdown_timeout)
+        self._runner._unreg_site(self)
+
+    async def _wait(self, parent_task: Optional["asyncio.Task[object]"]) -> None:
+        exclude = self._runner.starting_tasks | {asyncio.current_task(), parent_task}
+        while tasks := asyncio.all_tasks() - exclude:
+            await asyncio.wait(tasks)
+
+
+class TCPSite(BaseSite):
+    __slots__ = ("_host", "_port", "_reuse_address", "_reuse_port")
+
+    def __init__(
+        self,
+        runner: "BaseRunner",
+        host: Optional[str] = None,
+        port: Optional[int] = None,
+        *,
+        shutdown_timeout: float = 60.0,
+        ssl_context: Optional[SSLContext] = None,
+        backlog: int = 128,
+        reuse_address: Optional[bool] = None,
+        reuse_port: Optional[bool] = None,
+    ) -> None:
+        super().__init__(
+            runner,
+            shutdown_timeout=shutdown_timeout,
+            ssl_context=ssl_context,
+            backlog=backlog,
+        )
+        self._host = host
+        if port is None:
+            port = 8443 if self._ssl_context else 8080
+        self._port = port
+        self._reuse_address = reuse_address
+        self._reuse_port = reuse_port
+
+    @property
+    def name(self) -> str:
+        scheme = "https" if self._ssl_context else "http"
+        host = "0.0.0.0" if self._host is None else self._host
+        return str(URL.build(scheme=scheme, host=host, port=self._port))
+
+    async def start(self) -> None:
+        await super().start()
+        loop = asyncio.get_event_loop()
+        server = self._runner.server
+        assert server is not None
+        self._server = await loop.create_server(
+            server,
+            self._host,
+            self._port,
+            ssl=self._ssl_context,
+            backlog=self._backlog,
+            reuse_address=self._reuse_address,
+            reuse_port=self._reuse_port,
+        )
+
+
+class UnixSite(BaseSite):
+    __slots__ = ("_path",)
+
+    def __init__(
+        self,
+        runner: "BaseRunner",
+        path: PathLike,
+        *,
+        shutdown_timeout: float = 60.0,
+        ssl_context: Optional[SSLContext] = None,
+        backlog: int = 128,
+    ) -> None:
+        super().__init__(
+            runner,
+            shutdown_timeout=shutdown_timeout,
+            ssl_context=ssl_context,
+            backlog=backlog,
+        )
+        self._path = path
+
+    @property
+    def name(self) -> str:
+        scheme = "https" if self._ssl_context else "http"
+        return f"{scheme}://unix:{self._path}:"
+
+    async def start(self) -> None:
+        await super().start()
+        loop = asyncio.get_event_loop()
+        server = self._runner.server
+        assert server is not None
+        self._server = await loop.create_unix_server(
+            server,
+            self._path,
+            ssl=self._ssl_context,
+            backlog=self._backlog,
+        )
+
+
+class NamedPipeSite(BaseSite):
+    __slots__ = ("_path",)
+
+    def __init__(
+        self, runner: "BaseRunner", path: str, *, shutdown_timeout: float = 60.0
+    ) -> None:
+        loop = asyncio.get_event_loop()
+        if not isinstance(
+            loop, asyncio.ProactorEventLoop  # type: ignore[attr-defined]
+        ):
+            raise RuntimeError(
+                "Named Pipes only available in proactor" "loop under windows"
+            )
+        super().__init__(runner, shutdown_timeout=shutdown_timeout)
+        self._path = path
+
+    @property
+    def name(self) -> str:
+        return self._path
+
+    async def start(self) -> None:
+        await super().start()
+        loop = asyncio.get_event_loop()
+        server = self._runner.server
+        assert server is not None
+        _server = await loop.start_serving_pipe(  # type: ignore[attr-defined]
+            server, self._path
+        )
+        self._server = _server[0]
+
+
+class SockSite(BaseSite):
+    __slots__ = ("_sock", "_name")
+
+    def __init__(
+        self,
+        runner: "BaseRunner",
+        sock: socket.socket,
+        *,
+        shutdown_timeout: float = 60.0,
+        ssl_context: Optional[SSLContext] = None,
+        backlog: int = 128,
+    ) -> None:
+        super().__init__(
+            runner,
+            shutdown_timeout=shutdown_timeout,
+            ssl_context=ssl_context,
+            backlog=backlog,
+        )
+        self._sock = sock
+        scheme = "https" if self._ssl_context else "http"
+        if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX:
+            name = f"{scheme}://unix:{sock.getsockname()}:"
+        else:
+            host, port = sock.getsockname()[:2]
+            name = str(URL.build(scheme=scheme, host=host, port=port))
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    async def start(self) -> None:
+        await super().start()
+        loop = asyncio.get_event_loop()
+        server = self._runner.server
+        assert server is not None
+        self._server = await loop.create_server(
+            server, sock=self._sock, ssl=self._ssl_context, backlog=self._backlog
+        )
+
+
+class BaseRunner(ABC):
+    __slots__ = ("starting_tasks", "_handle_signals", "_kwargs", "_server", "_sites")
+
+    def __init__(self, *, handle_signals: bool = False, **kwargs: Any) -> None:
+        self._handle_signals = handle_signals
+        self._kwargs = kwargs
+        self._server: Optional[Server] = None
+        self._sites: List[BaseSite] = []
+
+    @property
+    def server(self) -> Optional[Server]:
+        return self._server
+
+    @property
+    def addresses(self) -> List[Any]:
+        ret: List[Any] = []
+        for site in self._sites:
+            server = site._server
+            if server is not None:
+                sockets = server.sockets  # type: ignore[attr-defined]
+                if sockets is not None:
+                    for sock in sockets:
+                        ret.append(sock.getsockname())
+        return ret
+
+    @property
+    def sites(self) -> Set[BaseSite]:
+        return set(self._sites)
+
+    async def setup(self) -> None:
+        loop = asyncio.get_event_loop()
+
+        if self._handle_signals:
+            try:
+                loop.add_signal_handler(signal.SIGINT, _raise_graceful_exit)
+                loop.add_signal_handler(signal.SIGTERM, _raise_graceful_exit)
+            except NotImplementedError:  # pragma: no cover
+                # add_signal_handler is not implemented on Windows
+                pass
+
+        self._server = await self._make_server()
+        # On shutdown we want to avoid waiting on tasks which run forever.
+        # It's very likely that all tasks which run forever will have been created by
+        # the time we have completed the application startup (in self._make_server()),
+        # so we just record all running tasks here and exclude them later.
+        self.starting_tasks = asyncio.all_tasks()
+
+    @abstractmethod
+    async def shutdown(self) -> None:
+        pass  # pragma: no cover
+
+    async def cleanup(self) -> None:
+        loop = asyncio.get_event_loop()
+
+        # The loop over sites is intentional, an exception on gather()
+        # leaves self._sites in unpredictable state.
+        # The loop guarantees that a site is either deleted on success or
+        # still present on failure
+        for site in list(self._sites):
+            await site.stop()
+        await self._cleanup_server()
+        self._server = None
+        if self._handle_signals:
+            try:
+                loop.remove_signal_handler(signal.SIGINT)
+                loop.remove_signal_handler(signal.SIGTERM)
+            except NotImplementedError:  # pragma: no cover
+                # remove_signal_handler is not implemented on Windows
+                pass
+
+    @abstractmethod
+    async def _make_server(self) -> Server:
+        pass  # pragma: no cover
+
+    @abstractmethod
+    async def _cleanup_server(self) -> None:
+        pass  # pragma: no cover
+
+    def _reg_site(self, site: BaseSite) -> None:
+        if site in self._sites:
+            raise RuntimeError(f"Site {site} is already registered in runner {self}")
+        self._sites.append(site)
+
+    def _check_site(self, site: BaseSite) -> None:
+        if site not in self._sites:
+            raise RuntimeError(f"Site {site} is not registered in runner {self}")
+
+    def _unreg_site(self, site: BaseSite) -> None:
+        if site not in self._sites:
+            raise RuntimeError(f"Site {site} is not registered in runner {self}")
+        self._sites.remove(site)
+
+
+class ServerRunner(BaseRunner):
+    """Low-level web server runner"""
+
+    __slots__ = ("_web_server",)
+
+    def __init__(
+        self, web_server: Server, *, handle_signals: bool = False, **kwargs: Any
+    ) -> None:
+        super().__init__(handle_signals=handle_signals, **kwargs)
+        self._web_server = web_server
+
+    async def shutdown(self) -> None:
+        pass
+
+    async def _make_server(self) -> Server:
+        return self._web_server
+
+    async def _cleanup_server(self) -> None:
+        pass
+
+
+class AppRunner(BaseRunner):
+    """Web Application runner"""
+
+    __slots__ = ("_app",)
+
+    def __init__(
+        self,
+        app: Application,
+        *,
+        handle_signals: bool = False,
+        access_log_class: Type[AbstractAccessLogger] = AccessLogger,
+        **kwargs: Any,
+    ) -> None:
+        if not isinstance(app, Application):
+            raise TypeError(
+                "The first argument should be web.Application "
+                "instance, got {!r}".format(app)
+            )
+        kwargs["access_log_class"] = access_log_class
+
+        if app._handler_args:
+            for k, v in app._handler_args.items():
+                kwargs[k] = v
+
+        if not issubclass(kwargs["access_log_class"], AbstractAccessLogger):
+            raise TypeError(
+                "access_log_class must be subclass of "
+                "aiohttp.abc.AbstractAccessLogger, got {}".format(
+                    kwargs["access_log_class"]
+                )
+            )
+
+        super().__init__(handle_signals=handle_signals, **kwargs)
+        self._app = app
+
+    @property
+    def app(self) -> Application:
+        return self._app
+
+    async def shutdown(self) -> None:
+        await self._app.shutdown()
+
+    async def _make_server(self) -> Server:
+        self._app.on_startup.freeze()
+        await self._app.startup()
+        self._app.freeze()
+
+        return Server(
+            self._app._handle,  # type: ignore[arg-type]
+            request_factory=self._make_request,
+            **self._kwargs,
+        )
+
+    def _make_request(
+        self,
+        message: RawRequestMessage,
+        payload: StreamReader,
+        protocol: RequestHandler,
+        writer: AbstractStreamWriter,
+        task: "asyncio.Task[None]",
+        _cls: Type[Request] = Request,
+    ) -> Request:
+        loop = asyncio.get_running_loop()
+        return _cls(
+            message,
+            payload,
+            protocol,
+            writer,
+            task,
+            loop,
+            client_max_size=self.app._client_max_size,
+        )
+
+    async def _cleanup_server(self) -> None:
+        await self._app.cleanup()

aiohttp/web_server.py

@@ -0,0 +1,79 @@
+"""Low level HTTP server."""
+import asyncio
+import warnings
+from typing import Any, Awaitable, Callable, Dict, List, Optional  # noqa
+
+from .abc import AbstractStreamWriter
+from .http_parser import RawRequestMessage
+from .streams import StreamReader
+from .web_protocol import RequestHandler, _RequestFactory, _RequestHandler
+from .web_request import BaseRequest
+
+__all__ = ("Server",)
+
+
+class Server:
+    def __init__(
+        self,
+        handler: _RequestHandler,
+        *,
+        request_factory: Optional[_RequestFactory] = None,
+        debug: Optional[bool] = None,
+        handler_cancellation: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        if debug is not None:
+            warnings.warn(
+                "debug argument is no-op since 4.0 " "and scheduled for removal in 5.0",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        self._loop = asyncio.get_running_loop()
+        self._connections: Dict[RequestHandler, asyncio.Transport] = {}
+        self._kwargs = kwargs
+        self.requests_count = 0
+        self.request_handler = handler
+        self.request_factory = request_factory or self._make_request
+        self.handler_cancellation = handler_cancellation
+
+    @property
+    def connections(self) -> List[RequestHandler]:
+        return list(self._connections.keys())
+
+    def connection_made(
+        self, handler: RequestHandler, transport: asyncio.Transport
+    ) -> None:
+        self._connections[handler] = transport
+
+    def connection_lost(
+        self, handler: RequestHandler, exc: Optional[BaseException] = None
+    ) -> None:
+        if handler in self._connections:
+            del self._connections[handler]
+
+    def _make_request(
+        self,
+        message: RawRequestMessage,
+        payload: StreamReader,
+        protocol: RequestHandler,
+        writer: AbstractStreamWriter,
+        task: "asyncio.Task[None]",
+    ) -> BaseRequest:
+        return BaseRequest(message, payload, protocol, writer, task, self._loop)
+
+    async def shutdown(self, timeout: Optional[float] = None) -> None:
+        coros = [conn.shutdown(timeout) for conn in self._connections]
+        await asyncio.gather(*coros)
+        self._connections.clear()
+
+    def __call__(self) -> RequestHandler:
+        try:
+            return RequestHandler(self, loop=self._loop, **self._kwargs)
+        except TypeError:
+            # Failsafe creation: remove all custom handler_args
+            kwargs = {
+                k: v
+                for k, v in self._kwargs.items()
+                if k in ["debug", "access_log_class"]
+            }
+            return RequestHandler(self, loop=self._loop, **kwargs)

aiohttp/web_ws.py

@@ -0,0 +1,525 @@
+import asyncio
+import base64
+import binascii
+import dataclasses
+import hashlib
+import json
+import sys
+from typing import Any, Final, Iterable, Optional, Tuple, cast
+
+from multidict import CIMultiDict
+
+from . import hdrs
+from .abc import AbstractStreamWriter
+from .helpers import call_later, set_result
+from .http import (
+    WS_CLOSED_MESSAGE,
+    WS_CLOSING_MESSAGE,
+    WS_KEY,
+    WebSocketError,
+    WebSocketReader,
+    WebSocketWriter,
+    WSCloseCode,
+    WSMessage,
+    WSMsgType as WSMsgType,
+    ws_ext_gen,
+    ws_ext_parse,
+)
+from .log import ws_logger
+from .streams import EofStream, FlowControlDataQueue
+from .typedefs import JSONDecoder, JSONEncoder
+from .web_exceptions import HTTPBadRequest, HTTPException
+from .web_request import BaseRequest
+from .web_response import StreamResponse
+
+if sys.version_info >= (3, 11):
+    import asyncio as async_timeout
+else:
+    import async_timeout
+
+__all__ = (
+    "WebSocketResponse",
+    "WebSocketReady",
+    "WSMsgType",
+)
+
+THRESHOLD_CONNLOST_ACCESS: Final[int] = 5
+
+
+@dataclasses.dataclass(frozen=True)
+class WebSocketReady:
+    ok: bool
+    protocol: Optional[str]
+
+    def __bool__(self) -> bool:
+        return self.ok
+
+
+class WebSocketResponse(StreamResponse):
+    __slots__ = (
+        "_protocols",
+        "_ws_protocol",
+        "_writer",
+        "_reader",
+        "_closed",
+        "_closing",
+        "_conn_lost",
+        "_close_code",
+        "_loop",
+        "_waiting",
+        "_exception",
+        "_timeout",
+        "_receive_timeout",
+        "_autoclose",
+        "_autoping",
+        "_heartbeat",
+        "_heartbeat_cb",
+        "_pong_heartbeat",
+        "_pong_response_cb",
+        "_compress",
+        "_max_msg_size",
+    )
+
+    def __init__(
+        self,
+        *,
+        timeout: float = 10.0,
+        receive_timeout: Optional[float] = None,
+        autoclose: bool = True,
+        autoping: bool = True,
+        heartbeat: Optional[float] = None,
+        protocols: Iterable[str] = (),
+        compress: bool = True,
+        max_msg_size: int = 4 * 1024 * 1024,
+    ) -> None:
+        super().__init__(status=101)
+        self._length_check = False
+        self._protocols = protocols
+        self._ws_protocol: Optional[str] = None
+        self._writer: Optional[WebSocketWriter] = None
+        self._reader: Optional[FlowControlDataQueue[WSMessage]] = None
+        self._closed = False
+        self._closing = False
+        self._conn_lost = 0
+        self._close_code: Optional[int] = None
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._waiting: Optional[asyncio.Future[bool]] = None
+        self._exception: Optional[BaseException] = None
+        self._timeout = timeout
+        self._receive_timeout = receive_timeout
+        self._autoclose = autoclose
+        self._autoping = autoping
+        self._heartbeat = heartbeat
+        self._heartbeat_cb: Optional[asyncio.TimerHandle] = None
+        if heartbeat is not None:
+            self._pong_heartbeat = heartbeat / 2.0
+        self._pong_response_cb: Optional[asyncio.TimerHandle] = None
+        self._compress = compress
+        self._max_msg_size = max_msg_size
+
+    def _cancel_heartbeat(self) -> None:
+        if self._pong_response_cb is not None:
+            self._pong_response_cb.cancel()
+            self._pong_response_cb = None
+
+        if self._heartbeat_cb is not None:
+            self._heartbeat_cb.cancel()
+            self._heartbeat_cb = None
+
+    def _reset_heartbeat(self) -> None:
+        self._cancel_heartbeat()
+
+        if self._heartbeat is not None:
+            assert self._loop is not None
+            self._heartbeat_cb = call_later(
+                self._send_heartbeat,
+                self._heartbeat,
+                self._loop,
+                timeout_ceil_threshold=self._req._protocol._timeout_ceil_threshold
+                if self._req is not None
+                else 5,
+            )
+
+    def _send_heartbeat(self) -> None:
+        if self._heartbeat is not None and not self._closed:
+            assert self._loop is not None and self._writer is not None
+            # fire-and-forget a task is not perfect but maybe ok for
+            # sending ping. Otherwise we need a long-living heartbeat
+            # task in the class.
+            self._loop.create_task(self._writer.ping())  # type: ignore[unused-awaitable]
+
+            if self._pong_response_cb is not None:
+                self._pong_response_cb.cancel()
+            self._pong_response_cb = call_later(
+                self._pong_not_received,
+                self._pong_heartbeat,
+                self._loop,
+                timeout_ceil_threshold=self._req._protocol._timeout_ceil_threshold
+                if self._req is not None
+                else 5,
+            )
+
+    def _pong_not_received(self) -> None:
+        if self._req is not None and self._req.transport is not None:
+            self._closed = True
+            self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+            self._exception = asyncio.TimeoutError()
+            self._req.transport.close()
+
+    async def prepare(self, request: BaseRequest) -> AbstractStreamWriter:
+        # make pre-check to don't hide it by do_handshake() exceptions
+        if self._payload_writer is not None:
+            return self._payload_writer
+
+        protocol, writer = self._pre_start(request)
+        payload_writer = await super().prepare(request)
+        assert payload_writer is not None
+        self._post_start(request, protocol, writer)
+        await payload_writer.drain()
+        return payload_writer
+
+    def _handshake(
+        self, request: BaseRequest
+    ) -> Tuple["CIMultiDict[str]", str, bool, bool]:
+        headers = request.headers
+        if "websocket" != headers.get(hdrs.UPGRADE, "").lower().strip():
+            raise HTTPBadRequest(
+                text=(
+                    "No WebSocket UPGRADE hdr: {}\n Can "
+                    '"Upgrade" only to "WebSocket".'
+                ).format(headers.get(hdrs.UPGRADE))
+            )
+
+        if "upgrade" not in headers.get(hdrs.CONNECTION, "").lower():
+            raise HTTPBadRequest(
+                text="No CONNECTION upgrade hdr: {}".format(
+                    headers.get(hdrs.CONNECTION)
+                )
+            )
+
+        # find common sub-protocol between client and server
+        protocol = None
+        if hdrs.SEC_WEBSOCKET_PROTOCOL in headers:
+            req_protocols = [
+                str(proto.strip())
+                for proto in headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(",")
+            ]
+
+            for proto in req_protocols:
+                if proto in self._protocols:
+                    protocol = proto
+                    break
+            else:
+                # No overlap found: Return no protocol as per spec
+                ws_logger.warning(
+                    "Client protocols %r don’t overlap server-known ones %r",
+                    req_protocols,
+                    self._protocols,
+                )
+
+        # check supported version
+        version = headers.get(hdrs.SEC_WEBSOCKET_VERSION, "")
+        if version not in ("13", "8", "7"):
+            raise HTTPBadRequest(text=f"Unsupported version: {version}")
+
+        # check client handshake for validity
+        key = headers.get(hdrs.SEC_WEBSOCKET_KEY)
+        try:
+            if not key or len(base64.b64decode(key)) != 16:
+                raise HTTPBadRequest(text=f"Handshake error: {key!r}")
+        except binascii.Error:
+            raise HTTPBadRequest(text=f"Handshake error: {key!r}") from None
+
+        accept_val = base64.b64encode(
+            hashlib.sha1(key.encode() + WS_KEY).digest()
+        ).decode()
+        response_headers = CIMultiDict(
+            {
+                hdrs.UPGRADE: "websocket",
+                hdrs.CONNECTION: "upgrade",
+                hdrs.SEC_WEBSOCKET_ACCEPT: accept_val,
+            }
+        )
+
+        notakeover = False
+        compress = 0
+        if self._compress:
+            extensions = headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS)
+            # Server side always get return with no exception.
+            # If something happened, just drop compress extension
+            compress, notakeover = ws_ext_parse(extensions, isserver=True)
+            if compress:
+                enabledext = ws_ext_gen(
+                    compress=compress, isserver=True, server_notakeover=notakeover
+                )
+                response_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = enabledext
+
+        if protocol:
+            response_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = protocol
+        return (
+            response_headers,
+            protocol,
+            compress,
+            notakeover,
+        )  # type: ignore[return-value]
+
+    def _pre_start(self, request: BaseRequest) -> Tuple[str, WebSocketWriter]:
+        self._loop = request._loop
+
+        headers, protocol, compress, notakeover = self._handshake(request)
+
+        self.set_status(101)
+        self.headers.update(headers)
+        self.force_close()
+        self._compress = compress
+        transport = request._protocol.transport
+        assert transport is not None
+        writer = WebSocketWriter(
+            request._protocol, transport, compress=compress, notakeover=notakeover
+        )
+
+        return protocol, writer
+
+    def _post_start(
+        self, request: BaseRequest, protocol: str, writer: WebSocketWriter
+    ) -> None:
+        self._ws_protocol = protocol
+        self._writer = writer
+
+        self._reset_heartbeat()
+
+        loop = self._loop
+        assert loop is not None
+        self._reader = FlowControlDataQueue(request._protocol, 2**16, loop=loop)
+        request.protocol.set_parser(
+            WebSocketReader(self._reader, self._max_msg_size, compress=self._compress)
+        )
+        # disable HTTP keepalive for WebSocket
+        request.protocol.keep_alive(False)
+
+    def can_prepare(self, request: BaseRequest) -> WebSocketReady:
+        if self._writer is not None:
+            raise RuntimeError("Already started")
+        try:
+            _, protocol, _, _ = self._handshake(request)
+        except HTTPException:
+            return WebSocketReady(False, None)
+        else:
+            return WebSocketReady(True, protocol)
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    @property
+    def close_code(self) -> Optional[int]:
+        return self._close_code
+
+    @property
+    def ws_protocol(self) -> Optional[str]:
+        return self._ws_protocol
+
+    @property
+    def compress(self) -> bool:
+        return self._compress
+
+    def exception(self) -> Optional[BaseException]:
+        return self._exception
+
+    async def ping(self, message: bytes = b"") -> None:
+        if self._writer is None:
+            raise RuntimeError("Call .prepare() first")
+        await self._writer.ping(message)
+
+    async def pong(self, message: bytes = b"") -> None:
+        # unsolicited pong
+        if self._writer is None:
+            raise RuntimeError("Call .prepare() first")
+        await self._writer.pong(message)
+
+    async def send_str(self, data: str, compress: Optional[bool] = None) -> None:
+        if self._writer is None:
+            raise RuntimeError("Call .prepare() first")
+        if not isinstance(data, str):
+            raise TypeError("data argument must be str (%r)" % type(data))
+        await self._writer.send(data, binary=False, compress=compress)
+
+    async def send_bytes(self, data: bytes, compress: Optional[bool] = None) -> None:
+        if self._writer is None:
+            raise RuntimeError("Call .prepare() first")
+        if not isinstance(data, (bytes, bytearray, memoryview)):
+            raise TypeError("data argument must be byte-ish (%r)" % type(data))
+        await self._writer.send(data, binary=True, compress=compress)
+
+    async def send_json(
+        self,
+        data: Any,
+        compress: Optional[bool] = None,
+        *,
+        dumps: JSONEncoder = json.dumps,
+    ) -> None:
+        await self.send_str(dumps(data), compress=compress)
+
+    async def write_eof(self) -> None:  # type: ignore[override]
+        if self._eof_sent:
+            return
+        if self._payload_writer is None:
+            raise RuntimeError("Response has not been started")
+
+        await self.close()
+        self._eof_sent = True
+
+    async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bool:
+        if self._writer is None:
+            raise RuntimeError("Call .prepare() first")
+
+        self._cancel_heartbeat()
+        reader = self._reader
+        assert reader is not None
+
+        # we need to break `receive()` cycle first,
+        # `close()` may be called from different task
+        if self._waiting is not None and not self._closed:
+            reader.feed_data(WS_CLOSING_MESSAGE, 0)
+            await self._waiting
+
+        if not self._closed:
+            self._closed = True
+            try:
+                await self._writer.close(code, message)
+                writer = self._payload_writer
+                assert writer is not None
+                await writer.drain()
+            except (asyncio.CancelledError, asyncio.TimeoutError):
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                raise
+            except Exception as exc:
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                self._exception = exc
+                return True
+
+            if self._closing:
+                return True
+
+            reader = self._reader
+            assert reader is not None
+            try:
+                async with async_timeout.timeout(self._timeout):
+                    msg = await reader.read()
+            except asyncio.CancelledError:
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                raise
+            except Exception as exc:
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                self._exception = exc
+                return True
+
+            if msg.type == WSMsgType.CLOSE:
+                self._close_code = msg.data
+                return True
+
+            self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+            self._exception = asyncio.TimeoutError()
+            return True
+        else:
+            return False
+
+    async def receive(self, timeout: Optional[float] = None) -> WSMessage:
+        if self._reader is None:
+            raise RuntimeError("Call .prepare() first")
+
+        loop = self._loop
+        assert loop is not None
+        while True:
+            if self._waiting is not None:
+                raise RuntimeError("Concurrent call to receive() is not allowed")
+
+            if self._closed:
+                self._conn_lost += 1
+                if self._conn_lost >= THRESHOLD_CONNLOST_ACCESS:
+                    raise RuntimeError("WebSocket connection is closed.")
+                return WS_CLOSED_MESSAGE
+            elif self._closing:
+                return WS_CLOSING_MESSAGE
+
+            try:
+                self._waiting = loop.create_future()
+                try:
+                    async with async_timeout.timeout(timeout or self._receive_timeout):
+                        msg = await self._reader.read()
+                    self._reset_heartbeat()
+                finally:
+                    waiter = self._waiting
+                    set_result(waiter, True)
+                    self._waiting = None
+            except (asyncio.CancelledError, asyncio.TimeoutError):
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                raise
+            except EofStream:
+                self._close_code = WSCloseCode.OK
+                await self.close()
+                return WSMessage(WSMsgType.CLOSED, None, None)
+            except WebSocketError as exc:
+                self._close_code = exc.code
+                await self.close(code=exc.code)
+                return WSMessage(WSMsgType.ERROR, exc, None)
+            except Exception as exc:
+                self._exception = exc
+                self._closing = True
+                self._close_code = WSCloseCode.ABNORMAL_CLOSURE
+                await self.close()
+                return WSMessage(WSMsgType.ERROR, exc, None)
+
+            if msg.type == WSMsgType.CLOSE:
+                self._closing = True
+                self._close_code = msg.data
+                # Could be closed while awaiting reader.
+                if not self._closed and self._autoclose:  # type: ignore[redundant-expr]
+                    await self.close()
+            elif msg.type == WSMsgType.CLOSING:
+                self._closing = True
+            elif msg.type == WSMsgType.PING and self._autoping:
+                await self.pong(msg.data)
+                continue
+            elif msg.type == WSMsgType.PONG and self._autoping:
+                continue
+
+            return msg
+
+    async def receive_str(self, *, timeout: Optional[float] = None) -> str:
+        msg = await self.receive(timeout)
+        if msg.type != WSMsgType.TEXT:
+            raise TypeError(
+                "Received message {}:{!r} is not WSMsgType.TEXT".format(
+                    msg.type, msg.data
+                )
+            )
+        return cast(str, msg.data)
+
+    async def receive_bytes(self, *, timeout: Optional[float] = None) -> bytes:
+        msg = await self.receive(timeout)
+        if msg.type != WSMsgType.BINARY:
+            raise TypeError(f"Received message {msg.type}:{msg.data!r} is not bytes")
+        return cast(bytes, msg.data)
+
+    async def receive_json(
+        self, *, loads: JSONDecoder = json.loads, timeout: Optional[float] = None
+    ) -> Any:
+        data = await self.receive_str(timeout=timeout)
+        return loads(data)
+
+    async def write(self, data: bytes) -> None:
+        raise RuntimeError("Cannot call .write() for websocket")
+
+    def __aiter__(self) -> "WebSocketResponse":
+        return self
+
+    async def __anext__(self) -> WSMessage:
+        msg = await self.receive()
+        if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED):
+            raise StopAsyncIteration
+        return msg
+
+    def _cancel(self, exc: BaseException) -> None:
+        if self._reader is not None:
+            self._reader.set_exception(exc)

aiohttp/worker.py

@@ -0,0 +1,239 @@
+"""Async gunicorn worker for aiohttp.web"""
+
+import asyncio
+import os
+import re
+import signal
+import sys
+from types import FrameType
+from typing import Any, Awaitable, Callable, Optional, Union  # noqa
+
+from gunicorn.config import AccessLogFormat as GunicornAccessLogFormat
+from gunicorn.workers import base
+
+from aiohttp import web
+
+from .helpers import set_result
+from .web_app import Application
+from .web_log import AccessLogger
+
+try:
+    import ssl
+
+    SSLContext = ssl.SSLContext
+except ImportError:  # pragma: no cover
+    ssl = None  # type: ignore[assignment]
+    SSLContext = object  # type: ignore[misc,assignment]
+
+
+__all__ = ("GunicornWebWorker", "GunicornUVLoopWebWorker")
+
+
+class GunicornWebWorker(base.Worker):  # type: ignore[misc,no-any-unimported]
+    DEFAULT_AIOHTTP_LOG_FORMAT = AccessLogger.LOG_FORMAT
+    DEFAULT_GUNICORN_LOG_FORMAT = GunicornAccessLogFormat.default
+
+    def __init__(self, *args: Any, **kw: Any) -> None:  # pragma: no cover
+        super().__init__(*args, **kw)
+
+        self._task: Optional[asyncio.Task[None]] = None
+        self.exit_code = 0
+        self._notify_waiter: Optional[asyncio.Future[bool]] = None
+
+    def init_process(self) -> None:
+        # create new event_loop after fork
+        self.loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self.loop)
+
+        super().init_process()
+
+    def run(self) -> None:
+        self._task = self.loop.create_task(self._run())
+
+        try:  # ignore all finalization problems
+            self.loop.run_until_complete(self._task)
+        except Exception:
+            self.log.exception("Exception in gunicorn worker")
+        self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+        self.loop.close()
+
+        sys.exit(self.exit_code)
+
+    async def _run(self) -> None:
+        runner = None
+        if isinstance(self.wsgi, Application):
+            app = self.wsgi
+        elif asyncio.iscoroutinefunction(self.wsgi):
+            wsgi = await self.wsgi()
+            if isinstance(wsgi, web.AppRunner):
+                runner = wsgi
+                app = runner.app
+            else:
+                app = wsgi
+        else:
+            raise RuntimeError(
+                "wsgi app should be either Application or "
+                "async function returning Application, got {}".format(self.wsgi)
+            )
+
+        if runner is None:
+            access_log = self.log.access_log if self.cfg.accesslog else None
+            runner = web.AppRunner(
+                app,
+                logger=self.log,
+                keepalive_timeout=self.cfg.keepalive,
+                access_log=access_log,
+                access_log_format=self._get_valid_log_format(
+                    self.cfg.access_log_format
+                ),
+            )
+        await runner.setup()
+
+        ctx = self._create_ssl_context(self.cfg) if self.cfg.is_ssl else None
+
+        assert runner is not None
+        server = runner.server
+        assert server is not None
+        for sock in self.sockets:
+            site = web.SockSite(
+                runner,
+                sock,
+                ssl_context=ctx,
+                shutdown_timeout=self.cfg.graceful_timeout / 100 * 95,
+            )
+            await site.start()
+
+        # If our parent changed then we shut down.
+        pid = os.getpid()
+        try:
+            while self.alive:  # type: ignore[has-type]
+                self.notify()
+
+                cnt = server.requests_count
+                if self.max_requests and cnt > self.max_requests:
+                    self.alive = False
+                    self.log.info("Max requests, shutting down: %s", self)
+
+                elif pid == os.getpid() and self.ppid != os.getppid():
+                    self.alive = False
+                    self.log.info("Parent changed, shutting down: %s", self)
+                else:
+                    await self._wait_next_notify()
+        except BaseException:
+            pass
+
+        await runner.cleanup()
+
+    def _wait_next_notify(self) -> "asyncio.Future[bool]":
+        self._notify_waiter_done()
+
+        loop = self.loop
+        assert loop is not None
+        self._notify_waiter = waiter = loop.create_future()
+        self.loop.call_later(1.0, self._notify_waiter_done, waiter)
+
+        return waiter
+
+    def _notify_waiter_done(
+        self, waiter: Optional["asyncio.Future[bool]"] = None
+    ) -> None:
+        if waiter is None:
+            waiter = self._notify_waiter
+        if waiter is not None:
+            set_result(waiter, True)
+
+        if waiter is self._notify_waiter:
+            self._notify_waiter = None
+
+    def init_signals(self) -> None:
+        # Set up signals through the event loop API.
+
+        self.loop.add_signal_handler(
+            signal.SIGQUIT, self.handle_quit, signal.SIGQUIT, None
+        )
+
+        self.loop.add_signal_handler(
+            signal.SIGTERM, self.handle_exit, signal.SIGTERM, None
+        )
+
+        self.loop.add_signal_handler(
+            signal.SIGINT, self.handle_quit, signal.SIGINT, None
+        )
+
+        self.loop.add_signal_handler(
+            signal.SIGWINCH, self.handle_winch, signal.SIGWINCH, None
+        )
+
+        self.loop.add_signal_handler(
+            signal.SIGUSR1, self.handle_usr1, signal.SIGUSR1, None
+        )
+
+        self.loop.add_signal_handler(
+            signal.SIGABRT, self.handle_abort, signal.SIGABRT, None
+        )
+
+        # Don't let SIGTERM and SIGUSR1 disturb active requests
+        # by interrupting system calls
+        signal.siginterrupt(signal.SIGTERM, False)
+        signal.siginterrupt(signal.SIGUSR1, False)
+        # Reset signals so Gunicorn doesn't swallow subprocess return codes
+        # See: https://github.com/aio-libs/aiohttp/issues/6130
+
+    def handle_quit(self, sig: int, frame: Optional[FrameType]) -> None:
+        self.alive = False
+
+        # worker_int callback
+        self.cfg.worker_int(self)
+
+        # wakeup closing process
+        self._notify_waiter_done()
+
+    def handle_abort(self, sig: int, frame: Optional[FrameType]) -> None:
+        self.alive = False
+        self.exit_code = 1
+        self.cfg.worker_abort(self)
+        sys.exit(1)
+
+    @staticmethod
+    def _create_ssl_context(cfg: Any) -> "SSLContext":
+        """Creates SSLContext instance for usage in asyncio.create_server.
+
+        See ssl.SSLSocket.__init__ for more details.
+        """
+        if ssl is None:  # pragma: no cover
+            raise RuntimeError("SSL is not supported.")
+
+        ctx = ssl.SSLContext(cfg.ssl_version)
+        ctx.load_cert_chain(cfg.certfile, cfg.keyfile)
+        ctx.verify_mode = cfg.cert_reqs
+        if cfg.ca_certs:
+            ctx.load_verify_locations(cfg.ca_certs)
+        if cfg.ciphers:
+            ctx.set_ciphers(cfg.ciphers)
+        return ctx
+
+    def _get_valid_log_format(self, source_format: str) -> str:
+        if source_format == self.DEFAULT_GUNICORN_LOG_FORMAT:
+            return self.DEFAULT_AIOHTTP_LOG_FORMAT
+        elif re.search(r"%\([^\)]+\)", source_format):
+            raise ValueError(
+                "Gunicorn's style options in form of `%(name)s` are not "
+                "supported for the log formatting. Please use aiohttp's "
+                "format specification to configure access log formatting: "
+                "http://docs.aiohttp.org/en/stable/logging.html"
+                "#format-specification"
+            )
+        else:
+            return source_format
+
+
+class GunicornUVLoopWebWorker(GunicornWebWorker):
+    def init_process(self) -> None:
+        import uvloop
+
+        # Setup uvloop policy, so that every
+        # asyncio.get_event_loop() will create an instance
+        # of uvloop event loop.
+        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+
+        super().init_process()

aiosignal/__init__.py

@@ -0,0 +1,36 @@
+from frozenlist import FrozenList
+
+__version__ = "1.3.1"
+
+__all__ = ("Signal",)
+
+
+class Signal(FrozenList):
+    """Coroutine-based signal implementation.
+
+    To connect a callback to a signal, use any list method.
+
+    Signals are fired using the send() coroutine, which takes named
+    arguments.
+    """
+
+    __slots__ = ("_owner",)
+
+    def __init__(self, owner):
+        super().__init__()
+        self._owner = owner
+
+    def __repr__(self):
+        return "<Signal owner={}, frozen={}, {!r}>".format(
+            self._owner, self.frozen, list(self)
+        )
+
+    async def send(self, *args, **kwargs):
+        """
+        Sends data to all registered receivers.
+        """
+        if not self.frozen:
+            raise RuntimeError("Cannot send non-frozen signal.")
+
+        for receiver in self:
+            await receiver(*args, **kwargs)  # type: ignore

aiosignal/__init__.pyi

@@ -0,0 +1,12 @@
+from typing import Any, Generic, TypeVar
+
+from frozenlist import FrozenList
+
+__all__ = ("Signal",)
+
+_T = TypeVar("_T")
+
+class Signal(FrozenList[_T], Generic[_T]):
+    def __init__(self, owner: Any) -> None: ...
+    def __repr__(self) -> str: ...
+    async def send(self, *args: Any, **kwargs: Any) -> None: ...

async_timeout/__init__.py

@@ -0,0 +1,239 @@
+import asyncio
+import enum
+import sys
+import warnings
+from types import TracebackType
+from typing import Optional, Type
+
+
+if sys.version_info >= (3, 8):
+    from typing import final
+else:
+    from typing_extensions import final
+
+
+if sys.version_info >= (3, 11):
+
+    def _uncancel_task(task: "asyncio.Task[object]") -> None:
+        task.uncancel()
+
+else:
+
+    def _uncancel_task(task: "asyncio.Task[object]") -> None:
+        pass
+
+
+__version__ = "4.0.3"
+
+
+__all__ = ("timeout", "timeout_at", "Timeout")
+
+
+def timeout(delay: Optional[float]) -> "Timeout":
+    """timeout context manager.
+
+    Useful in cases when you want to apply timeout logic around block
+    of code or in cases when asyncio.wait_for is not suitable. For example:
+
+    >>> async with timeout(0.001):
+    ...     async with aiohttp.get('https://github.com') as r:
+    ...         await r.text()
+
+
+    delay - value in seconds or None to disable timeout logic
+    """
+    loop = asyncio.get_running_loop()
+    if delay is not None:
+        deadline = loop.time() + delay  # type: Optional[float]
+    else:
+        deadline = None
+    return Timeout(deadline, loop)
+
+
+def timeout_at(deadline: Optional[float]) -> "Timeout":
+    """Schedule the timeout at absolute time.
+
+    deadline argument points on the time in the same clock system
+    as loop.time().
+
+    Please note: it is not POSIX time but a time with
+    undefined starting base, e.g. the time of the system power on.
+
+    >>> async with timeout_at(loop.time() + 10):
+    ...     async with aiohttp.get('https://github.com') as r:
+    ...         await r.text()
+
+
+    """
+    loop = asyncio.get_running_loop()
+    return Timeout(deadline, loop)
+
+
+class _State(enum.Enum):
+    INIT = "INIT"
+    ENTER = "ENTER"
+    TIMEOUT = "TIMEOUT"
+    EXIT = "EXIT"
+
+
+@final
+class Timeout:
+    # Internal class, please don't instantiate it directly
+    # Use timeout() and timeout_at() public factories instead.
+    #
+    # Implementation note: `async with timeout()` is preferred
+    # over `with timeout()`.
+    # While technically the Timeout class implementation
+    # doesn't need to be async at all,
+    # the `async with` statement explicitly points that
+    # the context manager should be used from async function context.
+    #
+    # This design allows to avoid many silly misusages.
+    #
+    # TimeoutError is raised immediately when scheduled
+    # if the deadline is passed.
+    # The purpose is to time out as soon as possible
+    # without waiting for the next await expression.
+
+    __slots__ = ("_deadline", "_loop", "_state", "_timeout_handler", "_task")
+
+    def __init__(
+        self, deadline: Optional[float], loop: asyncio.AbstractEventLoop
+    ) -> None:
+        self._loop = loop
+        self._state = _State.INIT
+
+        self._task: Optional["asyncio.Task[object]"] = None
+        self._timeout_handler = None  # type: Optional[asyncio.Handle]
+        if deadline is None:
+            self._deadline = None  # type: Optional[float]
+        else:
+            self.update(deadline)
+
+    def __enter__(self) -> "Timeout":
+        warnings.warn(
+            "with timeout() is deprecated, use async with timeout() instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self._do_enter()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> Optional[bool]:
+        self._do_exit(exc_type)
+        return None
+
+    async def __aenter__(self) -> "Timeout":
+        self._do_enter()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> Optional[bool]:
+        self._do_exit(exc_type)
+        return None
+
+    @property
+    def expired(self) -> bool:
+        """Is timeout expired during execution?"""
+        return self._state == _State.TIMEOUT
+
+    @property
+    def deadline(self) -> Optional[float]:
+        return self._deadline
+
+    def reject(self) -> None:
+        """Reject scheduled timeout if any."""
+        # cancel is maybe better name but
+        # task.cancel() raises CancelledError in asyncio world.
+        if self._state not in (_State.INIT, _State.ENTER):
+            raise RuntimeError(f"invalid state {self._state.value}")
+        self._reject()
+
+    def _reject(self) -> None:
+        self._task = None
+        if self._timeout_handler is not None:
+            self._timeout_handler.cancel()
+            self._timeout_handler = None
+
+    def shift(self, delay: float) -> None:
+        """Advance timeout on delay seconds.
+
+        The delay can be negative.
+
+        Raise RuntimeError if shift is called when deadline is not scheduled
+        """
+        deadline = self._deadline
+        if deadline is None:
+            raise RuntimeError("cannot shift timeout if deadline is not scheduled")
+        self.update(deadline + delay)
+
+    def update(self, deadline: float) -> None:
+        """Set deadline to absolute value.
+
+        deadline argument points on the time in the same clock system
+        as loop.time().
+
+        If new deadline is in the past the timeout is raised immediately.
+
+        Please note: it is not POSIX time but a time with
+        undefined starting base, e.g. the time of the system power on.
+        """
+        if self._state == _State.EXIT:
+            raise RuntimeError("cannot reschedule after exit from context manager")
+        if self._state == _State.TIMEOUT:
+            raise RuntimeError("cannot reschedule expired timeout")
+        if self._timeout_handler is not None:
+            self._timeout_handler.cancel()
+        self._deadline = deadline
+        if self._state != _State.INIT:
+            self._reschedule()
+
+    def _reschedule(self) -> None:
+        assert self._state == _State.ENTER
+        deadline = self._deadline
+        if deadline is None:
+            return
+
+        now = self._loop.time()
+        if self._timeout_handler is not None:
+            self._timeout_handler.cancel()
+
+        self._task = asyncio.current_task()
+        if deadline <= now:
+            self._timeout_handler = self._loop.call_soon(self._on_timeout)
+        else:
+            self._timeout_handler = self._loop.call_at(deadline, self._on_timeout)
+
+    def _do_enter(self) -> None:
+        if self._state != _State.INIT:
+            raise RuntimeError(f"invalid state {self._state.value}")
+        self._state = _State.ENTER
+        self._reschedule()
+
+    def _do_exit(self, exc_type: Optional[Type[BaseException]]) -> None:
+        if exc_type is asyncio.CancelledError and self._state == _State.TIMEOUT:
+            assert self._task is not None
+            _uncancel_task(self._task)
+            self._timeout_handler = None
+            self._task = None
+            raise asyncio.TimeoutError
+        # timeout has not expired
+        self._state = _State.EXIT
+        self._reject()
+        return None
+
+    def _on_timeout(self) -> None:
+        assert self._task is not None
+        self._task.cancel()
+        self._state = _State.TIMEOUT
+        # drop the reference early
+        self._timeout_handler = None

async_timeout/py.typed

@@ -0,0 +1 @@
+Placeholder

autoActions/README.md

@@ -0,0 +1,272 @@
+# AutoActions
+
+Automatiken, die im Web-UI unter „Automatismen" angelegt werden und hier
+ausgeführt werden: *wenn Bedingung, dann Kommando*.
+
+```
+Browser                     Datenbank homeMesh              Runner
+──────────────────────      ─────────────────────────       ────────────────────
+Karte „Automatismen"   ──▶  automations                ──▶  autoaction_runner.py
+ajax/AutoAction.php         automation_conditions           MQTT / HTTP / Tahoma
+restricted/automations.php  automation_actions              ──▶ Geräte
+js/solar/autoActionFuncs.js automation_action_params
+                            automation_log
+                            calendar_days              ◀──  fetch_calendar.py
+```
+
+## Wie eine Automatik aufgebaut ist
+
+Eine Automatik hat **Auslöser**, **Rahmenbedingungen** und **Aktionen**.
+
+Ein Auslöser vergleicht einen Messwert (`actor_states`) mit einer Schwelle.
+Mehrere Auslöser werden über `group_no` verknüpft: gleiche Nummer heißt UND,
+verschiedene Nummern heißen ODER — ausgewertet wird `any(all(gruppe))`. Im
+Editor ist eine Gruppe ein gerahmter Block mit eigenem „+ Bedingung", zwischen
+den Blöcken steht ein ODER. Die Klammerung ist damit gezeichnet und nicht bloß
+vereinbart, und darunter steht derselbe Ausdruck noch einmal als Satz.
+
+Eine Aktion ist ein Kommando (`actor_commands`) mit einem Wert je Parameter
+(`command_parameters`) — nicht vier feste Spalten „Wert 1" bis „Wert 4",
+sondern so viele Zeilen, wie das Gerät Parameter hat.
+
+Die Rahmenbedingungen (Wochentage, Zeitfenster, Ferien, Feiertage) sagen, wann
+die Automatik überhaupt hinsehen darf.
+
+## Warum ein Dauerläufer und kein Cronjob
+
+Zwei Gründe:
+
+* Schwellwert-Auslöser sollen greifen, wenn die MQTT-Nachricht hereinkommt,
+  nicht erst im nächsten Minutenraster.
+* `actor_states.current_value` wird sonst von niemandem fortgeschrieben — es
+  wird beim Geräte-Discovery einmal gesetzt und danach nie wieder. Ein
+  zustandsloser Cronjob hätte gar nichts, womit er vergleichen könnte. Der
+  Runner pflegt den Wert nebenbei mit — nur wenn er sich geändert hat und
+  höchstens einmal je Minute —, wovon auch der Editor profitiert: er zeigt
+  neben jedem Messwert den aktuellen Stand.
+
+## Was worauf wartet
+
+Die Hauptschleife wartet auf nichts. Alles, was ein Netz braucht, läuft
+daneben:
+
+| | wo | Takt |
+|---|---|---|
+| MQTT-Nachrichten | kommen von selbst, Zwischenspeicher im Transport | sofort |
+| Uhrzeit, Datum | im Runner gerechnet | jeder Takt |
+| Sonnenauf-/-untergang | `solarLog.daylight` | einmal je Tag |
+| HTTP- und WLED-Geräte | `Sammler`, eigener Faden | `poll_http`, `poll_wled` |
+| Tahoma | `Sammler`, eigener Faden | `poll_tahoma`, Vorgabe 5 Minuten |
+| Kommandos senden | `Versand`, ein Faden je Gerät | wenn etwas ansteht |
+
+Das war nicht immer so. Vorher wurden die Geräte mitten in der Schleife
+abgefragt — neunzehn Tahoma-Geräte nacheinander, jedes mit bis zu zehn
+Sekunden Zeitlimit. Eine Runde dauerte dadurch rund fünfundvierzig statt
+dreißig Sekunden, gepollt wurde erst jede zweite Runde, also alle neunzig
+Sekunden. Und weil die Uhr an derselben Abfrage hing, kam jede dritte Minute
+nie vor: `um 18:26` wurde nie wahr.
+
+Die Fäden fassen die Datenbank nicht an. Sie legen ihre Ergebnisse in eine
+Queue, geschrieben wird im Hauptfaden — eine pymysql-Verbindung gehört einem
+Faden.
+
+## Nur die steigende Flanke
+
+`automations.cond_met` hält fest, ob die Bedingung beim letzten Durchlauf schon
+erfüllt war. Ohne das würde „Temperatur über 22 Grad" bei jedem Takt erneut
+feuern. Verlässt die Automatik ihr Zeitfenster, wird die Flanke
+zurückgesetzt, damit sie im nächsten Fenster wieder steigen kann.
+
+Zeit-Auslöser gibt es in drei Formen:
+
+| | wahr, wenn |
+|---|---|
+| `um 16:30` | ab dieser Minute, noch `catchup_minutes` lang |
+| `ab 16:30` | von da an bis Mitternacht |
+| `vor 16:30` | bis dahin |
+
+Ausgelöst wird in allen drei Fällen nur einmal, eben wegen der Flanke.
+
+Das Nachholfenster bei `um` ist der Ersatz für die früher verlangte
+Punktgenauigkeit. Solange `um 16:30` nur in genau dieser Minute wahr war,
+kostete jeder Aussetzer die Automatik für den ganzen Tag — und Aussetzer gab
+es reichlich, weil die Uhr am Geräte-Poll hing und jede dritte Minute
+übersprang. Jetzt gilt die Bedingung fünf Minuten lang (einstellbar), die
+Flanke sorgt weiterhin für genau einen Lauf, und ein Neustart mitten im
+Fenster holt den Lauf nach. Die breiten Zeitfenster in `auto_watering.py`
+folgen derselben Überlegung.
+
+Beim Sonnenauf- und -untergang ist der Wert ein Versatz, und der kann davor
+oder danach liegen — deshalb dieselben drei Fälle mal zwei: `+ 00:30` eine
+halbe Stunde nach Sonnenaufgang, `ab - 00:30` ab einer halben Stunde davor,
+`vor + 00:30` bis eine halbe Stunde danach.
+
+Über den Tagesrand wird gerechnet, nicht abgeschnitten: „sechs Stunden vor
+Sonnenaufgang" landet am Vorabend, und das ist so gewollt — abgeschnitten
+wären solche Angaben gar nicht mehr formulierbar. Verglichen wird die Uhrzeit
+innerhalb des Tages; ein Ziel jenseits von Mitternacht gilt als diese Uhrzeit
+am selben Tag. Bei `+` und `-` ist das genau der gemeinte Zeitpunkt, bei `ab`
+und `vor` verschiebt sich der wahre Bereich entsprechend mit.
+
+`force_once` („am Ende des Zeitraums auf jeden Fall ausführen") greift, wenn
+das Fenster zugeht und in diesem Fenster noch nichts passiert ist.
+
+## Sperrzeit
+
+Die Flanke allein schützt nicht gegen einen Messwert, der um die Schwelle
+**pendelt**: „Temperatur > 22" bei 22,1 / 21,9 / 22,1 °C ist jedes Mal eine
+echte steigende Flanke, und über MQTT können die Werte im Sekundentakt
+hereinkommen. `automations.lockout_secs` sagt, wie lange nach einer Auslösung
+nicht wieder geschaltet wird. Der Editor bietet drei Stufen an:
+
+| | | gedacht für |
+|---|---|---|
+| Ohne | 0 s | volle Geschwindigkeit, jede Flanke schaltet |
+| Kurz | 60 s | Licht, Farbe, Dimmwert |
+| Lang | 900 s | Rollläden, Ventile, alles mit Motor |
+
+Gespeichert werden Sekunden, angeboten werden nur die drei Stufen — eine
+vierte ist damit eine Zeile in `lockoutChoices()` und keine Wanderung durch
+die Datenbank.
+
+Eine Flanke innerhalb der Sperrzeit wird **verworfen, nicht aufgehoben**. Ein
+Rollladen, der eine Viertelstunde später doch noch losfährt, weil vor langer
+Zeit einmal eine Schwelle gestreift wurde, wäre unangenehmer als einer, der
+gar nicht fährt — und der nächste echte Anlass nach Ablauf der Sperre kommt
+ohnehin durch. Verworfene Flanken stehen im Log auf `DEBUG`, nicht in
+`automation_log`; bei einem zappelnden Sensor wäre die Tabelle sonst voll
+davon.
+
+`force_once` ist von der Sperre nicht betroffen: es greift nur, wenn im
+Fenster gar nichts gelaufen ist — dann ist auch keine Sperre aktiv.
+
+## Transporte
+
+Welcher Weg zum Gerät führt, entscheidet die URL des Aktors in `actors`:
+
+| URL | Messwert (`actor_states.url`) | Kommando (`actor_commands.command_url`) |
+|---|---|---|
+| `mqtt://…` | vollständiges Topic, abonniert; bei mehreren Messwerten je Topic zusätzlich `value_path` | Nutzlast auf das Parameter-Topic |
+| `http://…` | Feldname in der JSON-Antwort, gepollt | Abfrageargumente an die Geräte-URL (`turn=on`) |
+| `wled://…` | Pfad in `/json/state` (`seg[0].col[0]`), gepollt | JSON-Vorlage mit Platzhaltern, als Ganzes gesendet |
+| Tahoma | Statusname (`core:ClosureState`), gepollt | `exec/apply` an die Box |
+| `Logic` | gerechnet: Uhrzeit, Datum, Sonne | – |
+
+Alle fünf stehen in `transports.py`. Eine sechste Geräteart kommt als weitere
+Klasse dazu; sie braucht `passt()`, `zustaende_lesen()` und `senden()`.
+
+Tahoma ist der einzige, der nicht am URL-Schema erkannt wird, sondern an der
+**Box-Kennung** in der URL. Das Schema beschreibt dort die Funkart, und
+dieselbe Box liefert `io://` für die Jalousien, `rts://` für die Dachfenster
+und `internal://` für die Alarmanlage. Ohne `pin` in der `config.ini` ist
+niemand zuständig — dann meldet der Runner beim Auslösen „kein Transport",
+statt still nichts zu tun.
+
+Mehrere Messwerte teilen sich oft **ein Topic**: der go-eCharger schickt
+sechzehn Zahlen als JSON-Feld auf `…/nrg`, und erst das `value_template` der
+Home-Assistant-Discovery sagt, dass „Strom L1" das fünfte Element ist. Diese
+Angabe steht in `actor_states.value_path` — in derselben Schreibweise, die
+auch WLED benutzt: `[4]`, `ssid`, `seg[0].col[0]`. Ohne Pfad gilt die ganze
+Nutzlast.
+
+Gelesen wird nur der einfache Fall aus dem Template: ein Zugriff auf
+`value_json` und was danach an Punkten und Klammern folgt.
+
+### Werttabellen
+
+Manche Geräte schicken eine Zahl und meinen einen Zustand:
+
+```
+{{ ['Unknown','Idle','Charging','WaitCar','Complete','Error'][value_json|int] }}
+{{ ['Default','Eco','NextTrip'][value_json|int-3] }}
+```
+
+Das ist kein Pfad, sondern eine Übersetzung von Zahl nach Text. Sie landet in
+`possible_values` — in der Schreibweise, die WLED für seine Effektliste schon
+benutzt: eine Liste aus `{Wert: Bezeichnung}`. Ein Versatz im Ausdruck wandert
+dabei in die Schlüssel, aus `[value_json|int-3]` wird also `{"3":"Default"}`.
+
+Der Runner übersetzt beim Lesen: aus der gesendeten `2` wird `Charging`. Eine
+Bedingung vergleicht damit genau den Klartext, den der Editor zur Auswahl
+stellt. Steht die Zahl nicht in der Tabelle, bleibt sie stehen — ein
+erfundener Name wäre schlimmer als ein roher Wert.
+
+Auf beiden Seiten des Editors steckt dieselbe Tabelle, aber der gespeicherte
+Wert ist ein anderer:
+
+| | angezeigt | gespeichert |
+|---|---|---|
+| **Messwert** (Bedingung) | `Charging` | `Charging` — der Runner hat schon übersetzt |
+| **Parameter** (Aktion) | `Blink` | `1` — das Gerät will die Zahl |
+
+Bei WLED trägt die Kommando-Vorlage alles: `{"seg":[{"col":[[%red%,%green%,%blue%]]}]}`
+wird mit den Parameterwerten gefüllt und am Stück geschickt. Deshalb haben die
+Parameter dort keine eigene URL — ihr Name *ist* der Platzhalter.
+
+## Voraussetzungen
+
+* Python 3 mit `pymysql`, `requests`, `paho-mqtt`
+* `homeMesh_automations.sql` einmal eingespielt
+* Im Web-Verzeichnis muss in `restricted/deviceDiscovery/config.ini`
+  **`clear_tables = false`** stehen.
+  Discovery schreibt mit `ON DUPLICATE KEY UPDATE` auf den URLs, das Leeren ist
+  unnötig — ein `TRUNCATE` würde dagegen die Geräte-IDs neu vergeben, und die
+  Automatiken zeigen per Fremdschlüssel genau auf diese IDs.
+
+## Einrichten
+
+```bash
+cp config.ini.example config.ini      # ausfüllen: Datenbank, MQTT, Tahoma
+python3 fetch_calendar.py             # Feiertage und Ferien holen
+python3 autoaction_runner.py --once --dry-run --verbose   # Probelauf
+```
+
+`--dry-run` schaltet nichts, protokolliert aber jedes Kommando, das geschickt
+würde. `--once` macht einen einzigen Durchlauf.
+
+## Wo das läuft
+
+Der Runner ist ein Hintergrundprozess und wohnt deshalb beim SolarManager,
+nicht im Web-Verzeichnis:
+
+```
+/volume1/homes/wagner/SolarManager/
+├── solarManager.py
+├── startSolarServer.sh          startet beide, siehe unten
+└── autoActions/
+    ├── autoaction_runner.py
+    ├── transports.py
+    ├── fetch_calendar.py
+    └── config.ini               Zugangsdaten, nicht im Git
+```
+
+Das Web-UI kennt diesen Pfad nicht — Browser und Runner reden ausschließlich
+über die Datenbank `homeMesh` miteinander. Der Runner lädt das Regelwerk nach,
+sobald im Browser etwas gespeichert wurde; ein Neustart nach jeder Änderung ist
+nicht nötig.
+
+`startSolarServer.sh` startet `solarManager.py` und den Runner gemeinsam und
+beendet vorher, was schon läuft. Aufgerufen wird es beim Booten (auf der
+Synology über den Aufgabenplaner, Ereignis „Hochfahren", als root); dasselbe
+Skript von Hand aufzurufen ist der normale Weg, den Runner neu zu starten.
+Zwei Instanzen dürfen nie gleichzeitig laufen — sie würden jedes Kommando
+doppelt schicken und sich gegenseitig vom MQTT-Broker werfen, weil beide
+dieselbe Client-Kennung benutzen. Genau davor schützt das Beenden am Anfang.
+
+Ausgabe landet in `autoActions.log` neben `solarOutput.log`. `SIGTERM` fängt
+der Runner ab und fährt geordnet herunter.
+
+`fetch_calendar.py` gehört einmal jährlich in den Cron:
+
+```
+0 4 1 1 * /usr/bin/python3 /volume1/homes/wagner/SolarManager/autoActions/fetch_calendar.py
+```
+
+Ein zusätzlicher Lauf im Herbst schadet nicht — die Ferientermine des
+übernächsten Schuljahres stehen erst später fest.
+
+## Nachsehen, was passiert ist
+
+`automation_log` hält je Auslösung fest, ob sie durchlief (`fired`), wegen
+`force_once` nachgeholt wurde (`forced`) oder scheiterte (`error`, mit Grund in
+`detail`). Einträge älter als 30 Tage räumt der Runner selbst weg.

autoActions/config.ini.example

@@ -0,0 +1,87 @@
+# ============================================================================
+# AutoAction-Runner - Konfiguration
+# ============================================================================
+# Kopieren nach config.ini und ausfuellen. config.ini ist nicht versioniert
+# (siehe .gitignore), weil hier Zugangsdaten stehen.
+
+# ============================================================================
+# DATENBANK - Geraete und Automatiken (homeMesh)
+# ============================================================================
+[database]
+host = nas.fritz.box
+port = 3310
+database = homeMesh
+user = homeMesh
+password =
+
+# ============================================================================
+# DATENBANK - Messwerte (solarLog)
+# ============================================================================
+# Nur fuer die Tabelle `daylight`, aus der Sonnenauf- und -untergang kommen.
+# Leer lassen, wenn dieselben Zugangsdaten wie oben gelten.
+[solar]
+database = solarLog
+user =
+password =
+
+# ============================================================================
+# MQTT
+# ============================================================================
+[mqtt]
+broker = nas.fritz.box
+port = 1883
+username =
+password =
+client_id = autoaction_runner
+
+# ============================================================================
+# TAHOMA
+# ============================================================================
+# Fuer Aktoren, deren URL mit io:// beginnt. Ohne Token bleiben sie stumm -
+# der Runner protokolliert das dann als Fehler, statt still nichts zu tun.
+[tahoma]
+pin =
+token =
+timeout = 10
+
+# ============================================================================
+# KALENDER
+# ============================================================================
+# Fuer fetch_calendar.py, das Feiertage und Schulferien in calendar_days
+# eintraegt. Regionscodes siehe openholidaysapi.org, Bayern ist DE-BY.
+[calendar]
+country = DE
+subdivision = DE-BY
+
+# ============================================================================
+# LAUFZEIT
+# ============================================================================
+[runner]
+# Abstand der Auswertung in Sekunden. Die Schleife wartet auf nichts mehr -
+# Geraete fragt der Sammler in eigenen Faeden ab, geschaltet wird im Versand -,
+# deshalb darf der Takt kurz sein. Er bestimmt nur noch, wie schnell eine
+# hereingekommene MQTT-Nachricht ausgewertet wird.
+tick_seconds = 10
+
+# Wie oft Geraete abgefragt werden, die nichts von sich aus melden. Getrennt
+# je Transport, weil sie unterschiedlich teuer sind: eine Shelly-Abfrage ist
+# in Millisekunden zurueck, die Tahoma-Box braucht fuer ihre Geraete
+# nacheinander gut zehn Sekunden. In Sekunden.
+poll_http = 60
+poll_wled = 60
+poll_tahoma = 300
+
+# Wie lange ein punktgenauer Ausloeser ("um 16:30", "Sonnenaufgang + 00:30")
+# nachtraeglich noch gilt, in Minuten. Faellt die Auswertung in dieser Minute
+# aus - Neustart, haengendes Geraet -, holt der naechste Takt sie nach.
+# Ausgeloest wird trotzdem nur einmal, weil nur die steigende Flanke zaehlt.
+catchup_minutes = 5
+
+# Wie oft der Runner nachsieht, ob sich das Regelwerk geaendert hat.
+reload_seconds = 30
+
+# Nichts wirklich schalten, nur protokollieren. Zum Ausprobieren neuer Regeln.
+dry_run = false
+
+# DEBUG, INFO, WARNING, ERROR
+log_level = INFO

autoActions/fetch_calendar.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+Fuellt die Tabelle `calendar_days` mit Feiertagen und Schulferien.
+
+Die Automatiken koennen mit "an Feiertagen ausfuehren" und "in den Ferien
+ausfuehren" auf besondere Tage reagieren; woher das Wissen kommt, steht hier.
+Nur besondere Tage werden eingetragen - ein fehlendes Datum ist ein
+gewoehnlicher Tag.
+
+Quelle ist openholidaysapi.org: ein offenes Verzeichnis der EU-Kommission, das
+gesetzliche Feiertage und Schulferien fuer alle deutschen Bundeslaender
+liefert. Ein Aufruf je Jahr genuegt, deshalb ist das ein Cronjob und kein
+Dauerlaeufer:
+
+    0 4 1 1 * /usr/bin/python3 /pfad/zu/fetch_calendar.py
+
+Ein zusaetzlicher Lauf im Herbst schadet nicht - die Ferientermine des
+uebernaechsten Schuljahres stehen erst spaeter fest.
+
+Bereits eingetragene Tage werden ueberschrieben, nie doppelt angelegt.
+"""
+
+import argparse
+import logging
+import sys
+from datetime import date, timedelta
+
+import pymysql
+import requests
+
+from autoaction_runner import Config, verbinden
+
+logger = logging.getLogger("kalender")
+
+BASIS = "https://openholidaysapi.org"
+
+
+def zeitraum(api, land, region, von, bis):
+    """Eintraege einer Art (PublicHolidays oder SchoolHolidays) abholen."""
+    antwort = requests.get(
+        BASIS + "/" + api,
+        params={"countryIsoCode": land, "subdivisionCode": region,
+                "languageIsoCode": "DE", "validFrom": von.isoformat(),
+                "validTo": bis.isoformat()},
+        headers={"Accept": "application/json"}, timeout=20)
+    antwort.raise_for_status()
+    return antwort.json()
+
+
+def name(eintrag):
+    for teil in eintrag.get("name", []):
+        if teil.get("text"):
+            return teil["text"][:80]
+    return "?"
+
+
+def tage(eintrag):
+    """Ferien erstrecken sich ueber Wochen; hier wird daraus Tag fuer Tag."""
+    start = date.fromisoformat(eintrag["startDate"])
+    ende = date.fromisoformat(eintrag["endDate"])
+    while start <= ende:
+        yield start
+        start += timedelta(days=1)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Feiertage und Ferien in calendar_days schreiben.")
+    parser.add_argument("--jahr", type=int, default=date.today().year)
+    parser.add_argument("--jahre", type=int, default=2, help="wie viele Jahre ab --jahr")
+    parser.add_argument("--config", default="config.ini")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+    config = Config(args.config)
+    land = config.text("calendar", "country", "DE")
+    region = config.text("calendar", "subdivision", "DE-BY")
+
+    von = date(args.jahr, 1, 1)
+    bis = date(args.jahr + args.jahre - 1, 12, 31)
+    logger.info("Hole %s bis %s fuer %s", von, bis, region)
+
+    gesammelt = {}
+    for eintrag in zeitraum("PublicHolidays", land, region, von, bis):
+        for tag in tage(eintrag):
+            gesammelt.setdefault(tag, {})["holiday"] = name(eintrag)
+    for eintrag in zeitraum("SchoolHolidays", land, region, von, bis):
+        for tag in tage(eintrag):
+            gesammelt.setdefault(tag, {})["vacation"] = name(eintrag)
+
+    db = verbinden(config)
+    with db.cursor() as c:
+        c.executemany(
+            """INSERT INTO calendar_days (date, holiday, vacation) VALUES (%s, %s, %s)
+               ON DUPLICATE KEY UPDATE holiday = VALUES(holiday), vacation = VALUES(vacation)""",
+            [(tag, eintrag.get("holiday"), eintrag.get("vacation"))
+             for tag, eintrag in sorted(gesammelt.items())])
+    logger.info("%d Tage eingetragen", len(gesammelt))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

autoActions/transports.py

@@ -0,0 +1,626 @@
+#!/usr/bin/env python3
+"""
+Transporte fuer den AutoAction-Runner.
+
+Ein Transport weiss, wie man mit einer Sorte Geraet redet - er liest deren
+Messwerte und schickt deren Kommandos. Welcher zustaendig ist, entscheidet die
+URL des Aktors in der Tabelle `actors`:
+
+    mqtt://...   MQTT-Geraete (Home-Assistant-Discovery)
+    http://...   Shelly und andere HTTP-Geraete
+    wled://...   WLED-Lampen
+    io://, rts://, internal://, ogp://   Tahoma - das Schema haengt an der
+                 Funkart des Geraets, deshalb wird dort nicht danach
+                 entschieden, sondern an der Box-Kennung in der URL
+    Logic        das gerechnete Geraet "Zeitpunkt" (Uhrzeit, Datum, Sonne)
+
+Alle liegen in einer Datei statt in einem Paket wie bei deviceDiscovery: es
+sind fuenf kurze Klassen, und wer eine sechste Geraeteart anschliesst, sieht
+hier auf einen Blick, was dafuer zu tun ist.
+
+Jeder Transport hat zwei Haelften:
+
+    zustaende_anmelden(states)  einmalig beim Start
+    zustaende_lesen()           liefert {state_id: wert} - nur was neu ist
+    senden(aktion)              fuehrt ein Kommando aus
+
+`states` ist eine Liste von Dicts mit actor_url, state_url, value_path und
+id, `aktion` ein Dict mit actor_url, command_url und params (Liste aus
+{url, name, wert}).
+"""
+
+import json
+import logging
+import re
+import time
+import urllib3
+from datetime import datetime
+from urllib.parse import quote
+
+logger = logging.getLogger("autoaction.transport")
+
+# Die Tahoma-Box hat ein selbst ausgestelltes Zertifikat auf einen Namen, den
+# nur das Heimnetz kennt. Die Pruefung ist dort bewusst aus (wie in
+# ajax/tahoma.php); ohne diese Zeile warnt urllib3 bei jeder einzelnen
+# Abfrage und uebertoent das eigentliche Protokoll.
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def wert_aus_pfad(daten, pfad):
+    """
+    Einen Teilwert aus einer Nutzlast holen: "[3]", "ssid", "seg[0].col[0]".
+    Ein leerer Pfad heisst: die Nutzlast selbst.
+
+    Gebraucht wird das an zwei Stellen. MQTT-Geraete legen mehrere Messwerte
+    auf ein Topic - der go-eCharger schickt sechzehn Zahlen als JSON-Feld -,
+    und WLED liefert seinen gesamten Zustand als ein Dokument.
+    """
+    if not pfad:
+        return daten
+    for teil in pfad.split("."):
+        treffer = re.match(r"^([^\[]*)((?:\[\d+\])*)$", teil)
+        if not treffer:
+            raise KeyError(pfad)
+        if treffer.group(1):
+            daten = daten[treffer.group(1)]
+        for index in re.findall(r"\[(\d+)\]", treffer.group(2)):
+            daten = daten[int(index)]
+    return daten
+
+
+def uebersetze(wert, tabelle):
+    """
+    Aus der gesendeten Zahl den Zustandsnamen machen: aus "2" wird "Charging".
+
+    Manche Geraete schicken einen Zahlencode und meinen einen Zustand. Welche
+    Zahl welchen Namen hat, steht in possible_values - derselben Spalte, aus
+    der auch der Editor seine Auswahlliste baut. Eine Bedingung vergleicht
+    damit genau den Klartext, den man dort ausgewaehlt hat.
+
+    Steht die Zahl nicht in der Tabelle, bleibt sie stehen: ein erfundener
+    Name waere schlimmer als ein roher Wert.
+    """
+    if not tabelle:
+        return wert
+    if wert in tabelle:
+        return tabelle[wert]
+    try:                      # "2.0" und "2" meinen dieselbe Stufe
+        ganz = str(int(float(wert)))
+    except (TypeError, ValueError):
+        return wert
+    return tabelle.get(ganz, wert)
+
+
+class Transport:
+    """Gemeinsame Form. Wer nichts zu lesen hat, erbt die leeren Methoden."""
+
+    schema = ""
+
+    def passt(self, actor_url):
+        return actor_url.startswith(self.schema)
+
+    def zustaende_anmelden(self, states):
+        pass
+
+    def zustaende_lesen(self):
+        return {}
+
+    def senden(self, aktion):
+        raise NotImplementedError
+
+
+# ---------------------------------------------------------------------------
+# MQTT
+# ---------------------------------------------------------------------------
+
+class MQTTTransport(Transport):
+    """
+    Die state_url ist hier das vollstaendige Topic, die parameter_url das
+    Kommando-Topic. Werte kommen von selbst herein und landen in einem
+    Zwischenspeicher, den der Runner im Takt abholt.
+    """
+
+    schema = "mqtt://"
+
+    def __init__(self, client, dry_run=False):
+        self.client = client
+        self.dry_run = dry_run
+        self.topics = {}      # topic -> [(state_id, value_path, wertetabelle), ...]
+        self.neu = {}         # state_id -> wert
+
+    def zustaende_anmelden(self, states):
+        self.topics = {}
+        for s in states:
+            if not s["state_url"]:
+                continue
+            self.topics.setdefault(s["state_url"], []).append(
+                (s["id"], s.get("value_path"), s.get("wertetabelle") or {}))
+        for topic in self.topics:
+            self.client.subscribe(topic)
+        mehrfach = sum(1 for e in self.topics.values() if len(e) > 1)
+        logger.info("MQTT: %d Topics abonniert, %d davon mit mehreren Messwerten",
+                    len(self.topics), mehrfach)
+
+    def nachricht(self, topic, payload):
+        """Wird vom Runner aus dem on_message-Rueckruf gerufen."""
+        eintraege = self.topics.get(topic, [])
+        if not eintraege:
+            return
+        text = payload.decode("utf-8", "replace").strip() if isinstance(payload, bytes) else str(payload)
+        try:
+            daten = json.loads(text)
+        except ValueError:
+            daten = None            # kein JSON - dann gilt der Rohtext
+        for state_id, pfad, tabelle in eintraege:
+            wert = self._wert(text, daten, pfad, tabelle)
+            if wert is not None:
+                self.neu[state_id] = wert
+
+    @staticmethod
+    def _wert(text, daten, pfad, tabelle=None):
+        """
+        Aus der Nutzlast den Wert eines einzelnen Messwerts machen.
+
+        Mehrere Messwerte teilen sich oft ein Topic; welcher Teil gemeint ist,
+        steht in value_path - gelesen aus dem value_template der
+        Home-Assistant-Discovery. Ohne Pfad gilt die ganze Nutzlast, und
+        JSON-Skalare werden ausgepackt: manche Geraete schicken 21.4 mit
+        Anfuehrungszeichen, andere true statt ON.
+
+        Steht eine Werttabelle dabei, wird aus der gesendeten Zahl der
+        Zustandsname: aus 2 wird "Charging".
+        """
+        if daten is None:
+            return uebersetze(text, tabelle)
+        try:
+            wert = wert_aus_pfad(daten, pfad)
+        except (KeyError, IndexError, TypeError):
+            logger.debug("Pfad %s nicht in der Nutzlast: %s", pfad, text[:80])
+            return None
+        if isinstance(wert, bool):
+            wert = "true" if wert else "false"
+        elif isinstance(wert, (int, float, str)):
+            wert = str(wert)
+        else:
+            return json.dumps(wert, ensure_ascii=False)
+        return uebersetze(wert, tabelle)
+
+    def zustaende_lesen(self):
+        werte, self.neu = self.neu, {}
+        return werte
+
+    def senden(self, aktion):
+        if not aktion["params"]:
+            # Kommando ohne Parameter: das Kommando selbst ist die Nutzlast.
+            self._publish(aktion["command_url"], "")
+            return
+        for p in aktion["params"]:
+            self._publish(p["url"] or aktion["command_url"], p["wert"])
+
+    def _publish(self, topic, nutzlast):
+        if not topic:
+            raise ValueError("Kommando ohne Topic")
+        if self.dry_run:
+            logger.info("[dry-run] MQTT %s <- %s", topic, nutzlast)
+            return
+        ergebnis = self.client.publish(topic, nutzlast, qos=1, retain=False)
+        ergebnis.wait_for_publish(timeout=5)
+
+
+# ---------------------------------------------------------------------------
+# HTTP (Shelly und Verwandte)
+# ---------------------------------------------------------------------------
+
+class HTTPTransport(Transport):
+    """
+    Die actor_url ist der Endpunkt, die state_url ein Feldname in dessen
+    JSON-Antwort ("tC", "a_voltage"). Solche Geraete melden sich nicht von
+    selbst, sie werden im Poll-Takt gefragt.
+
+    Beim Senden wird die command_url als Abfrageargument an die actor_url
+    gehaengt ("turn=on") und die Parameter mit ihrem eigenen Namen dazu.
+    Frueher stand hier eine Uebersetzungstabelle, weil die Shelly-Kommandos
+    ohne URL in der Datenbank landeten - das ist im Discovery behoben, die
+    Zuordnung gehoert dorthin und nicht in den Runner.
+    """
+
+    schema = "http"
+
+    def __init__(self, requests_modul, timeout=5, dry_run=False):
+        self.requests = requests_modul
+        self.timeout = timeout
+        self.dry_run = dry_run
+        self.states = []
+
+    def zustaende_anmelden(self, states):
+        self.states = [s for s in states if s["state_url"]]
+        logger.info("HTTP: %d Messwerte an %d Endpunkten",
+                    len(self.states), len({s["actor_url"] for s in self.states}))
+
+    def zustaende_lesen(self):
+        werte = {}
+        # Je Endpunkt eine Anfrage, auch wenn mehrere Messwerte daran haengen.
+        nach_url = {}
+        for s in self.states:
+            nach_url.setdefault(s["actor_url"], []).append(s)
+        for url, states in nach_url.items():
+            try:
+                antwort = self.requests.get(url, timeout=self.timeout)
+                daten = antwort.json()
+            except Exception as fehler:
+                logger.debug("HTTP %s nicht erreichbar: %s", url, fehler)
+                continue
+            for s in states:
+                if isinstance(daten, dict) and s["state_url"] in daten:
+                    werte[s["id"]] = str(daten[s["state_url"]])
+        return werte
+
+    def senden(self, aktion):
+        argumente = {}
+        for teil in (aktion["command_url"] or "").split("&"):
+            if "=" in teil:
+                schluessel, wert = teil.split("=", 1)
+                argumente[schluessel] = wert
+        for p in aktion["params"]:
+            if p["url"]:
+                argumente[p["url"]] = p["wert"]
+        if not argumente:
+            raise RuntimeError("Kommando ohne URL und ohne Parameter - im "
+                               "Geraetemodell fehlt die Angabe, was zu schicken ist")
+        if self.dry_run:
+            logger.info("[dry-run] HTTP %s %s", aktion["actor_url"], argumente)
+            return
+        antwort = self.requests.get(aktion["actor_url"], params=argumente, timeout=self.timeout)
+        if antwort.status_code >= 400:
+            raise RuntimeError("HTTP %d von %s" % (antwort.status_code, aktion["actor_url"]))
+
+
+# ---------------------------------------------------------------------------
+# WLED
+# ---------------------------------------------------------------------------
+
+class WLEDTransport(Transport):
+    """
+    WLED-Lampen sprechen ueber eine einzige JSON-Schnittstelle:
+    GET http://IP/json/state liefert den Zustand, POST dorthin setzt ihn.
+
+    Das Geraetemodell nutzt das elegant aus - die command_url ist eine
+    JSON-Vorlage mit Platzhaltern:
+
+        {"bri":%brightness%}
+        {"seg":[{"col":[[%red%,%green%,%blue%]]}]}
+
+    Gesendet wird also nicht Argument fuer Argument, sondern die ausgefuellte
+    Vorlage am Stueck. Deshalb haben die Parameter hier auch keine eigene URL:
+    ihr Name ist der Platzhalter.
+
+    Die state_url ist ein Pfad in die Antwort ("on", "bri",
+    "seg[0].col[0]") - dieselbe Schreibweise, die auch in current_value steht.
+    """
+
+    schema = "wled://"
+
+    def __init__(self, requests_modul, timeout=5, dry_run=False):
+        self.requests = requests_modul
+        self.timeout = timeout
+        self.dry_run = dry_run
+        self.states = []
+
+    @staticmethod
+    def _adresse(actor_url):
+        return "http://" + actor_url[len("wled://"):].rstrip("/")
+
+    def zustaende_anmelden(self, states):
+        self.states = [s for s in states if s["state_url"]]
+        logger.info("WLED: %d Messwerte an %d Lampen",
+                    len(self.states), len({s["actor_url"] for s in self.states}))
+
+    def zustaende_lesen(self):
+        werte = {}
+        nach_lampe = {}
+        for s in self.states:
+            nach_lampe.setdefault(s["actor_url"], []).append(s)
+        for actor_url, states in nach_lampe.items():
+            try:
+                antwort = self.requests.get(self._adresse(actor_url) + "/json/state",
+                                            timeout=self.timeout)
+                daten = antwort.json()
+            except Exception as fehler:
+                logger.debug("WLED %s nicht erreichbar: %s", actor_url, fehler)
+                continue
+            for s in states:
+                # Bei WLED ist die state_url selbst schon der Pfad. Ein
+                # gesetzter value_path hat trotzdem Vorrang, falls das
+                # Geraetemodell spaeter darauf umgestellt wird.
+                pfad = s.get("value_path") or s["state_url"]
+                try:
+                    werte[s["id"]] = str(wert_aus_pfad(daten, pfad))
+                except (KeyError, IndexError, TypeError):
+                    logger.debug("WLED %s: Pfad %s nicht gefunden", actor_url, pfad)
+        return werte
+
+    def senden(self, aktion):
+        vorlage = aktion["command_url"]
+        if not vorlage:
+            raise RuntimeError("WLED-Kommando ohne Vorlage")
+        for p in aktion["params"]:
+            vorlage = vorlage.replace("%" + p["name"] + "%", str(p["wert"]))
+        try:
+            rumpf = json.loads(vorlage)
+        except ValueError:
+            # Ein nicht ersetzter Platzhalter oder ein Textwert an einer
+            # Stelle, wo eine Zahl stehen muss. Lieber hier abbrechen als der
+            # Lampe etwas Unverstaendliches schicken.
+            raise RuntimeError("WLED-Vorlage ergibt kein gueltiges JSON: " + vorlage[:120])
+        if self.dry_run:
+            logger.info("[dry-run] WLED %s <- %s", aktion["actor_url"],
+                        json.dumps(rumpf, ensure_ascii=False))
+            return
+        antwort = self.requests.post(self._adresse(aktion["actor_url"]) + "/json/state",
+                                     json=rumpf, timeout=self.timeout)
+        if antwort.status_code >= 400:
+            raise RuntimeError("WLED antwortete mit %d" % antwort.status_code)
+
+
+# ---------------------------------------------------------------------------
+# Tahoma
+# ---------------------------------------------------------------------------
+
+class TahomaTransport(Transport):
+    """
+    Die actor_url ist die deviceURL, die state_url ein Statusname
+    ("core:ClosureState"), die command_url ein Kommandoname ("setClosure").
+    Geschickt wird ueber exec/apply - genauso wie in ajax/tahoma.php, nur ohne
+    die dortige Sonderbehandlung fuer "faehrt gerade".
+
+    Zustaendig ist dieser Transport fuer alles, was die Kennung der eigenen
+    Box in der URL traegt. Am Schema laesst sich das nicht festmachen: es
+    beschreibt die Funkart, und dieselbe Box liefert io:// fuer die
+    Jalousien, rts:// fuer die Dachfenster und internal:// fuer die Alarm-
+    anlage. Ohne PIN in der config.ini ist niemand zustaendig - dann meldet
+    der Runner beim Ausloesen "kein Transport", statt still nichts zu tun.
+    """
+
+    schema = "io://"
+
+    # Bis zu dieser Neigung fahren die Aussenjalousien direkt.
+    #
+    # Sie haben eine Kugelschreiber-Mechanik: beim Herunterfahren stehen die
+    # Lamellen bei etwa 30 %. In Richtung 0 % laesst sich von jeder Stellung
+    # aus direkt neigen; darueber hinaus muss die Mechanik erst einmal auf
+    # 0 % zurueck, sonst rastet sie nicht um - ein "Neigung 80 %" ohne
+    # diesen Umweg bleibt wirkungslos.
+    #
+    # Das gilt fuer jedes Kommando, das die Neigung setzt - "Neigung" ebenso
+    # wie "Position+Neigung". Erkannt wird es deshalb am Parameter und nicht
+    # am Kommandonamen: beide heissen ihren Neigungsparameter "Neigung".
+    #
+    # Dasselbe steht in restricted/commands.php: beide Versender brauchen es.
+    NEIGUNG_DIREKT_MAX = 30
+
+    # So heissen die beiden Parameter einer Jalousie im Geraetemodell.
+    NEIGUNG_PARAMETER = "Neigung"
+    POSITION_PARAMETER = "Position"
+
+    # So lange wird hoechstens auf das Ende einer Fahrt gewartet. Gemessen:
+    # eine Neigung von 100 % auf 0 % dauert gut fuenfzehn Sekunden, eine
+    # volle Fahrt von oben nach unten rund sechzig. Die Grenze ist die
+    # Notbremse, nicht die uebliche Dauer.
+    JALOUSIE_WARTE_SEKUNDEN = 120
+
+    # So lange gilt ein "faehrt nicht" direkt nach dem Absenden als noch
+    # nicht aussagekraeftig: die Box meldet core:MovingState traege, kurz
+    # nach einem Kommando steht dort noch der alte Wert.
+    JALOUSIE_VORLAUF_SEKUNDEN = 8
+
+    def __init__(self, requests_modul, pin, token, timeout=10, dry_run=False):
+        self.requests = requests_modul
+        self.pin = pin
+        self.token = token
+        self.timeout = timeout
+        self.dry_run = dry_run
+        self.states = []
+        self.kombigeraete = set()
+
+    def passt(self, actor_url):
+        return bool(self.pin) and ("://" + self.pin + "/") in actor_url
+
+    @property
+    def basis(self):
+        return "https://gateway-%s:8443/enduser-mobile-web/1/enduserAPI" % self.pin
+
+    def _kopf(self):
+        return {"Content-Type": "application/json",
+                "Authorization": "Bearer " + self.token}
+
+    def zustaende_anmelden(self, states):
+        self.states = [s for s in states if s["state_url"]]
+        logger.info("Tahoma: %d Messwerte an %d Geraeten",
+                    len(self.states), len({s["actor_url"] for s in self.states}))
+
+    def kombigeraete_setzen(self, urls):
+        """
+        Welche Geraete Position und Neigung zusammen koennen - vom Runner aus
+        dem Geraetemodell gesetzt, damit der Transport dafuer nicht selbst in
+        die Datenbank greifen muss.
+        """
+        self.kombigeraete = set(urls)
+
+    def zustaende_lesen(self):
+        if not self.token:
+            return {}
+        werte = {}
+        nach_geraet = {}
+        for s in self.states:
+            nach_geraet.setdefault(s["actor_url"], []).append(s)
+        for geraet, states in nach_geraet.items():
+            try:
+                antwort = self.requests.get(
+                    self.basis + "/setup/devices/" + quote(geraet, safe="") + "/states",
+                    headers=self._kopf(), timeout=self.timeout, verify=False)
+                zustaende = {z["name"]: z.get("value") for z in antwort.json()}
+            except Exception as fehler:
+                logger.debug("Tahoma %s nicht erreichbar: %s", geraet, fehler)
+                continue
+            for s in states:
+                if s["state_url"] in zustaende:
+                    werte[s["id"]] = str(zustaende[s["state_url"]])
+        return werte
+
+    def senden(self, aktion):
+        if not self.token:
+            raise RuntimeError("Kein Tahoma-Token in der config.ini")
+        # Die Reihenfolge der Parameter ist die aus command_parameters - bei
+        # setClosureAndOrientation also erst Position, dann Winkel.
+        befehl = aktion["command_url"]
+        parameter = [self._zahl(p["wert"]) for p in aktion["params"]]
+        neigung_index = None
+        position_index = None
+        for i, p in enumerate(aktion["params"]):
+            if p["name"] == self.NEIGUNG_PARAMETER:
+                neigung_index = i
+            elif p["name"] == self.POSITION_PARAMETER:
+                position_index = i
+
+        # "Zu" allein macht diese Jalousien nicht dicht: sie faehrt herunter,
+        # die Lamellen bleiben durch die Mechanik aber bei etwa 30 % offen.
+        # Gemeint ist "ganz unten, Lamellen geschlossen" - also dasselbe wie
+        # Position 100 mit Neigung 100, und damit ein Fall fuer die Regel.
+        if befehl == "down" and self._kannKombi(aktion["actor_url"]):
+            befehl = "setClosureAndOrientation"
+            parameter = [100, 100]
+            position_index, neigung_index = 0, 1
+
+        # Kugelschreiber-Mechanik, siehe NEIGUNG_DIREKT_MAX. Geschickt wird
+        # derselbe Befehl zweimal - erst mit Neigung 0, dann mit dem
+        # gewuenschten Wert. Die Position bleibt dabei stehen, die Jalousie
+        # faehrt also nur einmal.
+        umweg = (neigung_index is not None
+                 and isinstance(parameter[neigung_index], (int, float))
+                 and parameter[neigung_index] > self.NEIGUNG_DIREKT_MAX)
+        vorstufe = list(parameter)
+        if umweg:
+            vorstufe[neigung_index] = 0
+
+        if self.dry_run:
+            logger.info("[dry-run] Tahoma %s %s%s", befehl, parameter,
+                        " (zuerst %s, dann warten)" % vorstufe if umweg else "")
+            return
+
+        if umweg:
+            self._apply(aktion["actor_url"], befehl, vorstufe)
+            self._warteAufJalousie(aktion["actor_url"], 0,
+                                   None if position_index is None else int(parameter[position_index]))
+        self._apply(aktion["actor_url"], befehl, parameter)
+
+    def _kannKombi(self, actor_url):
+        """
+        Hat das Geraet ein Kommando fuer Position und Neigung zusammen?
+        Nur solche Geraete sind Jalousien mit der Kugelschreiber-Mechanik.
+        """
+        return actor_url in self.kombigeraete
+
+    def _apply(self, actor_url, name, parameter):
+        """Ein Kommando an die Box schicken."""
+        rumpf = {"label": "AutoAction",
+                 "actions": [{"deviceURL": actor_url,
+                              "commands": [{"name": name, "parameters": parameter}]}]}
+        antwort = self.requests.post(self.basis + "/exec/apply", headers=self._kopf(),
+                                     data=json.dumps(rumpf), timeout=self.timeout, verify=False)
+        if antwort.status_code >= 400:
+            raise RuntimeError("Tahoma antwortete mit %d: %s"
+                               % (antwort.status_code, antwort.text[:120]))
+
+    def _warteAufJalousie(self, actor_url, neigung_ziel, schliessung_ziel=None):
+        """
+        Wartet, bis die Jalousie ihre Fahrt beendet hat und die Ziele zeigt.
+
+        Zwei Auskuenfte zusammen, weil einzeln keine traegt:
+        core:MovingState taugt fuer die lange Fahrt hoch und runter, wird bei
+        kurzen Neigungsfahrten aber nie gesetzt; die Zustandswerte sind die
+        eigentliche Wahrheit, zeigen kurz nach dem Kommando aber noch den
+        alten Stand. Fertig ist die Fahrt, wenn nichts mehr faehrt, die Ziele
+        erreicht sind und entweder ein "faehrt" gesehen wurde oder der
+        Vorlauf um ist.
+        """
+        start = time.time()
+        gestartet = False
+        while time.time() - start < self.JALOUSIE_WARTE_SEKUNDEN:
+            time.sleep(2)
+            try:
+                antwort = self.requests.get(
+                    self.basis + "/setup/devices/" + quote(actor_url, safe="") + "/states",
+                    headers=self._kopf(), timeout=self.timeout, verify=False)
+                z = {x.get("name"): x.get("value") for x in antwort.json()}
+            except Exception as fehler:
+                logger.debug("Zustand nicht lesbar: %s", fehler)
+                continue
+            if z.get("core:MovingState") is True:
+                gestartet = True
+                continue
+            neigung = z.get("core:SlateOrientationState")
+            schliessung = z.get("core:ClosureState")
+            # Ein fehlendes Feld darf nicht als 0 durchgehen - das waere
+            # ausgerechnet beim Ziel 0 ein falsches Erfolgssignal.
+            neigung_ok = neigung is not None and int(neigung) == neigung_ziel
+            schliessung_ok = (schliessung_ziel is None
+                              or (schliessung is not None and int(schliessung) == schliessung_ziel))
+            if neigung_ok and schliessung_ok and (
+                    gestartet or time.time() - start >= self.JALOUSIE_VORLAUF_SEKUNDEN):
+                return True
+        logger.warning("%s hat Neigung %s%% nicht innerhalb von %d s erreicht",
+                       actor_url, neigung_ziel, self.JALOUSIE_WARTE_SEKUNDEN)
+        return False
+
+    @staticmethod
+    def _zahl(wert):
+        """Tahoma erwartet Zahlen als Zahlen, Text als Text."""
+        try:
+            return int(wert)
+        except (TypeError, ValueError):
+            pass
+        try:
+            return float(wert)
+        except (TypeError, ValueError):
+            return wert
+
+
+# ---------------------------------------------------------------------------
+# Logic - das gerechnete Geraet
+# ---------------------------------------------------------------------------
+
+class LogicTransport(Transport):
+    """
+    Uhrzeit, Datum, Sonnenauf- und -untergang. Es gibt nichts zu abonnieren und
+    nichts zu schalten, die Werte entstehen im Takt. Sonnenzeiten kommen aus
+    solarLog.daylight, dieselbe Tabelle, aus der auch ajax/getSunrise.php liest.
+    """
+
+    schema = "Logic"
+
+    def __init__(self, sonnenzeiten):
+        """sonnenzeiten: Funktion() -> (sonnenaufgang, sonnenuntergang) als "HH:MM"."""
+        self.sonnenzeiten = sonnenzeiten
+        self.states = []
+
+    def passt(self, actor_url):
+        return actor_url == "Logic"
+
+    def zustaende_anmelden(self, states):
+        self.states = states
+        logger.info("Logic: %d Messwerte", len(states))
+
+    def zustaende_lesen(self):
+        jetzt = datetime.now()
+        auf, unter = self.sonnenzeiten()
+        tabelle = {
+            "time":    jetzt.strftime("%H:%M"),
+            "date":    jetzt.strftime("%d.%m.%Y"),
+            "sunrise": auf,
+            "sunset":  unter,
+        }
+        return {s["id"]: tabelle[s["state_url"]]
+                for s in self.states if s["state_url"] in tabelle}
+
+    def senden(self, aktion):
+        raise RuntimeError("Das Geraet \"Zeitpunkt\" kann nichts schalten")

charger_goE.py

@@ -0,0 +1,132 @@
+import json
+import asyncio
+import aiohttp
+import sys
+import logging
+import math
+from dataclasses import dataclass
+_LOGGER = logging.getLogger(__name__)
+
+@dataclass
+class ChargerData:
+    p_l1ev:float = 0
+    p_l2ev:float = 0
+    p_l3ev:float = 0
+    Iset_ev:float = 0
+    Phase_set_ev:int = 0
+    AllowCharging_ev:bool = False
+    mode:str = ""
+    connected:bool = False
+    eto:int = 0             # Gesamtzaehlerstand der Wallbox in Wh
+
+
+async def gatherNeededStatus() -> ChargerData:
+    ret = ChargerData
+    #print("get charger status...");
+    timeout = aiohttp.ClientTimeout(total=3)
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get('http://192.168.179.122/api/status?filter=psm,fup,amp,frc,nrg,lmo,car,pgrid,pakku,ppv,eto',timeout=timeout) as response: #?filter=psm,fup,amp,frc,nrg,lmo,car
+                if response.status == 200:
+                    html = await response.text()
+                    #logging.warning(html)
+                    state = json.loads(html)
+                    # Gesamtzaehler der Wallbox in Wh. Genauer als die Summe
+                    # ueber die Fuenf-Minuten-Leistungswerte: die Differenz
+                    # zweier Staende trifft den Ladevorgang auf die
+                    # Wattstunde, waehrend eine Summe aus Mittelwerten an den
+                    # Raendern immer danebenliegt.
+                    ret.eto = int(state.get("eto",0) or 0)
+                    ret.p_l1ev = state["nrg"][7]/1000.0
+                    ret.p_l2ev = state["nrg"][8]/1000.0
+                    ret.p_l3ev = state["nrg"][9]/1000.0
+                    ret.Iset_ev = state["amp"]
+                    ret.Phase_set_ev = state["psm"]
+                    ret.AllowCharging_ev = state["frc"]
+                    if state["car"] == 2 or state["car"] == 4:
+                        ret.connected = True
+                    else:
+                        ret.connected = False
+                    if state["fup"] == True and state["lmo"] == 4:
+                        ret.mode = "Eco"
+                    elif state["fup"] == True and state["lmo"] == 5:
+                    	ret.mode = "Next Trip"
+                    else:
+                        ret.mode = "Default"
+                    
+    except:
+        return ret
+    #print("Body:", html)
+    return ret
+
+async def setPVparameters(pGrid:int,pPv:int,pAkku:int):
+    timeout = aiohttp.ClientTimeout(total=3)
+    #logging.warning('{"pGrid":'+str(round(pGrid))+',"pAkku":'+str(round(pAkku))+',"pPv":'+str(round(pPv))+'}')
+    #_LOGGER.error('{"pGrid":'+str(round(pGrid))+',"pAkku":'+str(round(pAkku))+',"pPv":'+str(round(pPv))+'}')
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get('http://192.168.179.122/api/set?ids={"pGrid":'+str(round(pGrid))+',"pAkku":'+str(round(pAkku))+',"pPv":'+str(round(pPv))+'}',timeout=timeout) as response:
+                if response.status != 200:
+                    #_LOGGER.error('goE Return: %s' % await response.text())
+                    return False
+                #else:
+                #    _LOGGER.error('goE Return: %s' % await response.text())
+    except Exception as error:
+    	_LOGGER.error(f"Error during sending GoE Parameters: %s",error)
+    	return False
+
+
+async def setChargePower(power:int, currPhases:int, currAmp:int, currCharging:bool):
+    timeout = aiohttp.ClientTimeout(total=3)
+  
+    if(power < 3700):
+        phases = 1
+        amp = math.floor(power/240)
+    elif(power > 4500):
+        phases = 2
+        amp = math.floor((power/3)/240)
+    else:
+        amp = 16
+
+    if(amp < 6):
+        amp = 0
+    elif(amp > 16):
+        amp = 16
+    if(phases != currPhases):
+        logging.warning("Change phases: "+str(phases))
+        #async with aiohttp.ClientSession() as session:
+        #    async with session.get('http://192.168.178.64/api/?psm='+phases,timeout=timeout) as response:
+        #        if response.status != 200:
+        #            return False
+    if(amp != currAmp and amp):
+        logging.warning("Change amp: "+str(amp))
+        if(currCharging == False):
+            logging.warning("start charging")
+        #async with aiohttp.ClientSession() as session:
+        #    async with session.get('http://192.168.178.64/api/set?amp='+amp,timeout=timeout) as response:
+        #        if response.status != 200:
+        #            return False
+    elif(currCharging):
+        logging.warning("stop charging")
+        #async with aiohttp.ClientSession() as session:
+        #    async with session.get('http://192.168.178.64/api/set?frc=1',timeout=timeout) as response:
+        #        if response.status != 200:
+        #            return False
+
+    #status = charger.get_status(status_type=goecharger_api_lite.GoeCharger.STATUS_FULL)
+    #print(json.dumps(status, indent=4))
+
+    #adjust amps
+#curl "http://1.2.3.4/api/set?amp=16"
+#current energy: nrg energy array, U (L1, L2, L3, N), I (L1, L2, L3), P (L1, L2, L3, N, Total), pf (L1, L2, L3, N)
+#set 1-phase
+#curl "http://1.2.3.4/api/set?psm=1"
+
+#set 3-phase
+#curl "http://1.2.3.4/api/set?psm=2"
+
+#start charging
+#curl "http://1.2.3.4/api/set?frc=0"
+
+#stop charging
+#curl "http://1.2.3.4/api/set?frc=1"

config.ini.example

@@ -0,0 +1,40 @@
+; Zugangsdaten des SolarManagers.
+;
+; Diese Datei ist die Vorlage und steht im Repository. Zum Aufsetzen einmal
+;
+;     cp config.ini.example config.ini
+;
+; und ausfuellen - config.ini selbst wird nicht eingecheckt (.gitignore).
+; Gelesen wird sie von konfig.py.
+
+; ===========================================================================
+; Messwerte (solarLog)
+; ===========================================================================
+; Benutzt von solarManager.py, gatherWaterData.py, gatherSkodaData.py,
+; skoda_testdaten.py und zeit.py.
+[database]
+host = localhost
+port = 3310
+user = solarLog
+password =
+database = solarLog
+
+; ===========================================================================
+; Wecker (wecker.py)
+; ===========================================================================
+[alarm]
+host = localhost
+port = 3310
+user = alarm
+password =
+database = alarm
+
+; ===========================================================================
+; Wallbox
+; ===========================================================================
+; Gelesen von startWattpilotMQTT.sh, das daraus die Umgebungsvariablen
+; WATTPILOT_HOST und WATTPILOT_PASSWORD setzt.
+[wattpilot]
+host =
+password =
+mqtt_host = nas.fritz.box

crcmod/__init__.py

@@ -0,0 +1,8 @@
+try:
+    from crcmod.crcmod import *
+    import crcmod.predefined
+except ImportError:
+    # Make this backward compatible
+    from crcmod import *
+    import predefined
+__doc__ = crcmod.__doc__

crcmod/_crcfunpy.py

@@ -0,0 +1,107 @@
+#-----------------------------------------------------------------------------
+# Low level CRC functions for use by crcmod.  This version is implemented in
+# Python for a couple of reasons.  1) Provide a reference implememtation.
+# 2) Provide a version that can be used on systems where a C compiler is not
+# available for building extension modules.
+#
+# Copyright (c) 2009  Raymond L. Buvel
+# Copyright (c) 2010  Craig McQueen
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#-----------------------------------------------------------------------------
+
+def _get_buffer_view(in_obj):
+    if isinstance(in_obj, str):
+        raise TypeError('Unicode-objects must be encoded before calculating a CRC')
+    mv = memoryview(in_obj)
+    if mv.ndim > 1:
+        raise BufferError('Buffer must be single dimension')
+    return mv
+
+
+def _crc8(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFF
+    for x in mv.tobytes():
+        crc = table[x ^ crc]
+    return crc
+
+def _crc8r(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFF
+    for x in mv.tobytes():
+        crc = table[x ^ crc]
+    return crc
+
+def _crc16(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFFFF
+    for x in mv.tobytes():
+        crc = table[x ^ ((crc>>8) & 0xFF)] ^ ((crc << 8) & 0xFF00)
+    return crc
+
+def _crc16r(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFFFF
+    for x in mv.tobytes():
+        crc = table[x ^ (crc & 0xFF)] ^ (crc >> 8)
+    return crc
+
+def _crc24(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFFFFFF
+    for x in mv.tobytes():
+        crc = table[x ^ (crc>>16 & 0xFF)] ^ ((crc << 8) & 0xFFFF00)
+    return crc
+
+def _crc24r(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFFFFFF
+    for x in mv.tobytes():
+        crc = table[x ^ (crc & 0xFF)] ^ (crc >> 8)
+    return crc
+
+def _crc32(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFFFFFFFF
+    for x in mv.tobytes():
+        crc = table[x ^ ((crc>>24) & 0xFF)] ^ ((crc << 8) & 0xFFFFFF00)
+    return crc
+
+def _crc32r(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFFFFFFFF
+    for x in mv.tobytes():
+        crc = table[x ^ (crc & 0xFF)] ^ (crc >> 8)
+    return crc
+
+def _crc64(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFFFFFFFFFFFFFFFF
+    for x in mv.tobytes():
+        crc = table[x ^ ((crc>>56) & 0xFF)] ^ ((crc << 8) & 0xFFFFFFFFFFFFFF00)
+    return crc
+
+def _crc64r(data, crc, table):
+    mv = _get_buffer_view(data)
+    crc = crc & 0xFFFFFFFFFFFFFFFF
+    for x in mv.tobytes():
+        crc = table[x ^ (crc & 0xFF)] ^ (crc >> 8)
+    return crc
+

crcmod/crcmod.py

@@ -0,0 +1,457 @@
+#-----------------------------------------------------------------------------
+# Copyright (c) 2010  Raymond L. Buvel
+# Copyright (c) 2010  Craig McQueen
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#-----------------------------------------------------------------------------
+'''crcmod is a Python module for gererating objects that compute the Cyclic
+Redundancy Check.  Any 8, 16, 24, 32, or 64 bit polynomial can be used.  
+
+The following are the public components of this module.
+
+Crc -- a class that creates instances providing the same interface as the
+algorithms in the hashlib module in the Python standard library.  These
+instances also provide a method for generating a C/C++ function to compute
+the CRC.
+
+mkCrcFun -- create a Python function to compute the CRC using the specified
+polynomial and initial value.  This provides a much simpler interface if
+all you need is a function for CRC calculation.
+'''
+
+__all__ = '''mkCrcFun Crc
+'''.split()
+
+# Select the appropriate set of low-level CRC functions for this installation.
+# If the extension module was not built, drop back to the Python implementation
+# even though it is significantly slower.
+try:
+    import crcmod._crcfunext as _crcfun
+    _usingExtension = True
+except ImportError:
+    import crcmod._crcfunpy as _crcfun
+    _usingExtension = False
+
+import sys, struct
+
+#-----------------------------------------------------------------------------
+class Crc:
+    '''Compute a Cyclic Redundancy Check (CRC) using the specified polynomial.
+
+    Instances of this class have the same interface as the algorithms in the
+    hashlib module in the Python standard library.  See the documentation of
+    this module for examples of how to use a Crc instance.
+
+    The string representation of a Crc instance identifies the polynomial,
+    initial value, XOR out value, and the current CRC value.  The print
+    statement can be used to output this information.
+
+    If you need to generate a C/C++ function for use in another application,
+    use the generateCode method.  If you need to generate code for another
+    language, subclass Crc and override the generateCode method.
+
+    The following are the parameters supplied to the constructor.
+
+    poly -- The generator polynomial to use in calculating the CRC.  The value
+    is specified as a Python integer.  The bits in this integer are the
+    coefficients of the polynomial.  The only polynomials allowed are those
+    that generate 8, 16, 24, 32, or 64 bit CRCs.
+
+    initCrc -- Initial value used to start the CRC calculation.  This initial
+    value should be the initial shift register value XORed with the final XOR
+    value.  That is equivalent to the CRC result the algorithm should return for
+    a zero-length string.  Defaults to all bits set because that starting value
+    will take leading zero bytes into account.  Starting with zero will ignore
+    all leading zero bytes.
+
+    rev -- A flag that selects a bit reversed algorithm when True.  Defaults to
+    True because the bit reversed algorithms are more efficient.
+
+    xorOut -- Final value to XOR with the calculated CRC value.  Used by some
+    CRC algorithms.  Defaults to zero.
+    '''
+    def __init__(self, poly, initCrc=~0, rev=True, xorOut=0, initialize=True):
+        if not initialize:
+            # Don't want to perform the initialization when using new or copy
+            # to create a new instance.
+            return
+
+        (sizeBits, initCrc, xorOut) = _verifyParams(poly, initCrc, xorOut)
+        self.digest_size = sizeBits//8
+        self.initCrc = initCrc
+        self.xorOut = xorOut
+
+        self.poly = poly
+        self.reverse = rev
+
+        (crcfun, table) = _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut)
+        self._crc = crcfun
+        self.table = table
+
+        self.crcValue = self.initCrc
+
+    def __str__(self):
+        lst = []
+        lst.append('poly = 0x%X' % self.poly)
+        lst.append('reverse = %s' % self.reverse)
+        fmt = '0x%%0%dX' % (self.digest_size*2)
+        lst.append('initCrc  = %s' % (fmt % self.initCrc))
+        lst.append('xorOut   = %s' % (fmt % self.xorOut))
+        lst.append('crcValue = %s' % (fmt % self.crcValue))
+        return '\n'.join(lst)
+
+    def new(self, arg=None):
+        '''Create a new instance of the Crc class initialized to the same
+        values as the original instance.  The current CRC is set to the initial
+        value.  If a string is provided in the optional arg parameter, it is
+        passed to the update method.
+        '''
+        n = Crc(poly=None, initialize=False)
+        n._crc = self._crc
+        n.digest_size = self.digest_size
+        n.initCrc = self.initCrc
+        n.xorOut = self.xorOut
+        n.table = self.table
+        n.crcValue = self.initCrc
+        n.reverse = self.reverse
+        n.poly = self.poly
+        if arg is not None:
+            n.update(arg)
+        return n
+
+    def copy(self):
+        '''Create a new instance of the Crc class initialized to the same
+        values as the original instance.  The current CRC is set to the current
+        value.  This allows multiple CRC calculations using a common initial
+        string.
+        '''
+        c = self.new()
+        c.crcValue = self.crcValue
+        return c
+
+    def update(self, data):
+        '''Update the current CRC value using the string specified as the data
+        parameter.
+        '''
+        self.crcValue = self._crc(data, self.crcValue)
+
+    def digest(self):
+        '''Return the current CRC value as a string of bytes.  The length of
+        this string is specified in the digest_size attribute.
+        '''
+        n = self.digest_size
+        crc = self.crcValue
+        lst = []
+        while n > 0:
+            lst.append(crc & 0xFF)
+            crc = crc >> 8
+            n -= 1
+        lst.reverse()
+        return bytes(lst)
+
+    def hexdigest(self):
+        '''Return the current CRC value as a string of hex digits.  The length
+        of this string is twice the digest_size attribute.
+        '''
+        n = self.digest_size
+        crc = self.crcValue
+        lst = []
+        while n > 0:
+            lst.append('%02X' % (crc & 0xFF))
+            crc = crc >> 8
+            n -= 1
+        lst.reverse()
+        return ''.join(lst)
+
+    def generateCode(self, functionName, out, dataType=None, crcType=None):
+        '''Generate a C/C++ function.
+
+        functionName -- String specifying the name of the function.
+
+        out -- An open file-like object with a write method.  This specifies
+        where the generated code is written.
+
+        dataType -- An optional parameter specifying the data type of the input
+        data to the function.  Defaults to UINT8.
+
+        crcType -- An optional parameter specifying the data type of the CRC
+        value.  Defaults to one of UINT8, UINT16, UINT32, or UINT64 depending
+        on the size of the CRC value.
+        '''
+        if dataType is None:
+            dataType = 'UINT8'
+
+        if crcType is None:
+            size = 8*self.digest_size
+            if size == 24:
+                size = 32
+            crcType = 'UINT%d' % size
+
+        if self.digest_size == 1:
+            # Both 8-bit CRC algorithms are the same
+            crcAlgor = 'table[*data ^ (%s)crc]'
+        elif self.reverse:
+            # The bit reverse algorithms are all the same except for the data
+            # type of the crc variable which is specified elsewhere.
+            crcAlgor = 'table[*data ^ (%s)crc] ^ (crc >> 8)'
+        else:
+            # The forward CRC algorithms larger than 8 bits have an extra shift
+            # operation to get the high byte.
+            shift = 8*(self.digest_size - 1)
+            crcAlgor = 'table[*data ^ (%%s)(crc >> %d)] ^ (crc << 8)' % shift
+
+        fmt = '0x%%0%dX' % (2*self.digest_size)
+        if self.digest_size <= 4:
+            fmt = fmt + 'U,'
+        else:
+            # Need the long long type identifier to keep gcc from complaining.
+            fmt = fmt + 'ULL,'
+
+        # Select the number of entries per row in the output code.
+        n = {1:8, 2:8, 3:4, 4:4, 8:2}[self.digest_size]
+
+        lst = []
+        for i, val in enumerate(self.table):
+            if (i % n) == 0:
+                lst.append('\n    ')
+            lst.append(fmt % val)
+
+        poly = 'polynomial: 0x%X' % self.poly
+        if self.reverse:
+            poly = poly + ', bit reverse algorithm'
+
+        if self.xorOut:
+            # Need to remove the comma from the format.
+            preCondition = '\n    crc = crc ^ %s;' % (fmt[:-1] % self.xorOut)
+            postCondition = preCondition
+        else:
+            preCondition = ''
+            postCondition = ''
+
+        if self.digest_size == 3:
+            # The 24-bit CRC needs to be conditioned so that only 24-bits are
+            # used from the 32-bit variable.
+            if self.reverse:
+                preCondition += '\n    crc = crc & 0xFFFFFFU;'
+            else:
+                postCondition += '\n    crc = crc & 0xFFFFFFU;'
+                
+
+        parms = {
+            'dataType' : dataType,
+            'crcType' : crcType,
+            'name' : functionName,
+            'crcAlgor' : crcAlgor % dataType,
+            'crcTable' : ''.join(lst),
+            'poly' : poly,
+            'preCondition' : preCondition,
+            'postCondition' : postCondition,
+        }
+        out.write(_codeTemplate % parms) 
+
+#-----------------------------------------------------------------------------
+def mkCrcFun(poly, initCrc=~0, rev=True, xorOut=0):
+    '''Return a function that computes the CRC using the specified polynomial.
+
+    poly -- integer representation of the generator polynomial
+    initCrc -- default initial CRC value
+    rev -- when true, indicates that the data is processed bit reversed.
+    xorOut -- the final XOR value
+
+    The returned function has the following user interface
+    def crcfun(data, crc=initCrc):
+    '''
+
+    # First we must verify the params
+    (sizeBits, initCrc, xorOut) = _verifyParams(poly, initCrc, xorOut)
+    # Make the function (and table), return the function
+    return _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut)[0]
+
+#-----------------------------------------------------------------------------
+# Naming convention:
+# All function names ending with r are bit reverse variants of the ones
+# without the r.
+
+#-----------------------------------------------------------------------------
+# Check the polynomial to make sure that it is acceptable and return the number
+# of bits in the CRC.
+
+def _verifyPoly(poly):
+    msg = 'The degree of the polynomial must be 8, 16, 24, 32 or 64'
+    for n in (8,16,24,32,64):
+        low = 1<<n
+        high = low*2
+        if low <= poly < high:
+            return n
+    raise ValueError(msg)
+
+#-----------------------------------------------------------------------------
+# Bit reverse the input value.
+
+def _bitrev(x, n):
+    y = 0
+    for i in range(n):
+        y = (y << 1) | (x & 1)
+        x = x >> 1
+    return y
+
+#-----------------------------------------------------------------------------
+# The following functions compute the CRC for a single byte.  These are used
+# to build up the tables needed in the CRC algorithm.  Assumes the high order
+# bit of the polynomial has been stripped off.
+
+def _bytecrc(crc, poly, n):
+    mask = 1<<(n-1)
+    for i in range(8):
+        if crc & mask:
+            crc = (crc << 1) ^ poly
+        else:
+            crc = crc << 1
+    mask = (1<<n) - 1
+    crc = crc & mask
+    return crc
+
+def _bytecrc_r(crc, poly, n):
+    for i in range(8):
+        if crc & 1:
+            crc = (crc >> 1) ^ poly
+        else:
+            crc = crc >> 1
+    mask = (1<<n) - 1
+    crc = crc & mask
+    return crc
+
+#-----------------------------------------------------------------------------
+# The following functions compute the table needed to compute the CRC.  The
+# table is returned as a list.  Note that the array module does not support
+# 64-bit integers on a 32-bit architecture as of Python 2.3.
+#
+# These routines assume that the polynomial and the number of bits in the CRC
+# have been checked for validity by the caller.
+
+def _mkTable(poly, n):
+    mask = (1<<n) - 1
+    poly = poly & mask
+    table = [_bytecrc(i<<(n-8),poly,n) for i in range(256)]
+    return table
+
+def _mkTable_r(poly, n):
+    mask = (1<<n) - 1
+    poly = _bitrev(poly & mask, n)
+    table = [_bytecrc_r(i,poly,n) for i in range(256)]
+    return table
+
+#-----------------------------------------------------------------------------
+# Map the CRC size onto the functions that handle these sizes.
+
+_sizeMap = {
+     8 : [_crcfun._crc8, _crcfun._crc8r],
+    16 : [_crcfun._crc16, _crcfun._crc16r],
+    24 : [_crcfun._crc24, _crcfun._crc24r],
+    32 : [_crcfun._crc32, _crcfun._crc32r],
+    64 : [_crcfun._crc64, _crcfun._crc64r],
+}
+
+#-----------------------------------------------------------------------------
+# Build a mapping of size to struct module type code.  This table is
+# constructed dynamically so that it has the best chance of picking the best
+# code to use for the platform we are running on.  This should properly adapt
+# to 32 and 64 bit machines.
+
+_sizeToTypeCode = {}
+
+for typeCode in 'B H I L Q'.split():
+    size = {1:8, 2:16, 4:32, 8:64}.get(struct.calcsize(typeCode),None)
+    if size is not None and size not in _sizeToTypeCode:
+        _sizeToTypeCode[size] = '256%s' % typeCode
+
+_sizeToTypeCode[24] = _sizeToTypeCode[32]
+
+del typeCode, size
+
+#-----------------------------------------------------------------------------
+# The following function validates the parameters of the CRC, namely,
+# poly, and initial/final XOR values.
+# It returns the size of the CRC (in bits), and "sanitized" initial/final XOR values.
+
+def _verifyParams(poly, initCrc, xorOut):
+    sizeBits = _verifyPoly(poly)
+
+    mask = (1<<sizeBits) - 1
+
+    # Adjust the initial CRC to the correct data type (unsigned value).
+    initCrc = initCrc & mask
+
+    # Similar for XOR-out value.
+    xorOut = xorOut & mask
+
+    return (sizeBits, initCrc, xorOut)
+
+#-----------------------------------------------------------------------------
+# The following function returns a Python function to compute the CRC.
+#
+# It must be passed parameters that are already verified & sanitized by
+# _verifyParams().
+#
+# The returned function calls a low level function that is written in C if the
+# extension module could be loaded.  Otherwise, a Python implementation is
+# used.
+#
+# In addition to this function, a list containing the CRC table is returned.
+
+def _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut):
+    if rev:
+        tableList = _mkTable_r(poly, sizeBits)
+        _fun = _sizeMap[sizeBits][1]
+    else:
+        tableList = _mkTable(poly, sizeBits)
+        _fun = _sizeMap[sizeBits][0]
+
+    _table = tableList
+    if _usingExtension:
+        _table = struct.pack(_sizeToTypeCode[sizeBits], *tableList)
+
+    if xorOut == 0:
+        def crcfun(data, crc=initCrc, table=_table, fun=_fun):
+            return fun(data, crc, table)
+    else:
+        def crcfun(data, crc=initCrc, table=_table, fun=_fun):
+            return xorOut ^ fun(data, xorOut ^ crc, table)
+
+    return crcfun, tableList
+
+#-----------------------------------------------------------------------------
+_codeTemplate = '''// Automatically generated CRC function
+// %(poly)s
+%(crcType)s
+%(name)s(%(dataType)s *data, int len, %(crcType)s crc)
+{
+    static const %(crcType)s table[256] = {%(crcTable)s
+    };
+    %(preCondition)s
+    while (len > 0)
+    {
+        crc = %(crcAlgor)s;
+        data++;
+        len--;
+    }%(postCondition)s
+    return crc;
+}
+'''
+

crcmod/predefined.py

@@ -0,0 +1,162 @@
+#-----------------------------------------------------------------------------
+# Copyright (c) 2010 Craig McQueen
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#-----------------------------------------------------------------------------
+'''
+crcmod.predefined defines some well-known CRC algorithms.
+
+To use it, e.g.:
+    import crcmod.predefined
+    
+    crc32func = crcmod.predefined.mkPredefinedCrcFun("crc-32")
+    crc32class = crcmod.predefined.PredefinedCrc("crc-32")
+
+crcmod.predefined.Crc is an alias for crcmod.predefined.PredefinedCrc
+But if doing 'from crc.predefined import *', only PredefinedCrc is imported.
+'''
+
+# local imports
+import crcmod
+
+__all__ = [
+    'PredefinedCrc',
+    'mkPredefinedCrcFun',
+]
+
+REVERSE = True
+NON_REVERSE = False
+
+# The following table defines the parameters of well-known CRC algorithms.
+# The "Check" value is the CRC for the ASCII byte sequence b"123456789". It
+# can be used for unit tests.
+_crc_definitions_table = [
+#       Name                Identifier-name,    Poly            Reverse         Init-value      XOR-out     Check
+    [   'crc-8',            'Crc8',             0x107,          NON_REVERSE,    0x00,           0x00,       0xF4,       ],
+    [   'crc-8-darc',       'Crc8Darc',         0x139,          REVERSE,        0x00,           0x00,       0x15,       ],
+    [   'crc-8-i-code',     'Crc8ICode',        0x11D,          NON_REVERSE,    0xFD,           0x00,       0x7E,       ],
+    [   'crc-8-itu',        'Crc8Itu',          0x107,          NON_REVERSE,    0x55,           0x55,       0xA1,       ],
+    [   'crc-8-maxim',      'Crc8Maxim',        0x131,          REVERSE,        0x00,           0x00,       0xA1,       ],
+    [   'crc-8-rohc',       'Crc8Rohc',         0x107,          REVERSE,        0xFF,           0x00,       0xD0,       ],
+    [   'crc-8-wcdma',      'Crc8Wcdma',        0x19B,          REVERSE,        0x00,           0x00,       0x25,       ],
+
+    [   'crc-16',           'Crc16',            0x18005,        REVERSE,        0x0000,         0x0000,     0xBB3D,     ],
+    [   'crc-16-buypass',   'Crc16Buypass',     0x18005,        NON_REVERSE,    0x0000,         0x0000,     0xFEE8,     ],
+    [   'crc-16-dds-110',   'Crc16Dds110',      0x18005,        NON_REVERSE,    0x800D,         0x0000,     0x9ECF,     ],
+    [   'crc-16-dect',      'Crc16Dect',        0x10589,        NON_REVERSE,    0x0001,         0x0001,     0x007E,     ],
+    [   'crc-16-dnp',       'Crc16Dnp',         0x13D65,        REVERSE,        0xFFFF,         0xFFFF,     0xEA82,     ],
+    [   'crc-16-en-13757',  'Crc16En13757',     0x13D65,        NON_REVERSE,    0xFFFF,         0xFFFF,     0xC2B7,     ],
+    [   'crc-16-genibus',   'Crc16Genibus',     0x11021,        NON_REVERSE,    0x0000,         0xFFFF,     0xD64E,     ],
+    [   'crc-16-maxim',     'Crc16Maxim',       0x18005,        REVERSE,        0xFFFF,         0xFFFF,     0x44C2,     ],
+    [   'crc-16-mcrf4xx',   'Crc16Mcrf4xx',     0x11021,        REVERSE,        0xFFFF,         0x0000,     0x6F91,     ],
+    [   'crc-16-riello',    'Crc16Riello',      0x11021,        REVERSE,        0x554D,         0x0000,     0x63D0,     ],
+    [   'crc-16-t10-dif',   'Crc16T10Dif',      0x18BB7,        NON_REVERSE,    0x0000,         0x0000,     0xD0DB,     ],
+    [   'crc-16-teledisk',  'Crc16Teledisk',    0x1A097,        NON_REVERSE,    0x0000,         0x0000,     0x0FB3,     ],
+    [   'crc-16-usb',       'Crc16Usb',         0x18005,        REVERSE,        0x0000,         0xFFFF,     0xB4C8,     ],
+    [   'x-25',             'CrcX25',           0x11021,        REVERSE,        0x0000,         0xFFFF,     0x906E,     ],
+    [   'xmodem',           'CrcXmodem',        0x11021,        NON_REVERSE,    0x0000,         0x0000,     0x31C3,     ],
+    [   'modbus',           'CrcModbus',        0x18005,        REVERSE,        0xFFFF,         0x0000,     0x4B37,     ],
+
+    # Note definitions of CCITT are disputable. See:
+    #    http://homepages.tesco.net/~rainstorm/crc-catalogue.htm
+    #    http://web.archive.org/web/20071229021252/http://www.joegeluso.com/software/articles/ccitt.htm
+    [   'kermit',           'CrcKermit',        0x11021,        REVERSE,        0x0000,         0x0000,     0x2189,     ],
+    [   'crc-ccitt-false',  'CrcCcittFalse',    0x11021,        NON_REVERSE,    0xFFFF,         0x0000,     0x29B1,     ],
+    [   'crc-aug-ccitt',    'CrcAugCcitt',      0x11021,        NON_REVERSE,    0x1D0F,         0x0000,     0xE5CC,     ],
+
+    [   'crc-24',           'Crc24',            0x1864CFB,      NON_REVERSE,    0xB704CE,       0x000000,   0x21CF02,   ],
+    [   'crc-24-flexray-a', 'Crc24FlexrayA',    0x15D6DCB,      NON_REVERSE,    0xFEDCBA,       0x000000,   0x7979BD,   ],
+    [   'crc-24-flexray-b', 'Crc24FlexrayB',    0x15D6DCB,      NON_REVERSE,    0xABCDEF,       0x000000,   0x1F23B8,   ],
+
+    [   'crc-32',           'Crc32',            0x104C11DB7,    REVERSE,        0x00000000,     0xFFFFFFFF, 0xCBF43926, ],
+    [   'crc-32-bzip2',     'Crc32Bzip2',       0x104C11DB7,    NON_REVERSE,    0x00000000,     0xFFFFFFFF, 0xFC891918, ],
+    [   'crc-32c',          'Crc32C',           0x11EDC6F41,    REVERSE,        0x00000000,     0xFFFFFFFF, 0xE3069283, ],
+    [   'crc-32d',          'Crc32D',           0x1A833982B,    REVERSE,        0x00000000,     0xFFFFFFFF, 0x87315576, ],
+    [   'crc-32-mpeg',      'Crc32Mpeg',        0x104C11DB7,    NON_REVERSE,    0xFFFFFFFF,     0x00000000, 0x0376E6E7, ],
+    [   'posix',            'CrcPosix',         0x104C11DB7,    NON_REVERSE,    0xFFFFFFFF,     0xFFFFFFFF, 0x765E7680, ],
+    [   'crc-32q',          'Crc32Q',           0x1814141AB,    NON_REVERSE,    0x00000000,     0x00000000, 0x3010BF7F, ],
+    [   'jamcrc',           'CrcJamCrc',        0x104C11DB7,    REVERSE,        0xFFFFFFFF,     0x00000000, 0x340BC6D9, ],
+    [   'xfer',             'CrcXfer',          0x1000000AF,    NON_REVERSE,    0x00000000,     0x00000000, 0xBD0BE338, ],
+
+# 64-bit
+#       Name                Identifier-name,    Poly                    Reverse         Init-value          XOR-out             Check
+    [   'crc-64',           'Crc64',            0x1000000000000001B,    REVERSE,        0x0000000000000000, 0x0000000000000000, 0x46A5A9388A5BEFFE, ],
+    [   'crc-64-we',        'Crc64We',          0x142F0E1EBA9EA3693,    NON_REVERSE,    0x0000000000000000, 0xFFFFFFFFFFFFFFFF, 0x62EC59E3F1A4F00A, ],
+    [   'crc-64-jones',     'Crc64Jones',       0x1AD93D23594C935A9,    REVERSE,        0xFFFFFFFFFFFFFFFF, 0x0000000000000000, 0xCAA717168609F281, ],
+]
+
+
+def _simplify_name(name):
+    """
+    Reduce CRC definition name to a simplified form:
+        * lowercase
+        * dashes removed
+        * spaces removed
+        * any initial "CRC" string removed
+    """
+    name = name.lower()
+    name = name.replace('-', '')
+    name = name.replace(' ', '')
+    if name.startswith('crc'):
+        name = name[len('crc'):]
+    return name
+
+
+_crc_definitions_by_name = {}
+_crc_definitions_by_identifier = {}
+_crc_definitions = []
+
+_crc_table_headings = [ 'name', 'identifier', 'poly', 'reverse', 'init', 'xor_out', 'check' ]
+
+for table_entry in _crc_definitions_table:
+    crc_definition = dict(zip(_crc_table_headings, table_entry))
+    _crc_definitions.append(crc_definition)
+    name = _simplify_name(table_entry[0])
+    if name in _crc_definitions_by_name:
+        raise Exception("Duplicate entry for '{0}' in CRC table".format(name))
+    _crc_definitions_by_name[name] = crc_definition
+    _crc_definitions_by_identifier[table_entry[1]] = crc_definition
+
+
+def _get_definition_by_name(crc_name):
+    definition = _crc_definitions_by_name.get(_simplify_name(crc_name), None)
+    if not definition:
+        definition = _crc_definitions_by_identifier.get(crc_name, None)
+    if not definition:
+        raise KeyError("Unkown CRC name '{0}'".format(crc_name))
+    return definition
+
+
+class PredefinedCrc(crcmod.Crc):
+    def __init__(self, crc_name):
+        definition = _get_definition_by_name(crc_name)
+        super().__init__(poly=definition['poly'], initCrc=definition['init'], rev=definition['reverse'], xorOut=definition['xor_out'])
+
+
+# crcmod.predefined.Crc is an alias for crcmod.predefined.PredefinedCrc
+Crc = PredefinedCrc
+
+
+def mkPredefinedCrcFun(crc_name):
+    definition = _get_definition_by_name(crc_name)
+    return crcmod.mkCrcFun(poly=definition['poly'], initCrc=definition['init'], rev=definition['reverse'], xorOut=definition['xor_out'])
+
+
+# crcmod.predefined.mkCrcFun is an alias for crcmod.predefined.mkPredefinedCrcFun
+mkCrcFun = mkPredefinedCrcFun

crcmod/test.py

@@ -0,0 +1,540 @@
+#-----------------------------------------------------------------------------
+# Copyright (c) 2010  Raymond L. Buvel
+# Copyright (c) 2010  Craig McQueen
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#-----------------------------------------------------------------------------
+'''Unit tests for crcmod functionality'''
+
+
+import unittest
+
+from array import array
+import binascii
+
+from .crcmod import mkCrcFun, Crc
+from .crcmod import _usingExtension
+from .predefined import PredefinedCrc
+from .predefined import mkPredefinedCrcFun
+from .predefined import _crc_definitions as _predefined_crc_definitions
+
+
+#-----------------------------------------------------------------------------
+# This polynomial was chosen because it is the product of two irreducible
+# polynomials.
+# g8 = (x^7+x+1)*(x+1)
+g8 = 0x185
+
+#-----------------------------------------------------------------------------
+# The following reproduces all of the entries in the Numerical Recipes table.
+# This is the standard CCITT polynomial.
+g16 = 0x11021
+
+#-----------------------------------------------------------------------------
+g24 = 0x15D6DCB
+
+#-----------------------------------------------------------------------------
+# This is the standard AUTODIN-II polynomial which appears to be used in a
+# wide variety of standards and applications.
+g32 = 0x104C11DB7
+
+
+#-----------------------------------------------------------------------------
+# I was able to locate a couple of 64-bit polynomials on the web.  To make it
+# easier to input the representation, define a function that builds a
+# polynomial from a list of the bits that need to be turned on.
+
+def polyFromBits(bits):
+    p = 0
+    for n in bits:
+        p = p | (1 << n)
+    return p
+
+# The following is from the paper "An Improved 64-bit Cyclic Redundancy Check
+# for Protein Sequences" by David T. Jones
+
+g64a = polyFromBits([64, 63, 61, 59, 58, 56, 55, 52, 49, 48, 47, 46, 44, 41,
+            37, 36, 34, 32, 31, 28, 26, 23, 22, 19, 16, 13, 12, 10, 9, 6, 4,
+            3, 0])
+
+# The following is from Standard ECMA-182 "Data Interchange on 12,7 mm 48-Track
+# Magnetic Tape Cartridges -DLT1 Format-", December 1992.
+
+g64b = polyFromBits([64, 62, 57, 55, 54, 53, 52, 47, 46, 45, 40, 39, 38, 37,
+            35, 33, 32, 31, 29, 27, 24, 23, 22, 21, 19, 17, 13, 12, 10, 9, 7,
+            4, 1, 0])
+
+#-----------------------------------------------------------------------------
+# This class is used to check the CRC calculations against a direct
+# implementation using polynomial division.
+
+class poly:
+    '''Class implementing polynomials over the field of integers mod 2'''
+    def __init__(self,p):
+        p = int(p)
+        if p < 0: raise ValueError('invalid polynomial')
+        self.p = p
+
+    def __int__(self):
+        return self.p
+
+    def __eq__(self,other):
+        return self.p == other.p
+
+    def __ne__(self,other):
+        return self.p != other.p
+
+    # To allow sorting of polynomials, use their long integer form for
+    # comparison
+    def __cmp__(self,other):
+        return cmp(self.p, other.p)
+
+    def __bool__(self):
+        return self.p != 0
+
+    def __neg__(self):
+        return self # These polynomials are their own inverse under addition
+
+    def __invert__(self):
+        n = max(self.deg() + 1, 1)
+        x = (1 << n) - 1
+        return poly(self.p ^ x)
+
+    def __add__(self,other):
+        return poly(self.p ^ other.p)
+
+    def __sub__(self,other):
+        return poly(self.p ^ other.p)
+
+    def __mul__(self,other):
+        a = self.p
+        b = other.p
+        if a == 0 or b == 0: return poly(0)
+        x = 0
+        while b:
+            if b&1:
+                x = x ^ a
+            a = a<<1
+            b = b>>1
+        return poly(x)
+
+    def __divmod__(self,other):
+        u = self.p
+        m = self.deg()
+        v = other.p
+        n = other.deg()
+        if v == 0: raise ZeroDivisionError('polynomial division by zero')
+        if n == 0: return (self,poly(0))
+        if m < n: return (poly(0),self)
+        k = m-n
+        a = 1 << m
+        v = v << k
+        q = 0
+        while k > 0:
+            if a & u:
+                u = u ^ v
+                q = q | 1
+            q = q << 1
+            a = a >> 1
+            v = v >> 1
+            k -= 1
+        if a & u:
+            u = u ^ v
+            q = q | 1
+        return (poly(q),poly(u))
+
+    def __div__(self,other):
+        return self.__divmod__(other)[0]
+
+    def __mod__(self,other):
+        return self.__divmod__(other)[1]
+
+    def __repr__(self):
+        return 'poly(0x%XL)' % self.p
+
+    def __str__(self):
+        p = self.p
+        if p == 0: return '0'
+        lst = { 0:[], 1:['1'], 2:['x'], 3:['1','x'] }[p&3]
+        p = p>>2
+        n = 2
+        while p:
+            if p&1: lst.append('x^%d' % n)
+            p = p>>1
+            n += 1
+        lst.reverse()
+        return '+'.join(lst)
+
+    def deg(self):
+        '''return the degree of the polynomial'''
+        a = self.p
+        if a == 0: return -1
+        n = 0
+        while a >= 0x10000:
+            n += 16
+            a = a >> 16
+        a = int(a)
+        while a > 1:
+            n += 1
+            a = a >> 1
+        return n
+
+#-----------------------------------------------------------------------------
+# The following functions compute the CRC using direct polynomial division.
+# These functions are checked against the result of the table driven
+# algorithms.
+
+g8p = poly(g8)
+x8p = poly(1<<8)
+def crc8p(d):
+    p = 0
+    for i in d:
+        p = p*256 + i
+    p = poly(p)
+    return int(p*x8p%g8p)
+
+g16p = poly(g16)
+x16p = poly(1<<16)
+def crc16p(d):
+    p = 0
+    for i in d:
+        p = p*256 + i
+    p = poly(p)
+    return int(p*x16p%g16p)
+
+g24p = poly(g24)
+x24p = poly(1<<24)
+def crc24p(d):
+    p = 0
+    for i in d:
+        p = p*256 + i
+    p = poly(p)
+    return int(p*x24p%g24p)
+
+g32p = poly(g32)
+x32p = poly(1<<32)
+def crc32p(d):
+    p = 0
+    for i in d:
+        p = p*256 + i
+    p = poly(p)
+    return int(p*x32p%g32p)
+
+g64ap = poly(g64a)
+x64p = poly(1<<64)
+def crc64ap(d):
+    p = 0
+    for i in d:
+        p = p*256 + i
+    p = poly(p)
+    return int(p*x64p%g64ap)
+
+g64bp = poly(g64b)
+def crc64bp(d):
+    p = 0
+    for i in d:
+        p = p*256 + i
+    p = poly(p)
+    return int(p*x64p%g64bp)
+
+
+class KnownAnswerTests(unittest.TestCase):
+    test_messages = [
+        b'T',
+        b'CatMouse987654321',
+    ]
+
+    known_answers = [
+        [ (g8,0,0),             (0xFE,          0x9D)           ],
+        [ (g8,-1,1),            (0x4F,          0x9B)           ],
+        [ (g8,0,1),             (0xFE,          0x62)           ],
+        [ (g16,0,0),            (0x1A71,        0xE556)         ],
+        [ (g16,-1,1),           (0x1B26,        0xF56E)         ],
+        [ (g16,0,1),            (0x14A1,        0xC28D)         ],
+        [ (g24,0,0),            (0xBCC49D,      0xC4B507)       ],
+        [ (g24,-1,1),           (0x59BD0E,      0x0AAA37)       ],
+        [ (g24,0,1),            (0xD52B0F,      0x1523AB)       ],
+        [ (g32,0,0),            (0x6B93DDDB,    0x12DCA0F4)     ],
+        [ (g32,0xFFFFFFFF,1),   (0x41FB859F,    0xF7B400A7)     ],
+        [ (g32,0,1),            (0x6C0695ED,    0xC1A40EE5)     ],
+        [ (g32,0,1,0xFFFFFFFF), (0xBE047A60,    0x084BFF58)     ],
+    ]
+
+    def test_known_answers(self):
+        for crcfun_params, v in self.known_answers:
+            crcfun = mkCrcFun(*crcfun_params)
+            self.assertEqual(crcfun(b'',0), 0, "Wrong answer for CRC parameters %s, input ''" % (crcfun_params,))
+            for i, msg in enumerate(self.test_messages):
+                self.assertEqual(crcfun(msg), v[i], "Wrong answer for CRC parameters %s, input '%s'" % (crcfun_params,msg))
+                self.assertEqual(crcfun(msg[4:], crcfun(msg[:4])), v[i], "Wrong answer for CRC parameters %s, input '%s'" % (crcfun_params,msg))
+                self.assertEqual(crcfun(msg[-1:], crcfun(msg[:-1])), v[i], "Wrong answer for CRC parameters %s, input '%s'" % (crcfun_params,msg))
+
+
+class CompareReferenceCrcTest(unittest.TestCase):
+    test_messages = [
+        b'',
+        b'T',
+        b'123456789',
+        b'CatMouse987654321',
+    ]
+
+    test_poly_crcs = [
+        [ (g8,0,0),     crc8p    ],
+        [ (g16,0,0),    crc16p   ],
+        [ (g24,0,0),    crc24p   ],
+        [ (g32,0,0),    crc32p   ],
+        [ (g64a,0,0),   crc64ap  ],
+        [ (g64b,0,0),   crc64bp  ],
+    ]
+
+    @staticmethod
+    def reference_crc32(d, crc=0):
+        """This function modifies the return value of binascii.crc32
+        to be an unsigned 32-bit value. I.e. in the range 0 to 2**32-1."""
+        # Work around the future warning on constants.
+        if crc > 0x7FFFFFFF:
+            x = int(crc & 0x7FFFFFFF)
+            crc = x | -2147483648
+        x = binascii.crc32(d,crc)
+        return int(x) & 0xFFFFFFFF
+
+    def test_compare_crc32(self):
+        """The binascii module has a 32-bit CRC function that is used in a wide range
+        of applications including the checksum used in the ZIP file format.
+        This test compares the CRC-32 implementation of this crcmod module to
+        that of binascii.crc32."""
+        # The following function should produce the same result as
+        # self.reference_crc32 which is derived from binascii.crc32.
+        crc32 = mkCrcFun(g32,0,1,0xFFFFFFFF)
+
+        for msg in self.test_messages:
+            self.assertEqual(crc32(msg), self.reference_crc32(msg))
+
+    def test_compare_poly(self):
+        """Compare various CRCs of this crcmod module to a pure
+        polynomial-based implementation."""
+        for crcfun_params, crc_poly_fun in self.test_poly_crcs:
+            # The following function should produce the same result as
+            # the associated polynomial CRC function.
+            crcfun = mkCrcFun(*crcfun_params)
+
+            for msg in self.test_messages:
+                self.assertEqual(crcfun(msg), crc_poly_fun(msg))
+
+
+class CrcClassTest(unittest.TestCase):
+    """Verify the Crc class"""
+
+    msg = b'CatMouse987654321'
+
+    def test_simple_crc32_class(self):
+        """Verify the CRC class when not using xorOut"""
+        crc = Crc(g32)
+
+        str_rep = \
+'''poly = 0x104C11DB7
+reverse = True
+initCrc  = 0xFFFFFFFF
+xorOut   = 0x00000000
+crcValue = 0xFFFFFFFF'''
+        self.assertEqual(str(crc), str_rep)
+        self.assertEqual(crc.digest(), b'\xff\xff\xff\xff')
+        self.assertEqual(crc.hexdigest(), 'FFFFFFFF')
+
+        crc.update(self.msg)
+        self.assertEqual(crc.crcValue, 0xF7B400A7)
+        self.assertEqual(crc.digest(), b'\xf7\xb4\x00\xa7')
+        self.assertEqual(crc.hexdigest(), 'F7B400A7')
+
+        # Verify the .copy() method
+        x = crc.copy()
+        self.assertTrue(x is not crc)
+        str_rep = \
+'''poly = 0x104C11DB7
+reverse = True
+initCrc  = 0xFFFFFFFF
+xorOut   = 0x00000000
+crcValue = 0xF7B400A7'''
+        self.assertEqual(str(crc), str_rep)
+        self.assertEqual(str(x), str_rep)
+
+    def test_full_crc32_class(self):
+        """Verify the CRC class when using xorOut"""
+
+        crc = Crc(g32, initCrc=0, xorOut= ~0)
+
+        str_rep = \
+'''poly = 0x104C11DB7
+reverse = True
+initCrc  = 0x00000000
+xorOut   = 0xFFFFFFFF
+crcValue = 0x00000000'''
+        self.assertEqual(str(crc), str_rep)
+        self.assertEqual(crc.digest(), b'\x00\x00\x00\x00')
+        self.assertEqual(crc.hexdigest(), '00000000')
+
+        crc.update(self.msg)
+        self.assertEqual(crc.crcValue, 0x84BFF58)
+        self.assertEqual(crc.digest(), b'\x08\x4b\xff\x58')
+        self.assertEqual(crc.hexdigest(), '084BFF58')
+
+        # Verify the .copy() method
+        x = crc.copy()
+        self.assertTrue(x is not crc)
+        str_rep = \
+'''poly = 0x104C11DB7
+reverse = True
+initCrc  = 0x00000000
+xorOut   = 0xFFFFFFFF
+crcValue = 0x084BFF58'''
+        self.assertEqual(str(crc), str_rep)
+        self.assertEqual(str(x), str_rep)
+
+        # Verify the .new() method
+        y = crc.new()
+        self.assertTrue(y is not crc)
+        self.assertTrue(y is not x)
+        str_rep = \
+'''poly = 0x104C11DB7
+reverse = True
+initCrc  = 0x00000000
+xorOut   = 0xFFFFFFFF
+crcValue = 0x00000000'''
+        self.assertEqual(str(y), str_rep)
+
+
+class PredefinedCrcTest(unittest.TestCase):
+    """Verify the predefined CRCs"""
+
+    test_messages_for_known_answers = [
+        b'',                           # Test cases below depend on this first entry being the empty string. 
+        b'T',
+        b'CatMouse987654321',
+    ]
+
+    known_answers = [
+        [ 'crc-aug-ccitt',  (0x1D0F,        0xD6ED,        0x5637)         ],
+        [ 'x-25',           (0x0000,        0xE4D9,        0x0A91)         ],
+        [ 'crc-32',         (0x00000000,    0xBE047A60,    0x084BFF58)     ],
+    ]
+
+    def test_known_answers(self):
+        for crcfun_name, v in self.known_answers:
+            crcfun = mkPredefinedCrcFun(crcfun_name)
+            self.assertEqual(crcfun(b'',0), 0, "Wrong answer for CRC '%s', input ''" % crcfun_name)
+            for i, msg in enumerate(self.test_messages_for_known_answers):
+                self.assertEqual(crcfun(msg), v[i], "Wrong answer for CRC %s, input '%s'" % (crcfun_name,msg))
+                self.assertEqual(crcfun(msg[4:], crcfun(msg[:4])), v[i], "Wrong answer for CRC %s, input '%s'" % (crcfun_name,msg))
+                self.assertEqual(crcfun(msg[-1:], crcfun(msg[:-1])), v[i], "Wrong answer for CRC %s, input '%s'" % (crcfun_name,msg))
+
+    def test_class_with_known_answers(self):
+        for crcfun_name, v in self.known_answers:
+            for i, msg in enumerate(self.test_messages_for_known_answers):
+                crc1 = PredefinedCrc(crcfun_name)
+                crc1.update(msg)
+                self.assertEqual(crc1.crcValue, v[i], "Wrong answer for crc1 %s, input '%s'" % (crcfun_name,msg))
+
+                crc2 = crc1.new()
+                # Check that crc1 maintains its same value, after .new() call.
+                self.assertEqual(crc1.crcValue, v[i], "Wrong state for crc1 %s, input '%s'" % (crcfun_name,msg))
+                # Check that the new class instance created by .new() contains the initialisation value.
+                # This depends on the first string in self.test_messages_for_known_answers being
+                # the empty string.
+                self.assertEqual(crc2.crcValue, v[0], "Wrong state for crc2 %s, input '%s'" % (crcfun_name,msg))
+
+                crc2.update(msg)
+                # Check that crc1 maintains its same value, after crc2 has called .update()
+                self.assertEqual(crc1.crcValue, v[i], "Wrong state for crc1 %s, input '%s'" % (crcfun_name,msg))
+                # Check that crc2 contains the right value after calling .update()
+                self.assertEqual(crc2.crcValue, v[i], "Wrong state for crc2 %s, input '%s'" % (crcfun_name,msg))
+
+    def test_function_predefined_table(self):
+        for table_entry in _predefined_crc_definitions:
+            # Check predefined function
+            crc_func = mkPredefinedCrcFun(table_entry['name'])
+            calc_value = crc_func(b"123456789")
+            self.assertEqual(calc_value, table_entry['check'], "Wrong answer for CRC '%s'" % table_entry['name'])
+
+    def test_class_predefined_table(self):
+        for table_entry in _predefined_crc_definitions:
+            # Check predefined class
+            crc1 = PredefinedCrc(table_entry['name'])
+            crc1.update(b"123456789")
+            self.assertEqual(crc1.crcValue, table_entry['check'], "Wrong answer for CRC '%s'" % table_entry['name'])
+
+
+class InputTypesTest(unittest.TestCase):
+    """Check the various input types that CRC functions can accept."""
+
+    msg = b'CatMouse987654321'
+
+    check_crc_names = [
+        'crc-aug-ccitt',
+        'x-25',
+        'crc-32',
+    ]
+    
+    array_check_types = [
+        'B',
+        'H',
+        'I',
+        'L',
+    ]
+
+    def test_bytearray_input(self):
+        """Test that bytearray inputs are accepted, as an example
+        of a type that implements the buffer protocol."""
+        for crc_name in self.check_crc_names:
+            crcfun = mkPredefinedCrcFun(crc_name)
+            for i in range(len(self.msg) + 1):
+                test_msg = self.msg[:i]
+                bytes_answer = crcfun(test_msg)
+                bytearray_answer = crcfun(bytearray(test_msg))
+                self.assertEqual(bytes_answer, bytearray_answer)
+
+    def test_array_input(self):
+        """Test that array inputs are accepted, as an example
+        of a type that implements the buffer protocol."""
+        for crc_name in self.check_crc_names:
+            crcfun = mkPredefinedCrcFun(crc_name)
+            for i in range(len(self.msg) + 1):
+                test_msg = self.msg[:i]
+                bytes_answer = crcfun(test_msg)
+                for array_type in self.array_check_types:
+                    if i % array(array_type).itemsize == 0:
+                        test_array = array(array_type, test_msg)
+                        array_answer = crcfun(test_array)
+                        self.assertEqual(bytes_answer, array_answer)
+
+    def test_unicode_input(self):
+        """Test that Unicode input raises TypeError"""
+        for crc_name in self.check_crc_names:
+            crcfun = mkPredefinedCrcFun(crc_name)
+            with self.assertRaises(TypeError):
+                crcfun("123456789")
+
+
+def runtests():
+    print("Using extension:", _usingExtension)
+    print()
+    unittest.main()
+
+
+if __name__ == '__main__':
+    runtests()

dateutil/__init__.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+import sys
+
+try:
+    from ._version import version as __version__
+except ImportError:
+    __version__ = 'unknown'
+
+__all__ = ['easter', 'parser', 'relativedelta', 'rrule', 'tz',
+           'utils', 'zoneinfo']
+
+def __getattr__(name):
+    import importlib
+
+    if name in __all__:
+        return importlib.import_module("." + name, __name__)
+    raise AttributeError(
+        "module {!r} has not attribute {!r}".format(__name__, name)
+    )
+
+
+def __dir__():
+    # __dir__ should include all the lazy-importable modules as well.
+    return [x for x in globals() if x not in sys.modules] + __all__

dateutil/_common.py

@@ -0,0 +1,43 @@
+"""
+Common code used in multiple modules.
+"""
+
+
+class weekday(object):
+    __slots__ = ["weekday", "n"]
+
+    def __init__(self, weekday, n=None):
+        self.weekday = weekday
+        self.n = n
+
+    def __call__(self, n):
+        if n == self.n:
+            return self
+        else:
+            return self.__class__(self.weekday, n)
+
+    def __eq__(self, other):
+        try:
+            if self.weekday != other.weekday or self.n != other.n:
+                return False
+        except AttributeError:
+            return False
+        return True
+
+    def __hash__(self):
+        return hash((
+          self.weekday,
+          self.n,
+        ))
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        s = ("MO", "TU", "WE", "TH", "FR", "SA", "SU")[self.weekday]
+        if not self.n:
+            return s
+        else:
+            return "%s(%+d)" % (s, self.n)
+
+# vim:ts=4:sw=4:et

dateutil/_version.py

@@ -0,0 +1,4 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+__version__ = version = '2.9.0.post0'
+__version_tuple__ = version_tuple = (2, 9, 0)

dateutil/easter.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+"""
+This module offers a generic Easter computing method for any given year, using
+Western, Orthodox or Julian algorithms.
+"""
+
+import datetime
+
+__all__ = ["easter", "EASTER_JULIAN", "EASTER_ORTHODOX", "EASTER_WESTERN"]
+
+EASTER_JULIAN = 1
+EASTER_ORTHODOX = 2
+EASTER_WESTERN = 3
+
+
+def easter(year, method=EASTER_WESTERN):
+    """
+    This method was ported from the work done by GM Arts,
+    on top of the algorithm by Claus Tondering, which was
+    based in part on the algorithm of Ouding (1940), as
+    quoted in "Explanatory Supplement to the Astronomical
+    Almanac", P.  Kenneth Seidelmann, editor.
+
+    This algorithm implements three different Easter
+    calculation methods:
+
+    1. Original calculation in Julian calendar, valid in
+       dates after 326 AD
+    2. Original method, with date converted to Gregorian
+       calendar, valid in years 1583 to 4099
+    3. Revised method, in Gregorian calendar, valid in
+       years 1583 to 4099 as well
+
+    These methods are represented by the constants:
+
+    * ``EASTER_JULIAN   = 1``
+    * ``EASTER_ORTHODOX = 2``
+    * ``EASTER_WESTERN  = 3``
+
+    The default method is method 3.
+
+    More about the algorithm may be found at:
+
+    `GM Arts: Easter Algorithms <http://www.gmarts.org/index.php?go=415>`_
+
+    and
+
+    `The Calendar FAQ: Easter <https://www.tondering.dk/claus/cal/easter.php>`_
+
+    """
+
+    if not (1 <= method <= 3):
+        raise ValueError("invalid method")
+
+    # g - Golden year - 1
+    # c - Century
+    # h - (23 - Epact) mod 30
+    # i - Number of days from March 21 to Paschal Full Moon
+    # j - Weekday for PFM (0=Sunday, etc)
+    # p - Number of days from March 21 to Sunday on or before PFM
+    #     (-6 to 28 methods 1 & 3, to 56 for method 2)
+    # e - Extra days to add for method 2 (converting Julian
+    #     date to Gregorian date)
+
+    y = year
+    g = y % 19
+    e = 0
+    if method < 3:
+        # Old method
+        i = (19*g + 15) % 30
+        j = (y + y//4 + i) % 7
+        if method == 2:
+            # Extra dates to convert Julian to Gregorian date
+            e = 10
+            if y > 1600:
+                e = e + y//100 - 16 - (y//100 - 16)//4
+    else:
+        # New method
+        c = y//100
+        h = (c - c//4 - (8*c + 13)//25 + 19*g + 15) % 30
+        i = h - (h//28)*(1 - (h//28)*(29//(h + 1))*((21 - g)//11))
+        j = (y + y//4 + i + 2 - c + c//4) % 7
+
+    # p can be from -6 to 56 corresponding to dates 22 March to 23 May
+    # (later dates apply to method 2, although 23 May never actually occurs)
+    p = i - j + e
+    d = 1 + (p + 27 + (p + 6)//40) % 31
+    m = 3 + (p + 26)//30
+    return datetime.date(int(y), int(m), int(d))

dateutil/parser/__init__.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+from ._parser import parse, parser, parserinfo, ParserError
+from ._parser import DEFAULTPARSER, DEFAULTTZPARSER
+from ._parser import UnknownTimezoneWarning
+
+from ._parser import __doc__
+
+from .isoparser import isoparser, isoparse
+
+__all__ = ['parse', 'parser', 'parserinfo',
+           'isoparse', 'isoparser',
+           'ParserError',
+           'UnknownTimezoneWarning']
+
+
+###
+# Deprecate portions of the private interface so that downstream code that
+# is improperly relying on it is given *some* notice.
+
+
+def __deprecated_private_func(f):
+    from functools import wraps
+    import warnings
+
+    msg = ('{name} is a private function and may break without warning, '
+           'it will be moved and or renamed in future versions.')
+    msg = msg.format(name=f.__name__)
+
+    @wraps(f)
+    def deprecated_func(*args, **kwargs):
+        warnings.warn(msg, DeprecationWarning)
+        return f(*args, **kwargs)
+
+    return deprecated_func
+
+def __deprecate_private_class(c):
+    import warnings
+
+    msg = ('{name} is a private class and may break without warning, '
+           'it will be moved and or renamed in future versions.')
+    msg = msg.format(name=c.__name__)
+
+    class private_class(c):
+        __doc__ = c.__doc__
+
+        def __init__(self, *args, **kwargs):
+            warnings.warn(msg, DeprecationWarning)
+            super(private_class, self).__init__(*args, **kwargs)
+
+    private_class.__name__ = c.__name__
+
+    return private_class
+
+
+from ._parser import _timelex, _resultbase
+from ._parser import _tzparser, _parsetz
+
+_timelex = __deprecate_private_class(_timelex)
+_tzparser = __deprecate_private_class(_tzparser)
+_resultbase = __deprecate_private_class(_resultbase)
+_parsetz = __deprecated_private_func(_parsetz)

dateutil/parser/isoparser.py

@@ -0,0 +1,416 @@
+# -*- coding: utf-8 -*-
+"""
+This module offers a parser for ISO-8601 strings
+
+It is intended to support all valid date, time and datetime formats per the
+ISO-8601 specification.
+
+..versionadded:: 2.7.0
+"""
+from datetime import datetime, timedelta, time, date
+import calendar
+from dateutil import tz
+
+from functools import wraps
+
+import re
+import six
+
+__all__ = ["isoparse", "isoparser"]
+
+
+def _takes_ascii(f):
+    @wraps(f)
+    def func(self, str_in, *args, **kwargs):
+        # If it's a stream, read the whole thing
+        str_in = getattr(str_in, 'read', lambda: str_in)()
+
+        # If it's unicode, turn it into bytes, since ISO-8601 only covers ASCII
+        if isinstance(str_in, six.text_type):
+            # ASCII is the same in UTF-8
+            try:
+                str_in = str_in.encode('ascii')
+            except UnicodeEncodeError as e:
+                msg = 'ISO-8601 strings should contain only ASCII characters'
+                six.raise_from(ValueError(msg), e)
+
+        return f(self, str_in, *args, **kwargs)
+
+    return func
+
+
+class isoparser(object):
+    def __init__(self, sep=None):
+        """
+        :param sep:
+            A single character that separates date and time portions. If
+            ``None``, the parser will accept any single character.
+            For strict ISO-8601 adherence, pass ``'T'``.
+        """
+        if sep is not None:
+            if (len(sep) != 1 or ord(sep) >= 128 or sep in '0123456789'):
+                raise ValueError('Separator must be a single, non-numeric ' +
+                                 'ASCII character')
+
+            sep = sep.encode('ascii')
+
+        self._sep = sep
+
+    @_takes_ascii
+    def isoparse(self, dt_str):
+        """
+        Parse an ISO-8601 datetime string into a :class:`datetime.datetime`.
+
+        An ISO-8601 datetime string consists of a date portion, followed
+        optionally by a time portion - the date and time portions are separated
+        by a single character separator, which is ``T`` in the official
+        standard. Incomplete date formats (such as ``YYYY-MM``) may *not* be
+        combined with a time portion.
+
+        Supported date formats are:
+
+        Common:
+
+        - ``YYYY``
+        - ``YYYY-MM``
+        - ``YYYY-MM-DD`` or ``YYYYMMDD``
+
+        Uncommon:
+
+        - ``YYYY-Www`` or ``YYYYWww`` - ISO week (day defaults to 0)
+        - ``YYYY-Www-D`` or ``YYYYWwwD`` - ISO week and day
+
+        The ISO week and day numbering follows the same logic as
+        :func:`datetime.date.isocalendar`.
+
+        Supported time formats are:
+
+        - ``hh``
+        - ``hh:mm`` or ``hhmm``
+        - ``hh:mm:ss`` or ``hhmmss``
+        - ``hh:mm:ss.ssssss`` (Up to 6 sub-second digits)
+
+        Midnight is a special case for `hh`, as the standard supports both
+        00:00 and 24:00 as a representation. The decimal separator can be
+        either a dot or a comma.
+
+
+        .. caution::
+
+            Support for fractional components other than seconds is part of the
+            ISO-8601 standard, but is not currently implemented in this parser.
+
+        Supported time zone offset formats are:
+
+        - `Z` (UTC)
+        - `±HH:MM`
+        - `±HHMM`
+        - `±HH`
+
+        Offsets will be represented as :class:`dateutil.tz.tzoffset` objects,
+        with the exception of UTC, which will be represented as
+        :class:`dateutil.tz.tzutc`. Time zone offsets equivalent to UTC (such
+        as `+00:00`) will also be represented as :class:`dateutil.tz.tzutc`.
+
+        :param dt_str:
+            A string or stream containing only an ISO-8601 datetime string
+
+        :return:
+            Returns a :class:`datetime.datetime` representing the string.
+            Unspecified components default to their lowest value.
+
+        .. warning::
+
+            As of version 2.7.0, the strictness of the parser should not be
+            considered a stable part of the contract. Any valid ISO-8601 string
+            that parses correctly with the default settings will continue to
+            parse correctly in future versions, but invalid strings that
+            currently fail (e.g. ``2017-01-01T00:00+00:00:00``) are not
+            guaranteed to continue failing in future versions if they encode
+            a valid date.
+
+        .. versionadded:: 2.7.0
+        """
+        components, pos = self._parse_isodate(dt_str)
+
+        if len(dt_str) > pos:
+            if self._sep is None or dt_str[pos:pos + 1] == self._sep:
+                components += self._parse_isotime(dt_str[pos + 1:])
+            else:
+                raise ValueError('String contains unknown ISO components')
+
+        if len(components) > 3 and components[3] == 24:
+            components[3] = 0
+            return datetime(*components) + timedelta(days=1)
+
+        return datetime(*components)
+
+    @_takes_ascii
+    def parse_isodate(self, datestr):
+        """
+        Parse the date portion of an ISO string.
+
+        :param datestr:
+            The string portion of an ISO string, without a separator
+
+        :return:
+            Returns a :class:`datetime.date` object
+        """
+        components, pos = self._parse_isodate(datestr)
+        if pos < len(datestr):
+            raise ValueError('String contains unknown ISO ' +
+                             'components: {!r}'.format(datestr.decode('ascii')))
+        return date(*components)
+
+    @_takes_ascii
+    def parse_isotime(self, timestr):
+        """
+        Parse the time portion of an ISO string.
+
+        :param timestr:
+            The time portion of an ISO string, without a separator
+
+        :return:
+            Returns a :class:`datetime.time` object
+        """
+        components = self._parse_isotime(timestr)
+        if components[0] == 24:
+            components[0] = 0
+        return time(*components)
+
+    @_takes_ascii
+    def parse_tzstr(self, tzstr, zero_as_utc=True):
+        """
+        Parse a valid ISO time zone string.
+
+        See :func:`isoparser.isoparse` for details on supported formats.
+
+        :param tzstr:
+            A string representing an ISO time zone offset
+
+        :param zero_as_utc:
+            Whether to return :class:`dateutil.tz.tzutc` for zero-offset zones
+
+        :return:
+            Returns :class:`dateutil.tz.tzoffset` for offsets and
+            :class:`dateutil.tz.tzutc` for ``Z`` and (if ``zero_as_utc`` is
+            specified) offsets equivalent to UTC.
+        """
+        return self._parse_tzstr(tzstr, zero_as_utc=zero_as_utc)
+
+    # Constants
+    _DATE_SEP = b'-'
+    _TIME_SEP = b':'
+    _FRACTION_REGEX = re.compile(b'[\\.,]([0-9]+)')
+
+    def _parse_isodate(self, dt_str):
+        try:
+            return self._parse_isodate_common(dt_str)
+        except ValueError:
+            return self._parse_isodate_uncommon(dt_str)
+
+    def _parse_isodate_common(self, dt_str):
+        len_str = len(dt_str)
+        components = [1, 1, 1]
+
+        if len_str < 4:
+            raise ValueError('ISO string too short')
+
+        # Year
+        components[0] = int(dt_str[0:4])
+        pos = 4
+        if pos >= len_str:
+            return components, pos
+
+        has_sep = dt_str[pos:pos + 1] == self._DATE_SEP
+        if has_sep:
+            pos += 1
+
+        # Month
+        if len_str - pos < 2:
+            raise ValueError('Invalid common month')
+
+        components[1] = int(dt_str[pos:pos + 2])
+        pos += 2
+
+        if pos >= len_str:
+            if has_sep:
+                return components, pos
+            else:
+                raise ValueError('Invalid ISO format')
+
+        if has_sep:
+            if dt_str[pos:pos + 1] != self._DATE_SEP:
+                raise ValueError('Invalid separator in ISO string')
+            pos += 1
+
+        # Day
+        if len_str - pos < 2:
+            raise ValueError('Invalid common day')
+        components[2] = int(dt_str[pos:pos + 2])
+        return components, pos + 2
+
+    def _parse_isodate_uncommon(self, dt_str):
+        if len(dt_str) < 4:
+            raise ValueError('ISO string too short')
+
+        # All ISO formats start with the year
+        year = int(dt_str[0:4])
+
+        has_sep = dt_str[4:5] == self._DATE_SEP
+
+        pos = 4 + has_sep       # Skip '-' if it's there
+        if dt_str[pos:pos + 1] == b'W':
+            # YYYY-?Www-?D?
+            pos += 1
+            weekno = int(dt_str[pos:pos + 2])
+            pos += 2
+
+            dayno = 1
+            if len(dt_str) > pos:
+                if (dt_str[pos:pos + 1] == self._DATE_SEP) != has_sep:
+                    raise ValueError('Inconsistent use of dash separator')
+
+                pos += has_sep
+
+                dayno = int(dt_str[pos:pos + 1])
+                pos += 1
+
+            base_date = self._calculate_weekdate(year, weekno, dayno)
+        else:
+            # YYYYDDD or YYYY-DDD
+            if len(dt_str) - pos < 3:
+                raise ValueError('Invalid ordinal day')
+
+            ordinal_day = int(dt_str[pos:pos + 3])
+            pos += 3
+
+            if ordinal_day < 1 or ordinal_day > (365 + calendar.isleap(year)):
+                raise ValueError('Invalid ordinal day' +
+                                 ' {} for year {}'.format(ordinal_day, year))
+
+            base_date = date(year, 1, 1) + timedelta(days=ordinal_day - 1)
+
+        components = [base_date.year, base_date.month, base_date.day]
+        return components, pos
+
+    def _calculate_weekdate(self, year, week, day):
+        """
+        Calculate the day of corresponding to the ISO year-week-day calendar.
+
+        This function is effectively the inverse of
+        :func:`datetime.date.isocalendar`.
+
+        :param year:
+            The year in the ISO calendar
+
+        :param week:
+            The week in the ISO calendar - range is [1, 53]
+
+        :param day:
+            The day in the ISO calendar - range is [1 (MON), 7 (SUN)]
+
+        :return:
+            Returns a :class:`datetime.date`
+        """
+        if not 0 < week < 54:
+            raise ValueError('Invalid week: {}'.format(week))
+
+        if not 0 < day < 8:     # Range is 1-7
+            raise ValueError('Invalid weekday: {}'.format(day))
+
+        # Get week 1 for the specific year:
+        jan_4 = date(year, 1, 4)   # Week 1 always has January 4th in it
+        week_1 = jan_4 - timedelta(days=jan_4.isocalendar()[2] - 1)
+
+        # Now add the specific number of weeks and days to get what we want
+        week_offset = (week - 1) * 7 + (day - 1)
+        return week_1 + timedelta(days=week_offset)
+
+    def _parse_isotime(self, timestr):
+        len_str = len(timestr)
+        components = [0, 0, 0, 0, None]
+        pos = 0
+        comp = -1
+
+        if len_str < 2:
+            raise ValueError('ISO time too short')
+
+        has_sep = False
+
+        while pos < len_str and comp < 5:
+            comp += 1
+
+            if timestr[pos:pos + 1] in b'-+Zz':
+                # Detect time zone boundary
+                components[-1] = self._parse_tzstr(timestr[pos:])
+                pos = len_str
+                break
+
+            if comp == 1 and timestr[pos:pos+1] == self._TIME_SEP:
+                has_sep = True
+                pos += 1
+            elif comp == 2 and has_sep:
+                if timestr[pos:pos+1] != self._TIME_SEP:
+                    raise ValueError('Inconsistent use of colon separator')
+                pos += 1
+
+            if comp < 3:
+                # Hour, minute, second
+                components[comp] = int(timestr[pos:pos + 2])
+                pos += 2
+
+            if comp == 3:
+                # Fraction of a second
+                frac = self._FRACTION_REGEX.match(timestr[pos:])
+                if not frac:
+                    continue
+
+                us_str = frac.group(1)[:6]  # Truncate to microseconds
+                components[comp] = int(us_str) * 10**(6 - len(us_str))
+                pos += len(frac.group())
+
+        if pos < len_str:
+            raise ValueError('Unused components in ISO string')
+
+        if components[0] == 24:
+            # Standard supports 00:00 and 24:00 as representations of midnight
+            if any(component != 0 for component in components[1:4]):
+                raise ValueError('Hour may only be 24 at 24:00:00.000')
+
+        return components
+
+    def _parse_tzstr(self, tzstr, zero_as_utc=True):
+        if tzstr == b'Z' or tzstr == b'z':
+            return tz.UTC
+
+        if len(tzstr) not in {3, 5, 6}:
+            raise ValueError('Time zone offset must be 1, 3, 5 or 6 characters')
+
+        if tzstr[0:1] == b'-':
+            mult = -1
+        elif tzstr[0:1] == b'+':
+            mult = 1
+        else:
+            raise ValueError('Time zone offset requires sign')
+
+        hours = int(tzstr[1:3])
+        if len(tzstr) == 3:
+            minutes = 0
+        else:
+            minutes = int(tzstr[(4 if tzstr[3:4] == self._TIME_SEP else 3):])
+
+        if zero_as_utc and hours == 0 and minutes == 0:
+            return tz.UTC
+        else:
+            if minutes > 59:
+                raise ValueError('Invalid minutes in time zone offset')
+
+            if hours > 23:
+                raise ValueError('Invalid hours in time zone offset')
+
+            return tz.tzoffset(None, mult * (hours * 60 + minutes) * 60)
+
+
+DEFAULT_ISOPARSER = isoparser()
+isoparse = DEFAULT_ISOPARSER.isoparse

dateutil/relativedelta.py

@@ -0,0 +1,599 @@
+# -*- coding: utf-8 -*-
+import datetime
+import calendar
+
+import operator
+from math import copysign
+
+from six import integer_types
+from warnings import warn
+
+from ._common import weekday
+
+MO, TU, WE, TH, FR, SA, SU = weekdays = tuple(weekday(x) for x in range(7))
+
+__all__ = ["relativedelta", "MO", "TU", "WE", "TH", "FR", "SA", "SU"]
+
+
+class relativedelta(object):
+    """
+    The relativedelta type is designed to be applied to an existing datetime and
+    can replace specific components of that datetime, or represents an interval
+    of time.
+
+    It is based on the specification of the excellent work done by M.-A. Lemburg
+    in his
+    `mx.DateTime <https://www.egenix.com/products/python/mxBase/mxDateTime/>`_ extension.
+    However, notice that this type does *NOT* implement the same algorithm as
+    his work. Do *NOT* expect it to behave like mx.DateTime's counterpart.
+
+    There are two different ways to build a relativedelta instance. The
+    first one is passing it two date/datetime classes::
+
+        relativedelta(datetime1, datetime2)
+
+    The second one is passing it any number of the following keyword arguments::
+
+        relativedelta(arg1=x,arg2=y,arg3=z...)
+
+        year, month, day, hour, minute, second, microsecond:
+            Absolute information (argument is singular); adding or subtracting a
+            relativedelta with absolute information does not perform an arithmetic
+            operation, but rather REPLACES the corresponding value in the
+            original datetime with the value(s) in relativedelta.
+
+        years, months, weeks, days, hours, minutes, seconds, microseconds:
+            Relative information, may be negative (argument is plural); adding
+            or subtracting a relativedelta with relative information performs
+            the corresponding arithmetic operation on the original datetime value
+            with the information in the relativedelta.
+
+        weekday:
+            One of the weekday instances (MO, TU, etc) available in the
+            relativedelta module. These instances may receive a parameter N,
+            specifying the Nth weekday, which could be positive or negative
+            (like MO(+1) or MO(-2)). Not specifying it is the same as specifying
+            +1. You can also use an integer, where 0=MO. This argument is always
+            relative e.g. if the calculated date is already Monday, using MO(1)
+            or MO(-1) won't change the day. To effectively make it absolute, use
+            it in combination with the day argument (e.g. day=1, MO(1) for first
+            Monday of the month).
+
+        leapdays:
+            Will add given days to the date found, if year is a leap
+            year, and the date found is post 28 of february.
+
+        yearday, nlyearday:
+            Set the yearday or the non-leap year day (jump leap days).
+            These are converted to day/month/leapdays information.
+
+    There are relative and absolute forms of the keyword
+    arguments. The plural is relative, and the singular is
+    absolute. For each argument in the order below, the absolute form
+    is applied first (by setting each attribute to that value) and
+    then the relative form (by adding the value to the attribute).
+
+    The order of attributes considered when this relativedelta is
+    added to a datetime is:
+
+    1. Year
+    2. Month
+    3. Day
+    4. Hours
+    5. Minutes
+    6. Seconds
+    7. Microseconds
+
+    Finally, weekday is applied, using the rule described above.
+
+    For example
+
+    >>> from datetime import datetime
+    >>> from dateutil.relativedelta import relativedelta, MO
+    >>> dt = datetime(2018, 4, 9, 13, 37, 0)
+    >>> delta = relativedelta(hours=25, day=1, weekday=MO(1))
+    >>> dt + delta
+    datetime.datetime(2018, 4, 2, 14, 37)
+
+    First, the day is set to 1 (the first of the month), then 25 hours
+    are added, to get to the 2nd day and 14th hour, finally the
+    weekday is applied, but since the 2nd is already a Monday there is
+    no effect.
+
+    """
+
+    def __init__(self, dt1=None, dt2=None,
+                 years=0, months=0, days=0, leapdays=0, weeks=0,
+                 hours=0, minutes=0, seconds=0, microseconds=0,
+                 year=None, month=None, day=None, weekday=None,
+                 yearday=None, nlyearday=None,
+                 hour=None, minute=None, second=None, microsecond=None):
+
+        if dt1 and dt2:
+            # datetime is a subclass of date. So both must be date
+            if not (isinstance(dt1, datetime.date) and
+                    isinstance(dt2, datetime.date)):
+                raise TypeError("relativedelta only diffs datetime/date")
+
+            # We allow two dates, or two datetimes, so we coerce them to be
+            # of the same type
+            if (isinstance(dt1, datetime.datetime) !=
+                    isinstance(dt2, datetime.datetime)):
+                if not isinstance(dt1, datetime.datetime):
+                    dt1 = datetime.datetime.fromordinal(dt1.toordinal())
+                elif not isinstance(dt2, datetime.datetime):
+                    dt2 = datetime.datetime.fromordinal(dt2.toordinal())
+
+            self.years = 0
+            self.months = 0
+            self.days = 0
+            self.leapdays = 0
+            self.hours = 0
+            self.minutes = 0
+            self.seconds = 0
+            self.microseconds = 0
+            self.year = None
+            self.month = None
+            self.day = None
+            self.weekday = None
+            self.hour = None
+            self.minute = None
+            self.second = None
+            self.microsecond = None
+            self._has_time = 0
+
+            # Get year / month delta between the two
+            months = (dt1.year - dt2.year) * 12 + (dt1.month - dt2.month)
+            self._set_months(months)
+
+            # Remove the year/month delta so the timedelta is just well-defined
+            # time units (seconds, days and microseconds)
+            dtm = self.__radd__(dt2)
+
+            # If we've overshot our target, make an adjustment
+            if dt1 < dt2:
+                compare = operator.gt
+                increment = 1
+            else:
+                compare = operator.lt
+                increment = -1
+
+            while compare(dt1, dtm):
+                months += increment
+                self._set_months(months)
+                dtm = self.__radd__(dt2)
+
+            # Get the timedelta between the "months-adjusted" date and dt1
+            delta = dt1 - dtm
+            self.seconds = delta.seconds + delta.days * 86400
+            self.microseconds = delta.microseconds
+        else:
+            # Check for non-integer values in integer-only quantities
+            if any(x is not None and x != int(x) for x in (years, months)):
+                raise ValueError("Non-integer years and months are "
+                                 "ambiguous and not currently supported.")
+
+            # Relative information
+            self.years = int(years)
+            self.months = int(months)
+            self.days = days + weeks * 7
+            self.leapdays = leapdays
+            self.hours = hours
+            self.minutes = minutes
+            self.seconds = seconds
+            self.microseconds = microseconds
+
+            # Absolute information
+            self.year = year
+            self.month = month
+            self.day = day
+            self.hour = hour
+            self.minute = minute
+            self.second = second
+            self.microsecond = microsecond
+
+            if any(x is not None and int(x) != x
+                   for x in (year, month, day, hour,
+                             minute, second, microsecond)):
+                # For now we'll deprecate floats - later it'll be an error.
+                warn("Non-integer value passed as absolute information. " +
+                     "This is not a well-defined condition and will raise " +
+                     "errors in future versions.", DeprecationWarning)
+
+            if isinstance(weekday, integer_types):
+                self.weekday = weekdays[weekday]
+            else:
+                self.weekday = weekday
+
+            yday = 0
+            if nlyearday:
+                yday = nlyearday
+            elif yearday:
+                yday = yearday
+                if yearday > 59:
+                    self.leapdays = -1
+            if yday:
+                ydayidx = [31, 59, 90, 120, 151, 181, 212,
+                           243, 273, 304, 334, 366]
+                for idx, ydays in enumerate(ydayidx):
+                    if yday <= ydays:
+                        self.month = idx+1
+                        if idx == 0:
+                            self.day = yday
+                        else:
+                            self.day = yday-ydayidx[idx-1]
+                        break
+                else:
+                    raise ValueError("invalid year day (%d)" % yday)
+
+        self._fix()
+
+    def _fix(self):
+        if abs(self.microseconds) > 999999:
+            s = _sign(self.microseconds)
+            div, mod = divmod(self.microseconds * s, 1000000)
+            self.microseconds = mod * s
+            self.seconds += div * s
+        if abs(self.seconds) > 59:
+            s = _sign(self.seconds)
+            div, mod = divmod(self.seconds * s, 60)
+            self.seconds = mod * s
+            self.minutes += div * s
+        if abs(self.minutes) > 59:
+            s = _sign(self.minutes)
+            div, mod = divmod(self.minutes * s, 60)
+            self.minutes = mod * s
+            self.hours += div * s
+        if abs(self.hours) > 23:
+            s = _sign(self.hours)
+            div, mod = divmod(self.hours * s, 24)
+            self.hours = mod * s
+            self.days += div * s
+        if abs(self.months) > 11:
+            s = _sign(self.months)
+            div, mod = divmod(self.months * s, 12)
+            self.months = mod * s
+            self.years += div * s
+        if (self.hours or self.minutes or self.seconds or self.microseconds
+                or self.hour is not None or self.minute is not None or
+                self.second is not None or self.microsecond is not None):
+            self._has_time = 1
+        else:
+            self._has_time = 0
+
+    @property
+    def weeks(self):
+        return int(self.days / 7.0)
+
+    @weeks.setter
+    def weeks(self, value):
+        self.days = self.days - (self.weeks * 7) + value * 7
+
+    def _set_months(self, months):
+        self.months = months
+        if abs(self.months) > 11:
+            s = _sign(self.months)
+            div, mod = divmod(self.months * s, 12)
+            self.months = mod * s
+            self.years = div * s
+        else:
+            self.years = 0
+
+    def normalized(self):
+        """
+        Return a version of this object represented entirely using integer
+        values for the relative attributes.
+
+        >>> relativedelta(days=1.5, hours=2).normalized()
+        relativedelta(days=+1, hours=+14)
+
+        :return:
+            Returns a :class:`dateutil.relativedelta.relativedelta` object.
+        """
+        # Cascade remainders down (rounding each to roughly nearest microsecond)
+        days = int(self.days)
+
+        hours_f = round(self.hours + 24 * (self.days - days), 11)
+        hours = int(hours_f)
+
+        minutes_f = round(self.minutes + 60 * (hours_f - hours), 10)
+        minutes = int(minutes_f)
+
+        seconds_f = round(self.seconds + 60 * (minutes_f - minutes), 8)
+        seconds = int(seconds_f)
+
+        microseconds = round(self.microseconds + 1e6 * (seconds_f - seconds))
+
+        # Constructor carries overflow back up with call to _fix()
+        return self.__class__(years=self.years, months=self.months,
+                              days=days, hours=hours, minutes=minutes,
+                              seconds=seconds, microseconds=microseconds,
+                              leapdays=self.leapdays, year=self.year,
+                              month=self.month, day=self.day,
+                              weekday=self.weekday, hour=self.hour,
+                              minute=self.minute, second=self.second,
+                              microsecond=self.microsecond)
+
+    def __add__(self, other):
+        if isinstance(other, relativedelta):
+            return self.__class__(years=other.years + self.years,
+                                 months=other.months + self.months,
+                                 days=other.days + self.days,
+                                 hours=other.hours + self.hours,
+                                 minutes=other.minutes + self.minutes,
+                                 seconds=other.seconds + self.seconds,
+                                 microseconds=(other.microseconds +
+                                               self.microseconds),
+                                 leapdays=other.leapdays or self.leapdays,
+                                 year=(other.year if other.year is not None
+                                       else self.year),
+                                 month=(other.month if other.month is not None
+                                        else self.month),
+                                 day=(other.day if other.day is not None
+                                      else self.day),
+                                 weekday=(other.weekday if other.weekday is not None
+                                          else self.weekday),
+                                 hour=(other.hour if other.hour is not None
+                                       else self.hour),
+                                 minute=(other.minute if other.minute is not None
+                                         else self.minute),
+                                 second=(other.second if other.second is not None
+                                         else self.second),
+                                 microsecond=(other.microsecond if other.microsecond
+                                              is not None else
+                                              self.microsecond))
+        if isinstance(other, datetime.timedelta):
+            return self.__class__(years=self.years,
+                                  months=self.months,
+                                  days=self.days + other.days,
+                                  hours=self.hours,
+                                  minutes=self.minutes,
+                                  seconds=self.seconds + other.seconds,
+                                  microseconds=self.microseconds + other.microseconds,
+                                  leapdays=self.leapdays,
+                                  year=self.year,
+                                  month=self.month,
+                                  day=self.day,
+                                  weekday=self.weekday,
+                                  hour=self.hour,
+                                  minute=self.minute,
+                                  second=self.second,
+                                  microsecond=self.microsecond)
+        if not isinstance(other, datetime.date):
+            return NotImplemented
+        elif self._has_time and not isinstance(other, datetime.datetime):
+            other = datetime.datetime.fromordinal(other.toordinal())
+        year = (self.year or other.year)+self.years
+        month = self.month or other.month
+        if self.months:
+            assert 1 <= abs(self.months) <= 12
+            month += self.months
+            if month > 12:
+                year += 1
+                month -= 12
+            elif month < 1:
+                year -= 1
+                month += 12
+        day = min(calendar.monthrange(year, month)[1],
+                  self.day or other.day)
+        repl = {"year": year, "month": month, "day": day}
+        for attr in ["hour", "minute", "second", "microsecond"]:
+            value = getattr(self, attr)
+            if value is not None:
+                repl[attr] = value
+        days = self.days
+        if self.leapdays and month > 2 and calendar.isleap(year):
+            days += self.leapdays
+        ret = (other.replace(**repl)
+               + datetime.timedelta(days=days,
+                                    hours=self.hours,
+                                    minutes=self.minutes,
+                                    seconds=self.seconds,
+                                    microseconds=self.microseconds))
+        if self.weekday:
+            weekday, nth = self.weekday.weekday, self.weekday.n or 1
+            jumpdays = (abs(nth) - 1) * 7
+            if nth > 0:
+                jumpdays += (7 - ret.weekday() + weekday) % 7
+            else:
+                jumpdays += (ret.weekday() - weekday) % 7
+                jumpdays *= -1
+            ret += datetime.timedelta(days=jumpdays)
+        return ret
+
+    def __radd__(self, other):
+        return self.__add__(other)
+
+    def __rsub__(self, other):
+        return self.__neg__().__radd__(other)
+
+    def __sub__(self, other):
+        if not isinstance(other, relativedelta):
+            return NotImplemented   # In case the other object defines __rsub__
+        return self.__class__(years=self.years - other.years,
+                             months=self.months - other.months,
+                             days=self.days - other.days,
+                             hours=self.hours - other.hours,
+                             minutes=self.minutes - other.minutes,
+                             seconds=self.seconds - other.seconds,
+                             microseconds=self.microseconds - other.microseconds,
+                             leapdays=self.leapdays or other.leapdays,
+                             year=(self.year if self.year is not None
+                                   else other.year),
+                             month=(self.month if self.month is not None else
+                                    other.month),
+                             day=(self.day if self.day is not None else
+                                  other.day),
+                             weekday=(self.weekday if self.weekday is not None else
+                                      other.weekday),
+                             hour=(self.hour if self.hour is not None else
+                                   other.hour),
+                             minute=(self.minute if self.minute is not None else
+                                     other.minute),
+                             second=(self.second if self.second is not None else
+                                     other.second),
+                             microsecond=(self.microsecond if self.microsecond
+                                          is not None else
+                                          other.microsecond))
+
+    def __abs__(self):
+        return self.__class__(years=abs(self.years),
+                              months=abs(self.months),
+                              days=abs(self.days),
+                              hours=abs(self.hours),
+                              minutes=abs(self.minutes),
+                              seconds=abs(self.seconds),
+                              microseconds=abs(self.microseconds),
+                              leapdays=self.leapdays,
+                              year=self.year,
+                              month=self.month,
+                              day=self.day,
+                              weekday=self.weekday,
+                              hour=self.hour,
+                              minute=self.minute,
+                              second=self.second,
+                              microsecond=self.microsecond)
+
+    def __neg__(self):
+        return self.__class__(years=-self.years,
+                             months=-self.months,
+                             days=-self.days,
+                             hours=-self.hours,
+                             minutes=-self.minutes,
+                             seconds=-self.seconds,
+                             microseconds=-self.microseconds,
+                             leapdays=self.leapdays,
+                             year=self.year,
+                             month=self.month,
+                             day=self.day,
+                             weekday=self.weekday,
+                             hour=self.hour,
+                             minute=self.minute,
+                             second=self.second,
+                             microsecond=self.microsecond)
+
+    def __bool__(self):
+        return not (not self.years and
+                    not self.months and
+                    not self.days and
+                    not self.hours and
+                    not self.minutes and
+                    not self.seconds and
+                    not self.microseconds and
+                    not self.leapdays and
+                    self.year is None and
+                    self.month is None and
+                    self.day is None and
+                    self.weekday is None and
+                    self.hour is None and
+                    self.minute is None and
+                    self.second is None and
+                    self.microsecond is None)
+    # Compatibility with Python 2.x
+    __nonzero__ = __bool__
+
+    def __mul__(self, other):
+        try:
+            f = float(other)
+        except TypeError:
+            return NotImplemented
+
+        return self.__class__(years=int(self.years * f),
+                             months=int(self.months * f),
+                             days=int(self.days * f),
+                             hours=int(self.hours * f),
+                             minutes=int(self.minutes * f),
+                             seconds=int(self.seconds * f),
+                             microseconds=int(self.microseconds * f),
+                             leapdays=self.leapdays,
+                             year=self.year,
+                             month=self.month,
+                             day=self.day,
+                             weekday=self.weekday,
+                             hour=self.hour,
+                             minute=self.minute,
+                             second=self.second,
+                             microsecond=self.microsecond)
+
+    __rmul__ = __mul__
+
+    def __eq__(self, other):
+        if not isinstance(other, relativedelta):
+            return NotImplemented
+        if self.weekday or other.weekday:
+            if not self.weekday or not other.weekday:
+                return False
+            if self.weekday.weekday != other.weekday.weekday:
+                return False
+            n1, n2 = self.weekday.n, other.weekday.n
+            if n1 != n2 and not ((not n1 or n1 == 1) and (not n2 or n2 == 1)):
+                return False
+        return (self.years == other.years and
+                self.months == other.months and
+                self.days == other.days and
+                self.hours == other.hours and
+                self.minutes == other.minutes and
+                self.seconds == other.seconds and
+                self.microseconds == other.microseconds and
+                self.leapdays == other.leapdays and
+                self.year == other.year and
+                self.month == other.month and
+                self.day == other.day and
+                self.hour == other.hour and
+                self.minute == other.minute and
+                self.second == other.second and
+                self.microsecond == other.microsecond)
+
+    def __hash__(self):
+        return hash((
+            self.weekday,
+            self.years,
+            self.months,
+            self.days,
+            self.hours,
+            self.minutes,
+            self.seconds,
+            self.microseconds,
+            self.leapdays,
+            self.year,
+            self.month,
+            self.day,
+            self.hour,
+            self.minute,
+            self.second,
+            self.microsecond,
+        ))
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __div__(self, other):
+        try:
+            reciprocal = 1 / float(other)
+        except TypeError:
+            return NotImplemented
+
+        return self.__mul__(reciprocal)
+
+    __truediv__ = __div__
+
+    def __repr__(self):
+        l = []
+        for attr in ["years", "months", "days", "leapdays",
+                     "hours", "minutes", "seconds", "microseconds"]:
+            value = getattr(self, attr)
+            if value:
+                l.append("{attr}={value:+g}".format(attr=attr, value=value))
+        for attr in ["year", "month", "day", "weekday",
+                     "hour", "minute", "second", "microsecond"]:
+            value = getattr(self, attr)
+            if value is not None:
+                l.append("{attr}={value}".format(attr=attr, value=repr(value)))
+        return "{classname}({attrs})".format(classname=self.__class__.__name__,
+                                             attrs=", ".join(l))
+
+
+def _sign(x):
+    return int(copysign(1, x))
+
+# vim:ts=4:sw=4:et

dateutil/tz/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+from .tz import *
+from .tz import __doc__
+
+__all__ = ["tzutc", "tzoffset", "tzlocal", "tzfile", "tzrange",
+           "tzstr", "tzical", "tzwin", "tzwinlocal", "gettz",
+           "enfold", "datetime_ambiguous", "datetime_exists",
+           "resolve_imaginary", "UTC", "DeprecatedTzFormatWarning"]
+
+
+class DeprecatedTzFormatWarning(Warning):
+    """Warning raised when time zones are parsed from deprecated formats."""

dateutil/tz/_common.py

@@ -0,0 +1,419 @@
+from six import PY2
+
+from functools import wraps
+
+from datetime import datetime, timedelta, tzinfo
+
+
+ZERO = timedelta(0)
+
+__all__ = ['tzname_in_python2', 'enfold']
+
+
+def tzname_in_python2(namefunc):
+    """Change unicode output into bytestrings in Python 2
+
+    tzname() API changed in Python 3. It used to return bytes, but was changed
+    to unicode strings
+    """
+    if PY2:
+        @wraps(namefunc)
+        def adjust_encoding(*args, **kwargs):
+            name = namefunc(*args, **kwargs)
+            if name is not None:
+                name = name.encode()
+
+            return name
+
+        return adjust_encoding
+    else:
+        return namefunc
+
+
+# The following is adapted from Alexander Belopolsky's tz library
+# https://github.com/abalkin/tz
+if hasattr(datetime, 'fold'):
+    # This is the pre-python 3.6 fold situation
+    def enfold(dt, fold=1):
+        """
+        Provides a unified interface for assigning the ``fold`` attribute to
+        datetimes both before and after the implementation of PEP-495.
+
+        :param fold:
+            The value for the ``fold`` attribute in the returned datetime. This
+            should be either 0 or 1.
+
+        :return:
+            Returns an object for which ``getattr(dt, 'fold', 0)`` returns
+            ``fold`` for all versions of Python. In versions prior to
+            Python 3.6, this is a ``_DatetimeWithFold`` object, which is a
+            subclass of :py:class:`datetime.datetime` with the ``fold``
+            attribute added, if ``fold`` is 1.
+
+        .. versionadded:: 2.6.0
+        """
+        return dt.replace(fold=fold)
+
+else:
+    class _DatetimeWithFold(datetime):
+        """
+        This is a class designed to provide a PEP 495-compliant interface for
+        Python versions before 3.6. It is used only for dates in a fold, so
+        the ``fold`` attribute is fixed at ``1``.
+
+        .. versionadded:: 2.6.0
+        """
+        __slots__ = ()
+
+        def replace(self, *args, **kwargs):
+            """
+            Return a datetime with the same attributes, except for those
+            attributes given new values by whichever keyword arguments are
+            specified. Note that tzinfo=None can be specified to create a naive
+            datetime from an aware datetime with no conversion of date and time
+            data.
+
+            This is reimplemented in ``_DatetimeWithFold`` because pypy3 will
+            return a ``datetime.datetime`` even if ``fold`` is unchanged.
+            """
+            argnames = (
+                'year', 'month', 'day', 'hour', 'minute', 'second',
+                'microsecond', 'tzinfo'
+            )
+
+            for arg, argname in zip(args, argnames):
+                if argname in kwargs:
+                    raise TypeError('Duplicate argument: {}'.format(argname))
+
+                kwargs[argname] = arg
+
+            for argname in argnames:
+                if argname not in kwargs:
+                    kwargs[argname] = getattr(self, argname)
+
+            dt_class = self.__class__ if kwargs.get('fold', 1) else datetime
+
+            return dt_class(**kwargs)
+
+        @property
+        def fold(self):
+            return 1
+
+    def enfold(dt, fold=1):
+        """
+        Provides a unified interface for assigning the ``fold`` attribute to
+        datetimes both before and after the implementation of PEP-495.
+
+        :param fold:
+            The value for the ``fold`` attribute in the returned datetime. This
+            should be either 0 or 1.
+
+        :return:
+            Returns an object for which ``getattr(dt, 'fold', 0)`` returns
+            ``fold`` for all versions of Python. In versions prior to
+            Python 3.6, this is a ``_DatetimeWithFold`` object, which is a
+            subclass of :py:class:`datetime.datetime` with the ``fold``
+            attribute added, if ``fold`` is 1.
+
+        .. versionadded:: 2.6.0
+        """
+        if getattr(dt, 'fold', 0) == fold:
+            return dt
+
+        args = dt.timetuple()[:6]
+        args += (dt.microsecond, dt.tzinfo)
+
+        if fold:
+            return _DatetimeWithFold(*args)
+        else:
+            return datetime(*args)
+
+
+def _validate_fromutc_inputs(f):
+    """
+    The CPython version of ``fromutc`` checks that the input is a ``datetime``
+    object and that ``self`` is attached as its ``tzinfo``.
+    """
+    @wraps(f)
+    def fromutc(self, dt):
+        if not isinstance(dt, datetime):
+            raise TypeError("fromutc() requires a datetime argument")
+        if dt.tzinfo is not self:
+            raise ValueError("dt.tzinfo is not self")
+
+        return f(self, dt)
+
+    return fromutc
+
+
+class _tzinfo(tzinfo):
+    """
+    Base class for all ``dateutil`` ``tzinfo`` objects.
+    """
+
+    def is_ambiguous(self, dt):
+        """
+        Whether or not the "wall time" of a given datetime is ambiguous in this
+        zone.
+
+        :param dt:
+            A :py:class:`datetime.datetime`, naive or time zone aware.
+
+
+        :return:
+            Returns ``True`` if ambiguous, ``False`` otherwise.
+
+        .. versionadded:: 2.6.0
+        """
+
+        dt = dt.replace(tzinfo=self)
+
+        wall_0 = enfold(dt, fold=0)
+        wall_1 = enfold(dt, fold=1)
+
+        same_offset = wall_0.utcoffset() == wall_1.utcoffset()
+        same_dt = wall_0.replace(tzinfo=None) == wall_1.replace(tzinfo=None)
+
+        return same_dt and not same_offset
+
+    def _fold_status(self, dt_utc, dt_wall):
+        """
+        Determine the fold status of a "wall" datetime, given a representation
+        of the same datetime as a (naive) UTC datetime. This is calculated based
+        on the assumption that ``dt.utcoffset() - dt.dst()`` is constant for all
+        datetimes, and that this offset is the actual number of hours separating
+        ``dt_utc`` and ``dt_wall``.
+
+        :param dt_utc:
+            Representation of the datetime as UTC
+
+        :param dt_wall:
+            Representation of the datetime as "wall time". This parameter must
+            either have a `fold` attribute or have a fold-naive
+            :class:`datetime.tzinfo` attached, otherwise the calculation may
+            fail.
+        """
+        if self.is_ambiguous(dt_wall):
+            delta_wall = dt_wall - dt_utc
+            _fold = int(delta_wall == (dt_utc.utcoffset() - dt_utc.dst()))
+        else:
+            _fold = 0
+
+        return _fold
+
+    def _fold(self, dt):
+        return getattr(dt, 'fold', 0)
+
+    def _fromutc(self, dt):
+        """
+        Given a timezone-aware datetime in a given timezone, calculates a
+        timezone-aware datetime in a new timezone.
+
+        Since this is the one time that we *know* we have an unambiguous
+        datetime object, we take this opportunity to determine whether the
+        datetime is ambiguous and in a "fold" state (e.g. if it's the first
+        occurrence, chronologically, of the ambiguous datetime).
+
+        :param dt:
+            A timezone-aware :class:`datetime.datetime` object.
+        """
+
+        # Re-implement the algorithm from Python's datetime.py
+        dtoff = dt.utcoffset()
+        if dtoff is None:
+            raise ValueError("fromutc() requires a non-None utcoffset() "
+                             "result")
+
+        # The original datetime.py code assumes that `dst()` defaults to
+        # zero during ambiguous times. PEP 495 inverts this presumption, so
+        # for pre-PEP 495 versions of python, we need to tweak the algorithm.
+        dtdst = dt.dst()
+        if dtdst is None:
+            raise ValueError("fromutc() requires a non-None dst() result")
+        delta = dtoff - dtdst
+
+        dt += delta
+        # Set fold=1 so we can default to being in the fold for
+        # ambiguous dates.
+        dtdst = enfold(dt, fold=1).dst()
+        if dtdst is None:
+            raise ValueError("fromutc(): dt.dst gave inconsistent "
+                             "results; cannot convert")
+        return dt + dtdst
+
+    @_validate_fromutc_inputs
+    def fromutc(self, dt):
+        """
+        Given a timezone-aware datetime in a given timezone, calculates a
+        timezone-aware datetime in a new timezone.
+
+        Since this is the one time that we *know* we have an unambiguous
+        datetime object, we take this opportunity to determine whether the
+        datetime is ambiguous and in a "fold" state (e.g. if it's the first
+        occurrence, chronologically, of the ambiguous datetime).
+
+        :param dt:
+            A timezone-aware :class:`datetime.datetime` object.
+        """
+        dt_wall = self._fromutc(dt)
+
+        # Calculate the fold status given the two datetimes.
+        _fold = self._fold_status(dt, dt_wall)
+
+        # Set the default fold value for ambiguous dates
+        return enfold(dt_wall, fold=_fold)
+
+
+class tzrangebase(_tzinfo):
+    """
+    This is an abstract base class for time zones represented by an annual
+    transition into and out of DST. Child classes should implement the following
+    methods:
+
+        * ``__init__(self, *args, **kwargs)``
+        * ``transitions(self, year)`` - this is expected to return a tuple of
+          datetimes representing the DST on and off transitions in standard
+          time.
+
+    A fully initialized ``tzrangebase`` subclass should also provide the
+    following attributes:
+        * ``hasdst``: Boolean whether or not the zone uses DST.
+        * ``_dst_offset`` / ``_std_offset``: :class:`datetime.timedelta` objects
+          representing the respective UTC offsets.
+        * ``_dst_abbr`` / ``_std_abbr``: Strings representing the timezone short
+          abbreviations in DST and STD, respectively.
+        * ``_hasdst``: Whether or not the zone has DST.
+
+    .. versionadded:: 2.6.0
+    """
+    def __init__(self):
+        raise NotImplementedError('tzrangebase is an abstract base class')
+
+    def utcoffset(self, dt):
+        isdst = self._isdst(dt)
+
+        if isdst is None:
+            return None
+        elif isdst:
+            return self._dst_offset
+        else:
+            return self._std_offset
+
+    def dst(self, dt):
+        isdst = self._isdst(dt)
+
+        if isdst is None:
+            return None
+        elif isdst:
+            return self._dst_base_offset
+        else:
+            return ZERO
+
+    @tzname_in_python2
+    def tzname(self, dt):
+        if self._isdst(dt):
+            return self._dst_abbr
+        else:
+            return self._std_abbr
+
+    def fromutc(self, dt):
+        """ Given a datetime in UTC, return local time """
+        if not isinstance(dt, datetime):
+            raise TypeError("fromutc() requires a datetime argument")
+
+        if dt.tzinfo is not self:
+            raise ValueError("dt.tzinfo is not self")
+
+        # Get transitions - if there are none, fixed offset
+        transitions = self.transitions(dt.year)
+        if transitions is None:
+            return dt + self.utcoffset(dt)
+
+        # Get the transition times in UTC
+        dston, dstoff = transitions
+
+        dston -= self._std_offset
+        dstoff -= self._std_offset
+
+        utc_transitions = (dston, dstoff)
+        dt_utc = dt.replace(tzinfo=None)
+
+        isdst = self._naive_isdst(dt_utc, utc_transitions)
+
+        if isdst:
+            dt_wall = dt + self._dst_offset
+        else:
+            dt_wall = dt + self._std_offset
+
+        _fold = int(not isdst and self.is_ambiguous(dt_wall))
+
+        return enfold(dt_wall, fold=_fold)
+
+    def is_ambiguous(self, dt):
+        """
+        Whether or not the "wall time" of a given datetime is ambiguous in this
+        zone.
+
+        :param dt:
+            A :py:class:`datetime.datetime`, naive or time zone aware.
+
+
+        :return:
+            Returns ``True`` if ambiguous, ``False`` otherwise.
+
+        .. versionadded:: 2.6.0
+        """
+        if not self.hasdst:
+            return False
+
+        start, end = self.transitions(dt.year)
+
+        dt = dt.replace(tzinfo=None)
+        return (end <= dt < end + self._dst_base_offset)
+
+    def _isdst(self, dt):
+        if not self.hasdst:
+            return False
+        elif dt is None:
+            return None
+
+        transitions = self.transitions(dt.year)
+
+        if transitions is None:
+            return False
+
+        dt = dt.replace(tzinfo=None)
+
+        isdst = self._naive_isdst(dt, transitions)
+
+        # Handle ambiguous dates
+        if not isdst and self.is_ambiguous(dt):
+            return not self._fold(dt)
+        else:
+            return isdst
+
+    def _naive_isdst(self, dt, transitions):
+        dston, dstoff = transitions
+
+        dt = dt.replace(tzinfo=None)
+
+        if dston < dstoff:
+            isdst = dston <= dt < dstoff
+        else:
+            isdst = not dstoff <= dt < dston
+
+        return isdst
+
+    @property
+    def _dst_base_offset(self):
+        return self._dst_offset - self._std_offset
+
+    __hash__ = None
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        return "%s(...)" % self.__class__.__name__
+
+    __reduce__ = object.__reduce__

dateutil/tz/_factories.py

@@ -0,0 +1,80 @@
+from datetime import timedelta
+import weakref
+from collections import OrderedDict
+
+from six.moves import _thread
+
+
+class _TzSingleton(type):
+    def __init__(cls, *args, **kwargs):
+        cls.__instance = None
+        super(_TzSingleton, cls).__init__(*args, **kwargs)
+
+    def __call__(cls):
+        if cls.__instance is None:
+            cls.__instance = super(_TzSingleton, cls).__call__()
+        return cls.__instance
+
+
+class _TzFactory(type):
+    def instance(cls, *args, **kwargs):
+        """Alternate constructor that returns a fresh instance"""
+        return type.__call__(cls, *args, **kwargs)
+
+
+class _TzOffsetFactory(_TzFactory):
+    def __init__(cls, *args, **kwargs):
+        cls.__instances = weakref.WeakValueDictionary()
+        cls.__strong_cache = OrderedDict()
+        cls.__strong_cache_size = 8
+
+        cls._cache_lock = _thread.allocate_lock()
+
+    def __call__(cls, name, offset):
+        if isinstance(offset, timedelta):
+            key = (name, offset.total_seconds())
+        else:
+            key = (name, offset)
+
+        instance = cls.__instances.get(key, None)
+        if instance is None:
+            instance = cls.__instances.setdefault(key,
+                                                  cls.instance(name, offset))
+
+        # This lock may not be necessary in Python 3. See GH issue #901
+        with cls._cache_lock:
+            cls.__strong_cache[key] = cls.__strong_cache.pop(key, instance)
+
+            # Remove an item if the strong cache is overpopulated
+            if len(cls.__strong_cache) > cls.__strong_cache_size:
+                cls.__strong_cache.popitem(last=False)
+
+        return instance
+
+
+class _TzStrFactory(_TzFactory):
+    def __init__(cls, *args, **kwargs):
+        cls.__instances = weakref.WeakValueDictionary()
+        cls.__strong_cache = OrderedDict()
+        cls.__strong_cache_size = 8
+
+        cls.__cache_lock = _thread.allocate_lock()
+
+    def __call__(cls, s, posix_offset=False):
+        key = (s, posix_offset)
+        instance = cls.__instances.get(key, None)
+
+        if instance is None:
+            instance = cls.__instances.setdefault(key,
+                cls.instance(s, posix_offset))
+
+        # This lock may not be necessary in Python 3. See GH issue #901
+        with cls.__cache_lock:
+            cls.__strong_cache[key] = cls.__strong_cache.pop(key, instance)
+
+            # Remove an item if the strong cache is overpopulated
+            if len(cls.__strong_cache) > cls.__strong_cache_size:
+                cls.__strong_cache.popitem(last=False)
+
+        return instance
+

dateutil/tz/win.py

@@ -0,0 +1,370 @@
+# -*- coding: utf-8 -*-
+"""
+This module provides an interface to the native time zone data on Windows,
+including :py:class:`datetime.tzinfo` implementations.
+
+Attempting to import this module on a non-Windows platform will raise an
+:py:obj:`ImportError`.
+"""
+# This code was originally contributed by Jeffrey Harris.
+import datetime
+import struct
+
+from six.moves import winreg
+from six import text_type
+
+try:
+    import ctypes
+    from ctypes import wintypes
+except ValueError:
+    # ValueError is raised on non-Windows systems for some horrible reason.
+    raise ImportError("Running tzwin on non-Windows system")
+
+from ._common import tzrangebase
+
+__all__ = ["tzwin", "tzwinlocal", "tzres"]
+
+ONEWEEK = datetime.timedelta(7)
+
+TZKEYNAMENT = r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Time Zones"
+TZKEYNAME9X = r"SOFTWARE\Microsoft\Windows\CurrentVersion\Time Zones"
+TZLOCALKEYNAME = r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation"
+
+
+def _settzkeyname():
+    handle = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE)
+    try:
+        winreg.OpenKey(handle, TZKEYNAMENT).Close()
+        TZKEYNAME = TZKEYNAMENT
+    except WindowsError:
+        TZKEYNAME = TZKEYNAME9X
+    handle.Close()
+    return TZKEYNAME
+
+
+TZKEYNAME = _settzkeyname()
+
+
+class tzres(object):
+    """
+    Class for accessing ``tzres.dll``, which contains timezone name related
+    resources.
+
+    .. versionadded:: 2.5.0
+    """
+    p_wchar = ctypes.POINTER(wintypes.WCHAR)        # Pointer to a wide char
+
+    def __init__(self, tzres_loc='tzres.dll'):
+        # Load the user32 DLL so we can load strings from tzres
+        user32 = ctypes.WinDLL('user32')
+
+        # Specify the LoadStringW function
+        user32.LoadStringW.argtypes = (wintypes.HINSTANCE,
+                                       wintypes.UINT,
+                                       wintypes.LPWSTR,
+                                       ctypes.c_int)
+
+        self.LoadStringW = user32.LoadStringW
+        self._tzres = ctypes.WinDLL(tzres_loc)
+        self.tzres_loc = tzres_loc
+
+    def load_name(self, offset):
+        """
+        Load a timezone name from a DLL offset (integer).
+
+        >>> from dateutil.tzwin import tzres
+        >>> tzr = tzres()
+        >>> print(tzr.load_name(112))
+        'Eastern Standard Time'
+
+        :param offset:
+            A positive integer value referring to a string from the tzres dll.
+
+        .. note::
+
+            Offsets found in the registry are generally of the form
+            ``@tzres.dll,-114``. The offset in this case is 114, not -114.
+
+        """
+        resource = self.p_wchar()
+        lpBuffer = ctypes.cast(ctypes.byref(resource), wintypes.LPWSTR)
+        nchar = self.LoadStringW(self._tzres._handle, offset, lpBuffer, 0)
+        return resource[:nchar]
+
+    def name_from_string(self, tzname_str):
+        """
+        Parse strings as returned from the Windows registry into the time zone
+        name as defined in the registry.
+
+        >>> from dateutil.tzwin import tzres
+        >>> tzr = tzres()
+        >>> print(tzr.name_from_string('@tzres.dll,-251'))
+        'Dateline Daylight Time'
+        >>> print(tzr.name_from_string('Eastern Standard Time'))
+        'Eastern Standard Time'
+
+        :param tzname_str:
+            A timezone name string as returned from a Windows registry key.
+
+        :return:
+            Returns the localized timezone string from tzres.dll if the string
+            is of the form `@tzres.dll,-offset`, else returns the input string.
+        """
+        if not tzname_str.startswith('@'):
+            return tzname_str
+
+        name_splt = tzname_str.split(',-')
+        try:
+            offset = int(name_splt[1])
+        except:
+            raise ValueError("Malformed timezone string.")
+
+        return self.load_name(offset)
+
+
+class tzwinbase(tzrangebase):
+    """tzinfo class based on win32's timezones available in the registry."""
+    def __init__(self):
+        raise NotImplementedError('tzwinbase is an abstract base class')
+
+    def __eq__(self, other):
+        # Compare on all relevant dimensions, including name.
+        if not isinstance(other, tzwinbase):
+            return NotImplemented
+
+        return  (self._std_offset == other._std_offset and
+                 self._dst_offset == other._dst_offset and
+                 self._stddayofweek == other._stddayofweek and
+                 self._dstdayofweek == other._dstdayofweek and
+                 self._stdweeknumber == other._stdweeknumber and
+                 self._dstweeknumber == other._dstweeknumber and
+                 self._stdhour == other._stdhour and
+                 self._dsthour == other._dsthour and
+                 self._stdminute == other._stdminute and
+                 self._dstminute == other._dstminute and
+                 self._std_abbr == other._std_abbr and
+                 self._dst_abbr == other._dst_abbr)
+
+    @staticmethod
+    def list():
+        """Return a list of all time zones known to the system."""
+        with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
+            with winreg.OpenKey(handle, TZKEYNAME) as tzkey:
+                result = [winreg.EnumKey(tzkey, i)
+                          for i in range(winreg.QueryInfoKey(tzkey)[0])]
+        return result
+
+    def display(self):
+        """
+        Return the display name of the time zone.
+        """
+        return self._display
+
+    def transitions(self, year):
+        """
+        For a given year, get the DST on and off transition times, expressed
+        always on the standard time side. For zones with no transitions, this
+        function returns ``None``.
+
+        :param year:
+            The year whose transitions you would like to query.
+
+        :return:
+            Returns a :class:`tuple` of :class:`datetime.datetime` objects,
+            ``(dston, dstoff)`` for zones with an annual DST transition, or
+            ``None`` for fixed offset zones.
+        """
+
+        if not self.hasdst:
+            return None
+
+        dston = picknthweekday(year, self._dstmonth, self._dstdayofweek,
+                               self._dsthour, self._dstminute,
+                               self._dstweeknumber)
+
+        dstoff = picknthweekday(year, self._stdmonth, self._stddayofweek,
+                                self._stdhour, self._stdminute,
+                                self._stdweeknumber)
+
+        # Ambiguous dates default to the STD side
+        dstoff -= self._dst_base_offset
+
+        return dston, dstoff
+
+    def _get_hasdst(self):
+        return self._dstmonth != 0
+
+    @property
+    def _dst_base_offset(self):
+        return self._dst_base_offset_
+
+
+class tzwin(tzwinbase):
+    """
+    Time zone object created from the zone info in the Windows registry
+
+    These are similar to :py:class:`dateutil.tz.tzrange` objects in that
+    the time zone data is provided in the format of a single offset rule
+    for either 0 or 2 time zone transitions per year.
+
+    :param: name
+        The name of a Windows time zone key, e.g. "Eastern Standard Time".
+        The full list of keys can be retrieved with :func:`tzwin.list`.
+    """
+
+    def __init__(self, name):
+        self._name = name
+
+        with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
+            tzkeyname = text_type("{kn}\\{name}").format(kn=TZKEYNAME, name=name)
+            with winreg.OpenKey(handle, tzkeyname) as tzkey:
+                keydict = valuestodict(tzkey)
+
+        self._std_abbr = keydict["Std"]
+        self._dst_abbr = keydict["Dlt"]
+
+        self._display = keydict["Display"]
+
+        # See http://ww_winreg.jsiinc.com/SUBA/tip0300/rh0398.htm
+        tup = struct.unpack("=3l16h", keydict["TZI"])
+        stdoffset = -tup[0]-tup[1]          # Bias + StandardBias * -1
+        dstoffset = stdoffset-tup[2]        # + DaylightBias * -1
+        self._std_offset = datetime.timedelta(minutes=stdoffset)
+        self._dst_offset = datetime.timedelta(minutes=dstoffset)
+
+        # for the meaning see the win32 TIME_ZONE_INFORMATION structure docs
+        # http://msdn.microsoft.com/en-us/library/windows/desktop/ms725481(v=vs.85).aspx
+        (self._stdmonth,
+         self._stddayofweek,   # Sunday = 0
+         self._stdweeknumber,  # Last = 5
+         self._stdhour,
+         self._stdminute) = tup[4:9]
+
+        (self._dstmonth,
+         self._dstdayofweek,   # Sunday = 0
+         self._dstweeknumber,  # Last = 5
+         self._dsthour,
+         self._dstminute) = tup[12:17]
+
+        self._dst_base_offset_ = self._dst_offset - self._std_offset
+        self.hasdst = self._get_hasdst()
+
+    def __repr__(self):
+        return "tzwin(%s)" % repr(self._name)
+
+    def __reduce__(self):
+        return (self.__class__, (self._name,))
+
+
+class tzwinlocal(tzwinbase):
+    """
+    Class representing the local time zone information in the Windows registry
+
+    While :class:`dateutil.tz.tzlocal` makes system calls (via the :mod:`time`
+    module) to retrieve time zone information, ``tzwinlocal`` retrieves the
+    rules directly from the Windows registry and creates an object like
+    :class:`dateutil.tz.tzwin`.
+
+    Because Windows does not have an equivalent of :func:`time.tzset`, on
+    Windows, :class:`dateutil.tz.tzlocal` instances will always reflect the
+    time zone settings *at the time that the process was started*, meaning
+    changes to the machine's time zone settings during the run of a program
+    on Windows will **not** be reflected by :class:`dateutil.tz.tzlocal`.
+    Because ``tzwinlocal`` reads the registry directly, it is unaffected by
+    this issue.
+    """
+    def __init__(self):
+        with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
+            with winreg.OpenKey(handle, TZLOCALKEYNAME) as tzlocalkey:
+                keydict = valuestodict(tzlocalkey)
+
+            self._std_abbr = keydict["StandardName"]
+            self._dst_abbr = keydict["DaylightName"]
+
+            try:
+                tzkeyname = text_type('{kn}\\{sn}').format(kn=TZKEYNAME,
+                                                          sn=self._std_abbr)
+                with winreg.OpenKey(handle, tzkeyname) as tzkey:
+                    _keydict = valuestodict(tzkey)
+                    self._display = _keydict["Display"]
+            except OSError:
+                self._display = None
+
+        stdoffset = -keydict["Bias"]-keydict["StandardBias"]
+        dstoffset = stdoffset-keydict["DaylightBias"]
+
+        self._std_offset = datetime.timedelta(minutes=stdoffset)
+        self._dst_offset = datetime.timedelta(minutes=dstoffset)
+
+        # For reasons unclear, in this particular key, the day of week has been
+        # moved to the END of the SYSTEMTIME structure.
+        tup = struct.unpack("=8h", keydict["StandardStart"])
+
+        (self._stdmonth,
+         self._stdweeknumber,  # Last = 5
+         self._stdhour,
+         self._stdminute) = tup[1:5]
+
+        self._stddayofweek = tup[7]
+
+        tup = struct.unpack("=8h", keydict["DaylightStart"])
+
+        (self._dstmonth,
+         self._dstweeknumber,  # Last = 5
+         self._dsthour,
+         self._dstminute) = tup[1:5]
+
+        self._dstdayofweek = tup[7]
+
+        self._dst_base_offset_ = self._dst_offset - self._std_offset
+        self.hasdst = self._get_hasdst()
+
+    def __repr__(self):
+        return "tzwinlocal()"
+
+    def __str__(self):
+        # str will return the standard name, not the daylight name.
+        return "tzwinlocal(%s)" % repr(self._std_abbr)
+
+    def __reduce__(self):
+        return (self.__class__, ())
+
+
+def picknthweekday(year, month, dayofweek, hour, minute, whichweek):
+    """ dayofweek == 0 means Sunday, whichweek 5 means last instance """
+    first = datetime.datetime(year, month, 1, hour, minute)
+
+    # This will work if dayofweek is ISO weekday (1-7) or Microsoft-style (0-6),
+    # Because 7 % 7 = 0
+    weekdayone = first.replace(day=((dayofweek - first.isoweekday()) % 7) + 1)
+    wd = weekdayone + ((whichweek - 1) * ONEWEEK)
+    if (wd.month != month):
+        wd -= ONEWEEK
+
+    return wd
+
+
+def valuestodict(key):
+    """Convert a registry key's values to a dictionary."""
+    dout = {}
+    size = winreg.QueryInfoKey(key)[1]
+    tz_res = None
+
+    for i in range(size):
+        key_name, value, dtype = winreg.EnumValue(key, i)
+        if dtype == winreg.REG_DWORD or dtype == winreg.REG_DWORD_LITTLE_ENDIAN:
+            # If it's a DWORD (32-bit integer), it's stored as unsigned - convert
+            # that to a proper signed integer
+            if value & (1 << 31):
+                value = value - (1 << 32)
+        elif dtype == winreg.REG_SZ:
+            # If it's a reference to the tzres DLL, load the actual string
+            if value.startswith('@tzres'):
+                tz_res = tz_res or tzres()
+                value = tz_res.name_from_string(value)
+
+            value = value.rstrip('\x00')    # Remove trailing nulls
+
+        dout[key_name] = value
+
+    return dout

dateutil/tzwin.py

@@ -0,0 +1,2 @@
+# tzwin has moved to dateutil.tz.win
+from .tz.win import *

dateutil/utils.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+"""
+This module offers general convenience and utility functions for dealing with
+datetimes.
+
+.. versionadded:: 2.7.0
+"""
+from __future__ import unicode_literals
+
+from datetime import datetime, time
+
+
+def today(tzinfo=None):
+    """
+    Returns a :py:class:`datetime` representing the current day at midnight
+
+    :param tzinfo:
+        The time zone to attach (also used to determine the current day).
+
+    :return:
+        A :py:class:`datetime.datetime` object representing the current day
+        at midnight.
+    """
+
+    dt = datetime.now(tzinfo)
+    return datetime.combine(dt.date(), time(0, tzinfo=tzinfo))
+
+
+def default_tzinfo(dt, tzinfo):
+    """
+    Sets the ``tzinfo`` parameter on naive datetimes only
+
+    This is useful for example when you are provided a datetime that may have
+    either an implicit or explicit time zone, such as when parsing a time zone
+    string.
+
+    .. doctest::
+
+        >>> from dateutil.tz import tzoffset
+        >>> from dateutil.parser import parse
+        >>> from dateutil.utils import default_tzinfo
+        >>> dflt_tz = tzoffset("EST", -18000)
+        >>> print(default_tzinfo(parse('2014-01-01 12:30 UTC'), dflt_tz))
+        2014-01-01 12:30:00+00:00
+        >>> print(default_tzinfo(parse('2014-01-01 12:30'), dflt_tz))
+        2014-01-01 12:30:00-05:00
+
+    :param dt:
+        The datetime on which to replace the time zone
+
+    :param tzinfo:
+        The :py:class:`datetime.tzinfo` subclass instance to assign to
+        ``dt`` if (and only if) it is naive.
+
+    :return:
+        Returns an aware :py:class:`datetime.datetime`.
+    """
+    if dt.tzinfo is not None:
+        return dt
+    else:
+        return dt.replace(tzinfo=tzinfo)
+
+
+def within_delta(dt1, dt2, delta):
+    """
+    Useful for comparing two datetimes that may have a negligible difference
+    to be considered equal.
+    """
+    delta = abs(delta)
+    difference = dt1 - dt2
+    return -delta <= difference <= delta

dateutil/zoneinfo/__init__.py

@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+import warnings
+import json
+
+from tarfile import TarFile
+from pkgutil import get_data
+from io import BytesIO
+
+from dateutil.tz import tzfile as _tzfile
+
+__all__ = ["get_zonefile_instance", "gettz", "gettz_db_metadata"]
+
+ZONEFILENAME = "dateutil-zoneinfo.tar.gz"
+METADATA_FN = 'METADATA'
+
+
+class tzfile(_tzfile):
+    def __reduce__(self):
+        return (gettz, (self._filename,))
+
+
+def getzoneinfofile_stream():
+    try:
+        return BytesIO(get_data(__name__, ZONEFILENAME))
+    except IOError as e:  # TODO  switch to FileNotFoundError?
+        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
+        return None
+
+
+class ZoneInfoFile(object):
+    def __init__(self, zonefile_stream=None):
+        if zonefile_stream is not None:
+            with TarFile.open(fileobj=zonefile_stream) as tf:
+                self.zones = {zf.name: tzfile(tf.extractfile(zf), filename=zf.name)
+                              for zf in tf.getmembers()
+                              if zf.isfile() and zf.name != METADATA_FN}
+                # deal with links: They'll point to their parent object. Less
+                # waste of memory
+                links = {zl.name: self.zones[zl.linkname]
+                         for zl in tf.getmembers() if
+                         zl.islnk() or zl.issym()}
+                self.zones.update(links)
+                try:
+                    metadata_json = tf.extractfile(tf.getmember(METADATA_FN))
+                    metadata_str = metadata_json.read().decode('UTF-8')
+                    self.metadata = json.loads(metadata_str)
+                except KeyError:
+                    # no metadata in tar file
+                    self.metadata = None
+        else:
+            self.zones = {}
+            self.metadata = None
+
+    def get(self, name, default=None):
+        """
+        Wrapper for :func:`ZoneInfoFile.zones.get`. This is a convenience method
+        for retrieving zones from the zone dictionary.
+
+        :param name:
+            The name of the zone to retrieve. (Generally IANA zone names)
+
+        :param default:
+            The value to return in the event of a missing key.
+
+        .. versionadded:: 2.6.0
+
+        """
+        return self.zones.get(name, default)
+
+
+# The current API has gettz as a module function, although in fact it taps into
+# a stateful class. So as a workaround for now, without changing the API, we
+# will create a new "global" class instance the first time a user requests a
+# timezone. Ugly, but adheres to the api.
+#
+# TODO: Remove after deprecation period.
+_CLASS_ZONE_INSTANCE = []
+
+
+def get_zonefile_instance(new_instance=False):
+    """
+    This is a convenience function which provides a :class:`ZoneInfoFile`
+    instance using the data provided by the ``dateutil`` package. By default, it
+    caches a single instance of the ZoneInfoFile object and returns that.
+
+    :param new_instance:
+        If ``True``, a new instance of :class:`ZoneInfoFile` is instantiated and
+        used as the cached instance for the next call. Otherwise, new instances
+        are created only as necessary.
+
+    :return:
+        Returns a :class:`ZoneInfoFile` object.
+
+    .. versionadded:: 2.6
+    """
+    if new_instance:
+        zif = None
+    else:
+        zif = getattr(get_zonefile_instance, '_cached_instance', None)
+
+    if zif is None:
+        zif = ZoneInfoFile(getzoneinfofile_stream())
+
+        get_zonefile_instance._cached_instance = zif
+
+    return zif
+
+
+def gettz(name):
+    """
+    This retrieves a time zone from the local zoneinfo tarball that is packaged
+    with dateutil.
+
+    :param name:
+        An IANA-style time zone name, as found in the zoneinfo file.
+
+    :return:
+        Returns a :class:`dateutil.tz.tzfile` time zone object.
+
+    .. warning::
+        It is generally inadvisable to use this function, and it is only
+        provided for API compatibility with earlier versions. This is *not*
+        equivalent to ``dateutil.tz.gettz()``, which selects an appropriate
+        time zone based on the inputs, favoring system zoneinfo. This is ONLY
+        for accessing the dateutil-specific zoneinfo (which may be out of
+        date compared to the system zoneinfo).
+
+    .. deprecated:: 2.6
+        If you need to use a specific zoneinfofile over the system zoneinfo,
+        instantiate a :class:`dateutil.zoneinfo.ZoneInfoFile` object and call
+        :func:`dateutil.zoneinfo.ZoneInfoFile.get(name)` instead.
+
+        Use :func:`get_zonefile_instance` to retrieve an instance of the
+        dateutil-provided zoneinfo.
+    """
+    warnings.warn("zoneinfo.gettz() will be removed in future versions, "
+                  "to use the dateutil-provided zoneinfo files, instantiate a "
+                  "ZoneInfoFile object and use ZoneInfoFile.zones.get() "
+                  "instead. See the documentation for details.",
+                  DeprecationWarning)
+
+    if len(_CLASS_ZONE_INSTANCE) == 0:
+        _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
+    return _CLASS_ZONE_INSTANCE[0].zones.get(name)
+
+
+def gettz_db_metadata():
+    """ Get the zonefile metadata
+
+    See `zonefile_metadata`_
+
+    :returns:
+        A dictionary with the database metadata
+
+    .. deprecated:: 2.6
+        See deprecation warning in :func:`zoneinfo.gettz`. To get metadata,
+        query the attribute ``zoneinfo.ZoneInfoFile.metadata``.
+    """
+    warnings.warn("zoneinfo.gettz_db_metadata() will be removed in future "
+                  "versions, to use the dateutil-provided zoneinfo files, "
+                  "ZoneInfoFile object and query the 'metadata' attribute "
+                  "instead. See the documentation for details.",
+                  DeprecationWarning)
+
+    if len(_CLASS_ZONE_INSTANCE) == 0:
+        _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
+    return _CLASS_ZONE_INSTANCE[0].metadata

dateutil/zoneinfo/rebuild.py

@@ -0,0 +1,75 @@
+import logging
+import os
+import tempfile
+import shutil
+import json
+from subprocess import check_call, check_output
+from tarfile import TarFile
+
+from dateutil.zoneinfo import METADATA_FN, ZONEFILENAME
+
+
+def rebuild(filename, tag=None, format="gz", zonegroups=[], metadata=None):
+    """Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar*
+
+    filename is the timezone tarball from ``ftp.iana.org/tz``.
+
+    """
+    tmpdir = tempfile.mkdtemp()
+    zonedir = os.path.join(tmpdir, "zoneinfo")
+    moduledir = os.path.dirname(__file__)
+    try:
+        with TarFile.open(filename) as tf:
+            for name in zonegroups:
+                tf.extract(name, tmpdir)
+            filepaths = [os.path.join(tmpdir, n) for n in zonegroups]
+
+            _run_zic(zonedir, filepaths)
+
+        # write metadata file
+        with open(os.path.join(zonedir, METADATA_FN), 'w') as f:
+            json.dump(metadata, f, indent=4, sort_keys=True)
+        target = os.path.join(moduledir, ZONEFILENAME)
+        with TarFile.open(target, "w:%s" % format) as tf:
+            for entry in os.listdir(zonedir):
+                entrypath = os.path.join(zonedir, entry)
+                tf.add(entrypath, entry)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def _run_zic(zonedir, filepaths):
+    """Calls the ``zic`` compiler in a compatible way to get a "fat" binary.
+
+    Recent versions of ``zic`` default to ``-b slim``, while older versions
+    don't even have the ``-b`` option (but default to "fat" binaries). The
+    current version of dateutil does not support Version 2+ TZif files, which
+    causes problems when used in conjunction with "slim" binaries, so this
+    function is used to ensure that we always get a "fat" binary.
+    """
+
+    try:
+        help_text = check_output(["zic", "--help"])
+    except OSError as e:
+        _print_on_nosuchfile(e)
+        raise
+
+    if b"-b " in help_text:
+        bloat_args = ["-b", "fat"]
+    else:
+        bloat_args = []
+
+    check_call(["zic"] + bloat_args + ["-d", zonedir] + filepaths)
+
+
+def _print_on_nosuchfile(e):
+    """Print helpful troubleshooting message
+
+    e is an exception raised by subprocess.check_call()
+
+    """
+    if e.errno == 2:
+        logging.error(
+            "Could not find zic. Perhaps you need to install "
+            "libc-bin or some other package that provides it, "
+            "or it's not in your PATH?")

frozenlist/__init__.py

@@ -0,0 +1,95 @@
+import os
+import sys
+import types
+from collections.abc import MutableSequence
+from functools import total_ordering
+from typing import Type
+
+__version__ = "1.4.0"
+
+__all__ = ("FrozenList", "PyFrozenList")  # type: Tuple[str, ...]
+
+
+NO_EXTENSIONS = bool(os.environ.get("FROZENLIST_NO_EXTENSIONS"))  # type: bool
+
+
+@total_ordering
+class FrozenList(MutableSequence):
+    __slots__ = ("_frozen", "_items")
+
+    if sys.version_info >= (3, 9):
+        __class_getitem__ = classmethod(types.GenericAlias)
+    else:
+
+        @classmethod
+        def __class_getitem__(cls: Type["FrozenList"]) -> Type["FrozenList"]:
+            return cls
+
+    def __init__(self, items=None):
+        self._frozen = False
+        if items is not None:
+            items = list(items)
+        else:
+            items = []
+        self._items = items
+
+    @property
+    def frozen(self):
+        return self._frozen
+
+    def freeze(self):
+        self._frozen = True
+
+    def __getitem__(self, index):
+        return self._items[index]
+
+    def __setitem__(self, index, value):
+        if self._frozen:
+            raise RuntimeError("Cannot modify frozen list.")
+        self._items[index] = value
+
+    def __delitem__(self, index):
+        if self._frozen:
+            raise RuntimeError("Cannot modify frozen list.")
+        del self._items[index]
+
+    def __len__(self):
+        return self._items.__len__()
+
+    def __iter__(self):
+        return self._items.__iter__()
+
+    def __reversed__(self):
+        return self._items.__reversed__()
+
+    def __eq__(self, other):
+        return list(self) == other
+
+    def __le__(self, other):
+        return list(self) <= other
+
+    def insert(self, pos, item):
+        if self._frozen:
+            raise RuntimeError("Cannot modify frozen list.")
+        self._items.insert(pos, item)
+
+    def __repr__(self):
+        return f"<FrozenList(frozen={self._frozen}, {self._items!r})>"
+
+    def __hash__(self):
+        if self._frozen:
+            return hash(tuple(self))
+        else:
+            raise RuntimeError("Cannot hash unfrozen list.")
+
+
+PyFrozenList = FrozenList
+
+
+try:
+    from ._frozenlist import FrozenList as CFrozenList  # type: ignore
+
+    if not NO_EXTENSIONS:  # pragma: no cover
+        FrozenList = CFrozenList  # type: ignore
+except ImportError:  # pragma: no cover
+    pass

frozenlist/__init__.pyi

@@ -0,0 +1,47 @@
+from typing import (
+    Generic,
+    Iterable,
+    Iterator,
+    List,
+    MutableSequence,
+    Optional,
+    TypeVar,
+    Union,
+    overload,
+)
+
+_T = TypeVar("_T")
+_Arg = Union[List[_T], Iterable[_T]]
+
+class FrozenList(MutableSequence[_T], Generic[_T]):
+    def __init__(self, items: Optional[_Arg[_T]] = None) -> None: ...
+    @property
+    def frozen(self) -> bool: ...
+    def freeze(self) -> None: ...
+    @overload
+    def __getitem__(self, i: int) -> _T: ...
+    @overload
+    def __getitem__(self, s: slice) -> FrozenList[_T]: ...
+    @overload
+    def __setitem__(self, i: int, o: _T) -> None: ...
+    @overload
+    def __setitem__(self, s: slice, o: Iterable[_T]) -> None: ...
+    @overload
+    def __delitem__(self, i: int) -> None: ...
+    @overload
+    def __delitem__(self, i: slice) -> None: ...
+    def __len__(self) -> int: ...
+    def __iter__(self) -> Iterator[_T]: ...
+    def __reversed__(self) -> Iterator[_T]: ...
+    def __eq__(self, other: object) -> bool: ...
+    def __le__(self, other: FrozenList[_T]) -> bool: ...
+    def __ne__(self, other: object) -> bool: ...
+    def __lt__(self, other: FrozenList[_T]) -> bool: ...
+    def __ge__(self, other: FrozenList[_T]) -> bool: ...
+    def __gt__(self, other: FrozenList[_T]) -> bool: ...
+    def insert(self, pos: int, item: _T) -> None: ...
+    def __repr__(self) -> str: ...
+    def __hash__(self) -> int: ...
+
+# types for C accelerators are the same
+CFrozenList = PyFrozenList = FrozenList

frozenlist/_frozenlist.pyx

@@ -0,0 +1,123 @@
+import sys
+import types
+from collections.abc import MutableSequence
+
+
+cdef class FrozenList:
+
+    if sys.version_info >= (3, 9):
+        __class_getitem__ = classmethod(types.GenericAlias)
+    else:
+        @classmethod
+        def __class_getitem__(cls):
+            return cls
+
+    cdef readonly bint frozen
+    cdef list _items
+
+    def __init__(self, items=None):
+        self.frozen = False
+        if items is not None:
+            items = list(items)
+        else:
+            items = []
+        self._items = items
+
+    cdef object _check_frozen(self):
+        if self.frozen:
+            raise RuntimeError("Cannot modify frozen list.")
+
+    cdef inline object _fast_len(self):
+        return len(self._items)
+
+    def freeze(self):
+        self.frozen = True
+
+    def __getitem__(self, index):
+        return self._items[index]
+
+    def __setitem__(self, index, value):
+        self._check_frozen()
+        self._items[index] = value
+
+    def __delitem__(self, index):
+        self._check_frozen()
+        del self._items[index]
+
+    def __len__(self):
+        return self._fast_len()
+
+    def __iter__(self):
+        return self._items.__iter__()
+
+    def __reversed__(self):
+        return self._items.__reversed__()
+
+    def __richcmp__(self, other, op):
+        if op == 0:  # <
+            return list(self) < other
+        if op == 1:  # <=
+            return list(self) <= other
+        if op == 2:  # ==
+            return list(self) == other
+        if op == 3:  # !=
+            return list(self) != other
+        if op == 4:  # >
+            return list(self) > other
+        if op == 5:  # =>
+            return list(self) >= other
+
+    def insert(self, pos, item):
+        self._check_frozen()
+        self._items.insert(pos, item)
+
+    def __contains__(self, item):
+        return item in self._items
+
+    def __iadd__(self, items):
+        self._check_frozen()
+        self._items += list(items)
+        return self
+
+    def index(self, item):
+        return self._items.index(item)
+
+    def remove(self, item):
+        self._check_frozen()
+        self._items.remove(item)
+
+    def clear(self):
+        self._check_frozen()
+        self._items.clear()
+
+    def extend(self, items):
+        self._check_frozen()
+        self._items += list(items)
+
+    def reverse(self):
+        self._check_frozen()
+        self._items.reverse()
+
+    def pop(self, index=-1):
+        self._check_frozen()
+        return self._items.pop(index)
+
+    def append(self, item):
+        self._check_frozen()
+        return self._items.append(item)
+
+    def count(self, item):
+        return self._items.count(item)
+
+    def __repr__(self):
+        return '<FrozenList(frozen={}, {!r})>'.format(self.frozen,
+                                                      self._items)
+
+    def __hash__(self):
+        if self.frozen:
+            return hash(tuple(self._items))
+        else:
+            raise RuntimeError("Cannot hash unfrozen list.")
+
+
+MutableSequence.register(FrozenList)

frozenlist/py.typed

@@ -0,0 +1 @@
+Marker

gatherDTUBIData.py

@@ -0,0 +1,59 @@
+import json
+from hoymiles_wifi.dtu import DTU
+import asyncio
+import aiohttp
+import logging
+import math
+import time
+from typing import List
+from dataclasses import dataclass
+from dataclasses import field
+UPDATE_INTERVAL = 31
+_LOGGER = logging.getLogger(__name__)
+
+dtu = DTU("192.168.179.184")
+
+@dataclass
+class DTUBIInvData:
+   p_PV:List[float] = field(default_factory=lambda: [0.0,0.0])
+   p_AC:float = 0.0
+   temp:float = 0.0
+   error:int = 0
+   name:str = ""
+
+@dataclass
+class DTUBIData:
+    inverter:List[DTUBIInvData] = field(default_factory=lambda: [DTUBIInvData()])#default_factory=lambda: [DTUInvData(), DTUInvData(),DTUInvData(),DTUInvData(),DTUInvData(),DTUInvData()])
+    p_AC_sum:float = 0
+    error = 0
+
+
+ret = DTUBIData()
+
+ret.lastUpdate = time.clock_gettime(0)-UPDATE_INTERVAL
+ret.inverter[0].name = "Veranda UG2"
+async def gatherData() -> DTUBIData:
+   if(time.clock_gettime(0) >= ret.lastUpdate + UPDATE_INTERVAL):
+      ret.lastUpdate = time.clock_gettime(0)
+      #print("get charger status...");
+      try:
+         state = await dtu.async_get_real_data_new()
+         #logging.warning(html)
+         #state = json.load(html)
+         ret.inverter[0].error = 0
+         ret.inverter[0].p_PV[0] = state.pv_data[0].power/10
+         ret.inverter[0].p_PV[1] = state.pv_data[1].power/10
+         ret.inverter[0].p_AC = state.sgs_data[0].active_power/10
+         ret.p_AC_sum = ret.inverter[0].p_AC
+         ret.inverter[0].temp = state.sgs_data[0].temperature/10
+         
+      except:
+         if(ret.inverter[0].error > 1):
+            ret.p_AC_sum = 0
+            ret.inverter[0].p_PV[0] = 0
+            ret.inverter[0].p_PV[1] = 0
+            ret.inverter[0].temp = 0
+            ret.inverter[0].p_AC = 0
+         else:
+            ret.inverter[0].error += 1
+   return ret

gatherHeaterData.py

@@ -0,0 +1,294 @@
+import json
+import asyncio
+import aiohttp
+import logging
+import math
+from typing import List
+from dataclasses import dataclass
+from dataclasses import field
+
+_LOGGER = logging.getLogger(__name__)
+
+@dataclass
+class HeaterData:
+   t_buffT:float = 0
+   t_buffM:float = 0
+   t_buffB:float = 0
+   t_heatVL:float = 0
+   t_heatRL:float = 0
+   t_gasVLu:float = 0
+   t_gasVLo:float = 0
+   t_gasRL:float = 0
+   t_fbVL:float = 0
+   t_fbRL:float = 0
+   t_triac:float = 0
+   p_heat:float = 0
+   error:int = 0
+   mode:str = ""
+
+ret = HeaterData()
+
+async def gatherData() -> HeaterData:
+   
+   #print("get charger status...");
+   timeout = aiohttp.ClientTimeout(total=3)
+   try:
+      async with aiohttp.ClientSession() as session:
+         async with session.get('http://192.168.179.169/data',timeout=timeout) as response: #?filter=psm,fup,amp,frc,nrg,lmo,car
+               if response.status == 200:
+                  html = await response.text()
+                  #logging.warning(html)
+                  state = json.loads(html)
+                  ret.error = 0
+                  ret.t_buffT = state["PufferO"]
+                  ret.t_buffM = state["PufferM"]
+                  ret.t_buffB = state["PufferU"]
+                  ret.t_heatVL = state["HeaterVL"]
+                  ret.t_heatRL = state["HeaterRL"]
+                  ret.t_gasVLu = state["ThermeVLu"]
+                  ret.t_gasVLo = state["ThermeVLo"]
+                  ret.t_gasRL = state["ThermeRL"]
+                  ret.t_triac = state["Triac"]
+                  ret.t_fbVL = state["fbVL"]
+                  ret.t_fbRL = state["fbRL"]
+                  ret.p_heat = state["heatPower"]
+                  ret.mode = state["mode"]
+   except:
+        if(ret.error > 5):
+            ret.p_heat = 0
+            _LOGGER.warning("heater data could not be fetched")
+        else:
+            ret.error += 1
+   return ret
+
+#async def main():
+#    await gatherNeededStatus()
+
+#loop = asyncio.get_event_loop()
+#loop.run_until_complete(main())
+
+
+"""
+{
+   "inverters":[
+      {
+         "serial":"116491626890",
+         "name":"Terrassendach",
+         "order":0,
+         "data_age":1,
+         "poll_enabled":true,
+         "reachable":true,
+         "producing":true,
+         "limit_relative":100,
+         "limit_absolute":1600,
+         "AC":{
+            "0":{
+               "Power":{
+                  "v":74.09999847,
+                  "u":"W",
+                  "d":1
+               },
+               "Voltage":{
+                  "v":235.3000031,
+                  "u":"V",
+                  "d":1
+               },
+               "Current":{
+                  "v":0.310000002,
+                  "u":"A",
+                  "d":2
+               },
+               "Power DC":{
+                  "v":77.80000305,
+                  "u":"W",
+                  "d":1
+               },
+               "YieldDay":{
+                  "v":179,
+                  "u":"Wh",
+                  "d":0
+               },
+               "YieldTotal":{
+                  "v":8.732000351,
+                  "u":"kWh",
+                  "d":3
+               },
+               "Frequency":{
+                  "v":50.00999832,
+                  "u":"Hz",
+                  "d":2
+               },
+               "PowerFactor":{
+                  "v":1,
+                  "u":"",
+                  "d":3
+               },
+               "ReactivePower":{
+                  "v":0,
+                  "u":"var",
+                  "d":1
+               },
+               "Efficiency":{
+                  "v":95.24420929,
+                  "u":"%",
+                  "d":3
+               }
+            }
+         },
+         "DC":{
+            "0":{
+               "name":{
+                  "u":""
+               },
+               "Power":{
+                  "v":12.80000019,
+                  "u":"W",
+                  "d":1
+               },
+               "Voltage":{
+                  "v":21.20000076,
+                  "u":"V",
+                  "d":1
+               },
+               "Current":{
+                  "v":0.610000014,
+                  "u":"A",
+                  "d":2
+               },
+               "YieldDay":{
+                  "v":31,
+                  "u":"Wh",
+                  "d":0
+               },
+               "YieldTotal":{
+                  "v":1.710000038,
+                  "u":"kWh",
+                  "d":3
+               }
+            },
+            "1":{
+               "name":{
+                  "u":""
+               },
+               "Power":{
+                  "v":26.60000038,
+                  "u":"W",
+                  "d":1
+               },
+               "Voltage":{
+                  "v":43.20000076,
+                  "u":"V",
+                  "d":1
+               },
+               "Current":{
+                  "v":0.610000014,
+                  "u":"A",
+                  "d":2
+               },
+               "YieldDay":{
+                  "v":62,
+                  "u":"Wh",
+                  "d":0
+               },
+               "YieldTotal":{
+                  "v":2.496000051,
+                  "u":"kWh",
+                  "d":3
+               }
+            },
+            "2":{
+               "name":{
+                  "u":""
+               },
+               "Power":{
+                  "v":12.5,
+                  "u":"W",
+                  "d":1
+               },
+               "Voltage":{
+                  "v":21.20000076,
+                  "u":"V",
+                  "d":1
+               },
+               "Current":{
+                  "v":0.589999974,
+                  "u":"A",
+                  "d":2
+               },
+               "YieldDay":{
+                  "v":26,
+                  "u":"Wh",
+                  "d":0
+               },
+               "YieldTotal":{
+                  "v":1.179999948,
+                  "u":"kWh",
+                  "d":3
+               }
+            },
+            "3":{
+               "name":{
+                  "u":""
+               },
+               "Power":{
+                  "v":25.89999962,
+                  "u":"W",
+                  "d":1
+               },
+               "Voltage":{
+                  "v":43,
+                  "u":"V",
+                  "d":1
+               },
+               "Current":{
+                  "v":0.600000024,
+                  "u":"A",
+                  "d":2
+               },
+               "YieldDay":{
+                  "v":60,
+                  "u":"Wh",
+                  "d":0
+               },
+               "YieldTotal":{
+                  "v":3.345999956,
+                  "u":"kWh",
+                  "d":3
+               }
+            }
+         },
+         "INV":{
+            "0":{
+               "Temperature":{
+                  "v":12.5,
+                  "u":"°C",
+                  "d":1
+               }
+            }
+         },
+         "events":1
+      }
+   ],
+   "total":{
+      "Power":{
+         "v":74.09999847,
+         "u":"W",
+         "d":1
+      },
+      "YieldDay":{
+         "v":179,
+         "u":"Wh",
+         "d":0
+      },
+      "YieldTotal":{
+         "v":8.732000351,
+         "u":"kWh",
+         "d":3
+      }
+   },
+   "hints":{
+      "time_sync":false,
+      "radio_problem":false,
+      "default_password":true
+   }
+}"""