summaryrefslogtreecommitdiff
path: root/searx/network/network.py
diff options
context:
space:
mode:
Diffstat (limited to 'searx/network/network.py')
-rw-r--r--searx/network/network.py143
1 files changed, 70 insertions, 73 deletions
diff --git a/searx/network/network.py b/searx/network/network.py
index eb53afb7f..42828ce41 100644
--- a/searx/network/network.py
+++ b/searx/network/network.py
@@ -13,11 +13,16 @@ import asyncio
import ipaddress
from itertools import cycle
-import httpx
+from curl_cffi import CurlHttpVersion
+from curl_cffi.requests.exceptions import (
+ ConnectionError as CurlConnectionError,
+ ProxyError,
+ RequestException,
+)
from searx import logger, sxng_debug
from searx.extended_types import SXNG_Response
-from .client import new_client, get_loop, AsyncHTTPTransportNoHttp
+from .client import DEFAULT_IMPERSONATE, AsyncClient, new_client, get_loop
from .raise_for_httperror import raise_for_httperror
@@ -48,9 +53,8 @@ class Network:
'enable_http',
'verify',
'enable_http2',
+ 'enable_http3',
'max_connections',
- 'max_keepalive_connections',
- 'keepalive_expiry',
'local_addresses',
'proxies',
'using_tor_proxy',
@@ -64,6 +68,7 @@ class Network:
)
_TOR_CHECK_RESULT = {}
+ _CLIENT_KWARGS = ('verify', 'max_redirects', 'impersonate', 'curl_options', 'enable_http3')
def __init__(
# pylint: disable=too-many-arguments
@@ -71,9 +76,8 @@ class Network:
enable_http: bool = True,
verify: bool = True,
enable_http2: bool = False,
+ enable_http3: bool = False,
max_connections: int = None, # pyright: ignore[reportArgumentType]
- max_keepalive_connections: int = None, # pyright: ignore[reportArgumentType]
- keepalive_expiry: float = None, # pyright: ignore[reportArgumentType]
proxies: str | dict[str, str] | None = None,
using_tor_proxy: bool = False,
local_addresses: str | list[str] | None = None,
@@ -86,9 +90,8 @@ class Network:
self.enable_http = enable_http
self.verify = verify
self.enable_http2 = enable_http2
+ self.enable_http3 = enable_http3
self.max_connections = max_connections
- self.max_keepalive_connections = max_keepalive_connections
- self.keepalive_expiry = keepalive_expiry
self.proxies = proxies
self.using_tor_proxy = using_tor_proxy
self.local_addresses = local_addresses
@@ -137,7 +140,6 @@ class Network:
def iter_proxies(self) -> Generator[tuple[str, list[str]]]:
if not self.proxies:
return
- # https://www.python-httpx.org/compatibility/#proxy-keys
if isinstance(self.proxies, str):
yield 'all://', [self.proxies]
else:
@@ -155,62 +157,73 @@ class Network:
# pylint: disable=stop-iteration-return
yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
- async def log_response(self, response: httpx.Response):
+ _HTTP_VERSION = {
+ int(CurlHttpVersion.V1_0): "HTTP/1.0",
+ int(CurlHttpVersion.V1_1): "HTTP/1.1",
+ int(CurlHttpVersion.V2_0): "HTTP/2",
+ int(CurlHttpVersion.V2TLS): "HTTP/2",
+ int(CurlHttpVersion.V2_PRIOR_KNOWLEDGE): "HTTP/2",
+ int(CurlHttpVersion.V3): "HTTP/3",
+ int(CurlHttpVersion.V3ONLY): "HTTP/3",
+ }
+
+ async def log_response(self, response: SXNG_Response):
request = response.request
- status = f"{response.status_code} {response.reason_phrase}"
- response_line = f"{response.http_version} {status}"
+ http_version = self._HTTP_VERSION.get(response.http_version, str(response.http_version))
+ status = f"{response.status_code} {response.reason}"
+ response_line = f"{http_version} {status}"
content_type = response.headers.get("Content-Type")
content_type = f' ({content_type})' if content_type else ''
- self._logger.debug(f'HTTP Request: {request.method} {request.url} "{response_line}"{content_type}')
+ method = request.method if request else "?"
+ url = request.url if request else response.url
+ self._logger.debug(f'HTTP Request: {method} {url} "{response_line}"{content_type}')
@staticmethod
- async def check_tor_proxy(client: httpx.AsyncClient, proxies) -> bool:
+ async def check_tor_proxy(client: AsyncClient, proxies) -> bool:
if proxies in Network._TOR_CHECK_RESULT:
return Network._TOR_CHECK_RESULT[proxies]
- result = True
- # ignore client._transport because it is not used with all://
- for transport in client._mounts.values(): # pylint: disable=protected-access
- if isinstance(transport, AsyncHTTPTransportNoHttp):
- continue
- if getattr(transport, "_pool") and getattr(
- # pylint: disable=protected-access
- transport._pool, # type: ignore
- "_rdns",
- False,
- ):
- continue
+ if not proxies or not all(url.startswith('socks5h://') for _, url in proxies):
+ Network._TOR_CHECK_RESULT[proxies] = False
return False
+
response = await client.get("https://check.torproject.org/api/ip", timeout=60)
- if not response.json()["IsTor"]:
- result = False
+ result = bool(response.json()["IsTor"])
Network._TOR_CHECK_RESULT[proxies] = result
return result
- async def get_client(self, verify: bool | None = None, max_redirects: int | None = None) -> httpx.AsyncClient:
+ async def get_client(
+ self,
+ verify: bool | None = None,
+ max_redirects: int | None = None,
+ impersonate: str | None = None,
+ curl_options: dict[int, t.Any] | None = None,
+ enable_http3: bool | None = None,
+ ) -> AsyncClient:
verify = self.verify if verify is None else verify
max_redirects = self.max_redirects if max_redirects is None else max_redirects
+ impersonate = impersonate or DEFAULT_IMPERSONATE
+ enable_http3 = self.enable_http3 if enable_http3 is None else enable_http3
local_address = next(self._local_addresses_cycle)
proxies = next(self._proxies_cycle) # is a tuple so it can be part of the key
- key = (verify, max_redirects, local_address, proxies)
- hook_log_response = self.log_response if sxng_debug else None
+ curl_key = tuple(sorted((int(k), v) for k, v in (curl_options or {}).items()))
+ key = (verify, max_redirects, local_address, proxies, impersonate, curl_key, enable_http3)
if key not in self._clients or self._clients[key].is_closed:
client = new_client(
self.enable_http,
verify,
self.enable_http2,
+ enable_http3,
self.max_connections,
- self.max_keepalive_connections,
- self.keepalive_expiry,
dict(proxies),
local_address,
- 0,
max_redirects,
- hook_log_response,
+ impersonate=impersonate,
+ curl_options=curl_options,
)
if self.using_tor_proxy and not await self.check_tor_proxy(client, proxies):
await client.aclose()
- raise httpx.ProxyError('Network configuration problem: not using Tor')
+ raise ProxyError('Network configuration problem: not using Tor')
self._clients[key] = client
return self._clients[key]
@@ -218,22 +231,14 @@ class Network:
async def close_client(client):
try:
await client.aclose()
- except httpx.HTTPError:
+ except RequestException:
pass
await asyncio.gather(*[close_client(client) for client in self._clients.values()], return_exceptions=False)
@staticmethod
def extract_kwargs_clients(kwargs: dict[str, t.Any]) -> dict[str, t.Any]:
- kwargs_clients: dict[str, t.Any] = {}
- if 'verify' in kwargs:
- kwargs_clients['verify'] = kwargs.pop('verify')
- if 'max_redirects' in kwargs:
- kwargs_clients['max_redirects'] = kwargs.pop('max_redirects')
- if 'allow_redirects' in kwargs:
- # see https://github.com/encode/httpx/pull/1808
- kwargs['follow_redirects'] = kwargs.pop('allow_redirects')
- return kwargs_clients
+ return {key: kwargs.pop(key) for key in Network._CLIENT_KWARGS if key in kwargs}
@staticmethod
def extract_do_raise_for_httperror(kwargs: dict[str, t.Any]):
@@ -243,23 +248,18 @@ class Network:
del kwargs['raise_for_httperror']
return do_raise_for_httperror
- def patch_response(self, response: httpx.Response, do_raise_for_httperror: bool) -> SXNG_Response:
- if isinstance(response, httpx.Response):
- response = t.cast(SXNG_Response, response)
- # requests compatibility (response is not streamed)
- # see also https://www.python-httpx.org/compatibility/#checking-for-4xx5xx-responses
- response.ok = not response.is_error
-
- # raise an exception
- if do_raise_for_httperror:
- try:
- raise_for_httperror(response)
- except:
- self._logger.warning(f"HTTP Request failed: {response.request.method} {response.request.url}")
- raise
+ def patch_response(self, response: SXNG_Response, do_raise_for_httperror: bool) -> SXNG_Response:
+ if do_raise_for_httperror:
+ try:
+ raise_for_httperror(response)
+ except:
+ method = response.request.method if response.request else "?"
+ url = response.request.url if response.request else response.url
+ self._logger.warning(f"HTTP Request failed: {method} {url}")
+ raise
return response
- def is_valid_response(self, response: httpx.Response):
+ def is_valid_response(self, response: SXNG_Response):
# pylint: disable=too-many-boolean-expressions
if (
(self.retry_on_http_error is True and 400 <= response.status_code <= 599)
@@ -276,26 +276,29 @@ class Network:
kwargs_clients = Network.extract_kwargs_clients(kwargs)
while retries >= 0: # pragma: no cover
client = await self.get_client(**kwargs_clients)
- cookies = kwargs.pop("cookies", None)
- client.cookies = httpx.Cookies(cookies)
try:
+ method = method.upper()
+ client.check_url(url)
if stream:
return client.stream(method, url, **kwargs)
response = await client.request(method, url, **kwargs)
+ if sxng_debug:
+ await self.log_response(response)
if self.is_valid_response(response) or retries <= 0:
return self.patch_response(response, do_raise_for_httperror)
- except httpx.RemoteProtocolError as e:
+ await client.aclose()
+ except CurlConnectionError as e:
if not was_disconnected:
# the server has closed the connection:
# try again without decreasing the retries variable & with a new HTTP client
was_disconnected = True
await client.aclose()
- self._logger.warning('httpx.RemoteProtocolError: the server has disconnected, retrying')
+ self._logger.warning('ConnectionError: the server has disconnected, retrying')
continue
if retries <= 0:
raise e
- except (httpx.RequestError, httpx.HTTPStatusError) as e:
+ except RequestException as e:
if retries <= 0:
raise e
retries -= 1
@@ -346,15 +349,12 @@ def initialize(
settings_engines = settings_engines or settings['engines']
settings_outgoing = settings_outgoing or settings['outgoing']
- # default parameters for AsyncHTTPTransport
- # see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long
default_params: dict[str, t.Any] = {
'enable_http': False,
'verify': settings_outgoing['verify'],
'enable_http2': settings_outgoing['enable_http2'],
+ 'enable_http3': False,
'max_connections': settings_outgoing['pool_connections'],
- 'max_keepalive_connections': settings_outgoing['pool_maxsize'],
- 'keepalive_expiry': settings_outgoing['keepalive_expiry'],
'local_addresses': settings_outgoing['source_ips'],
'using_tor_proxy': settings_outgoing['using_tor_proxy'],
'proxies': settings_outgoing['proxies'],
@@ -424,9 +424,6 @@ def initialize(
def done():
"""Close all HTTP client
- Avoid a warning at exit
- See https://github.com/encode/httpx/pull/2026
-
Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
So Network.aclose is called here using atexit.register
"""
generated by cgit v1.2.3 (git 2.46.0) at 2026-09-08 10:14:42 +0000