[mod] network: migrate to curl_cffi

4 files changed, 170 insertions, 288 deletions

diff --git a/searx/network/__init__.py b/searx/network/__init__.py
index 3a3b93d08..2257bcd31 100644
--- a/searx/network/__init__.py
+++ b/searx/network/__init__.py
@@ -14,8 +14,7 @@ from timeit import default_timer
 from collections.abc import Iterable
 from contextlib import contextmanager
 
-import httpx
-import anyio
+from curl_cffi.requests.exceptions import StreamConsumedError, Timeout
 
 from searx.extended_types import SXNG_Response
 from .network import get_network, initialize, check_network_configuration  # pylint:disable=cyclic-import
@@ -74,7 +73,6 @@ def _get_timeout(start_time: float, kwargs: t.Any) -> float:
     # pylint: disable=too-many-branches
 
     timeout: float | None
-    # timeout (httpx)
     if 'timeout' in kwargs:
         timeout = kwargs['timeout']
     else:
@@ -105,10 +103,10 @@ def request(method: str, url: str, **kwargs: t.Any) -> SXNG_Response:
         try:
             return future.result(timeout)
         except concurrent.futures.TimeoutError as e:
-            raise httpx.TimeoutException('Timeout', request=None) from e
+            raise Timeout('Timeout') from e
 
 
-def multi_requests(request_list: list["Request"]) -> list[httpx.Response | Exception]:
+def multi_requests(request_list: list["Request"]) -> list[SXNG_Response | Exception]:
     """send multiple HTTP requests in parallel. Wait for all requests to finish."""
     with _record_http_time() as start_time:
         # send the requests
@@ -128,7 +126,7 @@ def multi_requests(request_list: list["Request"]) -> list[httpx.Response | Excep
             try:
                 responses.append(future.result(timeout))
             except concurrent.futures.TimeoutError:
-                responses.append(httpx.TimeoutException('Timeout', request=None))
+                responses.append(Timeout('Timeout'))
             except Exception as e:  # pylint: disable=broad-except
                 responses.append(e)
         return responses
@@ -205,14 +203,12 @@ async def stream_chunk_to_queue(network, queue, method: str, url: str, **kwargs:
     try:
         async with await network.stream(method, url, **kwargs) as response:
             queue.put(response)
-            # aiter_raw: access the raw bytes on the response without applying any HTTP content decoding
-            # https://www.python-httpx.org/quickstart/#streaming-responses
-            async for chunk in response.aiter_raw(65536):
+            async for chunk in response.aiter_content():
                 if len(chunk) > 0:
                     queue.put(chunk)
-    except (httpx.StreamClosed, anyio.ClosedResourceError):
+    except StreamConsumedError:
         # the response was queued before the exception.
-        # the exception was raised on aiter_raw.
+        # the exception was raised on aiter_content.
         # we do nothing here: in the finally block, None will be queued
         # so stream(method, url, **kwargs) generator can stop
         pass
@@ -246,22 +242,19 @@ def _close_response_method(self):
     asyncio.run_coroutine_threadsafe(self.aclose(), get_loop())
     # reach the end of _self.generator ( _stream_generator ) to an avoid memory leak.
     # it makes sure that :
-    # * the httpx response is closed (see the stream_chunk_to_queue function)
+    # * the curl_cffi response is closed (see the stream_chunk_to_queue function)
     # * to call future.result() in _stream_generator
     for _ in self._generator:  # pylint: disable=protected-access
         continue
 
 
 def stream(method: str, url: str, **kwargs: t.Any) -> tuple[SXNG_Response, Iterable[bytes]]:
-    """Replace httpx.stream.
+    """Stream for the image proxy.
 
     Usage:
-    response, stream = poolrequests.stream(...)
+    response, stream = searx.network.stream(...)
     for chunk in stream:
         ...
-
-    httpx.Client.stream requires to write the httpx.HTTPTransport version of the
-    the httpx.AsyncHTTPTransport declared above.
     """
     generator = _stream_generator(method, url, **kwargs)
 
diff --git a/searx/network/client.py b/searx/network/client.py
index 4019e907c..9e6d213be 100644
--- a/searx/network/client.py
+++ b/searx/network/client.py
@@ -2,207 +2,114 @@
 # pylint: disable=missing-module-docstring, global-statement
 
 import typing as t
-from types import TracebackType
 
 import asyncio
 import logging
-import random
-from ssl import SSLContext
+import os
 import threading
 
-import httpx
-from httpx_socks import AsyncProxyTransport
-from python_socks import parse_proxy_url, ProxyConnectionError, ProxyTimeoutError, ProxyError
+from curl_cffi import AsyncSession, CurlHttpVersion, CurlOpt
+from curl_cffi.requests.exceptions import InvalidSchema, RequestException
 
-from searx import logger
+from searx.extended_types import SXNG_Response
 
-CertTypes = str | tuple[str, str] | tuple[str, str, str]
-SslContextKeyType = tuple[str | None, CertTypes | None, bool, bool]
-
-logger = logger.getChild('searx.network.client')
 LOOP: asyncio.AbstractEventLoop = None  # pyright: ignore[reportAssignmentType]
 
-SSLCONTEXTS: dict[SslContextKeyType, SSLContext] = {}
-
-
-def shuffle_ciphers(ssl_context: SSLContext):
-    """Shuffle httpx's default ciphers of a SSL context randomly.
-
-    From `What Is TLS Fingerprint and How to Bypass It`_
-
-    > When implementing TLS fingerprinting, servers can't operate based on a
-    > locked-in whitelist database of fingerprints.  New fingerprints appear
-    > when web clients or TLS libraries release new versions. So, they have to
-    > live off a blocklist database instead.
-    > ...
-    > It's safe to leave the first three as is but shuffle the remaining ciphers
-    > and you can bypass the TLS fingerprint check.
-
-    .. _What Is TLS Fingerprint and How to Bypass It:
-       https://www.zenrows.com/blog/what-is-tls-fingerprint#how-to-bypass-tls-fingerprinting
-
-    """
-    c_list = [cipher["name"] for cipher in ssl_context.get_ciphers()]
-    sc_list, c_list = c_list[:3], c_list[3:]
-    random.shuffle(c_list)
-    ssl_context.set_ciphers(":".join(sc_list + c_list))
-
+# chrome is used by default
+DEFAULT_IMPERSONATE = "chrome"
+NO_IMPERSONATE = "none"
 
-def get_sslcontexts(
-    proxy_url: str | None = None, cert: CertTypes | None = None, verify: bool = True, trust_env: bool = True
-) -> SSLContext:
-    key: SslContextKeyType = (proxy_url, cert, verify, trust_env)
-    if key not in SSLCONTEXTS:
-        SSLCONTEXTS[key] = httpx.create_ssl_context(verify, cert, trust_env)
-    shuffle_ciphers(SSLCONTEXTS[key])
-    return SSLCONTEXTS[key]
 
+class AsyncClient(AsyncSession):
+    """:class:`curl_cffi.AsyncSession` with ``aclose`` / ``is_closed``."""
 
-class AsyncHTTPTransportNoHttp(httpx.AsyncHTTPTransport):
-    """Block HTTP request
+    def __init__(self, enable_http: bool, **kwargs: t.Any):
+        self.enable_http = enable_http
+        self._closed = False
+        super().__init__(**kwargs)
 
-    The constructor is blank because httpx.AsyncHTTPTransport.__init__ creates an SSLContext unconditionally:
-    https://github.com/encode/httpx/blob/0f61aa58d66680c239ce43c8cdd453e7dc532bfc/httpx/_transports/default.py#L271
+    @property
+    def is_closed(self) -> bool:
+        return self._closed
 
-    Each SSLContext consumes more than 500kb of memory, since there is about one network per engine.
-
-    In consequence, this class overrides all public methods
-
-    For reference: https://github.com/encode/httpx/issues/2298
-    """
-
-    def __init__(self, *args, **kwargs):  # type: ignore
-        # pylint: disable=super-init-not-called
-        # this on purpose if the base class is not called
-        pass
-
-    async def handle_async_request(self, request: httpx.Request):
-        raise httpx.UnsupportedProtocol('HTTP protocol is disabled')
+    def check_url(self, url: str) -> None:
+        if not self.enable_http and str(url).startswith("http://"):
+            raise InvalidSchema("HTTP protocol is disabled")
 
     async def aclose(self) -> None:
-        pass
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(
-        self,
-        exc_type: type[BaseException] | None = None,
-        exc_value: BaseException | None = None,
-        traceback: TracebackType | None = None,
-    ) -> None:
-        pass
-
-
-class AsyncProxyTransportFixed(AsyncProxyTransport):
-    """Fix httpx_socks.AsyncProxyTransport
+        if self._closed:
+            return
+        self._closed = True
+        try:
+            await self.close()
+        except RequestException:
+            pass
 
-    Map python_socks exceptions to httpx.ProxyError exceptions
-    """
 
-    async def handle_async_request(self, request: httpx.Request):
-        try:
-            return await super().handle_async_request(request)
-        except ProxyConnectionError as e:
-            raise httpx.ProxyError("ProxyConnectionError: " + str(e.strerror), request=request) from e
-        except ProxyTimeoutError as e:
-            raise httpx.ProxyError("ProxyTimeoutError: " + str(e.args[0]), request=request) from e
-        except ProxyError as e:
-            raise httpx.ProxyError("ProxyError: " + str(e.args[0]), request=request) from e
-
-
-def get_transport_for_socks_proxy(
-    verify: bool, http2: bool, local_address: str, proxy_url: str, limit: httpx.Limits, retries: int
-):
-    # support socks5h (requests compatibility):
-    # https://requests.readthedocs.io/en/master/user/advanced/#socks
-    # socks5://   hostname is resolved on client side
-    # socks5h://  hostname is resolved on proxy side
-    rdns = False
-    socks5h = 'socks5h://'
-    if proxy_url.startswith(socks5h):
-        proxy_url = 'socks5://' + proxy_url[len(socks5h) :]
-        rdns = True
-
-    proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url)
-    _verify = get_sslcontexts(proxy_url, None, verify, True) if verify is True else verify
-    return AsyncProxyTransportFixed(
-        proxy_type=proxy_type,
-        proxy_host=proxy_host,
-        proxy_port=proxy_port,
-        username=proxy_username,
-        password=proxy_password,
-        rdns=rdns,
-        verify=_verify,  # pyright: ignore[reportArgumentType]
-        http2=http2,
-        local_address=local_address,
-        limits=limit,
-        retries=retries,
-    )
-
-
-def get_transport(
-    verify: bool, http2: bool, local_address: str, proxy_url: str | None, limit: httpx.Limits, retries: int
-):
-    _verify = get_sslcontexts(None, None, verify, True) if verify is True else verify
-    return httpx.AsyncHTTPTransport(
-        # pylint: disable=protected-access
-        verify=_verify,
-        http2=http2,
-        limits=limit,
-        proxy=httpx._config.Proxy(proxy_url) if proxy_url else None,  # pyright: ignore[reportPrivateUsage]
-        local_address=local_address,
-        retries=retries,
-    )
+def _proxy_kwargs(proxies: dict[str, str], enable_http: bool) -> dict[str, t.Any]:
+    """Map settings.yml proxy keys (``all://``, ``https://``) to curl_cffi."""
+    mapped: dict[str, str] = {}
+    all_proxy: str | None = None
+    for pattern, proxy_url in proxies.items():
+        if not enable_http and pattern.startswith("http://"):
+            continue
+        if pattern.startswith("https"):
+            mapped["https"] = proxy_url
+        elif pattern.startswith("http"):
+            mapped["http"] = proxy_url
+        else:
+            all_proxy = proxy_url
+    if all_proxy:
+        return {"proxy": all_proxy}
+    if mapped:
+        return {"proxies": mapped}
+    return {}
 
 
 def new_client(
     # pylint: disable=too-many-arguments
     enable_http: bool,
-    verify: bool,
+    verify: bool | str,
     enable_http2: bool,
+    enable_http3: bool,
     max_connections: int,
-    max_keepalive_connections: int,
-    keepalive_expiry: float,
     proxies: dict[str, str],
-    local_address: str,
-    retries: int,
+    local_address: str | None,
     max_redirects: int,
-    hook_log_response: t.Callable[..., t.Any] | None,
-) -> httpx.AsyncClient:
-    limit = httpx.Limits(
-        max_connections=max_connections,
-        max_keepalive_connections=max_keepalive_connections,
-        keepalive_expiry=keepalive_expiry,
-    )
-    # See https://www.python-httpx.org/advanced/#routing
-    mounts = {}
-    mounts: None | (dict[str, t.Any | None]) = {}
-    for pattern, proxy_url in proxies.items():
-        if not enable_http and pattern.startswith('http://'):
-            continue
-        if proxy_url.startswith('socks4://') or proxy_url.startswith('socks5://') or proxy_url.startswith('socks5h://'):
-            mounts[pattern] = get_transport_for_socks_proxy(
-                verify, enable_http2, local_address, proxy_url, limit, retries
-            )
-        else:
-            mounts[pattern] = get_transport(verify, enable_http2, local_address, proxy_url, limit, retries)
-
-    if not enable_http:
-        mounts['http://'] = AsyncHTTPTransportNoHttp()
-
-    transport = get_transport(verify, enable_http2, local_address, None, limit, retries)
-
-    event_hooks = None
-    if hook_log_response:
-        event_hooks = {'response': [hook_log_response]}
-
-    return httpx.AsyncClient(
-        transport=transport,
-        mounts=mounts,
-        max_redirects=max_redirects,
-        event_hooks=event_hooks,
-    )
+    impersonate: str = DEFAULT_IMPERSONATE,
+    curl_options: dict[int, t.Any] | None = None,
+) -> AsyncClient:
+    extra_curl = dict(curl_options or {})
+    cert_file = os.environ.get("SSL_CERT_FILE")
+    if cert_file:
+        extra_curl.setdefault(CurlOpt.CAINFO, cert_file)
+    cert_dir = os.environ.get("SSL_CERT_DIR")
+    if cert_dir:
+        extra_curl.setdefault(CurlOpt.CAPATH, cert_dir)
+    use_impersonate = impersonate not in ("", NO_IMPERSONATE)
+    kwargs: dict[str, t.Any] = {
+        "enable_http": enable_http,
+        "verify": verify,
+        "max_redirects": max_redirects,
+        "max_clients": max_connections or 10,
+        "response_class": SXNG_Response,
+        "discard_cookies": True,
+        **_proxy_kwargs(proxies, enable_http),
+    }
+    if use_impersonate:
+        kwargs["impersonate"] = impersonate
+        kwargs["default_headers"] = True
+    if local_address:
+        kwargs["interface"] = local_address
+    if not enable_http2:
+        kwargs["http_version"] = CurlHttpVersion.V1_1
+    elif enable_http3 and not proxies:
+        kwargs["http_version"] = CurlHttpVersion.V3
+    else:
+        kwargs["http_version"] = CurlHttpVersion.V2_0
+    if extra_curl:
+        kwargs["curl_options"] = extra_curl
+    return AsyncClient(**kwargs)
 
 
 def get_loop() -> asyncio.AbstractEventLoop:
@@ -210,30 +117,18 @@ def get_loop() -> asyncio.AbstractEventLoop:
 
 
 def init():
-    # log
-    for logger_name in (
-        'httpx',
-        'httpcore.proxy',
-        'httpcore.connection',
-        'httpcore.http11',
-        'httpcore.http2',
-        'hpack.hpack',
-        'hpack.table',
-    ):
-        logging.getLogger(logger_name).setLevel(logging.WARNING)
-
-    # loop
+    logging.getLogger("curl_cffi").setLevel(logging.WARNING)
+
+    ready = threading.Event()
+
     def loop_thread():
         global LOOP
         LOOP = asyncio.new_event_loop()
+        ready.set()
         LOOP.run_forever()
 
-    thread = threading.Thread(
-        target=loop_thread,
-        name='asyncio_loop',
-        daemon=True,
-    )
-    thread.start()
+    threading.Thread(target=loop_thread, name="asyncio_loop", daemon=True).start()
+    ready.wait()
 
 
 init()
diff --git a/searx/network/network.py b/searx/network/network.py
index eb53afb7f..42828ce41 100644
--- a/searx/network/network.py
+++ b/searx/network/network.py
@@ -13,11 +13,16 @@ import asyncio
 import ipaddress
 from itertools import cycle
 
-import httpx
+from curl_cffi import CurlHttpVersion
+from curl_cffi.requests.exceptions import (
+    ConnectionError as CurlConnectionError,
+    ProxyError,
+    RequestException,
+)
 
 from searx import logger, sxng_debug
 from searx.extended_types import SXNG_Response
-from .client import new_client, get_loop, AsyncHTTPTransportNoHttp
+from .client import DEFAULT_IMPERSONATE, AsyncClient, new_client, get_loop
 from .raise_for_httperror import raise_for_httperror
 
 
@@ -48,9 +53,8 @@ class Network:
         'enable_http',
         'verify',
         'enable_http2',
+        'enable_http3',
         'max_connections',
-        'max_keepalive_connections',
-        'keepalive_expiry',
         'local_addresses',
         'proxies',
         'using_tor_proxy',
@@ -64,6 +68,7 @@ class Network:
     )
 
     _TOR_CHECK_RESULT = {}
+    _CLIENT_KWARGS = ('verify', 'max_redirects', 'impersonate', 'curl_options', 'enable_http3')
 
     def __init__(
         # pylint: disable=too-many-arguments
@@ -71,9 +76,8 @@ class Network:
         enable_http: bool = True,
         verify: bool = True,
         enable_http2: bool = False,
+        enable_http3: bool = False,
         max_connections: int = None,  # pyright: ignore[reportArgumentType]
-        max_keepalive_connections: int = None,  # pyright: ignore[reportArgumentType]
-        keepalive_expiry: float = None,  # pyright: ignore[reportArgumentType]
         proxies: str | dict[str, str] | None = None,
         using_tor_proxy: bool = False,
         local_addresses: str | list[str] | None = None,
@@ -86,9 +90,8 @@ class Network:
         self.enable_http = enable_http
         self.verify = verify
         self.enable_http2 = enable_http2
+        self.enable_http3 = enable_http3
         self.max_connections = max_connections
-        self.max_keepalive_connections = max_keepalive_connections
-        self.keepalive_expiry = keepalive_expiry
         self.proxies = proxies
         self.using_tor_proxy = using_tor_proxy
         self.local_addresses = local_addresses
@@ -137,7 +140,6 @@ class Network:
     def iter_proxies(self) -> Generator[tuple[str, list[str]]]:
         if not self.proxies:
             return
-        # https://www.python-httpx.org/compatibility/#proxy-keys
         if isinstance(self.proxies, str):
             yield 'all://', [self.proxies]
         else:
@@ -155,62 +157,73 @@ class Network:
             # pylint: disable=stop-iteration-return
             yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
 
-    async def log_response(self, response: httpx.Response):
+    _HTTP_VERSION = {
+        int(CurlHttpVersion.V1_0): "HTTP/1.0",
+        int(CurlHttpVersion.V1_1): "HTTP/1.1",
+        int(CurlHttpVersion.V2_0): "HTTP/2",
+        int(CurlHttpVersion.V2TLS): "HTTP/2",
+        int(CurlHttpVersion.V2_PRIOR_KNOWLEDGE): "HTTP/2",
+        int(CurlHttpVersion.V3): "HTTP/3",
+        int(CurlHttpVersion.V3ONLY): "HTTP/3",
+    }
+
+    async def log_response(self, response: SXNG_Response):
         request = response.request
-        status = f"{response.status_code} {response.reason_phrase}"
-        response_line = f"{response.http_version} {status}"
+        http_version = self._HTTP_VERSION.get(response.http_version, str(response.http_version))
+        status = f"{response.status_code} {response.reason}"
+        response_line = f"{http_version} {status}"
         content_type = response.headers.get("Content-Type")
         content_type = f' ({content_type})' if content_type else ''
-        self._logger.debug(f'HTTP Request: {request.method} {request.url} "{response_line}"{content_type}')
+        method = request.method if request else "?"
+        url = request.url if request else response.url
+        self._logger.debug(f'HTTP Request: {method} {url} "{response_line}"{content_type}')
 
     @staticmethod
-    async def check_tor_proxy(client: httpx.AsyncClient, proxies) -> bool:
+    async def check_tor_proxy(client: AsyncClient, proxies) -> bool:
         if proxies in Network._TOR_CHECK_RESULT:
             return Network._TOR_CHECK_RESULT[proxies]
 
-        result = True
-        # ignore client._transport because it is not used with all://
-        for transport in client._mounts.values():  # pylint: disable=protected-access
-            if isinstance(transport, AsyncHTTPTransportNoHttp):
-                continue
-            if getattr(transport, "_pool") and getattr(
-                # pylint: disable=protected-access
-                transport._pool,  # type: ignore
-                "_rdns",
-                False,
-            ):
-                continue
+        if not proxies or not all(url.startswith('socks5h://') for _, url in proxies):
+            Network._TOR_CHECK_RESULT[proxies] = False
             return False
+
         response = await client.get("https://check.torproject.org/api/ip", timeout=60)
-        if not response.json()["IsTor"]:
-            result = False
+        result = bool(response.json()["IsTor"])
         Network._TOR_CHECK_RESULT[proxies] = result
         return result
 
-    async def get_client(self, verify: bool | None = None, max_redirects: int | None = None) -> httpx.AsyncClient:
+    async def get_client(
+        self,
+        verify: bool | None = None,
+        max_redirects: int | None = None,
+        impersonate: str | None = None,
+        curl_options: dict[int, t.Any] | None = None,
+        enable_http3: bool | None = None,
+    ) -> AsyncClient:
         verify = self.verify if verify is None else verify
         max_redirects = self.max_redirects if max_redirects is None else max_redirects
+        impersonate = impersonate or DEFAULT_IMPERSONATE
+        enable_http3 = self.enable_http3 if enable_http3 is None else enable_http3
         local_address = next(self._local_addresses_cycle)
         proxies = next(self._proxies_cycle)  # is a tuple so it can be part of the key
-        key = (verify, max_redirects, local_address, proxies)
-        hook_log_response = self.log_response if sxng_debug else None
+        curl_key = tuple(sorted((int(k), v) for k, v in (curl_options or {}).items()))
+        key = (verify, max_redirects, local_address, proxies, impersonate, curl_key, enable_http3)
         if key not in self._clients or self._clients[key].is_closed:
             client = new_client(
                 self.enable_http,
                 verify,
                 self.enable_http2,
+                enable_http3,
                 self.max_connections,
-                self.max_keepalive_connections,
-                self.keepalive_expiry,
                 dict(proxies),
                 local_address,
-                0,
                 max_redirects,
-                hook_log_response,
+                impersonate=impersonate,
+                curl_options=curl_options,
             )
             if self.using_tor_proxy and not await self.check_tor_proxy(client, proxies):
                 await client.aclose()
-                raise httpx.ProxyError('Network configuration problem: not using Tor')
+                raise ProxyError('Network configuration problem: not using Tor')
             self._clients[key] = client
         return self._clients[key]
 
@@ -218,22 +231,14 @@ class Network:
         async def close_client(client):
             try:
                 await client.aclose()
-            except httpx.HTTPError:
+            except RequestException:
                 pass
 
         await asyncio.gather(*[close_client(client) for client in self._clients.values()], return_exceptions=False)
 
     @staticmethod
     def extract_kwargs_clients(kwargs: dict[str, t.Any]) -> dict[str, t.Any]:
-        kwargs_clients: dict[str, t.Any] = {}
-        if 'verify' in kwargs:
-            kwargs_clients['verify'] = kwargs.pop('verify')
-        if 'max_redirects' in kwargs:
-            kwargs_clients['max_redirects'] = kwargs.pop('max_redirects')
-        if 'allow_redirects' in kwargs:
-            # see https://github.com/encode/httpx/pull/1808
-            kwargs['follow_redirects'] = kwargs.pop('allow_redirects')
-        return kwargs_clients
+        return {key: kwargs.pop(key) for key in Network._CLIENT_KWARGS if key in kwargs}
 
     @staticmethod
     def extract_do_raise_for_httperror(kwargs: dict[str, t.Any]):
@@ -243,23 +248,18 @@ class Network:
             del kwargs['raise_for_httperror']
         return do_raise_for_httperror
 
-    def patch_response(self, response: httpx.Response, do_raise_for_httperror: bool) -> SXNG_Response:
-        if isinstance(response, httpx.Response):
-            response = t.cast(SXNG_Response, response)
-            # requests compatibility (response is not streamed)
-            # see also https://www.python-httpx.org/compatibility/#checking-for-4xx5xx-responses
-            response.ok = not response.is_error
-
-            # raise an exception
-            if do_raise_for_httperror:
-                try:
-                    raise_for_httperror(response)
-                except:
-                    self._logger.warning(f"HTTP Request failed: {response.request.method} {response.request.url}")
-                    raise
+    def patch_response(self, response: SXNG_Response, do_raise_for_httperror: bool) -> SXNG_Response:
+        if do_raise_for_httperror:
+            try:
+                raise_for_httperror(response)
+            except:
+                method = response.request.method if response.request else "?"
+                url = response.request.url if response.request else response.url
+                self._logger.warning(f"HTTP Request failed: {method} {url}")
+                raise
         return response
 
-    def is_valid_response(self, response: httpx.Response):
+    def is_valid_response(self, response: SXNG_Response):
         # pylint: disable=too-many-boolean-expressions
         if (
             (self.retry_on_http_error is True and 400 <= response.status_code <= 599)
@@ -276,26 +276,29 @@ class Network:
         kwargs_clients = Network.extract_kwargs_clients(kwargs)
         while retries >= 0:  # pragma: no cover
             client = await self.get_client(**kwargs_clients)
-            cookies = kwargs.pop("cookies", None)
-            client.cookies = httpx.Cookies(cookies)
             try:
+                method = method.upper()
+                client.check_url(url)
                 if stream:
                     return client.stream(method, url, **kwargs)
 
                 response = await client.request(method, url, **kwargs)
+                if sxng_debug:
+                    await self.log_response(response)
                 if self.is_valid_response(response) or retries <= 0:
                     return self.patch_response(response, do_raise_for_httperror)
-            except httpx.RemoteProtocolError as e:
+                await client.aclose()
+            except CurlConnectionError as e:
                 if not was_disconnected:
                     # the server has closed the connection:
                     # try again without decreasing the retries variable & with a new HTTP client
                     was_disconnected = True
                     await client.aclose()
-                    self._logger.warning('httpx.RemoteProtocolError: the server has disconnected, retrying')
+                    self._logger.warning('ConnectionError: the server has disconnected, retrying')
                     continue
                 if retries <= 0:
                     raise e
-            except (httpx.RequestError, httpx.HTTPStatusError) as e:
+            except RequestException as e:
                 if retries <= 0:
                     raise e
             retries -= 1
@@ -346,15 +349,12 @@ def initialize(
     settings_engines = settings_engines or settings['engines']
     settings_outgoing = settings_outgoing or settings['outgoing']
 
-    # default parameters for AsyncHTTPTransport
-    # see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121  # pylint: disable=line-too-long
     default_params: dict[str, t.Any] = {
         'enable_http': False,
         'verify': settings_outgoing['verify'],
         'enable_http2': settings_outgoing['enable_http2'],
+        'enable_http3': False,
         'max_connections': settings_outgoing['pool_connections'],
-        'max_keepalive_connections': settings_outgoing['pool_maxsize'],
-        'keepalive_expiry': settings_outgoing['keepalive_expiry'],
         'local_addresses': settings_outgoing['source_ips'],
         'using_tor_proxy': settings_outgoing['using_tor_proxy'],
         'proxies': settings_outgoing['proxies'],
@@ -424,9 +424,6 @@ def initialize(
 def done():
     """Close all HTTP client
 
-    Avoid a warning at exit
-    See https://github.com/encode/httpx/pull/2026
-
     Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
     So Network.aclose is called here using atexit.register
     """
diff --git a/searx/network/raise_for_httperror.py b/searx/network/raise_for_httperror.py
index d50e650e2..f4f409c99 100644
--- a/searx/network/raise_for_httperror.py
+++ b/searx/network/raise_for_httperror.py
@@ -59,13 +59,10 @@ def raise_for_captcha(resp: "SXNG_Response"):
 
 
 def raise_for_httperror(resp: "SXNG_Response") -> None:
-    """Raise exception for an HTTP response is an error.
-
-    Args:
-        resp (requests.Response): Response to check
+    """Raise an exception if the HTTP response is an error.
 
     Raises:
-        requests.HTTPError: raise by resp.raise_for_status()
+        curl_cffi.requests.exceptions.HTTPError: raised by resp.raise_for_status()
         searx.exceptions.SearxEngineAccessDeniedException: raise when the HTTP status code is 402 or 403.
         searx.exceptions.SearxEngineTooManyRequestsException: raise when the HTTP status code is 429.
         searx.exceptions.SearxEngineCaptchaException: raise when if CATPCHA challenge is detected.