From 0cc374886647cf5b4fa33320573cc973efa2d453 Mon Sep 17 00:00:00 2001 From: jasonwitty Date: Tue, 11 Aug 2026 11:58:31 -0700 Subject: [PATCH] [fix] plugin: repeat the request on 5xx as well A server that is not ready does not always take its time about saying so. Ollama answers 5xx immediately while a model is still being loaded, and a gateway with no upstream yet does the same, so the failure arrives at once rather than as a timeout. Both mean "not right now" and are worth one more attempt. 4xx keeps being final -- a wrong API key or an unknown model name is not going to change between two requests. Co-Authored-By: Claude Opus 5 --- searx/plugins/ai_summary.py | 20 +++++++++++++++----- tests/unit/test_plugin_ai_summary.py | 26 +++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/searx/plugins/ai_summary.py b/searx/plugins/ai_summary.py index f3d22e366..d99386629 100644 --- a/searx/plugins/ai_summary.py +++ b/searx/plugins/ai_summary.py @@ -254,10 +254,13 @@ def _open_upstream(client: httpx.Client, payload: dict[str, t.Any]) -> tuple[t.A """Start the streaming completion on the LLM server. Returns the (already entered) stream context and the response, or - ``(None, None)`` if no usable response was received. A server that does not - answer in time is asked again (:py:obj:`UPSTREAM_RETRIES`); a server that - *did* answer, but with an error status, is not -- a wrong API key or an - unknown model name does not become right when asked twice.""" + ``(None, None)`` if no usable response was received. + + The request is repeated (:py:obj:`UPSTREAM_RETRIES`) when the server did not + answer in time, and when it answered ``5xx`` -- both mean *not right now*, + and the most common reason is a model that is still being loaded. A ``4xx`` + is not repeated: a wrong API key or an unknown model name does not become + right when asked twice.""" for attempt in range(UPSTREAM_RETRIES + 1): stream_ctx = client.stream("POST", "/chat/completions", json=payload) @@ -284,8 +287,15 @@ def _open_upstream(client: httpx.Client, payload: dict[str, t.Any]) -> tuple[t.A detail = resp.text.strip()[:200] except (httpx.HTTPError, UnicodeDecodeError): # pragma: no cover pass - log.warning("LLM server responded with HTTP %s %s", resp.status_code, detail) stream_ctx.__exit__(None, None, None) + + # 5xx is the server saying it is not able to answer *right now* -- a + # model still loading, a gateway with no upstream yet. 4xx is the + # server saying the request is wrong, which a second one would be too. + if resp.status_code >= 500 and attempt < UPSTREAM_RETRIES: + log.debug("LLM server replied HTTP %s (%s), asking again", resp.status_code, detail) + continue + log.warning("LLM server responded with HTTP %s %s", resp.status_code, detail) return None, None return None, None # pragma: no cover - the loop always returns diff --git a/tests/unit/test_plugin_ai_summary.py b/tests/unit/test_plugin_ai_summary.py index 3735dcc80..dec1807c5 100644 --- a/tests/unit/test_plugin_ai_summary.py +++ b/tests/unit/test_plugin_ai_summary.py @@ -473,7 +473,7 @@ class PluginAISummary(SearxTestCase): self.assertEqual(res.status_code, 502) self.assertEqual(2, len(attempts)) - def test_endpoint_does_not_retry_an_error_status(self): + def test_endpoint_does_not_retry_a_client_error(self): # a wrong API key or an unknown model does not fix itself attempts = [] bad = sse_stream_mock([], status_code=401) @@ -492,6 +492,30 @@ class PluginAISummary(SearxTestCase): self.assertEqual(res.status_code, 502) self.assertEqual(1, len(attempts)) + def test_endpoint_retries_a_server_error(self): + # 5xx means "not right now" -- e.g. a model that is still loading; + # this is the failure that arrives immediately instead of timing out + attempts = [] + ok = sse_stream_mock([{"choices": [{"delta": {"content": "hi"}}]}]) + bad = sse_stream_mock([], status_code=503) + + @contextmanager + def stream(*_args, **_kwargs): + attempts.append(1) + src = bad if len(attempts) == 1 else ok + with src.stream() as resp: + yield resp + + client_mock = Mock() + client_mock.stream = stream + self.mock_upstream(client_mock) + + res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]}) + self.assertEqual(res.status_code, 200) + self.assertEqual(2, len(attempts)) + lines = [json.loads(line) for line in res.data.decode().splitlines() if line] + self.assertEqual(lines[0], {"delta": "hi"}) + def test_endpoint_upstream_error(self): self.mock_upstream(sse_stream_mock([], status_code=500))