[fix] plugin: ask the LLM server twice before giving up
An idle LLM server unloads the model and loads it again on the next request. Nothing of the response is sent while that load runs, so a request that arrives on a cold server can exceed read_timeout and fail -- reliably making the first search after an idle period the one that does not get a summary. That same request is what starts the load, so asking again succeeds. Only a server that failed to answer is asked again. A server that did answer with an error status is not: a wrong API key or an unknown model name does not become right on a second attempt. The status and body of such an error response are now logged. Until now every upstream failure looked identical from the outside -- HTTP 502 with no indication of whether the key, the model name or the network was at fault. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
ce02bebf7b
commit
4a582c0a15
@@ -31,6 +31,8 @@ def sse_stream_mock(lines: list[dict], status_code: int = 200) -> Mock:
|
||||
|
||||
upstream = Mock(status_code=status_code)
|
||||
upstream.iter_lines.return_value = iter([f"data: {json.dumps(line)}" for line in lines] + ["data: [DONE]"])
|
||||
# httpx exposes the body of an error response as a str; the endpoint logs it
|
||||
upstream.text = ""
|
||||
|
||||
@contextmanager
|
||||
def stream(*_args, **_kwargs):
|
||||
@@ -425,6 +427,71 @@ class PluginAISummary(SearxTestCase):
|
||||
self.assertIn('tab-label-ai"', html)
|
||||
self.assertIn("ai_summary_api_key", html)
|
||||
|
||||
def test_endpoint_retries_a_server_that_does_not_answer(self):
|
||||
# a cold LLM server loads the model before it answers anything, which
|
||||
# can outlast read_timeout; the retry is what makes the first search
|
||||
# after an idle period work
|
||||
import httpx # pylint: disable=import-outside-toplevel
|
||||
|
||||
attempts = []
|
||||
ok = sse_stream_mock([{"choices": [{"delta": {"content": "hi"}}]}])
|
||||
|
||||
@contextmanager
|
||||
def stream(*_args, **_kwargs):
|
||||
attempts.append(1)
|
||||
if len(attempts) == 1:
|
||||
raise httpx.ReadTimeout("timed out")
|
||||
with ok.stream() as resp:
|
||||
yield resp
|
||||
|
||||
client_mock = Mock()
|
||||
client_mock.stream = stream
|
||||
self.mock_upstream(client_mock)
|
||||
|
||||
res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]})
|
||||
self.assertEqual(res.status_code, 200)
|
||||
self.assertEqual(2, len(attempts))
|
||||
lines = [json.loads(line) for line in res.data.decode().splitlines() if line]
|
||||
self.assertEqual(lines[0], {"delta": "hi"})
|
||||
|
||||
def test_endpoint_gives_up_after_the_retry(self):
|
||||
import httpx # pylint: disable=import-outside-toplevel
|
||||
|
||||
attempts = []
|
||||
|
||||
@contextmanager
|
||||
def stream(*_args, **_kwargs):
|
||||
attempts.append(1)
|
||||
raise httpx.ConnectTimeout("nope")
|
||||
yield # pylint: disable=unreachable
|
||||
|
||||
client_mock = Mock()
|
||||
client_mock.stream = stream
|
||||
self.mock_upstream(client_mock)
|
||||
|
||||
res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]})
|
||||
self.assertEqual(res.status_code, 502)
|
||||
self.assertEqual(2, len(attempts))
|
||||
|
||||
def test_endpoint_does_not_retry_an_error_status(self):
|
||||
# a wrong API key or an unknown model does not fix itself
|
||||
attempts = []
|
||||
bad = sse_stream_mock([], status_code=401)
|
||||
|
||||
@contextmanager
|
||||
def stream(*_args, **_kwargs):
|
||||
attempts.append(1)
|
||||
with bad.stream() as resp:
|
||||
yield resp
|
||||
|
||||
client_mock = Mock()
|
||||
client_mock.stream = stream
|
||||
self.mock_upstream(client_mock)
|
||||
|
||||
res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]})
|
||||
self.assertEqual(res.status_code, 502)
|
||||
self.assertEqual(1, len(attempts))
|
||||
|
||||
def test_endpoint_upstream_error(self):
|
||||
self.mock_upstream(sse_stream_mock([], status_code=500))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user