From 4abb7dba672a45dbfaa5dfb4fffe1a86195bcfd5 Mon Sep 17 00:00:00 2001 From: jasonwitty Date: Tue, 28 Jul 2026 13:01:24 -0700 Subject: [PATCH] [mod] ai_summary plugin: switch to the OpenAI chat completions API Talk to the LLM server via GET /v1/models and POST /v1/chat/completions (SSE) instead of Ollama's native API. Any OpenAI compatible server now works (Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...); Ollama serves this API natively, existing setups keep working unchanged. The Ollama specific keep_alive option is dropped, the ai_summary.grounding setting is added as instance wide default of the grounding preference. Co-Authored-By: Claude Fable 5 --- docs/admin/settings/settings_ai_summary.rst | 6 +- searx/ai_summary.py | 24 ++++--- searx/plugins/ai_summary.py | 65 +++++++++++-------- searx/settings.yml | 8 ++- .../simple/preferences/ai_summary.html | 4 +- tests/unit/test_plugin_ai_summary.py | 20 +++--- 6 files changed, 70 insertions(+), 57 deletions(-) diff --git a/docs/admin/settings/settings_ai_summary.rst b/docs/admin/settings/settings_ai_summary.rst index 027236f16..4de9fef1b 100644 --- a/docs/admin/settings/settings_ai_summary.rst +++ b/docs/admin/settings/settings_ai_summary.rst @@ -5,8 +5,10 @@ =============== Default configuration of the :ref:`AI summary plugin `. -Users configure the Ollama server URL and the model in the *AI Summary* tab of -their preferences; the values below only act as instance wide defaults. +Users configure the LLM server URL (any server implementing the OpenAI chat +completions API: Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...) +and the model in the *AI Summary* tab of their preferences; the values below +only act as instance wide defaults. .. code:: yaml diff --git a/searx/ai_summary.py b/searx/ai_summary.py index bab838b59..5d4a0b482 100644 --- a/searx/ai_summary.py +++ b/searx/ai_summary.py @@ -7,7 +7,7 @@ # - https://github.com/searxng/searxng/issues/5284 from __future__ import annotations -__all__ = ["SettingsAISummary", "MODELS", "model_choices", "build_ollama_messages"] +__all__ = ["SettingsAISummary", "MODELS", "model_choices", "build_chat_messages"] import msgspec @@ -43,9 +43,11 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True """ base_url: str = "" - """Default base URL of the Ollama server (e.g. ``http://127.0.0.1:11434``). - Users can set their own server URL in the preferences - (``ai_summary_server``) unless that preference is locked.""" + """Default base URL of the LLM server (e.g. ``http://127.0.0.1:11434`` + for Ollama). Any server that implements the OpenAI chat completions API + works (Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...). Users + can set their own server URL in the preferences (``ai_summary_server``) + unless that preference is locked.""" model: str = "" """Name of the default model (e.g. ``llama3.2:3b``). If empty, the first @@ -56,7 +58,7 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True models: list[str] = [] """List of model names suggested to the user in the preferences. If empty and :py:obj:`SettingsAISummary.base_url` is set, the list is requested - once at application setup from the Ollama server (``GET /api/tags``).""" + once at application setup from the LLM server (``GET /v1/models``).""" grounding: bool = False """Default of the ``ai_summary_grounding`` user preference: ground the @@ -64,7 +66,7 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True preferences unless that preference is locked.""" connect_timeout: float = 5.0 - """Timeout (seconds) to establish a TCP connection to the Ollama server.""" + """Timeout (seconds) to establish a TCP connection to the LLM server.""" read_timeout: float = 30.0 """Maximum gap (seconds) between two chunks of the token stream.""" @@ -72,10 +74,6 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True stream_timeout: float = 120.0 """Wall clock limit (seconds) for one completion.""" - keep_alive: str = "5m" - """How long the model stays loaded in memory after the request (passed - through to Ollama's ``keep_alive`` option).""" - max_context_items: int = 5 """Maximum number of search results accepted as grounding context.""" @@ -99,13 +97,13 @@ def model_choices() -> list[str]: return list(MODELS) -def build_ollama_messages( +def build_chat_messages( cfg: SettingsAISummary, messages: list[dict[str, str]], context: list[dict[str, str]] | None = None, ) -> list[dict[str, str]]: - """Build the message list for Ollama's ``/api/chat`` from the (already - validated) request ``messages``, prepending a system prompt. When + """Build the message list for the chat completions request from the + (already validated) request ``messages``, prepending a system prompt. When ``context`` items are given, the grounded system prompt is used and the context items are serialized into its ``{context}`` placeholder.""" diff --git a/searx/plugins/ai_summary.py b/searx/plugins/ai_summary.py index 4e6da0416..b4ae02f18 100644 --- a/searx/plugins/ai_summary.py +++ b/searx/plugins/ai_summary.py @@ -1,8 +1,10 @@ # SPDX-License-Identifier: AGPL-3.0-or-later """Plugin that displays an AI generated summary of the search query at the top -of the result page. The summary is generated by a (local) `Ollama`_ server. +of the result page. The summary is generated by a (local) LLM server that +implements the `OpenAI chat completions API`_ -- e.g. `Ollama`_, vLLM, +llama.cpp, LM Studio or Hugging Face TGI. -The Ollama server URL and the model are configured by the user in the *AI +The LLM server URL and the model are configured by the user in the *AI Summary* tab of the preferences (``ai_summary_server``, ``ai_summary_model``); the administrator can configure instance wide defaults in the ``ai_summary:`` section and lock the preferences via :ref:`settings preferences`. @@ -20,17 +22,17 @@ The result page is never delayed by this plugin: it only places an empty placeholder (:py:obj:`searx.result_types.AiSummary`) in the answer area, which is filled asynchronously by the client (``client/simple/src/js/plugin/ AiSummary.ts``) from the ``/ai_summary`` endpoint (registered in -:py:obj:`searx.webapp`). The endpoint streams the tokens from Ollama's -``/api/chat`` to the client as `NDJSON`_. +:py:obj:`searx.webapp`). The endpoint re-emits the SSE token stream of the +LLM server's ``/v1/chat/completions`` to the client as `NDJSON`_. A summary is only generated on the first page of a *general* search and only if no engine has contributed an infobox (e.g. wikipedia / wikidata) or an instant answer (e.g. ddg definitions) -- in these cases the query is most likely a lookup of a well known term that is already answered. -The requests to the Ollama server are sent directly (not via +The requests to the LLM server are sent directly (not via :py:obj:`searx.network`), an outgoing proxy configuration is deliberately not -applied to reach an Ollama server in the local network. +applied to reach an LLM server in the local network. Configuration of the defaults (:py:obj:`searx.ai_summary.SettingsAISummary`): @@ -47,6 +49,7 @@ Configuration of the defaults (:py:obj:`searx.ai_summary.SettingsAISummary`): active: false .. _Ollama: https://ollama.com/ +.. _OpenAI chat completions API: https://platform.openai.com/docs/api-reference/chat .. _NDJSON: https://github.com/ndjson/ndjson-spec .. _SSRF: https://owasp.org/www-community/attacks/Server_Side_Request_Forgery """ @@ -64,7 +67,7 @@ import httpx from flask_babel import gettext from searx import get_setting -from searx.ai_summary import SettingsAISummary, build_ollama_messages +from searx.ai_summary import SettingsAISummary, build_chat_messages from searx.extended_types import sxng_request from searx.result_types import EngineResults import searx.ai_summary @@ -81,7 +84,12 @@ MODEL_NAME_REGEXP = re.compile(r"[A-Za-z0-9._:/-]{1,128}") def _get_client(base_url: str, cfg: SettingsAISummary) -> httpx.Client: - """HTTP client for one request to the Ollama server at ``base_url``.""" + """HTTP client for one request to the LLM server at ``base_url``.""" + # the OpenAI API paths are prefixed with /v1, unless the base URL already + # points into an API prefix + base_url = base_url.rstrip("/") + if not base_url.endswith("/v1"): + base_url += "/v1" return httpx.Client( base_url=base_url, timeout=httpx.Timeout(connect=cfg.connect_timeout, read=cfg.read_timeout, write=10.0, pool=10.0), @@ -97,7 +105,7 @@ def _valid_server(url: str) -> bool: def _user_server(request: "SXNG_Request", cfg: SettingsAISummary) -> str: - """The Ollama server URL for this request: the user's ``ai_summary_server`` + """The LLM server URL for this request: the user's ``ai_summary_server`` preference, or the administrator's default.""" return str(request.preferences.get_value("ai_summary_server") or "").strip() or cfg.base_url @@ -116,7 +124,7 @@ class SXNGPlugin(Plugin): name=gettext("AI summary"), description=gettext( "Show an AI generated summary of the search query on top of the" - " result page (uses an Ollama server, see the settings below)." + " result page (uses a local LLM server, see the settings below)." ), preference_section="ai", ) @@ -134,14 +142,14 @@ class SXNGPlugin(Plugin): return True def _probe_models(self, cfg: SettingsAISummary) -> list[str]: - """Request the list of models from the Ollama server (``GET - /api/tags``). The server might not be up when SearXNG starts, a + """Request the list of models from the LLM server (``GET + /v1/models``). The server might not be up when SearXNG starts, a failing probe only leaves the model suggestion list empty.""" try: with _get_client(cfg.base_url, cfg) as client: - resp = client.get("/api/tags") + resp = client.get("/models") resp.raise_for_status() - models = [model["name"] for model in resp.json().get("models", [])] + models = [model["id"] for model in resp.json().get("data", [])] except (httpx.HTTPError, ValueError, KeyError) as exc: self.log.warning("can't request model list from %s: %s", cfg.base_url, exc) models = [] @@ -163,7 +171,7 @@ class SXNGPlugin(Plugin): or bool(search.result_container.infoboxes) or bool(search.result_container.answers) or not sq.query.strip() - # without an Ollama server (user preference or admin default) + # without an LLM server (user preference or admin default) # there is nothing to show or not _user_server(request, cfg) ) @@ -233,23 +241,22 @@ def ai_summary_view() -> flask.Response: server = _user_server(sxng_request, cfg) if not _valid_server(server): - return _bad_request("no valid Ollama server configured") + return _bad_request("no valid LLM server configured") model = str(sxng_request.preferences.get_value("ai_summary_model") or "").strip() or cfg.model if not MODEL_NAME_REGEXP.fullmatch(model): return _bad_request("no valid model configured") - ollama_payload = { + chat_payload = { "model": model, - "messages": build_ollama_messages(cfg, messages, context), + "messages": build_chat_messages(cfg, messages, context), "stream": True, - "keep_alive": cfg.keep_alive, } # open the upstream connection before streaming, a connection error is # reported as HTTP 502 instead of a line in an already started stream client = _get_client(server, cfg) - stream_ctx = client.stream("POST", "/api/chat", json=ollama_payload) + stream_ctx = client.stream("POST", "/chat/completions", json=chat_payload) upstream = None try: upstream = stream_ctx.__enter__() # pylint: disable=unnecessary-dunder-call @@ -273,22 +280,26 @@ def ai_summary_view() -> flask.Response: def generate(): start = time.monotonic() try: + # the upstream is a SSE stream: "data: {..}" lines, terminated by + # a "data: [DONE]" line for line in upstream.iter_lines(): if time.monotonic() - start > cfg.stream_timeout: yield ndjson({"done": True, "error": "timeout"}) return - if not line.strip(): + line = line.strip() + if not line or line.startswith(":") or not line.startswith("data:"): continue - data = json.loads(line) - if data.get("done"): - yield ndjson({"done": True, "model": model}) - return - delta = data.get("message", {}).get("content", "") + payload = line[len("data:") :].strip() + if payload == "[DONE]": + break + data = json.loads(payload) + choices = data.get("choices") or [{}] + delta = choices[0].get("delta", {}).get("content") or "" if delta: yield ndjson({"delta": delta}) yield ndjson({"done": True, "model": model}) except (httpx.HTTPError, ValueError) as exc: - log.warning("error while streaming from Ollama: %s", exc) + log.warning("error while streaming from the LLM server: %s", exc) yield ndjson({"done": True, "error": "upstream error"}) finally: stream_ctx.__exit__(None, None, None) diff --git a/searx/settings.yml b/searx/settings.yml index 63aec1085..5b1829d08 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -292,14 +292,16 @@ plugins: # # ai_summary: # -# # Base URL of the Ollama server; without this URL the plugin is inactive. +# # Base URL of an OpenAI compatible LLM server (Ollama, vLLM, LM Studio, +# # llama.cpp, Hugging Face TGI, ...), used as the default for the +# # ai_summary_server preference. # base_url: "http://127.0.0.1:11434" # # # Default model; if empty, the first entry of models: is used. # model: "llama3.2:3b" # -# # Models the user can select from in the preferences; if empty, the list -# # is requested from the Ollama server (GET /api/tags) at startup. +# # Models suggested to the user in the preferences; if empty, the list +# # is requested from the LLM server (GET /v1/models) at startup. # models: # - "llama3.2:3b" # - "gemma3:4b" diff --git a/searx/templates/simple/preferences/ai_summary.html b/searx/templates/simple/preferences/ai_summary.html index 2fca71bae..c53d589a0 100644 --- a/searx/templates/simple/preferences/ai_summary.html +++ b/searx/templates/simple/preferences/ai_summary.html @@ -1,6 +1,6 @@ {%- if 'ai_summary_server' not in locked_preferences -%}
{{- '' -}} - {{- _('Ollama server URL') -}}{{- '' -}} + {{- _('AI server URL') -}}{{- '' -}}
{{- '' -}} {{- '' -}}
{{- '' -}}
- {{- _('URL of the Ollama server that generates the summaries, e.g. http://192.168.1.10:11434.') -}} + {{- _('URL of the OpenAI compatible LLM server that generates the summaries (e.g. Ollama, LM Studio, vLLM), e.g. http://192.168.1.10:11434.') -}} {{- ' ' -}} {%- if ai_summary_default_server -%} {{- _('Leave empty to use the default of this instance.') -}} diff --git a/tests/unit/test_plugin_ai_summary.py b/tests/unit/test_plugin_ai_summary.py index 7bb1f9ffd..70ac82fc3 100644 --- a/tests/unit/test_plugin_ai_summary.py +++ b/tests/unit/test_plugin_ai_summary.py @@ -23,12 +23,12 @@ BASE_URL = "http://127.0.0.1:11434" MODEL = "test-model" -def ollama_stream_mock(lines: list[dict], status_code: int = 200) -> Mock: +def sse_stream_mock(lines: list[dict], status_code: int = 200) -> Mock: """A mock httpx client whose ``stream()`` context manager yields the given - (Ollama) NDJSON lines.""" + objects as a SSE stream (OpenAI chat completions format).""" upstream = Mock(status_code=status_code) - upstream.iter_lines.return_value = iter([json.dumps(line) for line in lines]) + upstream.iter_lines.return_value = iter([f"data: {json.dumps(line)}" for line in lines] + ["data: [DONE]"]) @contextmanager def stream(*_args, **_kwargs): @@ -218,11 +218,11 @@ class PluginAISummary(SearxTestCase): def test_endpoint_streams_ndjson(self): self.mock_upstream( - ollama_stream_mock( + sse_stream_mock( [ - {"message": {"content": "Hello "}, "done": False}, - {"message": {"content": "world"}, "done": False}, - {"done": True}, + {"choices": [{"delta": {"content": "Hello "}}]}, + {"choices": [{"delta": {"content": "world"}}]}, + {"choices": [{"delta": {}, "finish_reason": "stop"}]}, ] ) ) @@ -237,7 +237,7 @@ class PluginAISummary(SearxTestCase): self.assertEqual(lines[2], {"done": True, "model": MODEL}) def test_endpoint_model_pref_wins(self): - self.mock_upstream(ollama_stream_mock([{"done": True}])) + self.mock_upstream(sse_stream_mock([])) self.client.set_cookie("ai_summary_model", "my-own-model:7b") res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]}) @@ -245,7 +245,7 @@ class PluginAISummary(SearxTestCase): self.assertEqual(lines[-1], {"done": True, "model": "my-own-model:7b"}) def test_endpoint_upstream_error(self): - self.mock_upstream(ollama_stream_mock([], status_code=500)) + self.mock_upstream(sse_stream_mock([], status_code=500)) res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]}) self.assertEqual(res.status_code, 502) @@ -253,7 +253,7 @@ class PluginAISummary(SearxTestCase): def test_endpoint_error_while_streaming(self): upstream = Mock(status_code=200) upstream.iter_lines.return_value = iter( - [json.dumps({"message": {"content": "Hello"}, "done": False}), "this is not json"] + ['data: {"choices": [{"delta": {"content": "Hello"}}]}', "data: this is not json"] ) @contextmanager