[mod] ai_summary plugin: switch to the OpenAI chat completions API

Talk to the LLM server via GET /v1/models and POST /v1/chat/completions
(SSE) instead of Ollama's native API.  Any OpenAI compatible server now
works (Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...);
Ollama serves this API natively, existing setups keep working unchanged.

The Ollama specific keep_alive option is dropped, the ai_summary.grounding
setting is added as instance wide default of the grounding preference.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

docs/admin/settings/settings_ai_summary.rst

@@ -5,8 +5,10 @@
 ===============
 
 Default configuration of the :ref:`AI summary plugin <ai_summary plugin>`.
-Users configure the Ollama server URL and the model in the *AI Summary* tab of
+Users configure the LLM server URL (any server implementing the OpenAI chat
-their preferences; the values below only act as instance wide defaults.
+completions API: Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...)
+and the model in the *AI Summary* tab of their preferences; the values below
+only act as instance wide defaults.
 
 .. code:: yaml
 

searx/ai_summary.py

@@ -7,7 +7,7 @@
 # - https://github.com/searxng/searxng/issues/5284
 from __future__ import annotations
 
-__all__ = ["SettingsAISummary", "MODELS", "model_choices", "build_ollama_messages"]
+__all__ = ["SettingsAISummary", "MODELS", "model_choices", "build_chat_messages"]
 
 import msgspec
 
@@ -43,9 +43,11 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True
     """
 
     base_url: str = ""
-    """Default base URL of the Ollama server (e.g. ``http://127.0.0.1:11434``).
+    """Default base URL of the LLM server (e.g. ``http://127.0.0.1:11434``
-    Users can set their own server URL in the preferences
+    for Ollama).  Any server that implements the OpenAI chat completions API
-    (``ai_summary_server``) unless that preference is locked."""
+    works (Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...).  Users
+    can set their own server URL in the preferences (``ai_summary_server``)
+    unless that preference is locked."""
 
     model: str = ""
     """Name of the default model (e.g. ``llama3.2:3b``).  If empty, the first
@@ -56,7 +58,7 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True
     models: list[str] = []
     """List of model names suggested to the user in the preferences.  If empty
     and :py:obj:`SettingsAISummary.base_url` is set, the list is requested
-    once at application setup from the Ollama server (``GET /api/tags``)."""
+    once at application setup from the LLM server (``GET /v1/models``)."""
 
     grounding: bool = False
     """Default of the ``ai_summary_grounding`` user preference: ground the
@@ -64,7 +66,7 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True
     preferences unless that preference is locked."""
 
     connect_timeout: float = 5.0
-    """Timeout (seconds) to establish a TCP connection to the Ollama server."""
+    """Timeout (seconds) to establish a TCP connection to the LLM server."""
 
     read_timeout: float = 30.0
     """Maximum gap (seconds) between two chunks of the token stream."""
@@ -72,10 +74,6 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True
     stream_timeout: float = 120.0
     """Wall clock limit (seconds) for one completion."""
 
-    keep_alive: str = "5m"
-    """How long the model stays loaded in memory after the request (passed
-    through to Ollama's ``keep_alive`` option)."""
-
     max_context_items: int = 5
     """Maximum number of search results accepted as grounding context."""
 
@@ -99,13 +97,13 @@ def model_choices() -> list[str]:
     return list(MODELS)
 
 
-def build_ollama_messages(
+def build_chat_messages(
     cfg: SettingsAISummary,
     messages: list[dict[str, str]],
     context: list[dict[str, str]] | None = None,
 ) -> list[dict[str, str]]:
-    """Build the message list for Ollama's ``/api/chat`` from the (already
+    """Build the message list for the chat completions request from the
-    validated) request ``messages``, prepending a system prompt.  When
+    (already validated) request ``messages``, prepending a system prompt.  When
     ``context`` items are given, the grounded system prompt is used and the
     context items are serialized into its ``{context}`` placeholder."""
 

searx/plugins/ai_summary.py

@@ -1,8 +1,10 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Plugin that displays an AI generated summary of the search query at the top
-of the result page.  The summary is generated by a (local) `Ollama`_ server.
+of the result page.  The summary is generated by a (local) LLM server that
+implements the `OpenAI chat completions API`_ -- e.g. `Ollama`_, vLLM,
+llama.cpp, LM Studio or Hugging Face TGI.
 
-The Ollama server URL and the model are configured by the user in the *AI
+The LLM server URL and the model are configured by the user in the *AI
 Summary* tab of the preferences (``ai_summary_server``, ``ai_summary_model``);
 the administrator can configure instance wide defaults in the ``ai_summary:``
 section and lock the preferences via :ref:`settings preferences`.
@@ -20,17 +22,17 @@ The result page is never delayed by this plugin: it only places an empty
 placeholder (:py:obj:`searx.result_types.AiSummary`) in the answer area, which
 is filled asynchronously by the client (``client/simple/src/js/plugin/
 AiSummary.ts``) from the ``/ai_summary`` endpoint (registered in
-:py:obj:`searx.webapp`).  The endpoint streams the tokens from Ollama's
+:py:obj:`searx.webapp`).  The endpoint re-emits the SSE token stream of the
-``/api/chat`` to the client as `NDJSON`_.
+LLM server's ``/v1/chat/completions`` to the client as `NDJSON`_.
 
 A summary is only generated on the first page of a *general* search and only
 if no engine has contributed an infobox (e.g. wikipedia / wikidata) or an
 instant answer (e.g. ddg definitions) -- in these cases the query is most
 likely a lookup of a well known term that is already answered.
 
-The requests to the Ollama server are sent directly (not via
+The requests to the LLM server are sent directly (not via
 :py:obj:`searx.network`), an outgoing proxy configuration is deliberately not
-applied to reach an Ollama server in the local network.
+applied to reach an LLM server in the local network.
 
 Configuration of the defaults (:py:obj:`searx.ai_summary.SettingsAISummary`):
 
@@ -47,6 +49,7 @@ Configuration of the defaults (:py:obj:`searx.ai_summary.SettingsAISummary`):
        active: false
 
 .. _Ollama: https://ollama.com/
+.. _OpenAI chat completions API: https://platform.openai.com/docs/api-reference/chat
 .. _NDJSON: https://github.com/ndjson/ndjson-spec
 .. _SSRF: https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
 """
@@ -64,7 +67,7 @@ import httpx
 from flask_babel import gettext
 
 from searx import get_setting
-from searx.ai_summary import SettingsAISummary, build_ollama_messages
+from searx.ai_summary import SettingsAISummary, build_chat_messages
 from searx.extended_types import sxng_request
 from searx.result_types import EngineResults
 import searx.ai_summary
@@ -81,7 +84,12 @@ MODEL_NAME_REGEXP = re.compile(r"[A-Za-z0-9._:/-]{1,128}")
 
 
 def _get_client(base_url: str, cfg: SettingsAISummary) -> httpx.Client:
-    """HTTP client for one request to the Ollama server at ``base_url``."""
+    """HTTP client for one request to the LLM server at ``base_url``."""
+    # the OpenAI API paths are prefixed with /v1, unless the base URL already
+    # points into an API prefix
+    base_url = base_url.rstrip("/")
+    if not base_url.endswith("/v1"):
+        base_url += "/v1"
     return httpx.Client(
         base_url=base_url,
         timeout=httpx.Timeout(connect=cfg.connect_timeout, read=cfg.read_timeout, write=10.0, pool=10.0),
@@ -97,7 +105,7 @@ def _valid_server(url: str) -> bool:
 
 
 def _user_server(request: "SXNG_Request", cfg: SettingsAISummary) -> str:
-    """The Ollama server URL for this request: the user's ``ai_summary_server``
+    """The LLM server URL for this request: the user's ``ai_summary_server``
     preference, or the administrator's default."""
     return str(request.preferences.get_value("ai_summary_server") or "").strip() or cfg.base_url
 
@@ -116,7 +124,7 @@ class SXNGPlugin(Plugin):
             name=gettext("AI summary"),
             description=gettext(
                 "Show an AI generated summary of the search query on top of the"
-                " result page (uses an Ollama server, see the settings below)."
+                " result page (uses a local LLM server, see the settings below)."
             ),
             preference_section="ai",
         )
@@ -134,14 +142,14 @@ class SXNGPlugin(Plugin):
         return True
 
     def _probe_models(self, cfg: SettingsAISummary) -> list[str]:
-        """Request the list of models from the Ollama server (``GET
+        """Request the list of models from the LLM server (``GET
-        /api/tags``).  The server might not be up when SearXNG starts, a
+        /v1/models``).  The server might not be up when SearXNG starts, a
         failing probe only leaves the model suggestion list empty."""
         try:
             with _get_client(cfg.base_url, cfg) as client:
-                resp = client.get("/api/tags")
+                resp = client.get("/models")
                 resp.raise_for_status()
-                models = [model["name"] for model in resp.json().get("models", [])]
+                models = [model["id"] for model in resp.json().get("data", [])]
         except (httpx.HTTPError, ValueError, KeyError) as exc:
             self.log.warning("can't request model list from %s: %s", cfg.base_url, exc)
             models = []
@@ -163,7 +171,7 @@ class SXNGPlugin(Plugin):
             or bool(search.result_container.infoboxes)
             or bool(search.result_container.answers)
             or not sq.query.strip()
-            # without an Ollama server (user preference or admin default)
+            # without an LLM server (user preference or admin default)
             # there is nothing to show
             or not _user_server(request, cfg)
         )
@@ -233,23 +241,22 @@ def ai_summary_view() -> flask.Response:
 
     server = _user_server(sxng_request, cfg)
     if not _valid_server(server):
-        return _bad_request("no valid Ollama server configured")
+        return _bad_request("no valid LLM server configured")
 
     model = str(sxng_request.preferences.get_value("ai_summary_model") or "").strip() or cfg.model
     if not MODEL_NAME_REGEXP.fullmatch(model):
         return _bad_request("no valid model configured")
 
-    ollama_payload = {
+    chat_payload = {
         "model": model,
-        "messages": build_ollama_messages(cfg, messages, context),
+        "messages": build_chat_messages(cfg, messages, context),
         "stream": True,
-        "keep_alive": cfg.keep_alive,
     }
 
     # open the upstream connection before streaming, a connection error is
     # reported as HTTP 502 instead of a line in an already started stream
     client = _get_client(server, cfg)
-    stream_ctx = client.stream("POST", "/api/chat", json=ollama_payload)
+    stream_ctx = client.stream("POST", "/chat/completions", json=chat_payload)
     upstream = None
     try:
         upstream = stream_ctx.__enter__()  # pylint: disable=unnecessary-dunder-call
@@ -273,22 +280,26 @@ def ai_summary_view() -> flask.Response:
     def generate():
         start = time.monotonic()
         try:
+            # the upstream is a SSE stream: "data: {..}" lines, terminated by
+            # a "data: [DONE]" line
             for line in upstream.iter_lines():
                 if time.monotonic() - start > cfg.stream_timeout:
                     yield ndjson({"done": True, "error": "timeout"})
                     return
-                if not line.strip():
+                line = line.strip()
+                if not line or line.startswith(":") or not line.startswith("data:"):
                     continue
-                data = json.loads(line)
+                payload = line[len("data:") :].strip()
-                if data.get("done"):
+                if payload == "[DONE]":
-                    yield ndjson({"done": True, "model": model})
+                    break
-                    return
+                data = json.loads(payload)
-                delta = data.get("message", {}).get("content", "")
+                choices = data.get("choices") or [{}]
+                delta = choices[0].get("delta", {}).get("content") or ""
                 if delta:
                     yield ndjson({"delta": delta})
             yield ndjson({"done": True, "model": model})
         except (httpx.HTTPError, ValueError) as exc:
-            log.warning("error while streaming from Ollama: %s", exc)
+            log.warning("error while streaming from the LLM server: %s", exc)
             yield ndjson({"done": True, "error": "upstream error"})
         finally:
             stream_ctx.__exit__(None, None, None)

searx/settings.yml

@@ -292,14 +292,16 @@ plugins:
 #
 # ai_summary:
 #
-#   # Base URL of the Ollama server; without this URL the plugin is inactive.
+#   # Base URL of an OpenAI compatible LLM server (Ollama, vLLM, LM Studio,
+#   # llama.cpp, Hugging Face TGI, ...), used as the default for the
+#   # ai_summary_server preference.
 #   base_url: "http://127.0.0.1:11434"
 #
 #   # Default model; if empty, the first entry of models: is used.
 #   model: "llama3.2:3b"
 #
-#   # Models the user can select from in the preferences; if empty, the list
+#   # Models suggested to the user in the preferences; if empty, the list
-#   # is requested from the Ollama server (GET /api/tags) at startup.
+#   # is requested from the LLM server (GET /v1/models) at startup.
 #   models:
 #     - "llama3.2:3b"
 #     - "gemma3:4b"

searx/templates/simple/preferences/ai_summary.html

@@ -1,6 +1,6 @@
 {%- if 'ai_summary_server' not in locked_preferences -%}
 <fieldset>{{- '' -}}
-  <legend id="pref_ai_summary_server">{{- _('Ollama server URL') -}}</legend>{{- '' -}}
+  <legend id="pref_ai_summary_server">{{- _('AI server URL') -}}</legend>{{- '' -}}
   <div class="value">{{- '' -}}
     <input name="ai_summary_server" aria-labelledby="pref_ai_summary_server" type="text"
            autocomplete="off" spellcheck="false" autocorrect="off"
@@ -8,7 +8,7 @@
            value="{{ preferences.get_value('ai_summary_server') }}">{{- '' -}}
   </div>{{- '' -}}
   <div class="description">
-    {{- _('URL of the Ollama server that generates the summaries, e.g. http://192.168.1.10:11434.') -}}
+    {{- _('URL of the OpenAI compatible LLM server that generates the summaries (e.g. Ollama, LM Studio, vLLM), e.g. http://192.168.1.10:11434.') -}}
     {{- ' ' -}}
     {%- if ai_summary_default_server -%}
       {{- _('Leave empty to use the default of this instance.') -}}

tests/unit/test_plugin_ai_summary.py

@@ -23,12 +23,12 @@ BASE_URL = "http://127.0.0.1:11434"
 MODEL = "test-model"
 
 
-def ollama_stream_mock(lines: list[dict], status_code: int = 200) -> Mock:
+def sse_stream_mock(lines: list[dict], status_code: int = 200) -> Mock:
     """A mock httpx client whose ``stream()`` context manager yields the given
-    (Ollama) NDJSON lines."""
+    objects as a SSE stream (OpenAI chat completions format)."""
 
     upstream = Mock(status_code=status_code)
-    upstream.iter_lines.return_value = iter([json.dumps(line) for line in lines])
+    upstream.iter_lines.return_value = iter([f"data: {json.dumps(line)}" for line in lines] + ["data: [DONE]"])
 
     @contextmanager
     def stream(*_args, **_kwargs):
@@ -218,11 +218,11 @@ class PluginAISummary(SearxTestCase):
 
     def test_endpoint_streams_ndjson(self):
         self.mock_upstream(
-            ollama_stream_mock(
+            sse_stream_mock(
                 [
-                    {"message": {"content": "Hello "}, "done": False},
+                    {"choices": [{"delta": {"content": "Hello "}}]},
-                    {"message": {"content": "world"}, "done": False},
+                    {"choices": [{"delta": {"content": "world"}}]},
-                    {"done": True},
+                    {"choices": [{"delta": {}, "finish_reason": "stop"}]},
                 ]
             )
         )
@@ -237,7 +237,7 @@ class PluginAISummary(SearxTestCase):
         self.assertEqual(lines[2], {"done": True, "model": MODEL})
 
     def test_endpoint_model_pref_wins(self):
-        self.mock_upstream(ollama_stream_mock([{"done": True}]))
+        self.mock_upstream(sse_stream_mock([]))
         self.client.set_cookie("ai_summary_model", "my-own-model:7b")
 
         res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]})
@@ -245,7 +245,7 @@ class PluginAISummary(SearxTestCase):
         self.assertEqual(lines[-1], {"done": True, "model": "my-own-model:7b"})
 
     def test_endpoint_upstream_error(self):
-        self.mock_upstream(ollama_stream_mock([], status_code=500))
+        self.mock_upstream(sse_stream_mock([], status_code=500))
 
         res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]})
         self.assertEqual(res.status_code, 502)
@@ -253,7 +253,7 @@ class PluginAISummary(SearxTestCase):
     def test_endpoint_error_while_streaming(self):
         upstream = Mock(status_code=200)
         upstream.iter_lines.return_value = iter(
-            [json.dumps({"message": {"content": "Hello"}, "done": False}), "this is not json"]
+            ['data: {"choices": [{"delta": {"content": "Hello"}}]}', "data: this is not json"]
         )
 
         @contextmanager