jason/searxng
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[mod] ai_summary plugin: switch to the OpenAI chat completions API

Browse Source 
Talk to the LLM server via GET /v1/models and POST /v1/chat/completions
(SSE) instead of Ollama's native API.  Any OpenAI compatible server now
works (Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...);
Ollama serves this API natively, existing setups keep working unchanged.

The Ollama specific keep_alive option is dropped, the ai_summary.grounding
setting is added as instance wide default of the grounding preference.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jasonwitty
2026-07-28 13:01:24 -07:00
parent 112541db28
commit 4abb7dba67
6 changed files with 70 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/admin/settings/settings_ai_summary.rst
+4 -2
View File
@@ -5,8 +5,10 @@
===============
Default configuration of the :ref:`AI summary plugin <ai_summary plugin>`.
Users configure the Ollama server URL and the model in the *AI Summary* tab of
their preferences; the values below only act as instance wide defaults.
Users configure the LLM server URL (any server implementing the OpenAI chat
completions API: Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...)
and the model in the *AI Summary* tab of their preferences; the values below
only act as instance wide defaults.
.. code:: yaml
searx/ai_summary.py
+11 -13
View File
@@ -7,7 +7,7 @@
# - https://github.com/searxng/searxng/issues/5284
from __future__ import annotations
__all__ = ["SettingsAISummary", "MODELS", "model_choices", "build_ollama_messages"]
__all__ = ["SettingsAISummary", "MODELS", "model_choices", "build_chat_messages"]
import msgspec
@@ -43,9 +43,11 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True
"""
base_url: str = ""
"""Default base URL of the Ollama server (e.g. ``http://127.0.0.1:11434``).
Users can set their own server URL in the preferences
(``ai_summary_server``) unless that preference is locked."""
"""Default base URL of the LLM server (e.g. ``http://127.0.0.1:11434``
for Ollama). Any server that implements the OpenAI chat completions API
works (Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...). Users
can set their own server URL in the preferences (``ai_summary_server``)
unless that preference is locked."""
model: str = ""
"""Name of the default model (e.g. ``llama3.2:3b``). If empty, the first
@@ -56,7 +58,7 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True
models: list[str] = []
"""List of model names suggested to the user in the preferences. If empty
and :py:obj:`SettingsAISummary.base_url` is set, the list is requested
once at application setup from the Ollama server (``GET /api/tags``)."""
once at application setup from the LLM server (``GET /v1/models``)."""
grounding: bool = False
"""Default of the ``ai_summary_grounding`` user preference: ground the
@@ -64,7 +66,7 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True
preferences unless that preference is locked."""
connect_timeout: float = 5.0
"""Timeout (seconds) to establish a TCP connection to the Ollama server."""
"""Timeout (seconds) to establish a TCP connection to the LLM server."""
read_timeout: float = 30.0
"""Maximum gap (seconds) between two chunks of the token stream."""
@@ -72,10 +74,6 @@ class SettingsAISummary(msgspec.Struct, kw_only=True, forbid_unknown_fields=True
stream_timeout: float = 120.0
"""Wall clock limit (seconds) for one completion."""
keep_alive: str = "5m"
"""How long the model stays loaded in memory after the request (passed
through to Ollama's ``keep_alive`` option)."""
max_context_items: int = 5
"""Maximum number of search results accepted as grounding context."""
@@ -99,13 +97,13 @@ def model_choices() -> list[str]:
return list(MODELS)
def build_ollama_messages(
def build_chat_messages(
cfg: SettingsAISummary,
messages: list[dict[str, str]],
context: list[dict[str, str]] | None = None,
) -> list[dict[str, str]]:
"""Build the message list for Ollama's ``/api/chat`` from the (already
validated) request ``messages``, prepending a system prompt. When
"""Build the message list for the chat completions request from the
(already validated) request ``messages``, prepending a system prompt. When
``context`` items are given, the grounded system prompt is used and the
context items are serialized into its ``{context}`` placeholder."""
searx/plugins/ai_summary.py
+38 -27
View File
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Plugin that displays an AI generated summary of the search query at the top
of the result page. The summary is generated by a (local) `Ollama`_ server.
of the result page. The summary is generated by a (local) LLM server that
implements the `OpenAI chat completions API`_ -- e.g. `Ollama`_, vLLM,
llama.cpp, LM Studio or Hugging Face TGI.
The Ollama server URL and the model are configured by the user in the *AI
The LLM server URL and the model are configured by the user in the *AI
Summary* tab of the preferences (``ai_summary_server``, ``ai_summary_model``);
the administrator can configure instance wide defaults in the ``ai_summary:``
section and lock the preferences via :ref:`settings preferences`.
@@ -20,17 +22,17 @@ The result page is never delayed by this plugin: it only places an empty
placeholder (:py:obj:`searx.result_types.AiSummary`) in the answer area, which
is filled asynchronously by the client (``client/simple/src/js/plugin/
AiSummary.ts``) from the ``/ai_summary`` endpoint (registered in
:py:obj:`searx.webapp`). The endpoint streams the tokens from Ollama's
``/api/chat`` to the client as `NDJSON`_.
:py:obj:`searx.webapp`). The endpoint re-emits the SSE token stream of the
LLM server's ``/v1/chat/completions`` to the client as `NDJSON`_.
A summary is only generated on the first page of a *general* search and only
if no engine has contributed an infobox (e.g. wikipedia / wikidata) or an
instant answer (e.g. ddg definitions) -- in these cases the query is most
likely a lookup of a well known term that is already answered.
The requests to the Ollama server are sent directly (not via
The requests to the LLM server are sent directly (not via
:py:obj:`searx.network`), an outgoing proxy configuration is deliberately not
applied to reach an Ollama server in the local network.
applied to reach an LLM server in the local network.
Configuration of the defaults (:py:obj:`searx.ai_summary.SettingsAISummary`):
@@ -47,6 +49,7 @@ Configuration of the defaults (:py:obj:`searx.ai_summary.SettingsAISummary`):
active: false
.. _Ollama: https://ollama.com/
.. _OpenAI chat completions API: https://platform.openai.com/docs/api-reference/chat
.. _NDJSON: https://github.com/ndjson/ndjson-spec
.. _SSRF: https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
"""
@@ -64,7 +67,7 @@ import httpx
from flask_babel import gettext
from searx import get_setting
from searx.ai_summary import SettingsAISummary, build_ollama_messages
from searx.ai_summary import SettingsAISummary, build_chat_messages
from searx.extended_types import sxng_request
from searx.result_types import EngineResults
import searx.ai_summary
@@ -81,7 +84,12 @@ MODEL_NAME_REGEXP = re.compile(r"[A-Za-z0-9._:/-]{1,128}")
def _get_client(base_url: str, cfg: SettingsAISummary) -> httpx.Client:
"""HTTP client for one request to the Ollama server at ``base_url``."""
"""HTTP client for one request to the LLM server at ``base_url``."""
# the OpenAI API paths are prefixed with /v1, unless the base URL already
# points into an API prefix
base_url = base_url.rstrip("/")
if not base_url.endswith("/v1"):
base_url += "/v1"
return httpx.Client(
base_url=base_url,
timeout=httpx.Timeout(connect=cfg.connect_timeout, read=cfg.read_timeout, write=10.0, pool=10.0),
@@ -97,7 +105,7 @@ def _valid_server(url: str) -> bool:
def _user_server(request: "SXNG_Request", cfg: SettingsAISummary) -> str:
"""The Ollama server URL for this request: the user's ``ai_summary_server``
"""The LLM server URL for this request: the user's ``ai_summary_server``
preference, or the administrator's default."""
return str(request.preferences.get_value("ai_summary_server") or "").strip() or cfg.base_url
@@ -116,7 +124,7 @@ class SXNGPlugin(Plugin):
name=gettext("AI summary"),
description=gettext(
"Show an AI generated summary of the search query on top of the"
" result page (uses an Ollama server, see the settings below)."
" result page (uses a local LLM server, see the settings below)."
),
preference_section="ai",
)
@@ -134,14 +142,14 @@ class SXNGPlugin(Plugin):
return True
def _probe_models(self, cfg: SettingsAISummary) -> list[str]:
"""Request the list of models from the Ollama server (``GET
/api/tags``). The server might not be up when SearXNG starts, a
"""Request the list of models from the LLM server (``GET
/v1/models``). The server might not be up when SearXNG starts, a
failing probe only leaves the model suggestion list empty."""
try:
with _get_client(cfg.base_url, cfg) as client:
resp = client.get("/api/tags")
resp = client.get("/models")
resp.raise_for_status()
models = [model["name"] for model in resp.json().get("models", [])]
models = [model["id"] for model in resp.json().get("data", [])]
except (httpx.HTTPError, ValueError, KeyError) as exc:
self.log.warning("can't request model list from %s: %s", cfg.base_url, exc)
models = []
@@ -163,7 +171,7 @@ class SXNGPlugin(Plugin):
or bool(search.result_container.infoboxes)
or bool(search.result_container.answers)
or not sq.query.strip()
# without an Ollama server (user preference or admin default)
# without an LLM server (user preference or admin default)
# there is nothing to show
or not _user_server(request, cfg)
)
@@ -233,23 +241,22 @@ def ai_summary_view() -> flask.Response:
server = _user_server(sxng_request, cfg)
if not _valid_server(server):
return _bad_request("no valid Ollama server configured")
return _bad_request("no valid LLM server configured")
model = str(sxng_request.preferences.get_value("ai_summary_model") or "").strip() or cfg.model
if not MODEL_NAME_REGEXP.fullmatch(model):
return _bad_request("no valid model configured")
ollama_payload = {
chat_payload = {
"model": model,
"messages": build_ollama_messages(cfg, messages, context),
"messages": build_chat_messages(cfg, messages, context),
"stream": True,
"keep_alive": cfg.keep_alive,
}
# open the upstream connection before streaming, a connection error is
# reported as HTTP 502 instead of a line in an already started stream
client = _get_client(server, cfg)
stream_ctx = client.stream("POST", "/api/chat", json=ollama_payload)
stream_ctx = client.stream("POST", "/chat/completions", json=chat_payload)
upstream = None
try:
upstream = stream_ctx.__enter__() # pylint: disable=unnecessary-dunder-call
@@ -273,22 +280,26 @@ def ai_summary_view() -> flask.Response:
def generate():
start = time.monotonic()
try:
# the upstream is a SSE stream: "data: {..}" lines, terminated by
# a "data: [DONE]" line
for line in upstream.iter_lines():
if time.monotonic() - start > cfg.stream_timeout:
yield ndjson({"done": True, "error": "timeout"})
return
if not line.strip():
line = line.strip()
if not line or line.startswith(":") or not line.startswith("data:"):
continue
data = json.loads(line)
if data.get("done"):
yield ndjson({"done": True, "model": model})
return
delta = data.get("message", {}).get("content", "")
payload = line[len("data:") :].strip()
if payload == "[DONE]":
break
data = json.loads(payload)
choices = data.get("choices") or [{}]
delta = choices[0].get("delta", {}).get("content") or ""
if delta:
yield ndjson({"delta": delta})
yield ndjson({"done": True, "model": model})
except (httpx.HTTPError, ValueError) as exc:
log.warning("error while streaming from Ollama: %s", exc)
log.warning("error while streaming from the LLM server: %s", exc)
yield ndjson({"done": True, "error": "upstream error"})
finally:
stream_ctx.__exit__(None, None, None)
searx/settings.yml
+5 -3
View File
@@ -292,14 +292,16 @@ plugins:
#
# ai_summary:
#
# # Base URL of the Ollama server; without this URL the plugin is inactive.
# # Base URL of an OpenAI compatible LLM server (Ollama, vLLM, LM Studio,
# # llama.cpp, Hugging Face TGI, ...), used as the default for the
# # ai_summary_server preference.
# base_url: "http://127.0.0.1:11434"
#
# # Default model; if empty, the first entry of models: is used.
# model: "llama3.2:3b"
#
# # Models the user can select from in the preferences; if empty, the list
# # is requested from the Ollama server (GET /api/tags) at startup.
# # Models suggested to the user in the preferences; if empty, the list
# # is requested from the LLM server (GET /v1/models) at startup.
# models:
# - "llama3.2:3b"
# - "gemma3:4b"
searx/templates/simple/preferences/ai_summary.html
+2 -2
View File
@@ -1,6 +1,6 @@
{%- if 'ai_summary_server' not in locked_preferences -%}
<fieldset>{{- '' -}}
<legend id="pref_ai_summary_server">{{- _('Ollama server URL') -}}</legend>{{- '' -}}
<legend id="pref_ai_summary_server">{{- _('AI server URL') -}}</legend>{{- '' -}}
<div class="value">{{- '' -}}
<input name="ai_summary_server" aria-labelledby="pref_ai_summary_server" type="text"
autocomplete="off" spellcheck="false" autocorrect="off"
@@ -8,7 +8,7 @@
value="{{ preferences.get_value('ai_summary_server') }}">{{- '' -}}
</div>{{- '' -}}
<div class="description">
{{- _('URL of the Ollama server that generates the summaries, e.g. http://192.168.1.10:11434.') -}}
{{- _('URL of the OpenAI compatible LLM server that generates the summaries (e.g. Ollama, LM Studio, vLLM), e.g. http://192.168.1.10:11434.') -}}
{{- ' ' -}}
{%- if ai_summary_default_server -%}
{{- _('Leave empty to use the default of this instance.') -}}
tests/unit/test_plugin_ai_summary.py
+10 -10
View File
@@ -23,12 +23,12 @@ BASE_URL = "http://127.0.0.1:11434"
MODEL = "test-model"
def ollama_stream_mock(lines: list[dict], status_code: int = 200) -> Mock:
def sse_stream_mock(lines: list[dict], status_code: int = 200) -> Mock:
"""A mock httpx client whose ``stream()`` context manager yields the given
(Ollama) NDJSON lines."""
objects as a SSE stream (OpenAI chat completions format)."""
upstream = Mock(status_code=status_code)
upstream.iter_lines.return_value = iter([json.dumps(line) for line in lines])
upstream.iter_lines.return_value = iter([f"data: {json.dumps(line)}" for line in lines] + ["data: [DONE]"])
@contextmanager
def stream(*_args, **_kwargs):
@@ -218,11 +218,11 @@ class PluginAISummary(SearxTestCase):
def test_endpoint_streams_ndjson(self):
self.mock_upstream(
ollama_stream_mock(
sse_stream_mock(
[
{"message": {"content": "Hello "}, "done": False},
{"message": {"content": "world"}, "done": False},
{"done": True},
{"choices": [{"delta": {"content": "Hello "}}]},
{"choices": [{"delta": {"content": "world"}}]},
{"choices": [{"delta": {}, "finish_reason": "stop"}]},
]
)
)
@@ -237,7 +237,7 @@ class PluginAISummary(SearxTestCase):
self.assertEqual(lines[2], {"done": True, "model": MODEL})
def test_endpoint_model_pref_wins(self):
self.mock_upstream(ollama_stream_mock([{"done": True}]))
self.mock_upstream(sse_stream_mock([]))
self.client.set_cookie("ai_summary_model", "my-own-model:7b")
res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]})
@@ -245,7 +245,7 @@ class PluginAISummary(SearxTestCase):
self.assertEqual(lines[-1], {"done": True, "model": "my-own-model:7b"})
def test_endpoint_upstream_error(self):
self.mock_upstream(ollama_stream_mock([], status_code=500))
self.mock_upstream(sse_stream_mock([], status_code=500))
res = self.client.post("/ai_summary", json={"messages": [{"role": "user", "content": "hi"}]})
self.assertEqual(res.status_code, 502)
@@ -253,7 +253,7 @@ class PluginAISummary(SearxTestCase):
def test_endpoint_error_while_streaming(self):
upstream = Mock(status_code=200)
upstream.iter_lines.return_value = iter(
[json.dumps({"message": {"content": "Hello"}, "done": False}), "this is not json"]
['data: {"choices": [{"delta": {"content": "Hello"}}]}', "data: this is not json"]
)
@contextmanager