jason/searxng
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[mod] ai_summary plugin: switch to the OpenAI chat completions API

Browse Source 
Talk to the LLM server via GET /v1/models and POST /v1/chat/completions
(SSE) instead of Ollama's native API.  Any OpenAI compatible server now
works (Ollama, vLLM, llama.cpp, LM Studio, Hugging Face TGI, ...);
Ollama serves this API natively, existing setups keep working unchanged.

The Ollama specific keep_alive option is dropped, the ai_summary.grounding
setting is added as instance wide default of the grounding preference.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jasonwitty
2026-07-28 13:01:24 -07:00
parent 112541db28
commit 4abb7dba67
6 changed files with 70 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
searx/plugins/ai_summary.py
+38 -27
View File
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Plugin that displays an AI generated summary of the search query at the top
of the result page. The summary is generated by a (local) `Ollama`_ server.
of the result page. The summary is generated by a (local) LLM server that
implements the `OpenAI chat completions API`_ -- e.g. `Ollama`_, vLLM,
llama.cpp, LM Studio or Hugging Face TGI.
The Ollama server URL and the model are configured by the user in the *AI
The LLM server URL and the model are configured by the user in the *AI
Summary* tab of the preferences (``ai_summary_server``, ``ai_summary_model``);
the administrator can configure instance wide defaults in the ``ai_summary:``
section and lock the preferences via :ref:`settings preferences`.
@@ -20,17 +22,17 @@ The result page is never delayed by this plugin: it only places an empty
placeholder (:py:obj:`searx.result_types.AiSummary`) in the answer area, which
is filled asynchronously by the client (``client/simple/src/js/plugin/
AiSummary.ts``) from the ``/ai_summary`` endpoint (registered in
:py:obj:`searx.webapp`). The endpoint streams the tokens from Ollama's
``/api/chat`` to the client as `NDJSON`_.
:py:obj:`searx.webapp`). The endpoint re-emits the SSE token stream of the
LLM server's ``/v1/chat/completions`` to the client as `NDJSON`_.
A summary is only generated on the first page of a *general* search and only
if no engine has contributed an infobox (e.g. wikipedia / wikidata) or an
instant answer (e.g. ddg definitions) -- in these cases the query is most
likely a lookup of a well known term that is already answered.
The requests to the Ollama server are sent directly (not via
The requests to the LLM server are sent directly (not via
:py:obj:`searx.network`), an outgoing proxy configuration is deliberately not
applied to reach an Ollama server in the local network.
applied to reach an LLM server in the local network.
Configuration of the defaults (:py:obj:`searx.ai_summary.SettingsAISummary`):
@@ -47,6 +49,7 @@ Configuration of the defaults (:py:obj:`searx.ai_summary.SettingsAISummary`):
active: false
.. _Ollama: https://ollama.com/
.. _OpenAI chat completions API: https://platform.openai.com/docs/api-reference/chat
.. _NDJSON: https://github.com/ndjson/ndjson-spec
.. _SSRF: https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
"""
@@ -64,7 +67,7 @@ import httpx
from flask_babel import gettext
from searx import get_setting
from searx.ai_summary import SettingsAISummary, build_ollama_messages
from searx.ai_summary import SettingsAISummary, build_chat_messages
from searx.extended_types import sxng_request
from searx.result_types import EngineResults
import searx.ai_summary
@@ -81,7 +84,12 @@ MODEL_NAME_REGEXP = re.compile(r"[A-Za-z0-9._:/-]{1,128}")
def _get_client(base_url: str, cfg: SettingsAISummary) -> httpx.Client:
"""HTTP client for one request to the Ollama server at ``base_url``."""
"""HTTP client for one request to the LLM server at ``base_url``."""
# the OpenAI API paths are prefixed with /v1, unless the base URL already
# points into an API prefix
base_url = base_url.rstrip("/")
if not base_url.endswith("/v1"):
base_url += "/v1"
return httpx.Client(
base_url=base_url,
timeout=httpx.Timeout(connect=cfg.connect_timeout, read=cfg.read_timeout, write=10.0, pool=10.0),
@@ -97,7 +105,7 @@ def _valid_server(url: str) -> bool:
def _user_server(request: "SXNG_Request", cfg: SettingsAISummary) -> str:
"""The Ollama server URL for this request: the user's ``ai_summary_server``
"""The LLM server URL for this request: the user's ``ai_summary_server``
preference, or the administrator's default."""
return str(request.preferences.get_value("ai_summary_server") or "").strip() or cfg.base_url
@@ -116,7 +124,7 @@ class SXNGPlugin(Plugin):
name=gettext("AI summary"),
description=gettext(
"Show an AI generated summary of the search query on top of the"
" result page (uses an Ollama server, see the settings below)."
" result page (uses a local LLM server, see the settings below)."
),
preference_section="ai",
)
@@ -134,14 +142,14 @@ class SXNGPlugin(Plugin):
return True
def _probe_models(self, cfg: SettingsAISummary) -> list[str]:
"""Request the list of models from the Ollama server (``GET
/api/tags``). The server might not be up when SearXNG starts, a
"""Request the list of models from the LLM server (``GET
/v1/models``). The server might not be up when SearXNG starts, a
failing probe only leaves the model suggestion list empty."""
try:
with _get_client(cfg.base_url, cfg) as client:
resp = client.get("/api/tags")
resp = client.get("/models")
resp.raise_for_status()
models = [model["name"] for model in resp.json().get("models", [])]
models = [model["id"] for model in resp.json().get("data", [])]
except (httpx.HTTPError, ValueError, KeyError) as exc:
self.log.warning("can't request model list from %s: %s", cfg.base_url, exc)
models = []
@@ -163,7 +171,7 @@ class SXNGPlugin(Plugin):
or bool(search.result_container.infoboxes)
or bool(search.result_container.answers)
or not sq.query.strip()
# without an Ollama server (user preference or admin default)
# without an LLM server (user preference or admin default)
# there is nothing to show
or not _user_server(request, cfg)
)
@@ -233,23 +241,22 @@ def ai_summary_view() -> flask.Response:
server = _user_server(sxng_request, cfg)
if not _valid_server(server):
return _bad_request("no valid Ollama server configured")
return _bad_request("no valid LLM server configured")
model = str(sxng_request.preferences.get_value("ai_summary_model") or "").strip() or cfg.model
if not MODEL_NAME_REGEXP.fullmatch(model):
return _bad_request("no valid model configured")
ollama_payload = {
chat_payload = {
"model": model,
"messages": build_ollama_messages(cfg, messages, context),
"messages": build_chat_messages(cfg, messages, context),
"stream": True,
"keep_alive": cfg.keep_alive,
}
# open the upstream connection before streaming, a connection error is
# reported as HTTP 502 instead of a line in an already started stream
client = _get_client(server, cfg)
stream_ctx = client.stream("POST", "/api/chat", json=ollama_payload)
stream_ctx = client.stream("POST", "/chat/completions", json=chat_payload)
upstream = None
try:
upstream = stream_ctx.__enter__() # pylint: disable=unnecessary-dunder-call
@@ -273,22 +280,26 @@ def ai_summary_view() -> flask.Response:
def generate():
start = time.monotonic()
try:
# the upstream is a SSE stream: "data: {..}" lines, terminated by
# a "data: [DONE]" line
for line in upstream.iter_lines():
if time.monotonic() - start > cfg.stream_timeout:
yield ndjson({"done": True, "error": "timeout"})
return
if not line.strip():
line = line.strip()
if not line or line.startswith(":") or not line.startswith("data:"):
continue
data = json.loads(line)
if data.get("done"):
yield ndjson({"done": True, "model": model})
return
delta = data.get("message", {}).get("content", "")
payload = line[len("data:") :].strip()
if payload == "[DONE]":
break
data = json.loads(payload)
choices = data.get("choices") or [{}]
delta = choices[0].get("delta", {}).get("content") or ""
if delta:
yield ndjson({"delta": delta})
yield ndjson({"done": True, "model": model})
except (httpx.HTTPError, ValueError) as exc:
log.warning("error while streaming from Ollama: %s", exc)
log.warning("error while streaming from the LLM server: %s", exc)
yield ndjson({"done": True, "error": "upstream error"})
finally:
stream_ctx.__exit__(None, None, None)