[feat] plugin: AI summary of search results via local Ollama server
Add an optional, disabled-by-default plugin that shows an AI generated summary at the top of the result page, generated by a (local) Ollama server: - async: the result page is never delayed; a client plugin streams the answer (NDJSON over a new /ai_summary endpoint) into a placeholder answer with a typing indicator, collapsed behind a More button, with an inline follow-up chat - trigger: first page of general searches only, skipped when an infobox or instant answer already answers the query - grounding (per-user preference): send the top result snippets as context, the model answers from them instead of its own knowledge - configuration: new AI Summary preferences tab (server URL, model, grounding) with instance defaults in a new ai_summary: settings section; all three preferences can be locked for public instances Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jasonwitty
co-authored by
Claude Fable 5
Claude Fable 5
parent
b060c780d0
commit
b47fd08cb7
@@ -42,7 +42,7 @@ class PluginInfo:
|
||||
description: str
|
||||
"""Short description of the *answerer*."""
|
||||
|
||||
preference_section: t.Literal["general", "ui", "privacy", "query"] | None = "general"
|
||||
preference_section: t.Literal["general", "ui", "privacy", "query", "ai"] | None = "general"
|
||||
"""Section (tab/group) in the preferences where this plugin is shown to the
|
||||
user.
|
||||
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Plugin that displays an AI generated summary of the search query at the top
|
||||
of the result page. The summary is generated by a (local) `Ollama`_ server.
|
||||
|
||||
The Ollama server URL and the model are configured by the user in the *AI
|
||||
Summary* tab of the preferences (``ai_summary_server``, ``ai_summary_model``);
|
||||
the administrator can configure instance wide defaults in the ``ai_summary:``
|
||||
section and lock the preferences via :ref:`settings preferences`.
|
||||
|
||||
.. attention::
|
||||
|
||||
A user configurable server URL allows any user of the instance to make the
|
||||
SearXNG server send requests to a URL of their choice (`SSRF`_), and each
|
||||
summary is real LLM work. This plugin is intended for private instances --
|
||||
on a public instance, lock the ``ai_summary_server``, ``ai_summary_model``
|
||||
and ``ai_summary_grounding`` preferences and configure the ``ai_summary:``
|
||||
section instead.
|
||||
|
||||
The result page is never delayed by this plugin: it only places an empty
|
||||
placeholder (:py:obj:`searx.result_types.AiSummary`) in the answer area, which
|
||||
is filled asynchronously by the client (``client/simple/src/js/plugin/
|
||||
AiSummary.ts``) from the ``/ai_summary`` endpoint (registered in
|
||||
:py:obj:`searx.webapp`). The endpoint streams the tokens from Ollama's
|
||||
``/api/chat`` to the client as `NDJSON`_.
|
||||
|
||||
A summary is only generated on the first page of a *general* search and only
|
||||
if no engine has contributed an infobox (e.g. wikipedia / wikidata) or an
|
||||
instant answer (e.g. ddg definitions) -- in these cases the query is most
|
||||
likely a lookup of a well known term that is already answered.
|
||||
|
||||
The requests to the Ollama server are sent directly (not via
|
||||
:py:obj:`searx.network`), an outgoing proxy configuration is deliberately not
|
||||
applied to reach an Ollama server in the local network.
|
||||
|
||||
Configuration of the defaults (:py:obj:`searx.ai_summary.SettingsAISummary`):
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
ai_summary:
|
||||
base_url: "http://127.0.0.1:11434"
|
||||
model: "llama3.2:3b"
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
plugins:
|
||||
searx.plugins.ai_summary.SXNGPlugin:
|
||||
active: false
|
||||
|
||||
.. _Ollama: https://ollama.com/
|
||||
.. _NDJSON: https://github.com/ndjson/ndjson-spec
|
||||
.. _SSRF: https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import flask
|
||||
import httpx
|
||||
from flask_babel import gettext
|
||||
|
||||
from searx import get_setting
|
||||
from searx.ai_summary import SettingsAISummary, build_ollama_messages
|
||||
from searx.extended_types import sxng_request
|
||||
from searx.result_types import EngineResults
|
||||
import searx.ai_summary
|
||||
|
||||
from . import Plugin, PluginInfo
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.search import SearchWithPlugins
|
||||
from searx.extended_types import SXNG_Request
|
||||
from . import PluginCfg
|
||||
|
||||
VALID_ROLES = ("user", "assistant")
|
||||
MODEL_NAME_REGEXP = re.compile(r"[A-Za-z0-9._:/-]{1,128}")
|
||||
|
||||
|
||||
def _get_client(base_url: str, cfg: SettingsAISummary) -> httpx.Client:
|
||||
"""HTTP client for one request to the Ollama server at ``base_url``."""
|
||||
return httpx.Client(
|
||||
base_url=base_url,
|
||||
timeout=httpx.Timeout(connect=cfg.connect_timeout, read=cfg.read_timeout, write=10.0, pool=10.0),
|
||||
)
|
||||
|
||||
|
||||
def _valid_server(url: str) -> bool:
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except ValueError:
|
||||
return False
|
||||
return parsed.scheme in ("http", "https") and bool(parsed.netloc) and len(url) <= 256
|
||||
|
||||
|
||||
def _user_server(request: "SXNG_Request", cfg: SettingsAISummary) -> str:
|
||||
"""The Ollama server URL for this request: the user's ``ai_summary_server``
|
||||
preference, or the administrator's default."""
|
||||
return str(request.preferences.get_value("ai_summary_server") or "").strip() or cfg.base_url
|
||||
|
||||
|
||||
class SXNGPlugin(Plugin):
|
||||
"""Plugin that adds the AI summary placeholder to the result page, the
|
||||
``/ai_summary`` endpoint itself is registered in :py:obj:`searx.webapp`."""
|
||||
|
||||
id = "ai_summary"
|
||||
|
||||
def __init__(self, plg_cfg: "PluginCfg"):
|
||||
super().__init__(plg_cfg)
|
||||
|
||||
self.info = PluginInfo(
|
||||
id=self.id,
|
||||
name=gettext("AI summary"),
|
||||
description=gettext(
|
||||
"Show an AI generated summary of the search query on top of the"
|
||||
" result page (uses an Ollama server, see the settings below)."
|
||||
),
|
||||
preference_section="ai",
|
||||
)
|
||||
|
||||
def init(self, app: "flask.Flask") -> bool:
|
||||
cfg: SettingsAISummary = get_setting("ai_summary")
|
||||
|
||||
if cfg.base_url:
|
||||
searx.ai_summary.MODELS = list(cfg.models) or self._probe_models(cfg)
|
||||
if not cfg.model and searx.ai_summary.MODELS:
|
||||
cfg.model = searx.ai_summary.MODELS[0]
|
||||
if cfg.model and cfg.model not in searx.ai_summary.MODELS:
|
||||
searx.ai_summary.MODELS.insert(0, cfg.model)
|
||||
|
||||
return True
|
||||
|
||||
def _probe_models(self, cfg: SettingsAISummary) -> list[str]:
|
||||
"""Request the list of models from the Ollama server (``GET
|
||||
/api/tags``). The server might not be up when SearXNG starts, a
|
||||
failing probe only leaves the model suggestion list empty."""
|
||||
try:
|
||||
with _get_client(cfg.base_url, cfg) as client:
|
||||
resp = client.get("/api/tags")
|
||||
resp.raise_for_status()
|
||||
models = [model["name"] for model in resp.json().get("models", [])]
|
||||
except (httpx.HTTPError, ValueError, KeyError) as exc:
|
||||
self.log.warning("can't request model list from %s: %s", cfg.base_url, exc)
|
||||
models = []
|
||||
return models or ([cfg.model] if cfg.model else [])
|
||||
|
||||
def post_search(self, request: "SXNG_Request", search: "SearchWithPlugins") -> EngineResults | None:
|
||||
results = EngineResults()
|
||||
sq = search.search_query
|
||||
cfg: SettingsAISummary = get_setting("ai_summary")
|
||||
|
||||
skip = (
|
||||
sq.pageno > 1
|
||||
# post_search is also called for the json, csv and rss formats,
|
||||
# the placeholder is only useful on the HTML result page
|
||||
or request.form.get("format", "html") != "html"
|
||||
or "general" not in sq.categories
|
||||
# an infobox (e.g. wikipedia / wikidata) or an instant answer
|
||||
# (e.g. ddg definitions) most likely already answers the query
|
||||
or bool(search.result_container.infoboxes)
|
||||
or bool(search.result_container.answers)
|
||||
or not sq.query.strip()
|
||||
# without an Ollama server (user preference or admin default)
|
||||
# there is nothing to show
|
||||
or not _user_server(request, cfg)
|
||||
)
|
||||
if skip:
|
||||
return None
|
||||
|
||||
grounding = bool(request.preferences.get_value("ai_summary_grounding"))
|
||||
results.add(results.types.AiSummary(query=sq.query, grounding=grounding))
|
||||
return results
|
||||
|
||||
|
||||
def _bad_request(msg: str) -> flask.Response:
|
||||
return flask.Response(json.dumps({"error": msg}), status=400, mimetype="application/json")
|
||||
|
||||
|
||||
def _validate_messages(messages: t.Any, cfg: SettingsAISummary) -> list[dict[str, str]]:
|
||||
if not isinstance(messages, list) or not messages or len(messages) > cfg.max_history_messages:
|
||||
raise ValueError("invalid messages")
|
||||
for msg in messages:
|
||||
if not isinstance(msg, dict) or msg.keys() != {"role", "content"}:
|
||||
raise ValueError("invalid message")
|
||||
if msg["role"] not in VALID_ROLES or not isinstance(msg["content"], str):
|
||||
raise ValueError("invalid message")
|
||||
if not msg["content"].strip() or len(msg["content"]) > cfg.max_message_length:
|
||||
raise ValueError("invalid message")
|
||||
if messages[-1]["role"] != "user":
|
||||
raise ValueError("last message is not a user message")
|
||||
return messages
|
||||
|
||||
|
||||
def _validate_context(context: t.Any, cfg: SettingsAISummary) -> list[dict[str, str]]:
|
||||
if not isinstance(context, list) or len(context) > cfg.max_context_items:
|
||||
raise ValueError("invalid context")
|
||||
for item in context:
|
||||
if not isinstance(item, dict) or not item.keys() <= {"title", "url", "snippet"}:
|
||||
raise ValueError("invalid context item")
|
||||
for val in item.values():
|
||||
if not isinstance(val, str) or len(val) > cfg.max_message_length:
|
||||
raise ValueError("invalid context item")
|
||||
return context
|
||||
|
||||
|
||||
def _validate_payload(payload: t.Any, cfg: SettingsAISummary) -> tuple[list[dict[str, str]], list[dict[str, str]]]:
|
||||
"""Validate the request body of the ``/ai_summary`` endpoint and return
|
||||
the ``messages`` and ``context`` lists. Raises a :py:obj:`ValueError` for
|
||||
any malformed payload."""
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("payload is not an object")
|
||||
return _validate_messages(payload.get("messages"), cfg), _validate_context(payload.get("context", []), cfg)
|
||||
|
||||
|
||||
def ai_summary_view() -> flask.Response:
|
||||
"""Stream an AI generated answer for the messages in the request body,
|
||||
response is NDJSON: ``{"delta": ..}`` lines followed by one final
|
||||
``{"done": true, ..}`` line."""
|
||||
|
||||
cfg: SettingsAISummary = get_setting("ai_summary")
|
||||
|
||||
if SXNGPlugin.id not in sxng_request.user_plugins:
|
||||
return flask.Response(json.dumps({"error": "plugin is not enabled"}), status=403, mimetype="application/json")
|
||||
|
||||
try:
|
||||
messages, context = _validate_payload(sxng_request.get_json(force=True, silent=True), cfg)
|
||||
except ValueError as exc:
|
||||
return _bad_request(str(exc))
|
||||
|
||||
server = _user_server(sxng_request, cfg)
|
||||
if not _valid_server(server):
|
||||
return _bad_request("no valid Ollama server configured")
|
||||
|
||||
model = str(sxng_request.preferences.get_value("ai_summary_model") or "").strip() or cfg.model
|
||||
if not MODEL_NAME_REGEXP.fullmatch(model):
|
||||
return _bad_request("no valid model configured")
|
||||
|
||||
ollama_payload = {
|
||||
"model": model,
|
||||
"messages": build_ollama_messages(cfg, messages, context),
|
||||
"stream": True,
|
||||
"keep_alive": cfg.keep_alive,
|
||||
}
|
||||
|
||||
# open the upstream connection before streaming, a connection error is
|
||||
# reported as HTTP 502 instead of a line in an already started stream
|
||||
client = _get_client(server, cfg)
|
||||
stream_ctx = client.stream("POST", "/api/chat", json=ollama_payload)
|
||||
upstream = None
|
||||
try:
|
||||
upstream = stream_ctx.__enter__() # pylint: disable=unnecessary-dunder-call
|
||||
if upstream.status_code != 200:
|
||||
stream_ctx.__exit__(None, None, None)
|
||||
upstream = None
|
||||
except httpx.HTTPError:
|
||||
upstream = None
|
||||
if upstream is None:
|
||||
client.close()
|
||||
return flask.Response(json.dumps({"error": "upstream error"}), status=502, mimetype="application/json")
|
||||
|
||||
# from here on nothing must be read from the request context, the
|
||||
# generator runs after the request context has been torn down
|
||||
log = logging.getLogger("searx.plugins.ai_summary")
|
||||
|
||||
def ndjson(obj: dict[str, t.Any]) -> bytes:
|
||||
# the generator bypasses flask's response encoding (direct_passthrough)
|
||||
return (json.dumps(obj) + "\n").encode()
|
||||
|
||||
def generate():
|
||||
start = time.monotonic()
|
||||
try:
|
||||
for line in upstream.iter_lines():
|
||||
if time.monotonic() - start > cfg.stream_timeout:
|
||||
yield ndjson({"done": True, "error": "timeout"})
|
||||
return
|
||||
if not line.strip():
|
||||
continue
|
||||
data = json.loads(line)
|
||||
if data.get("done"):
|
||||
yield ndjson({"done": True, "model": model})
|
||||
return
|
||||
delta = data.get("message", {}).get("content", "")
|
||||
if delta:
|
||||
yield ndjson({"delta": delta})
|
||||
yield ndjson({"done": True, "model": model})
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
log.warning("error while streaming from Ollama: %s", exc)
|
||||
yield ndjson({"done": True, "error": "upstream error"})
|
||||
finally:
|
||||
stream_ctx.__exit__(None, None, None)
|
||||
client.close()
|
||||
|
||||
return flask.Response(
|
||||
generate(),
|
||||
mimetype="application/x-ndjson",
|
||||
headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"},
|
||||
direct_passthrough=True,
|
||||
)
|
||||
Reference in New Issue
Block a user