ct-public/isparkclaw-webui
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: BYOK/custom provider models missing from WebUI model dropdown (#815)

Browse Source 
Closes #815.

Three root causes fixed:

1. Provider aliases (z.ai/x.ai/google/grok/claude/aws-bedrock/dashscope/~25 more) not
   normalized before _PROVIDER_MODELS lookup — provider fell to empty else-branch while
   TUI worked (it normalizes at startup). Fixed via _resolve_provider_alias() + inlined
   _PROVIDER_ALIASES table in api/config.py.

2. Silent ImportError in original normalization: 'from hermes_cli.models import
   _PROVIDER_ALIASES' inside try/except silently failed without hermes-agent on sys.path
   (CI, minimal installs). The inlined table fixes this — normalization now works
   regardless of whether hermes-agent is installed.

3. /api/models/live?provider=custom now falls back to custom_providers entries from
   config.yaml when provider_model_ids() returns empty.

Also: provider_id on every group in /api/models response for deterministic JS optgroup
matching (no substring false positives). 17 targeted tests, 1725/1725 full suite.
This commit is contained in:
nesquena-hermes
2026-04-21 17:24:54 -07:00
committed by GitHub
parent a4d59b9e6c
commit 8f1f582caf
5 changed files with 496 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

378
tests/test_byok_model_dropdown.py Normal file
View File

@@ -0,0 +1,378 @@
"""Tests for #815 — BYOK/custom provider models missing from WebUI model dropdown.
Root causes fixed:
1. active_provider alias not normalized in get_available_models()
('z.ai' -> 'zai', 'x.ai' -> 'xai', 'google' -> 'gemini', etc.)
causing the provider to fall to the 'else/unknown' branch with no models.
2. /api/models/live didn't normalize the provider query param, so
provider_model_ids() received the un-aliased form and returned [].
3. /api/models/live returned empty for provider='custom' even when
custom_providers entries exist in config.yaml — the live enrichment
step never added those models.
"""
import pathlib
import re
import sys
import unittest.mock as mock
import pytest
REPO = pathlib.Path(__file__).parent.parent
sys.path.insert(0, str(REPO))
sys.path.insert(0, str(REPO.parent / ".hermes" / "hermes-agent"))
def read(rel):
return (REPO / rel).read_text(encoding="utf-8")
@pytest.fixture(autouse=True)
def _isolate_models_cache():
"""Invalidate the TTL model cache before AND after every test.
``get_available_models()`` caches its result keyed on config.yaml mtime.
Tests in this file repoint ``_get_config_path`` to a tmp_path, populate
the cache there, then let monkeypatch restore the original path. The
cache, keyed on the tmp_path's mtime, then poisons downstream tests
(e.g. test_model_resolver) which see stale data and never hit their
mocks. Clearing the cache around each test breaks that linkage.
"""
import api.config as c
try:
c.invalidate_models_cache()
except Exception:
pass
yield
try:
c.invalidate_models_cache()
except Exception:
pass
# ── api/config.py — active_provider normalization ─────────────────────────────
class TestActiveProviderNormalization:
"""get_available_models() must normalize active_provider aliases before lookup."""
def _run(self, tmp_path, provider_str, monkeypatch):
"""Return get_available_models() output for a given provider string."""
import api.config as c
cfgfile = tmp_path / "config.yaml"
cfgfile.write_text(
f"model:\n provider: {provider_str}\n default: test-model\n",
encoding="utf-8",
)
monkeypatch.setattr(c, "_get_config_path", lambda: cfgfile)
c.reload_config()
# Patch list_available_providers to avoid real network calls
fake_prov = mock.MagicMock()
fake_prov.return_value = []
try:
import hermes_cli.models as hm
monkeypatch.setattr(hm, "list_available_providers", fake_prov)
except Exception:
pass
result = c.get_available_models()
c.reload_config()
return result
def test_z_dot_ai_normalized_to_zai(self, tmp_path, monkeypatch):
result = self._run(tmp_path, "z.ai", monkeypatch)
# active_provider returned to browser must be canonical 'zai' or
# at minimum must not be 'z.ai' (which would miss the _PROVIDER_MODELS lookup)
ap = result.get("active_provider", "")
assert ap in ("zai", ""), f"active_provider should be 'zai', got {ap!r}"
def test_x_dot_ai_normalized_to_xai(self, tmp_path, monkeypatch):
result = self._run(tmp_path, "x.ai", monkeypatch)
ap = result.get("active_provider", "")
assert ap in ("xai", ""), f"active_provider should be 'xai', got {ap!r}"
def test_google_normalized_to_gemini(self, tmp_path, monkeypatch):
result = self._run(tmp_path, "google", monkeypatch)
ap = result.get("active_provider", "")
assert ap in ("gemini", ""), f"active_provider should be 'gemini', got {ap!r}"
def test_normalization_code_present(self):
"""Source-level check: config.py must call _PROVIDER_ALIASES for active_provider."""
src = read("api/config.py")
# Must alias-normalize active_provider before the group-builder runs
assert "_PROVIDER_ALIASES" in src, (
"api/config.py must import _PROVIDER_ALIASES to normalize active_provider"
)
# The normalization must happen before the group builder loop
alias_pos = src.index("_PROVIDER_ALIASES")
group_builder_pos = src.index("for pid in sorted(detected_providers)")
assert alias_pos < group_builder_pos, (
"active_provider normalization must occur before the group-builder loop"
)
# ── api/routes.py — /api/models/live provider normalization ───────────────────
class TestLiveModelsProviderNormalization:
"""_handle_live_models must normalize the provider query param."""
def test_live_models_normalizes_provider_alias(self):
src = read("api/routes.py")
# Find _handle_live_models function
m = re.search(
r"def _handle_live_models\(.*?\ndef ",
src,
re.DOTALL,
)
assert m, "_handle_live_models not found"
fn = m.group(0)
assert "_resolve_provider_alias" in fn, (
"_handle_live_models must normalize provider via "
"api.config._resolve_provider_alias so 'z.ai' -> 'zai' "
"before calling provider_model_ids()"
)
def test_live_models_normalization_before_provider_model_ids(self):
"""Normalization call must appear before the provider_model_ids call site."""
src = read("api/routes.py")
alias_match = re.search(
r"provider\s*=\s*_resolve_provider_alias\(provider\)",
src,
)
pmi_call_match = re.search(
r"ids\s*=\s*_pmi\(provider\)",
src,
)
assert alias_match, "_resolve_provider_alias call not found in routes.py"
assert pmi_call_match, "ids = _pmi(provider) call not found"
assert alias_match.start() < pmi_call_match.start(), (
"alias normalization must occur before ids = _pmi(provider)"
)
def test_alias_resolver_works_without_hermes_cli(self):
"""Normalization must work even when hermes_cli is not importable —
CI and installs without the agent cloned alongside the WebUI.
The WebUI ships its own _PROVIDER_ALIASES table; the agent's table
is merged only when available."""
import api.config as c
# Core CLI aliases from #815's bug report
assert c._resolve_provider_alias('z.ai') == 'zai'
assert c._resolve_provider_alias('x.ai') == 'xai'
assert c._resolve_provider_alias('google') == 'gemini'
assert c._resolve_provider_alias('grok') == 'xai'
# Case / whitespace insensitive
assert c._resolve_provider_alias(' Z.AI ') == 'zai'
# Canonical names pass through unchanged
assert c._resolve_provider_alias('openrouter') == 'openrouter'
assert c._resolve_provider_alias('anthropic') == 'anthropic'
assert c._resolve_provider_alias('custom') == 'custom'
# Empty / None pass through
assert c._resolve_provider_alias('') == ''
assert c._resolve_provider_alias(None) is None
# ── api/routes.py — /api/models/live custom_providers fallback ────────────────
class TestLiveModelsCustomProviderFallback:
"""When provider='custom' and provider_model_ids() returns [],
/api/models/live must fall back to custom_providers entries from config.yaml."""
def test_custom_fallback_code_present(self):
src = read("api/routes.py")
m = re.search(
r"def _handle_live_models\(.*?\ndef ",
src,
re.DOTALL,
)
assert m, "_handle_live_models not found"
fn = m.group(0)
assert "custom_providers" in fn, (
"_handle_live_models must read custom_providers from config "
"as fallback when provider='custom' and provider_model_ids() returns []"
)
assert 'provider == "custom"' in fn or "provider=='custom'" in fn, (
"_handle_live_models must check provider == 'custom' before fallback"
)
def test_custom_fallback_returns_configured_models(self, tmp_path, monkeypatch):
"""End-to-end: /api/models/live?provider=custom returns custom_providers models."""
import api.config as c
import api.routes as r
cfgfile = tmp_path / "config.yaml"
cfgfile.write_text(
"model:\n provider: custom\n default: my-byok-model\n"
"custom_providers:\n"
" - model: my-byok-model\n"
" api_base: https://my-llm.example.com/v1\n"
" api_key: sk-test\n",
encoding="utf-8",
)
monkeypatch.setattr(c, "_get_config_path", lambda: cfgfile)
c.reload_config()
# Mock handler and parsed URL
handler = mock.MagicMock()
responses = []
def fake_j(h, data, **kw):
responses.append(data)
return True
monkeypatch.setattr(r, "j", fake_j)
from urllib.parse import urlparse
parsed = mock.MagicMock()
parsed.query = "provider=custom"
# Mock provider_model_ids to return [] (simulating no live endpoint)
try:
import hermes_cli.models as hm
monkeypatch.setattr(hm, "provider_model_ids", lambda p: [])
except Exception:
pass
r._handle_live_models(handler, parsed)
assert responses, "handler must produce a response"
resp = responses[-1]
assert "models" in resp
model_ids = [m["id"] for m in resp.get("models", [])]
assert "my-byok-model" in model_ids, (
f"custom_providers model 'my-byok-model' must appear in live response; "
f"got {model_ids}"
)
# ── Regression: known-good providers still work ───────────────────────────────
class TestKnownProvidersUnaffected:
"""Normalization must not break providers whose names are already canonical."""
def test_openrouter_unaffected(self):
src = read("api/config.py")
# _PROVIDER_ALIASES lookup: 'openrouter' -> 'openrouter' (no change)
assert "openrouter" in src, "openrouter must still exist in config"
def test_anthropic_unaffected(self):
src = read("api/config.py")
assert "anthropic" in src
def test_custom_unaffected(self):
"""'custom' is not in _PROVIDER_ALIASES so normalization is a no-op."""
try:
from hermes_cli.models import _PROVIDER_ALIASES
assert "custom" not in _PROVIDER_ALIASES, (
"'custom' must not be aliased to anything — it's a special sentinel"
)
except ImportError:
pass # hermes-agent not available in this env — skip
# ── Source-level: active_provider returned to browser is canonical ─────────────
class TestProviderIdInGroupResponse:
"""get_available_models() must include provider_id on every group so the JS
_fetchLiveModels can match optgroups exactly rather than by substring."""
def test_groups_include_provider_id(self, tmp_path, monkeypatch):
import api.config as c
cfgfile = tmp_path / "config.yaml"
cfgfile.write_text(
"model:\n provider: zai\n default: glm-5\n",
encoding="utf-8",
)
monkeypatch.setattr(c, "_get_config_path", lambda: cfgfile)
c.reload_config()
try:
import hermes_cli.models as hm
monkeypatch.setattr(hm, "list_available_providers", lambda: [
{"id": "zai", "authenticated": True}
])
import hermes_cli.auth as ha
monkeypatch.setattr(ha, "get_auth_status", lambda p: {"key_source": "env"})
except Exception:
pass
result = c.get_available_models()
c.reload_config()
for g in result.get("groups", []):
assert "provider_id" in g, (
f"group {g.get('provider')!r} missing provider_id — "
"JS _fetchLiveModels needs it to match optgroups exactly"
)
def test_provider_id_in_static_ui_js_optgroup(self):
src = read("static/ui.js")
assert "og.dataset.provider" in src, (
"populateModelDropdown must set og.dataset.provider from g.provider_id "
"so _fetchLiveModels can match by exact provider_id"
)
def test_fetch_live_models_prefers_data_provider_match(self):
src = read("static/ui.js")
m = re.search(r'function _fetchLiveModels\b.*?\n\}', src, re.DOTALL)
assert m, "_fetchLiveModels not found"
fn = m.group(0)
assert 'og.dataset.provider' in fn, (
"_fetchLiveModels must check og.dataset.provider===provider before "
"falling back to label substring match"
)
# The data-provider check must come before the label.includes check
dp_pos = fn.index('og.dataset.provider')
label_pos = fn.index('og.label')
assert dp_pos < label_pos, (
"data-provider exact match must be attempted before label substring match"
)
# ── Opus-identified edge case: 'ollama' normalizes to 'custom' ────────────────
class TestOllamaAliasEdgeCase:
"""Opus review found: 'ollama' -> 'custom' via _PROVIDER_ALIASES.
This is better behaviour (custom_providers fallback catches it) but worth
documenting and not regressing."""
def test_ollama_not_in_provider_aliases_as_ollama(self):
"""'ollama' maps to 'custom' in _PROVIDER_ALIASES — verify this is the
intended behavior post-normalization (not a silent breakage)."""
try:
from hermes_cli.models import _PROVIDER_ALIASES
# 'ollama' -> 'custom' means ollama users hit the custom_providers path
# This is fine — ollama models appear via base_url auto-detection (step 3)
# in get_available_models, not via _PROVIDER_MODELS lookup.
ollama_target = _PROVIDER_ALIASES.get("ollama", "ollama")
# Acceptable outcomes: either unchanged (not in aliases) or 'custom'/'ollama-cloud'
assert ollama_target in ("ollama", "custom", "ollama-cloud"), (
f"Unexpected ollama alias: {ollama_target}"
)
except ImportError:
pass # hermes-agent not available
class TestGetAvailableModelsReturnsCanonicalProvider:
"""get_available_models() must return normalized active_provider in its response
so the browser sends the right value to /api/models/live."""
def test_active_provider_in_response_is_normalized(self, tmp_path, monkeypatch):
import api.config as c
cfgfile = tmp_path / "config.yaml"
cfgfile.write_text(
"model:\n provider: z.ai\n default: glm-5\n",
encoding="utf-8",
)
monkeypatch.setattr(c, "_get_config_path", lambda: cfgfile)
c.reload_config()
try:
import hermes_cli.models as hm
monkeypatch.setattr(hm, "list_available_providers", lambda: [])
except Exception:
pass
result = c.get_available_models()
c.reload_config()
ap = result.get("active_provider", "")
# The browser will pass this value to /api/models/live?provider=<ap>
# It must be 'zai' so optgroup matching works in _fetchLiveModels
assert ap != "z.ai", (
"active_provider 'z.ai' must be normalized to 'zai' before being "
"returned to the browser (browser passes it back to /api/models/live)"
)