Code / services/llm_provider_service/api/llm_routes.py
services/llm_provider_service/api/llm_routes.py
1
"""LLM provider API routes."""
2
3
from uuid import uuid4
4
5
from dishka import FromDishka
6
from quart import Blueprint, Response, jsonify, request
7
from quart_dishka import inject
8
9
from common_core import (
10
CircuitBreakerState,
11
LLMComparisonRequest,
12
LLMProviderType,
13
LLMQueuedResponse,
14
)
15
from huleedu_service_libs.logging_utils import create_service_logger
16
from huleedu_service_libs.resilience import CircuitBreakerError, CircuitBreakerRegistry
17
from services.llm_provider_service.api_models import (
18
LLMProviderListResponse,
19
LLMProviderStatus,
20
ModelInfoResponse,
21
ModelManifestResponse,
22
)
23
from services.llm_provider_service.config import Settings
24
from services.llm_provider_service.exceptions import HuleEduError
25
from services.llm_provider_service.implementations.connection_pool_manager_impl import (
26
ConnectionPoolManagerImpl,
27
)
28
from services.llm_provider_service.implementations.trace_context_manager_impl import (
29
TraceContextManagerImpl,
30
)
31
from services.llm_provider_service.internal_models import LLMQueuedResult
32
from services.llm_provider_service.llama_cpp_status import read_llama_cpp_status
33
from services.llm_provider_service.manifest import ProviderName, list_models
34
from services.llm_provider_service.metrics import get_llm_metrics
35
from services.llm_provider_service.protocols import LLMOrchestratorProtocol
36
37
logger = create_service_logger("llm_provider_service.api")
38
39
llm_bp = Blueprint("llm", __name__)
40
41
42
@llm_bp.route("/comparison", methods=["POST"])
43
@inject
44
async def generate_comparison(
45
orchestrator: FromDishka[LLMOrchestratorProtocol],
46
tracer: FromDishka[TraceContextManagerImpl],
47
) -> Response | tuple[Response, int]:
48
"""Generate essay comparison using configured LLM provider."""
49
metrics = get_llm_metrics()
50
51
try:
52
# Parse request first to get correlation_id
53
data = await request.get_json()
54
if not data:
55
return jsonify({"error": "No JSON data provided"}), 400
56
57
try:
58
comparison_request = LLMComparisonRequest(**data)
59
except Exception as e:
60
return jsonify({"error": f"Invalid request format: {str(e)}"}), 400
61
62
# Use correlation_id from request if provided, otherwise generate a new one
63
correlation_id = comparison_request.correlation_id or uuid4()
64
65
# Start tracing for the API request
66
with tracer.start_api_request_span("comparison", correlation_id):
67
tracer.add_span_event("request_started", {"correlation_id": str(correlation_id)})
68
69
# Extract configuration overrides
70
provider_override = None
71
model_override = None
72
endpoint_profile = None
73
temperature_override = None
74
max_tokens_override = None
75
system_prompt_override = None
76
reasoning_effort = None
77
output_verbosity = None
78
allowance_mode = "free_allowance_only"
79
80
if comparison_request.llm_config_overrides:
81
provider_override = comparison_request.llm_config_overrides.provider_override
82
model_override = comparison_request.llm_config_overrides.model_override
83
endpoint_profile = comparison_request.llm_config_overrides.endpoint_profile
84
temperature_override = comparison_request.llm_config_overrides.temperature_override
85
max_tokens_override = comparison_request.llm_config_overrides.max_tokens_override
86
system_prompt_override = (
87
comparison_request.llm_config_overrides.system_prompt_override
88
)
89
reasoning_effort = comparison_request.llm_config_overrides.reasoning_effort
90
output_verbosity = comparison_request.llm_config_overrides.output_verbosity
91
allowance_mode = comparison_request.llm_config_overrides.allowance_mode
92
93
# Require explicit provider configuration
94
if not provider_override:
95
logger.warning(
96
"Request missing required provider configuration, correlation_id: %s",
97
correlation_id,
98
)
99
return jsonify(
100
{
101
"error": "Provider configuration required",
102
"details": "llm_config_overrides.provider_override must be specified",
103
"correlation_id": str(correlation_id),
104
}
105
), 400
106
107
if endpoint_profile is not None and provider_override is not LLMProviderType.OPENROUTER:
108
return jsonify(
109
{
110
"error": "Endpoint profile is only valid for OpenRouter",
111
"correlation_id": str(correlation_id),
112
}
113
), 400
114
115
# Add provider info to span
116
tracer.set_span_attributes(
117
{
118
"llm.provider": provider_override.value if provider_override else "unknown",
119
"llm.model": str(model_override) if model_override is not None else "default",
120
}
121
)
122
123
# Call orchestrator - HuleEduError handled by app-level error handler
124
result = await orchestrator.perform_comparison(
125
provider=provider_override,
126
user_prompt=comparison_request.user_prompt,
127
prompt_blocks=comparison_request.prompt_blocks,
128
correlation_id=correlation_id,
129
model_override=model_override,
130
endpoint_profile=endpoint_profile,
131
temperature_override=temperature_override,
132
max_tokens_override=max_tokens_override,
133
system_prompt_override=system_prompt_override,
134
callback_topic=comparison_request.callback_topic,
135
request_metadata=comparison_request.metadata,
136
reasoning_effort=reasoning_effort,
137
output_verbosity=output_verbosity,
138
allowance_mode=allowance_mode,
139
)
140
141
# Success - result should not be None
142
if not result:
143
logger.error("Orchestrator returned no result")
144
return jsonify(
145
{
146
"error": "Internal error: No result from orchestrator",
147
"correlation_id": str(correlation_id),
148
}
149
), 500
150
151
# ARCHITECTURAL TRUTH: ALL LLM requests are queued - no immediate responses
152
# Always expect LLMQueuedResult as orchestrator should never return immediate responses
153
if not isinstance(result, LLMQueuedResult):
154
logger.error(f"Orchestrator returned non-queued result: {type(result)}")
155
return jsonify(
156
{
157
"error": "Internal error: Expected queued result",
158
"correlation_id": str(correlation_id),
159
}
160
), 500
161
162
metrics["llm_requests_total"].labels(
163
provider=result.provider,
164
model=model_override or "default",
165
request_type="comparison",
166
status="queued",
167
).inc()
168
169
# Build queued response - callback will be delivered to specified topic
170
queued_response = LLMQueuedResponse(
171
queue_id=result.queue_id,
172
status=result.status,
173
message=(
174
f"Request queued for processing. "
175
f"Result will be delivered via callback to topic: "
176
f"{comparison_request.callback_topic}"
177
),
178
estimated_wait_minutes=result.estimated_wait_minutes,
179
)
180
181
logger.debug(f"Request queued with ID: {result.queue_id}")
182
tracer.add_span_event("request_completed", {"status": "queued"})
183
return jsonify(queued_response.model_dump()), 202
184
185
except CircuitBreakerError as e:
186
logger.error(f"Circuit breaker open: {e}")
187
return jsonify(
188
{
189
"error": "Service temporarily unavailable due to provider issues",
190
"details": str(e),
191
}
192
), 503
193
194
except HuleEduError:
195
raise
196
197
except Exception as e:
198
logger.exception("Unexpected error in comparison endpoint")
199
return jsonify(
200
{
201
"error": "Internal server error",
202
"details": str(e),
203
}
204
), 500
205
206
207
@llm_bp.route("/providers", methods=["GET"])
208
@inject
209
async def list_providers(
210
settings: FromDishka[Settings],
211
circuit_breaker_registry: FromDishka[CircuitBreakerRegistry],
212
pool_manager: FromDishka[ConnectionPoolManagerImpl],
213
) -> Response | tuple[Response, int]:
214
"""List all configured LLM providers and their status."""
215
try:
216
providers = []
217
218
# Check each provider
219
for provider_name in ["anthropic", "openai", "google", "openrouter", "llama_cpp"]:
220
# Check if enabled
221
enabled = getattr(settings, f"{provider_name.upper()}_ENABLED", False)
222
223
# Check circuit breaker status
224
circuit_breaker = circuit_breaker_registry.get(f"llm_{provider_name}")
225
circuit_breaker_state = CircuitBreakerState.CLOSED.value
226
if circuit_breaker:
227
state_info = circuit_breaker.get_state()
228
circuit_breaker_state = state_info.get("state", CircuitBreakerState.CLOSED.value)
229
230
# Check if API key is configured
231
api_key = getattr(settings, f"{provider_name.upper()}_API_KEY", "")
232
configured = (
233
bool(settings.LLAMA_CPP_BASE_URL) if provider_name == "llama_cpp" else bool(api_key)
234
)
235
runtime = None
236
if provider_name == "llama_cpp":
237
session = await pool_manager.get_session("llama_cpp")
238
runtime = await read_llama_cpp_status(session=session, settings=settings)
239
240
providers.append(
241
LLMProviderStatus(
242
name=provider_name,
243
enabled=enabled,
244
available=(
245
enabled
246
and configured
247
and circuit_breaker_state == CircuitBreakerState.CLOSED.value
248
and (runtime is None or runtime.reachable)
249
),
250
circuit_breaker_state=circuit_breaker_state,
251
default_model=getattr(settings, f"{provider_name.upper()}_DEFAULT_MODEL", None),
252
runtime=runtime,
253
)
254
)
255
256
response = LLMProviderListResponse(
257
providers=providers,
258
default_provider=settings.DEFAULT_LLM_PROVIDER,
259
selection_strategy=settings.PROVIDER_SELECTION_STRATEGY,
260
)
261
262
return jsonify(response.model_dump()), 200
263
264
except Exception as e:
265
logger.exception("Error listing providers")
266
return jsonify(
267
{
268
"error": "Failed to list providers",
269
"details": str(e),
270
}
271
), 500
272
273
274
@llm_bp.route("/providers/<string:provider>/test", methods=["POST"])
275
@inject
276
async def test_provider(
277
provider: str,
278
orchestrator: FromDishka[LLMOrchestratorProtocol],
279
) -> Response | tuple[Response, int]:
280
"""Test a specific LLM provider."""
281
try:
282
# Convert string to enum
283
try:
284
provider_enum = LLMProviderType(provider)
285
except ValueError:
286
return jsonify({"error": f"Invalid provider: {provider}"}), 400
287
288
correlation_id = uuid4()
289
success = await orchestrator.test_provider(
290
provider=provider_enum, correlation_id=correlation_id
291
)
292
293
return jsonify(
294
{
295
"success": success,
296
"provider": provider,
297
"message": "Provider operational" if success else "Provider unavailable",
298
}
299
), 200
300
301
except Exception as e:
302
logger.exception(f"Error testing provider {provider}")
303
return jsonify(
304
{
305
"error": f"Failed to test provider {provider}",
306
"details": str(e),
307
}
308
), 500
309
310
311
@llm_bp.route("/models", methods=["GET"])
312
async def get_models() -> Response | tuple[Response, int]:
313
"""Get available LLM models grouped by provider.
314
315
Query Parameters:
316
provider (optional): Filter by specific provider (e.g., 'anthropic', 'openai')
317
include_deprecated (optional): Include deprecated models (default: false)
318
319
Returns:
320
200: ModelManifestResponse with models grouped by provider
321
400: Invalid provider name
322
500: Internal error
323
"""
324
try:
325
# Parse query parameters
326
provider_filter = request.args.get("provider")
327
include_deprecated = request.args.get("include_deprecated", "false").lower() == "true"
328
329
# Validate provider if specified
330
provider_enum = None
331
if provider_filter:
332
try:
333
provider_enum = ProviderName(provider_filter.lower())
334
except ValueError:
335
valid_providers = [p.value for p in ProviderName if p != ProviderName.MOCK]
336
return jsonify(
337
{
338
"error": f"Invalid provider: {provider_filter}",
339
"valid_providers": valid_providers,
340
}
341
), 400
342
343
# Get models from manifest
344
models = list_models(provider_enum)
345
346
# Filter out deprecated models unless explicitly requested
347
if not include_deprecated:
348
models = [m for m in models if not m.is_deprecated]
349
350
# Filter out MOCK provider (internal testing only)
351
models = [m for m in models if m.provider != ProviderName.MOCK]
352
353
# Group models by provider
354
providers_dict: dict[str, list[ModelInfoResponse]] = {}
355
for model in models:
356
provider_name = model.provider.value
357
if provider_name not in providers_dict:
358
providers_dict[provider_name] = []
359
360
# Convert ModelConfig to ModelInfoResponse
361
model_info = ModelInfoResponse(
362
model_id=model.model_id,
363
provider=model.provider.value,
364
display_name=model.display_name,
365
model_family=model.model_family,
366
max_tokens=model.max_tokens,
367
context_window=model.context_window,
368
supports_streaming=model.supports_streaming,
369
capabilities=model.capabilities,
370
cost_per_1k_input_tokens=model.cost_per_1k_input_tokens,
371
cost_per_1k_output_tokens=model.cost_per_1k_output_tokens,
372
is_deprecated=model.is_deprecated,
373
release_date=model.release_date,
374
recommended_for=model.recommended_for,
375
)
376
providers_dict[provider_name].append(model_info)
377
378
# Build response
379
response = ModelManifestResponse(
380
providers=providers_dict,
381
total_models=len(models),
382
)
383
384
logger.info(
385
f"Manifest query returned {len(models)} models "
386
f"(provider_filter={provider_filter}, include_deprecated={include_deprecated})"
387
)
388
389
return jsonify(response.model_dump(mode="json")), 200
390
391
except Exception as e:
392
logger.exception("Error retrieving model manifest")
393
return jsonify(
394
{
395
"error": "Failed to retrieve model manifest",
396
"details": str(e),
397
}
398
), 500
399
400
401
@llm_bp.route("/status/<string:queue_id>", methods=["GET"])
402
@inject
403
async def get_queue_status(
404
queue_id: str,
405
tracer: FromDishka[TraceContextManagerImpl],
406
) -> Response | tuple[Response, int]:
407
"""Get status of a queued request."""
408
correlation_id = uuid4()
409
410
try:
411
with tracer.start_api_request_span("get_queue_status", correlation_id):
412
# For now, return 404 for all queue IDs since queue persistence isn't implemented
413
return jsonify({"error": f"Queue ID {queue_id} not found"}), 404
414
except Exception as e:
415
logger.error(
416
"Error getting queue status",
417
error=str(e),
418
queue_id=queue_id,
419
correlation_id=str(correlation_id),
420
)
421
return jsonify({"error": f"Failed to get status: {e}"}), 500
422
423
424
@llm_bp.route("/results/<string:queue_id>", methods=["GET"])
425
@inject
426
async def get_queue_results(
427
queue_id: str,
428
tracer: FromDishka[TraceContextManagerImpl],
429
) -> Response | tuple[Response, int]:
430
"""Get results of a queued request."""
431
correlation_id = uuid4()
432
433
try:
434
with tracer.start_api_request_span("get_queue_results", correlation_id):
435
# For now, return 404 for all queue IDs since queue persistence isn't implemented
436
return jsonify({"error": f"Queue ID {queue_id} not found"}), 404
437
except Exception as e:
438
logger.error(
439
"Error getting queue results",
440
error=str(e),
441
queue_id=queue_id,
442
correlation_id=str(correlation_id),
443
)
444
return jsonify({"error": f"Failed to get results: {e}"}), 500