Upload folder using huggingface_hub
#7
by petter2025 - opened
- Dockerfile +1 -5
- README.md +3 -8
- app/api/routes_incidents.py +1 -1
- app/core/usage_tracker.py +92 -159
- app/database/models_intents.py +194 -36
Dockerfile
CHANGED
|
@@ -1,11 +1,7 @@
|
|
| 1 |
FROM python:3.12-slim
|
| 2 |
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
|
| 3 |
WORKDIR /app
|
| 4 |
-
|
| 5 |
-
ARG CLASSIC_TOKEN=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
|
| 6 |
-
RUN git config --global url."https://x-access-token:${CLASSIC_TOKEN}@github.com/".insteadOf "https://github.com/"
|
| 7 |
-
|
| 8 |
COPY requirements.txt .
|
| 9 |
RUN pip install --no-cache-dir -r requirements.txt
|
| 10 |
COPY . .
|
| 11 |
-
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
|
|
|
|
| 1 |
FROM python:3.12-slim
|
| 2 |
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
|
| 3 |
WORKDIR /app
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
COPY requirements.txt .
|
| 5 |
RUN pip install --no-cache-dir -r requirements.txt
|
| 6 |
COPY . .
|
| 7 |
+
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
CHANGED
|
@@ -1,9 +1,3 @@
|
|
| 1 |
-
---
|
| 2 |
-
title: ARF API Control Plane
|
| 3 |
-
sdk: docker
|
| 4 |
-
colorFrom: blue
|
| 5 |
-
colorTo: green
|
| 6 |
-
---
|
| 7 |
# arf-api
|
| 8 |
|
| 9 |
ARF API Control Plane (FastAPI)
|
|
@@ -87,7 +81,7 @@ curl -X POST "http://localhost:8000/api/v1/v1/incidents/evaluate" -H "Content-
|
|
| 87 |
"justification": "Causal: If we apply restart_container instead of no_action, latency would change from 600.00 to 510.00 (Δ = -90.00). Based on heuristic causal model.",
|
| 88 |
"confidence": 0.85,
|
| 89 |
"risk_score": 0.54,
|
| 90 |
-
"status": "
|
| 91 |
},
|
| 92 |
"causal_explanation": {
|
| 93 |
"factual_outcome": 600,
|
|
@@ -123,4 +117,5 @@ Notes
|
|
| 123 |
-----
|
| 124 |
|
| 125 |
- The governance endpoints use an in-process `RiskEngine` initialized at startup.
|
| 126 |
-
- The outcome recording endpoint is not implemented in this repository and returns HTTP 501.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
# arf-api
|
| 2 |
|
| 3 |
ARF API Control Plane (FastAPI)
|
|
|
|
| 81 |
"justification": "Causal: If we apply restart_container instead of no_action, latency would change from 600.00 to 510.00 (Δ = -90.00). Based on heuristic causal model.",
|
| 82 |
"confidence": 0.85,
|
| 83 |
"risk_score": 0.54,
|
| 84 |
+
"status": "success"
|
| 85 |
},
|
| 86 |
"causal_explanation": {
|
| 87 |
"factual_outcome": 600,
|
|
|
|
| 117 |
-----
|
| 118 |
|
| 119 |
- The governance endpoints use an in-process `RiskEngine` initialized at startup.
|
| 120 |
+
- The outcome recording endpoint is not implemented in this repository and returns HTTP 501.
|
| 121 |
+
|
app/api/routes_incidents.py
CHANGED
|
@@ -198,7 +198,7 @@ async def evaluate_incident(
|
|
| 198 |
),
|
| 199 |
"confidence": 1.0 - result.get("uncertainty", 0.0),
|
| 200 |
"risk_score": result["risk_score"],
|
| 201 |
-
"status": "
|
| 202 |
}
|
| 203 |
|
| 204 |
response_data = {
|
|
|
|
| 198 |
),
|
| 199 |
"confidence": 1.0 - result.get("uncertainty", 0.0),
|
| 200 |
"risk_score": result["risk_score"],
|
| 201 |
+
"status": "success",
|
| 202 |
}
|
| 203 |
|
| 204 |
response_data = {
|
app/core/usage_tracker.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
| 1 |
"""
|
| 2 |
Usage Tracker for ARF API – quotas, tiers, and audit logging.
|
| 3 |
Thread‑safe, atomic quota consumption, idempotent, fail‑closed.
|
| 4 |
-
"""
|
| 5 |
|
|
|
|
|
|
|
|
|
|
| 6 |
import json
|
| 7 |
import sqlite3
|
| 8 |
import threading
|
|
@@ -10,7 +12,7 @@ import time
|
|
| 10 |
from contextlib import contextmanager
|
| 11 |
from datetime import datetime, timedelta
|
| 12 |
from dataclasses import dataclass
|
| 13 |
-
from typing import Dict, Any, Optional, List, Tuple
|
| 14 |
from enum import Enum
|
| 15 |
from fastapi import BackgroundTasks, HTTPException, Request
|
| 16 |
|
|
@@ -24,6 +26,7 @@ except ImportError:
|
|
| 24 |
|
| 25 |
|
| 26 |
class Tier(str, Enum):
|
|
|
|
| 27 |
FREE = "free"
|
| 28 |
PRO = "pro"
|
| 29 |
PREMIUM = "premium"
|
|
@@ -31,16 +34,18 @@ class Tier(str, Enum):
|
|
| 31 |
|
| 32 |
@property
|
| 33 |
def monthly_evaluation_limit(self) -> Optional[int]:
|
|
|
|
| 34 |
limits = {
|
| 35 |
Tier.FREE: 1000,
|
| 36 |
Tier.PRO: 10_000,
|
| 37 |
Tier.PREMIUM: 50_000,
|
| 38 |
-
Tier.ENTERPRISE: None,
|
| 39 |
}
|
| 40 |
return limits[self]
|
| 41 |
|
| 42 |
@property
|
| 43 |
def audit_log_retention_days(self) -> int:
|
|
|
|
| 44 |
retention = {
|
| 45 |
Tier.FREE: 7,
|
| 46 |
Tier.PRO: 30,
|
|
@@ -52,7 +57,7 @@ class Tier(str, Enum):
|
|
| 52 |
|
| 53 |
@dataclass
|
| 54 |
class UsageRecord:
|
| 55 |
-
"""Single
|
| 56 |
api_key: str
|
| 57 |
tier: Tier
|
| 58 |
timestamp: float
|
|
@@ -66,6 +71,7 @@ class UsageRecord:
|
|
| 66 |
class UsageTracker:
|
| 67 |
"""
|
| 68 |
Thread‑safe usage tracker with atomic quota consumption and idempotency.
|
|
|
|
| 69 |
"""
|
| 70 |
|
| 71 |
def __init__(self, db_path: str = "arf_usage.db",
|
|
@@ -78,12 +84,11 @@ class UsageTracker:
|
|
| 78 |
if redis_url and REDIS_AVAILABLE:
|
| 79 |
self._redis_client = redis.from_url(redis_url)
|
| 80 |
elif redis_url:
|
| 81 |
-
raise ImportError(
|
| 82 |
-
"Redis client not installed. Run: pip install redis")
|
| 83 |
|
| 84 |
@contextmanager
|
| 85 |
def _get_conn(self):
|
| 86 |
-
"""Get a thread‑local SQLite connection with
|
| 87 |
if not hasattr(self._local, "conn"):
|
| 88 |
self._local.conn = sqlite3.connect(
|
| 89 |
self.db_path, check_same_thread=False, isolation_level=None)
|
|
@@ -92,10 +97,13 @@ class UsageTracker:
|
|
| 92 |
yield self._local.conn
|
| 93 |
|
| 94 |
def _init_db(self):
|
|
|
|
| 95 |
with self._get_conn() as conn:
|
|
|
|
| 96 |
conn.execute("""
|
| 97 |
CREATE TABLE IF NOT EXISTS api_keys (
|
| 98 |
key TEXT PRIMARY KEY,
|
|
|
|
| 99 |
tier TEXT NOT NULL,
|
| 100 |
created_at REAL NOT NULL,
|
| 101 |
last_used_at REAL,
|
|
@@ -139,16 +147,31 @@ class UsageTracker:
|
|
| 139 |
def _get_month_key(self) -> str:
|
| 140 |
return datetime.now().strftime("%Y-%m")
|
| 141 |
|
| 142 |
-
def get_or_create_api_key(self, key: str, tier: Tier = Tier.FREE) -> bool:
|
| 143 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
with self._get_conn() as conn:
|
| 145 |
row = conn.execute(
|
| 146 |
"SELECT key FROM api_keys WHERE key = ?", (key,)).fetchone()
|
| 147 |
if row:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
return True
|
| 149 |
conn.execute(
|
| 150 |
-
"INSERT INTO api_keys (key, tier, created_at, is_active) VALUES (?, ?, ?, ?)",
|
| 151 |
-
(key, tier.value, time.time(), 1)
|
| 152 |
)
|
| 153 |
conn.commit()
|
| 154 |
return True
|
|
@@ -164,6 +187,17 @@ class UsageTracker:
|
|
| 164 |
return None
|
| 165 |
return Tier(row["tier"])
|
| 166 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
def update_api_key_tier(self, api_key: str, new_tier: Tier) -> bool:
|
| 168 |
"""Update the tier of an existing API key. Returns True if successful."""
|
| 169 |
with self._get_conn() as conn:
|
|
@@ -173,41 +207,28 @@ class UsageTracker:
|
|
| 173 |
return False
|
| 174 |
conn.execute(
|
| 175 |
"UPDATE api_keys SET tier = ? WHERE key = ?",
|
| 176 |
-
(new_tier.value,
|
| 177 |
-
api_key))
|
| 178 |
conn.commit()
|
| 179 |
return True
|
| 180 |
|
| 181 |
# --------------------------------------------------------------------------
|
| 182 |
-
# Atomic quota consumption
|
| 183 |
# --------------------------------------------------------------------------
|
| 184 |
-
def _consume_quota_atomic_sqlite(
|
| 185 |
-
self,
|
| 186 |
-
api_key: str,
|
| 187 |
-
tier: Tier,
|
| 188 |
-
month: str) -> bool: # noqa: E501
|
| 189 |
-
"""
|
| 190 |
-
Atomically increment counter only if under limit.
|
| 191 |
-
Returns True if quota was consumed, False if limit reached.
|
| 192 |
-
"""
|
| 193 |
limit = tier.monthly_evaluation_limit
|
| 194 |
if limit is None:
|
| 195 |
-
# Unlimited – still increment for tracking but always succeed
|
| 196 |
with self._get_conn() as conn:
|
| 197 |
conn.execute(
|
| 198 |
-
"
|
| 199 |
-
|
| 200 |
-
ON CONFLICT(api_key, year_month) DO UPDATE SET count = count + 1""",
|
| 201 |
(api_key, month)
|
| 202 |
)
|
| 203 |
conn.commit()
|
| 204 |
return True
|
| 205 |
|
| 206 |
-
# Use BEGIN IMMEDIATE to lock the database for the transaction
|
| 207 |
with self._get_conn() as conn:
|
| 208 |
conn.execute("BEGIN IMMEDIATE")
|
| 209 |
try:
|
| 210 |
-
# Get current count (or 0)
|
| 211 |
row = conn.execute(
|
| 212 |
"SELECT count FROM monthly_counts WHERE api_key = ? AND year_month = ?",
|
| 213 |
(api_key, month)
|
|
@@ -216,11 +237,9 @@ class UsageTracker:
|
|
| 216 |
if current >= limit:
|
| 217 |
conn.rollback()
|
| 218 |
return False
|
| 219 |
-
# Increment
|
| 220 |
conn.execute(
|
| 221 |
-
"
|
| 222 |
-
|
| 223 |
-
ON CONFLICT(api_key, year_month) DO UPDATE SET count = count + 1""",
|
| 224 |
(api_key, month)
|
| 225 |
)
|
| 226 |
conn.commit()
|
|
@@ -229,15 +248,9 @@ class UsageTracker:
|
|
| 229 |
conn.rollback()
|
| 230 |
raise
|
| 231 |
|
| 232 |
-
def _consume_quota_atomic_redis(
|
| 233 |
-
self,
|
| 234 |
-
api_key: str,
|
| 235 |
-
tier: Tier,
|
| 236 |
-
month: str) -> bool:
|
| 237 |
-
"""Atomic Lua script for Redis: INCR only if below limit."""
|
| 238 |
limit = tier.monthly_evaluation_limit
|
| 239 |
if limit is None:
|
| 240 |
-
# Unlimited – just increment and return True
|
| 241 |
redis_key = f"arf:quota:{api_key}:{month}"
|
| 242 |
self._redis_client.incr(redis_key)
|
| 243 |
self._redis_client.expire(redis_key, timedelta(days=31))
|
|
@@ -251,7 +264,7 @@ class UsageTracker:
|
|
| 251 |
return 0
|
| 252 |
end
|
| 253 |
local new = redis.call('INCR', key)
|
| 254 |
-
redis.call('EXPIRE', key, 2678400)
|
| 255 |
return 1
|
| 256 |
"""
|
| 257 |
redis_key = f"arf:quota:{api_key}:{month}"
|
|
@@ -259,144 +272,83 @@ class UsageTracker:
|
|
| 259 |
return result == 1
|
| 260 |
|
| 261 |
# --------------------------------------------------------------------------
|
| 262 |
-
# Idempotency handling
|
| 263 |
# --------------------------------------------------------------------------
|
| 264 |
def _is_idempotent_key_used(self, key: str) -> bool:
|
| 265 |
-
"""Check if idempotency key already processed."""
|
| 266 |
with self._get_conn() as conn:
|
| 267 |
row = conn.execute(
|
| 268 |
"SELECT 1 FROM idempotency_keys WHERE key = ?", (key,)).fetchone()
|
| 269 |
return row is not None
|
| 270 |
|
| 271 |
def _mark_idempotent_key_used(self, key: str, ttl_seconds: int = 86400):
|
| 272 |
-
"""Store idempotency key with expiration (cleanup later)."""
|
| 273 |
with self._get_conn() as conn:
|
| 274 |
conn.execute(
|
| 275 |
"INSERT INTO idempotency_keys (key, consumed_at) VALUES (?, ?)",
|
| 276 |
(key, time.time())
|
| 277 |
)
|
| 278 |
conn.commit()
|
| 279 |
-
# Optionally schedule cleanup of old keys (can be done in a background
|
| 280 |
-
# thread)
|
| 281 |
|
| 282 |
# --------------------------------------------------------------------------
|
| 283 |
-
# Core usage recording (atomic + idempotent)
|
| 284 |
# --------------------------------------------------------------------------
|
| 285 |
-
def consume_quota_and_log(
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
) -> Tuple[bool, Optional[Dict[str, Any]]]:
|
| 290 |
-
"""
|
| 291 |
-
Atomically consume quota and insert audit log.
|
| 292 |
-
Returns (success, existing_response) where existing_response is not None
|
| 293 |
-
only when idempotency_key matched a previous successful call.
|
| 294 |
-
"""
|
| 295 |
-
# Idempotency check (if key provided)
|
| 296 |
-
if idempotency_key:
|
| 297 |
-
if self._is_idempotent_key_used(idempotency_key):
|
| 298 |
-
# Retrieve previous response from audit log (simplified – you may cache full response)
|
| 299 |
-
# For full idempotency, we would store the response body in idempotency table.
|
| 300 |
-
# Here we return a marker that caller should use cached
|
| 301 |
-
# response.
|
| 302 |
-
return False, {"idempotent": True,
|
| 303 |
-
"message": "Already processed"}
|
| 304 |
|
| 305 |
month = self._get_month_key()
|
| 306 |
-
# Atomic quota consumption
|
| 307 |
if self._redis_client:
|
| 308 |
-
quota_ok = self._consume_quota_atomic_redis(
|
| 309 |
-
record.api_key, record.tier, month)
|
| 310 |
else:
|
| 311 |
-
quota_ok = self._consume_quota_atomic_sqlite(
|
| 312 |
-
record.api_key, record.tier, month)
|
| 313 |
|
| 314 |
if not quota_ok:
|
| 315 |
return False, None
|
| 316 |
|
| 317 |
-
# Insert audit log (with idempotency key as unique constraint)
|
| 318 |
try:
|
| 319 |
with self._get_conn() as conn:
|
| 320 |
conn.execute(
|
| 321 |
"""INSERT INTO usage_log
|
| 322 |
-
(api_key, tier, timestamp, endpoint,
|
| 323 |
-
request_body, response, error, processing_ms,
|
| 324 |
-
idempotency_key)
|
| 325 |
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
| 326 |
-
(record.api_key,
|
| 327 |
-
record.
|
| 328 |
-
record.
|
| 329 |
-
record.
|
| 330 |
-
|
| 331 |
-
record.request_body) if record.request_body else None,
|
| 332 |
-
json.dumps(
|
| 333 |
-
record.response) if record.response else None,
|
| 334 |
-
record.error,
|
| 335 |
-
record.processing_ms,
|
| 336 |
-
idempotency_key,
|
| 337 |
-
))
|
| 338 |
conn.commit()
|
| 339 |
except sqlite3.IntegrityError as e:
|
| 340 |
-
# Duplicate idempotency_key – already inserted by another
|
| 341 |
-
# concurrent request
|
| 342 |
if "UNIQUE constraint failed: usage_log.idempotency_key" in str(e):
|
| 343 |
-
return False, {"idempotent": True,
|
| 344 |
-
"message": "Already processed"}
|
| 345 |
raise
|
| 346 |
|
| 347 |
if idempotency_key:
|
| 348 |
self._mark_idempotent_key_used(idempotency_key)
|
| 349 |
-
# Removed stray # noqa: E501 comment that was wrongly indented here
|
| 350 |
return True, None
|
| 351 |
|
| 352 |
# --------------------------------------------------------------------------
|
| 353 |
-
# Legacy interface (kept for compatibility
|
| 354 |
# --------------------------------------------------------------------------
|
| 355 |
-
def increment_usage_sync(
|
| 356 |
-
self,
|
| 357 |
-
record: UsageRecord,
|
| 358 |
-
idempotency_key: Optional[str] = None) -> bool:
|
| 359 |
-
"""
|
| 360 |
-
Synchronously record usage and increment counter.
|
| 361 |
-
Returns True if within quota and recorded, False otherwise.
|
| 362 |
-
This method now uses the atomic implementation.
|
| 363 |
-
"""
|
| 364 |
success, _ = self.consume_quota_and_log(record, idempotency_key)
|
| 365 |
return success
|
| 366 |
|
| 367 |
-
async def increment_usage_async(
|
| 368 |
-
|
| 369 |
-
record: UsageRecord,
|
| 370 |
-
background_tasks: BackgroundTasks,
|
| 371 |
-
idempotency_key: Optional[str] = None
|
| 372 |
-
) -> bool:
|
| 373 |
-
"""
|
| 374 |
-
Asynchronously record usage using FastAPI BackgroundTasks.
|
| 375 |
-
Still does the atomic check synchronously, then schedules the insert.
|
| 376 |
-
"""
|
| 377 |
-
# First, do atomic quota check (synchronous) – we must ensure we don't double-consume.
|
| 378 |
-
# Because background tasks may run later, we still need to reserve quota now.
|
| 379 |
-
# Simplified: we call consume_quota_and_log synchronously – that defeats async benefit.
|
| 380 |
-
# Better to use a queue or Redis with background processing.
|
| 381 |
-
# For this fix, we'll use the sync method (blocking) but still support
|
| 382 |
-
# idempotency.
|
| 383 |
return self.increment_usage_sync(record, idempotency_key)
|
| 384 |
|
| 385 |
# --------------------------------------------------------------------------
|
| 386 |
-
# Quota inspection
|
| 387 |
# --------------------------------------------------------------------------
|
| 388 |
def get_remaining_quota(self, api_key: str, tier: Tier) -> Optional[int]:
|
| 389 |
-
"""Return remaining evaluations for the month (non‑atomic, for info only)."""
|
| 390 |
limit = tier.monthly_evaluation_limit
|
| 391 |
if limit is None:
|
| 392 |
return None
|
| 393 |
-
|
| 394 |
month = self._get_month_key()
|
| 395 |
if self._redis_client:
|
| 396 |
redis_key = f"arf:quota:{api_key}:{month}"
|
| 397 |
count = int(self._redis_client.get(redis_key) or 0)
|
| 398 |
return max(0, limit - count)
|
| 399 |
-
|
| 400 |
with self._get_conn() as conn:
|
| 401 |
row = conn.execute(
|
| 402 |
"SELECT count FROM monthly_counts WHERE api_key = ? AND year_month = ?",
|
|
@@ -406,16 +358,10 @@ class UsageTracker:
|
|
| 406 |
return max(0, limit - count)
|
| 407 |
|
| 408 |
# --------------------------------------------------------------------------
|
| 409 |
-
# Audit and maintenance
|
| 410 |
# --------------------------------------------------------------------------
|
| 411 |
-
def get_audit_logs(
|
| 412 |
-
|
| 413 |
-
api_key: str,
|
| 414 |
-
start_date: Optional[datetime] = None,
|
| 415 |
-
end_date: Optional[datetime] = None,
|
| 416 |
-
limit: int = 100,
|
| 417 |
-
) -> List[Dict[str, Any]]:
|
| 418 |
-
"""Retrieve audit logs for a given API key."""
|
| 419 |
query = "SELECT * FROM usage_log WHERE api_key = ?"
|
| 420 |
params = [api_key]
|
| 421 |
if start_date:
|
|
@@ -426,47 +372,36 @@ class UsageTracker:
|
|
| 426 |
params.append(end_date.timestamp())
|
| 427 |
query += " ORDER BY timestamp DESC LIMIT ?"
|
| 428 |
params.append(limit)
|
| 429 |
-
|
| 430 |
with self._get_conn() as conn:
|
| 431 |
rows = conn.execute(query, params).fetchall()
|
| 432 |
return [dict(row) for row in rows]
|
| 433 |
|
| 434 |
def clean_old_logs(self):
|
| 435 |
-
"""Delete logs older than retention period for each tier, and old idempotency keys."""
|
| 436 |
with self._get_conn() as conn:
|
| 437 |
-
# Delete old usage logs
|
| 438 |
for tier in Tier:
|
| 439 |
retention_days = tier.audit_log_retention_days
|
| 440 |
-
if retention_days is None:
|
| 441 |
-
continue
|
| 442 |
cutoff = time.time() - retention_days * 86400
|
| 443 |
conn.execute(
|
| 444 |
"DELETE FROM usage_log WHERE tier = ? AND timestamp < ?",
|
| 445 |
(tier.value, cutoff)
|
| 446 |
)
|
| 447 |
-
# Delete idempotency keys older than 7 days
|
| 448 |
cutoff = time.time() - 7 * 86400
|
| 449 |
-
conn.execute(
|
| 450 |
-
"DELETE FROM idempotency_keys WHERE consumed_at < ?", (cutoff,))
|
| 451 |
conn.commit()
|
| 452 |
|
| 453 |
|
| 454 |
# --------------------------------------------------------------------------
|
| 455 |
-
# Global instance and FastAPI dependency
|
| 456 |
# --------------------------------------------------------------------------
|
| 457 |
tracker: Optional[UsageTracker] = None
|
| 458 |
|
| 459 |
|
| 460 |
-
def init_tracker(
|
| 461 |
-
db_path: str = "arf_usage.db",
|
| 462 |
-
redis_url: Optional[str] = None):
|
| 463 |
-
"""Initialize the global tracker. Must be called before enforce_quota."""
|
| 464 |
global tracker
|
| 465 |
tracker = UsageTracker(db_path, redis_url)
|
| 466 |
|
| 467 |
|
| 468 |
def update_key_tier(api_key: str, new_tier: Tier) -> bool:
|
| 469 |
-
"""Globally accessible helper to update API key tier."""
|
| 470 |
if tracker is None:
|
| 471 |
return False
|
| 472 |
return tracker.update_api_key_tier(api_key, new_tier)
|
|
@@ -474,16 +409,11 @@ def update_key_tier(api_key: str, new_tier: Tier) -> bool:
|
|
| 474 |
|
| 475 |
async def enforce_quota(request: Request, api_key: str = None):
|
| 476 |
"""
|
| 477 |
-
|
| 478 |
-
FAILS CLOSED: if tracker not initialised, raises HTTP 503.
|
| 479 |
"""
|
| 480 |
-
# P0 fix: No fallback that allows all requests
|
| 481 |
if tracker is None:
|
| 482 |
-
raise HTTPException(
|
| 483 |
-
status_code=503,
|
| 484 |
-
detail="Usage tracking service not initialised. Please contact administrator.")
|
| 485 |
|
| 486 |
-
# Extract API key from header or query
|
| 487 |
if api_key is None:
|
| 488 |
auth_header = request.headers.get("Authorization")
|
| 489 |
if auth_header and auth_header.startswith("Bearer "):
|
|
@@ -496,16 +426,19 @@ async def enforce_quota(request: Request, api_key: str = None):
|
|
| 496 |
|
| 497 |
tier = tracker.get_tier(api_key)
|
| 498 |
if tier is None:
|
| 499 |
-
raise HTTPException(
|
| 500 |
-
status_code=403,
|
| 501 |
-
detail="Invalid or inactive API key")
|
| 502 |
|
| 503 |
remaining = tracker.get_remaining_quota(api_key, tier)
|
| 504 |
if remaining is not None and remaining <= 0:
|
| 505 |
-
raise HTTPException(status_code=429,
|
| 506 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 507 |
|
| 508 |
-
# Store in request state for later logging (optional)
|
| 509 |
request.state.api_key = api_key
|
| 510 |
request.state.tier = tier
|
| 511 |
-
|
|
|
|
|
|
|
|
|
| 1 |
"""
|
| 2 |
Usage Tracker for ARF API – quotas, tiers, and audit logging.
|
| 3 |
Thread‑safe, atomic quota consumption, idempotent, fail‑closed.
|
|
|
|
| 4 |
|
| 5 |
+
Extended for multi‑tenancy: each API key is linked to a tenant ID.
|
| 6 |
+
Tenant ID is stored in the `api_keys` table and used for resource isolation.
|
| 7 |
+
"""
|
| 8 |
import json
|
| 9 |
import sqlite3
|
| 10 |
import threading
|
|
|
|
| 12 |
from contextlib import contextmanager
|
| 13 |
from datetime import datetime, timedelta
|
| 14 |
from dataclasses import dataclass
|
| 15 |
+
from typing import Dict, Any, Optional, List, Tuple, Callable
|
| 16 |
from enum import Enum
|
| 17 |
from fastapi import BackgroundTasks, HTTPException, Request
|
| 18 |
|
|
|
|
| 26 |
|
| 27 |
|
| 28 |
class Tier(str, Enum):
|
| 29 |
+
"""Pricing tiers with associated quota limits and audit retention."""
|
| 30 |
FREE = "free"
|
| 31 |
PRO = "pro"
|
| 32 |
PREMIUM = "premium"
|
|
|
|
| 34 |
|
| 35 |
@property
|
| 36 |
def monthly_evaluation_limit(self) -> Optional[int]:
|
| 37 |
+
"""Monthly evaluation quota. None = unlimited."""
|
| 38 |
limits = {
|
| 39 |
Tier.FREE: 1000,
|
| 40 |
Tier.PRO: 10_000,
|
| 41 |
Tier.PREMIUM: 50_000,
|
| 42 |
+
Tier.ENTERPRISE: None,
|
| 43 |
}
|
| 44 |
return limits[self]
|
| 45 |
|
| 46 |
@property
|
| 47 |
def audit_log_retention_days(self) -> int:
|
| 48 |
+
"""How many days to keep usage and decision audit logs."""
|
| 49 |
retention = {
|
| 50 |
Tier.FREE: 7,
|
| 51 |
Tier.PRO: 30,
|
|
|
|
| 57 |
|
| 58 |
@dataclass
|
| 59 |
class UsageRecord:
|
| 60 |
+
"""Single API call usage record (for quota and debugging)."""
|
| 61 |
api_key: str
|
| 62 |
tier: Tier
|
| 63 |
timestamp: float
|
|
|
|
| 71 |
class UsageTracker:
|
| 72 |
"""
|
| 73 |
Thread‑safe usage tracker with atomic quota consumption and idempotency.
|
| 74 |
+
Extended to support tenant isolation: each API key is linked to a tenant.
|
| 75 |
"""
|
| 76 |
|
| 77 |
def __init__(self, db_path: str = "arf_usage.db",
|
|
|
|
| 84 |
if redis_url and REDIS_AVAILABLE:
|
| 85 |
self._redis_client = redis.from_url(redis_url)
|
| 86 |
elif redis_url:
|
| 87 |
+
raise ImportError("Redis client not installed. Run: pip install redis")
|
|
|
|
| 88 |
|
| 89 |
@contextmanager
|
| 90 |
def _get_conn(self):
|
| 91 |
+
"""Get a thread‑local SQLite connection with WAL and immediate transactions."""
|
| 92 |
if not hasattr(self._local, "conn"):
|
| 93 |
self._local.conn = sqlite3.connect(
|
| 94 |
self.db_path, check_same_thread=False, isolation_level=None)
|
|
|
|
| 97 |
yield self._local.conn
|
| 98 |
|
| 99 |
def _init_db(self):
|
| 100 |
+
"""Initialise SQLite tables with tenant_id support."""
|
| 101 |
with self._get_conn() as conn:
|
| 102 |
+
# Modified: api_keys now has tenant_id column
|
| 103 |
conn.execute("""
|
| 104 |
CREATE TABLE IF NOT EXISTS api_keys (
|
| 105 |
key TEXT PRIMARY KEY,
|
| 106 |
+
tenant_id TEXT NOT NULL,
|
| 107 |
tier TEXT NOT NULL,
|
| 108 |
created_at REAL NOT NULL,
|
| 109 |
last_used_at REAL,
|
|
|
|
| 147 |
def _get_month_key(self) -> str:
|
| 148 |
return datetime.now().strftime("%Y-%m")
|
| 149 |
|
| 150 |
+
def get_or_create_api_key(self, key: str, tenant_id: str, tier: Tier = Tier.FREE) -> bool:
|
| 151 |
+
"""
|
| 152 |
+
Register a new API key for a given tenant.
|
| 153 |
+
|
| 154 |
+
Args:
|
| 155 |
+
key: The API key (plain text, will be hashed in production).
|
| 156 |
+
tenant_id: UUID of the tenant (must already exist in main DB).
|
| 157 |
+
tier: Initial tier for the key.
|
| 158 |
+
|
| 159 |
+
Returns:
|
| 160 |
+
True if key was created (or already exists for the same tenant).
|
| 161 |
+
"""
|
| 162 |
with self._get_conn() as conn:
|
| 163 |
row = conn.execute(
|
| 164 |
"SELECT key FROM api_keys WHERE key = ?", (key,)).fetchone()
|
| 165 |
if row:
|
| 166 |
+
# Key already exists – ensure it belongs to the same tenant
|
| 167 |
+
existing_tenant = conn.execute(
|
| 168 |
+
"SELECT tenant_id FROM api_keys WHERE key = ?", (key,)).fetchone()
|
| 169 |
+
if existing_tenant["tenant_id"] != tenant_id:
|
| 170 |
+
raise ValueError(f"Key {key[:8]}... already belongs to a different tenant.")
|
| 171 |
return True
|
| 172 |
conn.execute(
|
| 173 |
+
"INSERT INTO api_keys (key, tenant_id, tier, created_at, is_active) VALUES (?, ?, ?, ?, ?)",
|
| 174 |
+
(key, tenant_id, tier.value, time.time(), 1)
|
| 175 |
)
|
| 176 |
conn.commit()
|
| 177 |
return True
|
|
|
|
| 187 |
return None
|
| 188 |
return Tier(row["tier"])
|
| 189 |
|
| 190 |
+
def get_tenant_id(self, api_key: str) -> Optional[str]:
|
| 191 |
+
"""Return the tenant ID associated with the API key, or None if key invalid."""
|
| 192 |
+
with self._get_conn() as conn:
|
| 193 |
+
row = conn.execute(
|
| 194 |
+
"SELECT tenant_id FROM api_keys WHERE key = ? AND is_active = 1",
|
| 195 |
+
(api_key,)
|
| 196 |
+
).fetchone()
|
| 197 |
+
if not row:
|
| 198 |
+
return None
|
| 199 |
+
return row["tenant_id"]
|
| 200 |
+
|
| 201 |
def update_api_key_tier(self, api_key: str, new_tier: Tier) -> bool:
|
| 202 |
"""Update the tier of an existing API key. Returns True if successful."""
|
| 203 |
with self._get_conn() as conn:
|
|
|
|
| 207 |
return False
|
| 208 |
conn.execute(
|
| 209 |
"UPDATE api_keys SET tier = ? WHERE key = ?",
|
| 210 |
+
(new_tier.value, api_key))
|
|
|
|
| 211 |
conn.commit()
|
| 212 |
return True
|
| 213 |
|
| 214 |
# --------------------------------------------------------------------------
|
| 215 |
+
# Atomic quota consumption (unchanged, but uses api_key which links to tenant)
|
| 216 |
# --------------------------------------------------------------------------
|
| 217 |
+
def _consume_quota_atomic_sqlite(self, api_key: str, tier: Tier, month: str) -> bool:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 218 |
limit = tier.monthly_evaluation_limit
|
| 219 |
if limit is None:
|
|
|
|
| 220 |
with self._get_conn() as conn:
|
| 221 |
conn.execute(
|
| 222 |
+
"INSERT INTO monthly_counts (api_key, year_month, count) VALUES (?, ?, 1) "
|
| 223 |
+
"ON CONFLICT(api_key, year_month) DO UPDATE SET count = count + 1",
|
|
|
|
| 224 |
(api_key, month)
|
| 225 |
)
|
| 226 |
conn.commit()
|
| 227 |
return True
|
| 228 |
|
|
|
|
| 229 |
with self._get_conn() as conn:
|
| 230 |
conn.execute("BEGIN IMMEDIATE")
|
| 231 |
try:
|
|
|
|
| 232 |
row = conn.execute(
|
| 233 |
"SELECT count FROM monthly_counts WHERE api_key = ? AND year_month = ?",
|
| 234 |
(api_key, month)
|
|
|
|
| 237 |
if current >= limit:
|
| 238 |
conn.rollback()
|
| 239 |
return False
|
|
|
|
| 240 |
conn.execute(
|
| 241 |
+
"INSERT INTO monthly_counts (api_key, year_month, count) VALUES (?, ?, 1) "
|
| 242 |
+
"ON CONFLICT(api_key, year_month) DO UPDATE SET count = count + 1",
|
|
|
|
| 243 |
(api_key, month)
|
| 244 |
)
|
| 245 |
conn.commit()
|
|
|
|
| 248 |
conn.rollback()
|
| 249 |
raise
|
| 250 |
|
| 251 |
+
def _consume_quota_atomic_redis(self, api_key: str, tier: Tier, month: str) -> bool:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
limit = tier.monthly_evaluation_limit
|
| 253 |
if limit is None:
|
|
|
|
| 254 |
redis_key = f"arf:quota:{api_key}:{month}"
|
| 255 |
self._redis_client.incr(redis_key)
|
| 256 |
self._redis_client.expire(redis_key, timedelta(days=31))
|
|
|
|
| 264 |
return 0
|
| 265 |
end
|
| 266 |
local new = redis.call('INCR', key)
|
| 267 |
+
redis.call('EXPIRE', key, 2678400)
|
| 268 |
return 1
|
| 269 |
"""
|
| 270 |
redis_key = f"arf:quota:{api_key}:{month}"
|
|
|
|
| 272 |
return result == 1
|
| 273 |
|
| 274 |
# --------------------------------------------------------------------------
|
| 275 |
+
# Idempotency handling (unchanged)
|
| 276 |
# --------------------------------------------------------------------------
|
| 277 |
def _is_idempotent_key_used(self, key: str) -> bool:
|
|
|
|
| 278 |
with self._get_conn() as conn:
|
| 279 |
row = conn.execute(
|
| 280 |
"SELECT 1 FROM idempotency_keys WHERE key = ?", (key,)).fetchone()
|
| 281 |
return row is not None
|
| 282 |
|
| 283 |
def _mark_idempotent_key_used(self, key: str, ttl_seconds: int = 86400):
|
|
|
|
| 284 |
with self._get_conn() as conn:
|
| 285 |
conn.execute(
|
| 286 |
"INSERT INTO idempotency_keys (key, consumed_at) VALUES (?, ?)",
|
| 287 |
(key, time.time())
|
| 288 |
)
|
| 289 |
conn.commit()
|
|
|
|
|
|
|
| 290 |
|
| 291 |
# --------------------------------------------------------------------------
|
| 292 |
+
# Core usage recording (atomic + idempotent) – unchanged
|
| 293 |
# --------------------------------------------------------------------------
|
| 294 |
+
def consume_quota_and_log(self, record: UsageRecord, idempotency_key: Optional[str] = None
|
| 295 |
+
) -> Tuple[bool, Optional[Dict[str, Any]]]:
|
| 296 |
+
if idempotency_key and self._is_idempotent_key_used(idempotency_key):
|
| 297 |
+
return False, {"idempotent": True, "message": "Already processed"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
|
| 299 |
month = self._get_month_key()
|
|
|
|
| 300 |
if self._redis_client:
|
| 301 |
+
quota_ok = self._consume_quota_atomic_redis(record.api_key, record.tier, month)
|
|
|
|
| 302 |
else:
|
| 303 |
+
quota_ok = self._consume_quota_atomic_sqlite(record.api_key, record.tier, month)
|
|
|
|
| 304 |
|
| 305 |
if not quota_ok:
|
| 306 |
return False, None
|
| 307 |
|
|
|
|
| 308 |
try:
|
| 309 |
with self._get_conn() as conn:
|
| 310 |
conn.execute(
|
| 311 |
"""INSERT INTO usage_log
|
| 312 |
+
(api_key, tier, timestamp, endpoint, request_body, response, error, processing_ms, idempotency_key)
|
|
|
|
|
|
|
| 313 |
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
| 314 |
+
(record.api_key, record.tier.value, record.timestamp, record.endpoint,
|
| 315 |
+
json.dumps(record.request_body) if record.request_body else None,
|
| 316 |
+
json.dumps(record.response) if record.response else None,
|
| 317 |
+
record.error, record.processing_ms, idempotency_key)
|
| 318 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
conn.commit()
|
| 320 |
except sqlite3.IntegrityError as e:
|
|
|
|
|
|
|
| 321 |
if "UNIQUE constraint failed: usage_log.idempotency_key" in str(e):
|
| 322 |
+
return False, {"idempotent": True, "message": "Already processed"}
|
|
|
|
| 323 |
raise
|
| 324 |
|
| 325 |
if idempotency_key:
|
| 326 |
self._mark_idempotent_key_used(idempotency_key)
|
|
|
|
| 327 |
return True, None
|
| 328 |
|
| 329 |
# --------------------------------------------------------------------------
|
| 330 |
+
# Legacy interface (kept for compatibility)
|
| 331 |
# --------------------------------------------------------------------------
|
| 332 |
+
def increment_usage_sync(self, record: UsageRecord, idempotency_key: Optional[str] = None) -> bool:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
success, _ = self.consume_quota_and_log(record, idempotency_key)
|
| 334 |
return success
|
| 335 |
|
| 336 |
+
async def increment_usage_async(self, record: UsageRecord, background_tasks: BackgroundTasks,
|
| 337 |
+
idempotency_key: Optional[str] = None) -> bool:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
return self.increment_usage_sync(record, idempotency_key)
|
| 339 |
|
| 340 |
# --------------------------------------------------------------------------
|
| 341 |
+
# Quota inspection
|
| 342 |
# --------------------------------------------------------------------------
|
| 343 |
def get_remaining_quota(self, api_key: str, tier: Tier) -> Optional[int]:
|
|
|
|
| 344 |
limit = tier.monthly_evaluation_limit
|
| 345 |
if limit is None:
|
| 346 |
return None
|
|
|
|
| 347 |
month = self._get_month_key()
|
| 348 |
if self._redis_client:
|
| 349 |
redis_key = f"arf:quota:{api_key}:{month}"
|
| 350 |
count = int(self._redis_client.get(redis_key) or 0)
|
| 351 |
return max(0, limit - count)
|
|
|
|
| 352 |
with self._get_conn() as conn:
|
| 353 |
row = conn.execute(
|
| 354 |
"SELECT count FROM monthly_counts WHERE api_key = ? AND year_month = ?",
|
|
|
|
| 358 |
return max(0, limit - count)
|
| 359 |
|
| 360 |
# --------------------------------------------------------------------------
|
| 361 |
+
# Audit and maintenance (kept for usage_log)
|
| 362 |
# --------------------------------------------------------------------------
|
| 363 |
+
def get_audit_logs(self, api_key: str, start_date: Optional[datetime] = None,
|
| 364 |
+
end_date: Optional[datetime] = None, limit: int = 100) -> List[Dict[str, Any]]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 365 |
query = "SELECT * FROM usage_log WHERE api_key = ?"
|
| 366 |
params = [api_key]
|
| 367 |
if start_date:
|
|
|
|
| 372 |
params.append(end_date.timestamp())
|
| 373 |
query += " ORDER BY timestamp DESC LIMIT ?"
|
| 374 |
params.append(limit)
|
|
|
|
| 375 |
with self._get_conn() as conn:
|
| 376 |
rows = conn.execute(query, params).fetchall()
|
| 377 |
return [dict(row) for row in rows]
|
| 378 |
|
| 379 |
def clean_old_logs(self):
|
|
|
|
| 380 |
with self._get_conn() as conn:
|
|
|
|
| 381 |
for tier in Tier:
|
| 382 |
retention_days = tier.audit_log_retention_days
|
|
|
|
|
|
|
| 383 |
cutoff = time.time() - retention_days * 86400
|
| 384 |
conn.execute(
|
| 385 |
"DELETE FROM usage_log WHERE tier = ? AND timestamp < ?",
|
| 386 |
(tier.value, cutoff)
|
| 387 |
)
|
|
|
|
| 388 |
cutoff = time.time() - 7 * 86400
|
| 389 |
+
conn.execute("DELETE FROM idempotency_keys WHERE consumed_at < ?", (cutoff,))
|
|
|
|
| 390 |
conn.commit()
|
| 391 |
|
| 392 |
|
| 393 |
# --------------------------------------------------------------------------
|
| 394 |
+
# Global instance and FastAPI dependency
|
| 395 |
# --------------------------------------------------------------------------
|
| 396 |
tracker: Optional[UsageTracker] = None
|
| 397 |
|
| 398 |
|
| 399 |
+
def init_tracker(db_path: str = "arf_usage.db", redis_url: Optional[str] = None):
|
|
|
|
|
|
|
|
|
|
| 400 |
global tracker
|
| 401 |
tracker = UsageTracker(db_path, redis_url)
|
| 402 |
|
| 403 |
|
| 404 |
def update_key_tier(api_key: str, new_tier: Tier) -> bool:
|
|
|
|
| 405 |
if tracker is None:
|
| 406 |
return False
|
| 407 |
return tracker.update_api_key_tier(api_key, new_tier)
|
|
|
|
| 409 |
|
| 410 |
async def enforce_quota(request: Request, api_key: str = None):
|
| 411 |
"""
|
| 412 |
+
FastAPI dependency that enforces quota and attaches tenant_id to request state.
|
|
|
|
| 413 |
"""
|
|
|
|
| 414 |
if tracker is None:
|
| 415 |
+
raise HTTPException(status_code=503, detail="Usage tracking service not initialised.")
|
|
|
|
|
|
|
| 416 |
|
|
|
|
| 417 |
if api_key is None:
|
| 418 |
auth_header = request.headers.get("Authorization")
|
| 419 |
if auth_header and auth_header.startswith("Bearer "):
|
|
|
|
| 426 |
|
| 427 |
tier = tracker.get_tier(api_key)
|
| 428 |
if tier is None:
|
| 429 |
+
raise HTTPException(status_code=403, detail="Invalid or inactive API key")
|
|
|
|
|
|
|
| 430 |
|
| 431 |
remaining = tracker.get_remaining_quota(api_key, tier)
|
| 432 |
if remaining is not None and remaining <= 0:
|
| 433 |
+
raise HTTPException(status_code=429, detail="Monthly evaluation quota exceeded")
|
| 434 |
+
|
| 435 |
+
# Retrieve tenant_id
|
| 436 |
+
tenant_id = tracker.get_tenant_id(api_key)
|
| 437 |
+
if not tenant_id:
|
| 438 |
+
raise HTTPException(status_code=403, detail="API key not associated with a tenant")
|
| 439 |
|
|
|
|
| 440 |
request.state.api_key = api_key
|
| 441 |
request.state.tier = tier
|
| 442 |
+
request.state.tenant_id = tenant_id
|
| 443 |
+
|
| 444 |
+
return {"api_key": api_key, "tier": tier, "tenant_id": tenant_id, "remaining": remaining}
|
app/database/models_intents.py
CHANGED
|
@@ -1,50 +1,149 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
from sqlalchemy.orm import relationship
|
| 3 |
import datetime
|
| 4 |
from .base import Base
|
| 5 |
|
| 6 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
class IntentDB(Base):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
__tablename__ = "intents"
|
|
|
|
| 9 |
id = Column(Integer, primary_key=True, index=True)
|
| 10 |
-
deterministic_id = Column(
|
| 11 |
-
|
| 12 |
-
unique=True,
|
| 13 |
-
index=True,
|
| 14 |
-
nullable=False)
|
| 15 |
intent_type = Column(String(64), nullable=False)
|
| 16 |
payload = Column(JSON, nullable=False)
|
| 17 |
oss_payload = Column(JSON, nullable=True)
|
| 18 |
environment = Column(String(32), nullable=True)
|
| 19 |
-
created_at = Column(
|
| 20 |
-
DateTime,
|
| 21 |
-
default=datetime.datetime.utcnow,
|
| 22 |
-
nullable=False)
|
| 23 |
evaluated_at = Column(DateTime, nullable=True)
|
| 24 |
risk_score = Column(String(32), nullable=True)
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
|
| 30 |
|
| 31 |
class OutcomeDB(Base):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
__tablename__ = "intent_outcomes"
|
|
|
|
| 33 |
id = Column(Integer, primary_key=True, index=True)
|
| 34 |
-
intent_id = Column(
|
| 35 |
-
Integer,
|
| 36 |
-
ForeignKey(
|
| 37 |
-
"intents.id",
|
| 38 |
-
ondelete="CASCADE"),
|
| 39 |
-
nullable=False)
|
| 40 |
success = Column(Boolean, nullable=False)
|
| 41 |
recorded_by = Column(String(128), nullable=True)
|
| 42 |
notes = Column(Text, nullable=True)
|
| 43 |
-
recorded_at = Column(
|
| 44 |
-
DateTime,
|
| 45 |
-
default=datetime.datetime.utcnow,
|
| 46 |
-
nullable=False)
|
| 47 |
idempotency_key = Column(String(128), unique=True, nullable=True)
|
|
|
|
| 48 |
intent = relationship("IntentDB", back_populates="outcomes")
|
| 49 |
|
| 50 |
__table_args__ = (
|
|
@@ -52,24 +151,83 @@ class OutcomeDB(Base):
|
|
| 52 |
)
|
| 53 |
|
| 54 |
|
| 55 |
-
#
|
| 56 |
-
#
|
| 57 |
-
#
|
|
|
|
| 58 |
class BetaStateDB(Base):
|
| 59 |
"""
|
| 60 |
-
Stores the
|
| 61 |
-
|
|
|
|
| 62 |
|
| 63 |
-
|
| 64 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
"""
|
| 66 |
__tablename__ = "beta_state"
|
| 67 |
|
| 68 |
id = Column(Integer, primary_key=True, index=True)
|
| 69 |
-
|
|
|
|
| 70 |
alpha = Column(Float, nullable=False)
|
| 71 |
beta = Column(Float, nullable=False)
|
| 72 |
-
updated_at = Column(
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Database models for the ARF API Control Plane.
|
| 3 |
+
|
| 4 |
+
This module defines the SQLAlchemy ORM models for:
|
| 5 |
+
- Intents (InfrastructureIntent evaluations)
|
| 6 |
+
- Outcomes (recorded results of executed intents)
|
| 7 |
+
- Beta state (conjugate Bayesian posteriors per tenant and category)
|
| 8 |
+
- Audit logs (immutable decision records for compliance)
|
| 9 |
+
- Tenants (multi‑tenant isolation)
|
| 10 |
+
|
| 11 |
+
All tables include a `tenant_id` column to enforce data partitioning.
|
| 12 |
+
The BetaStateDB now stores parameters per (tenant, category) pair.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from sqlalchemy import (
|
| 16 |
+
Column, Integer, String, DateTime, Boolean, Text, JSON,
|
| 17 |
+
Float, ForeignKey, UniqueConstraint, Index
|
| 18 |
+
)
|
| 19 |
from sqlalchemy.orm import relationship
|
| 20 |
import datetime
|
| 21 |
from .base import Base
|
| 22 |
|
| 23 |
|
| 24 |
+
# ============================================================================
|
| 25 |
+
# Tenant table – root of multi‑tenancy
|
| 26 |
+
# ============================================================================
|
| 27 |
+
|
| 28 |
+
class TenantDB(Base):
|
| 29 |
+
"""
|
| 30 |
+
Represents a customer tenant (organisation). All other tables
|
| 31 |
+
reference this table via a foreign key `tenant_id`.
|
| 32 |
+
|
| 33 |
+
Attributes:
|
| 34 |
+
id (str): UUID of the tenant (primary key).
|
| 35 |
+
name (str): Human‑readable organisation name.
|
| 36 |
+
created_at (datetime): UTC timestamp of creation.
|
| 37 |
+
created_by (str, optional): Email or user ID of the creator.
|
| 38 |
+
"""
|
| 39 |
+
__tablename__ = "tenants"
|
| 40 |
+
|
| 41 |
+
id = Column(String(64), primary_key=True, index=True)
|
| 42 |
+
name = Column(String(256), nullable=False)
|
| 43 |
+
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
|
| 44 |
+
created_by = Column(String(128), nullable=True)
|
| 45 |
+
|
| 46 |
+
# Relationships
|
| 47 |
+
api_keys = relationship("APIKeyDB", back_populates="tenant", cascade="all, delete-orphan")
|
| 48 |
+
intents = relationship("IntentDB", back_populates="tenant")
|
| 49 |
+
beta_states = relationship("BetaStateDB", back_populates="tenant")
|
| 50 |
+
audit_logs = relationship("DecisionAuditLogDB", back_populates="tenant")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
# ============================================================================
|
| 54 |
+
# API keys (extended with tenant_id)
|
| 55 |
+
# ============================================================================
|
| 56 |
+
|
| 57 |
+
class APIKeyDB(Base):
|
| 58 |
+
"""
|
| 59 |
+
Stores API keys for authentication and tiered quota. Each key belongs
|
| 60 |
+
to exactly one tenant. The `tier` determines monthly evaluation limits.
|
| 61 |
+
|
| 62 |
+
Attributes:
|
| 63 |
+
key (str): The hashed API key (primary key).
|
| 64 |
+
tenant_id (str): Foreign key to `tenants.id`.
|
| 65 |
+
tier (str): Tier enumeration value (free, pro, premium, enterprise).
|
| 66 |
+
created_at (datetime): UTC creation time.
|
| 67 |
+
last_used_at (datetime, optional): Timestamp of last successful request.
|
| 68 |
+
is_active (bool): Soft‑delete flag.
|
| 69 |
+
"""
|
| 70 |
+
__tablename__ = "api_keys"
|
| 71 |
+
|
| 72 |
+
key = Column(String(256), primary_key=True, index=True)
|
| 73 |
+
tenant_id = Column(String(64), ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
|
| 74 |
+
tier = Column(String(32), nullable=False)
|
| 75 |
+
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
|
| 76 |
+
last_used_at = Column(DateTime, nullable=True)
|
| 77 |
+
is_active = Column(Boolean, default=True, nullable=False)
|
| 78 |
+
|
| 79 |
+
# Relationships
|
| 80 |
+
tenant = relationship("TenantDB", back_populates="api_keys")
|
| 81 |
+
usage_logs = relationship("UsageLogDB", back_populates="api_key")
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
# ============================================================================
|
| 85 |
+
# Intents (evaluations) – now tenant‑scoped
|
| 86 |
+
# ============================================================================
|
| 87 |
+
|
| 88 |
class IntentDB(Base):
|
| 89 |
+
"""
|
| 90 |
+
Stores each InfrastructureIntent evaluation request and its resulting
|
| 91 |
+
risk score. One‑to‑many with OutcomeDB.
|
| 92 |
+
|
| 93 |
+
Attributes:
|
| 94 |
+
id (int): Auto‑increment primary key.
|
| 95 |
+
deterministic_id (str): Client‑provided idempotency identifier (unique).
|
| 96 |
+
tenant_id (str): Tenant that owns this intent.
|
| 97 |
+
intent_type (str): Type of intent (e.g., "provision_resource").
|
| 98 |
+
payload (JSON): Original API request payload.
|
| 99 |
+
oss_payload (JSON): Canonical OSS intent representation.
|
| 100 |
+
environment (str, optional): Environment label (prod, staging, etc.).
|
| 101 |
+
created_at (datetime): UTC timestamp of evaluation.
|
| 102 |
+
evaluated_at (datetime, optional): When the risk engine processed it.
|
| 103 |
+
risk_score (str, optional): String representation of the risk score.
|
| 104 |
+
"""
|
| 105 |
__tablename__ = "intents"
|
| 106 |
+
|
| 107 |
id = Column(Integer, primary_key=True, index=True)
|
| 108 |
+
deterministic_id = Column(String(64), unique=True, index=True, nullable=False)
|
| 109 |
+
tenant_id = Column(String(64), ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
|
|
|
|
|
|
|
|
|
|
| 110 |
intent_type = Column(String(64), nullable=False)
|
| 111 |
payload = Column(JSON, nullable=False)
|
| 112 |
oss_payload = Column(JSON, nullable=True)
|
| 113 |
environment = Column(String(32), nullable=True)
|
| 114 |
+
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
|
|
|
|
|
|
|
|
|
|
| 115 |
evaluated_at = Column(DateTime, nullable=True)
|
| 116 |
risk_score = Column(String(32), nullable=True)
|
| 117 |
+
|
| 118 |
+
# Relationships
|
| 119 |
+
tenant = relationship("TenantDB", back_populates="intents")
|
| 120 |
+
outcomes = relationship("OutcomeDB", back_populates="intent", cascade="all, delete-orphan")
|
| 121 |
|
| 122 |
|
| 123 |
class OutcomeDB(Base):
|
| 124 |
+
"""
|
| 125 |
+
Records the outcome (success/failure) of a previously evaluated intent.
|
| 126 |
+
Only one outcome per intent is allowed (unique constraint on intent_id).
|
| 127 |
+
|
| 128 |
+
Attributes:
|
| 129 |
+
id (int): Primary key.
|
| 130 |
+
intent_id (int): Foreign key to `intents.id`.
|
| 131 |
+
success (bool): Whether the executed action succeeded.
|
| 132 |
+
recorded_by (str, optional): Identity of the caller (e.g., API key owner).
|
| 133 |
+
notes (str, optional): Free‑text notes.
|
| 134 |
+
recorded_at (datetime): UTC timestamp.
|
| 135 |
+
idempotency_key (str, optional): Unique idempotency key for this outcome.
|
| 136 |
+
"""
|
| 137 |
__tablename__ = "intent_outcomes"
|
| 138 |
+
|
| 139 |
id = Column(Integer, primary_key=True, index=True)
|
| 140 |
+
intent_id = Column(Integer, ForeignKey("intents.id", ondelete="CASCADE"), nullable=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
success = Column(Boolean, nullable=False)
|
| 142 |
recorded_by = Column(String(128), nullable=True)
|
| 143 |
notes = Column(Text, nullable=True)
|
| 144 |
+
recorded_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
|
|
|
|
|
|
|
|
|
|
| 145 |
idempotency_key = Column(String(128), unique=True, nullable=True)
|
| 146 |
+
|
| 147 |
intent = relationship("IntentDB", back_populates="outcomes")
|
| 148 |
|
| 149 |
__table_args__ = (
|
|
|
|
| 151 |
)
|
| 152 |
|
| 153 |
|
| 154 |
+
# ============================================================================
|
| 155 |
+
# Bayesian conjugate state – now per tenant and per category
|
| 156 |
+
# ============================================================================
|
| 157 |
+
|
| 158 |
class BetaStateDB(Base):
|
| 159 |
"""
|
| 160 |
+
Stores the posterior parameters (α, β) of the conjugate Beta model
|
| 161 |
+
for each (tenant, category) pair. This allows online learning to be
|
| 162 |
+
isolated per customer.
|
| 163 |
|
| 164 |
+
Attributes:
|
| 165 |
+
id (int): Primary key.
|
| 166 |
+
tenant_id (str): Tenant that owns this state.
|
| 167 |
+
category (str): ActionCategory value (e.g., "database", "compute").
|
| 168 |
+
alpha (float): α parameter of the Beta distribution.
|
| 169 |
+
beta (float): β parameter of the Beta distribution.
|
| 170 |
+
updated_at (datetime): Last update timestamp (auto‑set).
|
| 171 |
"""
|
| 172 |
__tablename__ = "beta_state"
|
| 173 |
|
| 174 |
id = Column(Integer, primary_key=True, index=True)
|
| 175 |
+
tenant_id = Column(String(64), ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
|
| 176 |
+
category = Column(String(32), nullable=False, index=True)
|
| 177 |
alpha = Column(Float, nullable=False)
|
| 178 |
beta = Column(Float, nullable=False)
|
| 179 |
+
updated_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
|
| 180 |
+
|
| 181 |
+
# Composite unique constraint: (tenant_id, category)
|
| 182 |
+
__table_args__ = (
|
| 183 |
+
UniqueConstraint("tenant_id", "category", name="uq_beta_state_tenant_category"),
|
| 184 |
+
)
|
| 185 |
+
|
| 186 |
+
# Relationships
|
| 187 |
+
tenant = relationship("TenantDB", back_populates="beta_states")
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
# ============================================================================
|
| 191 |
+
# NEW: Audit log for compliance (immutable decision records)
|
| 192 |
+
# ============================================================================
|
| 193 |
+
|
| 194 |
+
class DecisionAuditLogDB(Base):
|
| 195 |
+
"""
|
| 196 |
+
Immutable, tamper‑evident record of every governance decision.
|
| 197 |
+
Designed for compliance (SOC2, ISO) and forensic analysis.
|
| 198 |
+
|
| 199 |
+
Attributes:
|
| 200 |
+
id (str): UUID primary key.
|
| 201 |
+
tenant_id (str): Tenant that owns the decision.
|
| 202 |
+
deterministic_id (str): Intent identifier (idempotency key).
|
| 203 |
+
timestamp (datetime): UTC decision time.
|
| 204 |
+
risk_score (float): Fused Bayesian risk score (0‑1).
|
| 205 |
+
action (str): Selected action (approve, deny, escalate).
|
| 206 |
+
justification (str): Human‑readable explanation.
|
| 207 |
+
memory_success_rate (float, optional): Memory‑based correction value.
|
| 208 |
+
memory_weight (float, optional): Weight assigned to memory.
|
| 209 |
+
counterfactual (JSON, optional): Structured counterfactual explanation.
|
| 210 |
+
trace_id (str, optional): OpenTelemetry trace ID for debugging.
|
| 211 |
+
signature (str, optional): Ed25519 signature for tamper‑proofing.
|
| 212 |
+
"""
|
| 213 |
+
__tablename__ = "decision_audit_log"
|
| 214 |
+
|
| 215 |
+
id = Column(String(64), primary_key=True, default=lambda: str(uuid.uuid4()))
|
| 216 |
+
tenant_id = Column(String(64), ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
|
| 217 |
+
deterministic_id = Column(String(64), nullable=False, index=True)
|
| 218 |
+
timestamp = Column(DateTime, default=datetime.datetime.utcnow, nullable=False, index=True)
|
| 219 |
+
risk_score = Column(Float, nullable=False)
|
| 220 |
+
action = Column(String(32), nullable=False)
|
| 221 |
+
justification = Column(Text, nullable=False)
|
| 222 |
+
memory_success_rate = Column(Float, nullable=True)
|
| 223 |
+
memory_weight = Column(Float, nullable=True)
|
| 224 |
+
counterfactual = Column(JSON, nullable=True)
|
| 225 |
+
trace_id = Column(String(128), nullable=True)
|
| 226 |
+
signature = Column(String(256), nullable=True)
|
| 227 |
+
|
| 228 |
+
# Composite index for fast filtered queries
|
| 229 |
+
__table_args__ = (
|
| 230 |
+
Index("idx_audit_tenant_time", "tenant_id", "timestamp"),
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
tenant = relationship("TenantDB", back_populates="audit_logs")
|