Upload folder using huggingface_hub

#7
Dockerfile CHANGED
@@ -1,11 +1,7 @@
1
  FROM python:3.12-slim
2
  RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
3
  WORKDIR /app
4
-
5
- ARG CLASSIC_TOKEN=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
6
- RUN git config --global url."https://x-access-token:${CLASSIC_TOKEN}@github.com/".insteadOf "https://github.com/"
7
-
8
  COPY requirements.txt .
9
  RUN pip install --no-cache-dir -r requirements.txt
10
  COPY . .
11
- CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
 
1
  FROM python:3.12-slim
2
  RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
3
  WORKDIR /app
 
 
 
 
4
  COPY requirements.txt .
5
  RUN pip install --no-cache-dir -r requirements.txt
6
  COPY . .
7
+ CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
README.md CHANGED
@@ -1,9 +1,3 @@
1
- ---
2
- title: ARF API Control Plane
3
- sdk: docker
4
- colorFrom: blue
5
- colorTo: green
6
- ---
7
  # arf-api
8
 
9
  ARF API Control Plane (FastAPI)
@@ -87,7 +81,7 @@ curl -X POST "http://localhost:8000/api/v1/v1/incidents/evaluate" -H "Content-
87
  "justification": "Causal: If we apply restart_container instead of no_action, latency would change from 600.00 to 510.00 (Δ = -90.00). Based on heuristic causal model.",
88
  "confidence": 0.85,
89
  "risk_score": 0.54,
90
- "status": "oss_advisory_only"
91
  },
92
  "causal_explanation": {
93
  "factual_outcome": 600,
@@ -123,4 +117,5 @@ Notes
123
  -----
124
 
125
  - The governance endpoints use an in-process `RiskEngine` initialized at startup.
126
- - The outcome recording endpoint is not implemented in this repository and returns HTTP 501.
 
 
 
 
 
 
 
 
1
  # arf-api
2
 
3
  ARF API Control Plane (FastAPI)
 
81
  "justification": "Causal: If we apply restart_container instead of no_action, latency would change from 600.00 to 510.00 (Δ = -90.00). Based on heuristic causal model.",
82
  "confidence": 0.85,
83
  "risk_score": 0.54,
84
+ "status": "success"
85
  },
86
  "causal_explanation": {
87
  "factual_outcome": 600,
 
117
  -----
118
 
119
  - The governance endpoints use an in-process `RiskEngine` initialized at startup.
120
+ - The outcome recording endpoint is not implemented in this repository and returns HTTP 501.
121
+
app/api/routes_incidents.py CHANGED
@@ -198,7 +198,7 @@ async def evaluate_incident(
198
  ),
199
  "confidence": 1.0 - result.get("uncertainty", 0.0),
200
  "risk_score": result["risk_score"],
201
- "status": "oss_advisory_only",
202
  }
203
 
204
  response_data = {
 
198
  ),
199
  "confidence": 1.0 - result.get("uncertainty", 0.0),
200
  "risk_score": result["risk_score"],
201
+ "status": "success",
202
  }
203
 
204
  response_data = {
app/core/usage_tracker.py CHANGED
@@ -1,8 +1,10 @@
1
  """
2
  Usage Tracker for ARF API – quotas, tiers, and audit logging.
3
  Thread‑safe, atomic quota consumption, idempotent, fail‑closed.
4
- """
5
 
 
 
 
6
  import json
7
  import sqlite3
8
  import threading
@@ -10,7 +12,7 @@ import time
10
  from contextlib import contextmanager
11
  from datetime import datetime, timedelta
12
  from dataclasses import dataclass
13
- from typing import Dict, Any, Optional, List, Tuple
14
  from enum import Enum
15
  from fastapi import BackgroundTasks, HTTPException, Request
16
 
@@ -24,6 +26,7 @@ except ImportError:
24
 
25
 
26
  class Tier(str, Enum):
 
27
  FREE = "free"
28
  PRO = "pro"
29
  PREMIUM = "premium"
@@ -31,16 +34,18 @@ class Tier(str, Enum):
31
 
32
  @property
33
  def monthly_evaluation_limit(self) -> Optional[int]:
 
34
  limits = {
35
  Tier.FREE: 1000,
36
  Tier.PRO: 10_000,
37
  Tier.PREMIUM: 50_000,
38
- Tier.ENTERPRISE: None, # unlimited
39
  }
40
  return limits[self]
41
 
42
  @property
43
  def audit_log_retention_days(self) -> int:
 
44
  retention = {
45
  Tier.FREE: 7,
46
  Tier.PRO: 30,
@@ -52,7 +57,7 @@ class Tier(str, Enum):
52
 
53
  @dataclass
54
  class UsageRecord:
55
- """Single evaluation usage record."""
56
  api_key: str
57
  tier: Tier
58
  timestamp: float
@@ -66,6 +71,7 @@ class UsageRecord:
66
  class UsageTracker:
67
  """
68
  Thread‑safe usage tracker with atomic quota consumption and idempotency.
 
69
  """
70
 
71
  def __init__(self, db_path: str = "arf_usage.db",
@@ -78,12 +84,11 @@ class UsageTracker:
78
  if redis_url and REDIS_AVAILABLE:
79
  self._redis_client = redis.from_url(redis_url)
80
  elif redis_url:
81
- raise ImportError(
82
- "Redis client not installed. Run: pip install redis")
83
 
84
  @contextmanager
85
  def _get_conn(self):
86
- """Get a thread‑local SQLite connection with write‑ahead logging and immediate transactions."""
87
  if not hasattr(self._local, "conn"):
88
  self._local.conn = sqlite3.connect(
89
  self.db_path, check_same_thread=False, isolation_level=None)
@@ -92,10 +97,13 @@ class UsageTracker:
92
  yield self._local.conn
93
 
94
  def _init_db(self):
 
95
  with self._get_conn() as conn:
 
96
  conn.execute("""
97
  CREATE TABLE IF NOT EXISTS api_keys (
98
  key TEXT PRIMARY KEY,
 
99
  tier TEXT NOT NULL,
100
  created_at REAL NOT NULL,
101
  last_used_at REAL,
@@ -139,16 +147,31 @@ class UsageTracker:
139
  def _get_month_key(self) -> str:
140
  return datetime.now().strftime("%Y-%m")
141
 
142
- def get_or_create_api_key(self, key: str, tier: Tier = Tier.FREE) -> bool:
143
- """Register a new API key. Returns True if key exists or was created."""
 
 
 
 
 
 
 
 
 
 
144
  with self._get_conn() as conn:
145
  row = conn.execute(
146
  "SELECT key FROM api_keys WHERE key = ?", (key,)).fetchone()
147
  if row:
 
 
 
 
 
148
  return True
149
  conn.execute(
150
- "INSERT INTO api_keys (key, tier, created_at, is_active) VALUES (?, ?, ?, ?)",
151
- (key, tier.value, time.time(), 1)
152
  )
153
  conn.commit()
154
  return True
@@ -164,6 +187,17 @@ class UsageTracker:
164
  return None
165
  return Tier(row["tier"])
166
 
 
 
 
 
 
 
 
 
 
 
 
167
  def update_api_key_tier(self, api_key: str, new_tier: Tier) -> bool:
168
  """Update the tier of an existing API key. Returns True if successful."""
169
  with self._get_conn() as conn:
@@ -173,41 +207,28 @@ class UsageTracker:
173
  return False
174
  conn.execute(
175
  "UPDATE api_keys SET tier = ? WHERE key = ?",
176
- (new_tier.value,
177
- api_key))
178
  conn.commit()
179
  return True
180
 
181
  # --------------------------------------------------------------------------
182
- # Atomic quota consumption
183
  # --------------------------------------------------------------------------
184
- def _consume_quota_atomic_sqlite(
185
- self,
186
- api_key: str,
187
- tier: Tier,
188
- month: str) -> bool: # noqa: E501
189
- """
190
- Atomically increment counter only if under limit.
191
- Returns True if quota was consumed, False if limit reached.
192
- """
193
  limit = tier.monthly_evaluation_limit
194
  if limit is None:
195
- # Unlimited – still increment for tracking but always succeed
196
  with self._get_conn() as conn:
197
  conn.execute(
198
- """INSERT INTO monthly_counts (api_key, year_month, count)
199
- VALUES (?, ?, 1)
200
- ON CONFLICT(api_key, year_month) DO UPDATE SET count = count + 1""",
201
  (api_key, month)
202
  )
203
  conn.commit()
204
  return True
205
 
206
- # Use BEGIN IMMEDIATE to lock the database for the transaction
207
  with self._get_conn() as conn:
208
  conn.execute("BEGIN IMMEDIATE")
209
  try:
210
- # Get current count (or 0)
211
  row = conn.execute(
212
  "SELECT count FROM monthly_counts WHERE api_key = ? AND year_month = ?",
213
  (api_key, month)
@@ -216,11 +237,9 @@ class UsageTracker:
216
  if current >= limit:
217
  conn.rollback()
218
  return False
219
- # Increment
220
  conn.execute(
221
- """INSERT INTO monthly_counts (api_key, year_month, count)
222
- VALUES (?, ?, 1)
223
- ON CONFLICT(api_key, year_month) DO UPDATE SET count = count + 1""",
224
  (api_key, month)
225
  )
226
  conn.commit()
@@ -229,15 +248,9 @@ class UsageTracker:
229
  conn.rollback()
230
  raise
231
 
232
- def _consume_quota_atomic_redis(
233
- self,
234
- api_key: str,
235
- tier: Tier,
236
- month: str) -> bool:
237
- """Atomic Lua script for Redis: INCR only if below limit."""
238
  limit = tier.monthly_evaluation_limit
239
  if limit is None:
240
- # Unlimited – just increment and return True
241
  redis_key = f"arf:quota:{api_key}:{month}"
242
  self._redis_client.incr(redis_key)
243
  self._redis_client.expire(redis_key, timedelta(days=31))
@@ -251,7 +264,7 @@ class UsageTracker:
251
  return 0
252
  end
253
  local new = redis.call('INCR', key)
254
- redis.call('EXPIRE', key, 2678400) -- 31 days
255
  return 1
256
  """
257
  redis_key = f"arf:quota:{api_key}:{month}"
@@ -259,144 +272,83 @@ class UsageTracker:
259
  return result == 1
260
 
261
  # --------------------------------------------------------------------------
262
- # Idempotency handling
263
  # --------------------------------------------------------------------------
264
  def _is_idempotent_key_used(self, key: str) -> bool:
265
- """Check if idempotency key already processed."""
266
  with self._get_conn() as conn:
267
  row = conn.execute(
268
  "SELECT 1 FROM idempotency_keys WHERE key = ?", (key,)).fetchone()
269
  return row is not None
270
 
271
  def _mark_idempotent_key_used(self, key: str, ttl_seconds: int = 86400):
272
- """Store idempotency key with expiration (cleanup later)."""
273
  with self._get_conn() as conn:
274
  conn.execute(
275
  "INSERT INTO idempotency_keys (key, consumed_at) VALUES (?, ?)",
276
  (key, time.time())
277
  )
278
  conn.commit()
279
- # Optionally schedule cleanup of old keys (can be done in a background
280
- # thread)
281
 
282
  # --------------------------------------------------------------------------
283
- # Core usage recording (atomic + idempotent)
284
  # --------------------------------------------------------------------------
285
- def consume_quota_and_log(
286
- self,
287
- record: UsageRecord,
288
- idempotency_key: Optional[str] = None,
289
- ) -> Tuple[bool, Optional[Dict[str, Any]]]:
290
- """
291
- Atomically consume quota and insert audit log.
292
- Returns (success, existing_response) where existing_response is not None
293
- only when idempotency_key matched a previous successful call.
294
- """
295
- # Idempotency check (if key provided)
296
- if idempotency_key:
297
- if self._is_idempotent_key_used(idempotency_key):
298
- # Retrieve previous response from audit log (simplified – you may cache full response)
299
- # For full idempotency, we would store the response body in idempotency table.
300
- # Here we return a marker that caller should use cached
301
- # response.
302
- return False, {"idempotent": True,
303
- "message": "Already processed"}
304
 
305
  month = self._get_month_key()
306
- # Atomic quota consumption
307
  if self._redis_client:
308
- quota_ok = self._consume_quota_atomic_redis(
309
- record.api_key, record.tier, month)
310
  else:
311
- quota_ok = self._consume_quota_atomic_sqlite(
312
- record.api_key, record.tier, month)
313
 
314
  if not quota_ok:
315
  return False, None
316
 
317
- # Insert audit log (with idempotency key as unique constraint)
318
  try:
319
  with self._get_conn() as conn:
320
  conn.execute(
321
  """INSERT INTO usage_log
322
- (api_key, tier, timestamp, endpoint,
323
- request_body, response, error, processing_ms,
324
- idempotency_key)
325
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
326
- (record.api_key,
327
- record.tier.value,
328
- record.timestamp,
329
- record.endpoint,
330
- json.dumps(
331
- record.request_body) if record.request_body else None,
332
- json.dumps(
333
- record.response) if record.response else None,
334
- record.error,
335
- record.processing_ms,
336
- idempotency_key,
337
- ))
338
  conn.commit()
339
  except sqlite3.IntegrityError as e:
340
- # Duplicate idempotency_key – already inserted by another
341
- # concurrent request
342
  if "UNIQUE constraint failed: usage_log.idempotency_key" in str(e):
343
- return False, {"idempotent": True,
344
- "message": "Already processed"}
345
  raise
346
 
347
  if idempotency_key:
348
  self._mark_idempotent_key_used(idempotency_key)
349
- # Removed stray # noqa: E501 comment that was wrongly indented here
350
  return True, None
351
 
352
  # --------------------------------------------------------------------------
353
- # Legacy interface (kept for compatibility but deprecated)
354
  # --------------------------------------------------------------------------
355
- def increment_usage_sync(
356
- self,
357
- record: UsageRecord,
358
- idempotency_key: Optional[str] = None) -> bool:
359
- """
360
- Synchronously record usage and increment counter.
361
- Returns True if within quota and recorded, False otherwise.
362
- This method now uses the atomic implementation.
363
- """
364
  success, _ = self.consume_quota_and_log(record, idempotency_key)
365
  return success
366
 
367
- async def increment_usage_async(
368
- self,
369
- record: UsageRecord,
370
- background_tasks: BackgroundTasks,
371
- idempotency_key: Optional[str] = None
372
- ) -> bool:
373
- """
374
- Asynchronously record usage using FastAPI BackgroundTasks.
375
- Still does the atomic check synchronously, then schedules the insert.
376
- """
377
- # First, do atomic quota check (synchronous) – we must ensure we don't double-consume.
378
- # Because background tasks may run later, we still need to reserve quota now.
379
- # Simplified: we call consume_quota_and_log synchronously – that defeats async benefit.
380
- # Better to use a queue or Redis with background processing.
381
- # For this fix, we'll use the sync method (blocking) but still support
382
- # idempotency.
383
  return self.increment_usage_sync(record, idempotency_key)
384
 
385
  # --------------------------------------------------------------------------
386
- # Quota inspection (non‑atomic, for display only)
387
  # --------------------------------------------------------------------------
388
  def get_remaining_quota(self, api_key: str, tier: Tier) -> Optional[int]:
389
- """Return remaining evaluations for the month (non‑atomic, for info only)."""
390
  limit = tier.monthly_evaluation_limit
391
  if limit is None:
392
  return None
393
-
394
  month = self._get_month_key()
395
  if self._redis_client:
396
  redis_key = f"arf:quota:{api_key}:{month}"
397
  count = int(self._redis_client.get(redis_key) or 0)
398
  return max(0, limit - count)
399
-
400
  with self._get_conn() as conn:
401
  row = conn.execute(
402
  "SELECT count FROM monthly_counts WHERE api_key = ? AND year_month = ?",
@@ -406,16 +358,10 @@ class UsageTracker:
406
  return max(0, limit - count)
407
 
408
  # --------------------------------------------------------------------------
409
- # Audit and maintenance
410
  # --------------------------------------------------------------------------
411
- def get_audit_logs(
412
- self,
413
- api_key: str,
414
- start_date: Optional[datetime] = None,
415
- end_date: Optional[datetime] = None,
416
- limit: int = 100,
417
- ) -> List[Dict[str, Any]]:
418
- """Retrieve audit logs for a given API key."""
419
  query = "SELECT * FROM usage_log WHERE api_key = ?"
420
  params = [api_key]
421
  if start_date:
@@ -426,47 +372,36 @@ class UsageTracker:
426
  params.append(end_date.timestamp())
427
  query += " ORDER BY timestamp DESC LIMIT ?"
428
  params.append(limit)
429
-
430
  with self._get_conn() as conn:
431
  rows = conn.execute(query, params).fetchall()
432
  return [dict(row) for row in rows]
433
 
434
  def clean_old_logs(self):
435
- """Delete logs older than retention period for each tier, and old idempotency keys."""
436
  with self._get_conn() as conn:
437
- # Delete old usage logs
438
  for tier in Tier:
439
  retention_days = tier.audit_log_retention_days
440
- if retention_days is None:
441
- continue
442
  cutoff = time.time() - retention_days * 86400
443
  conn.execute(
444
  "DELETE FROM usage_log WHERE tier = ? AND timestamp < ?",
445
  (tier.value, cutoff)
446
  )
447
- # Delete idempotency keys older than 7 days
448
  cutoff = time.time() - 7 * 86400
449
- conn.execute(
450
- "DELETE FROM idempotency_keys WHERE consumed_at < ?", (cutoff,))
451
  conn.commit()
452
 
453
 
454
  # --------------------------------------------------------------------------
455
- # Global instance and FastAPI dependency (fail‑closed)
456
  # --------------------------------------------------------------------------
457
  tracker: Optional[UsageTracker] = None
458
 
459
 
460
- def init_tracker(
461
- db_path: str = "arf_usage.db",
462
- redis_url: Optional[str] = None):
463
- """Initialize the global tracker. Must be called before enforce_quota."""
464
  global tracker
465
  tracker = UsageTracker(db_path, redis_url)
466
 
467
 
468
  def update_key_tier(api_key: str, new_tier: Tier) -> bool:
469
- """Globally accessible helper to update API key tier."""
470
  if tracker is None:
471
  return False
472
  return tracker.update_api_key_tier(api_key, new_tier)
@@ -474,16 +409,11 @@ def update_key_tier(api_key: str, new_tier: Tier) -> bool:
474
 
475
  async def enforce_quota(request: Request, api_key: str = None):
476
  """
477
- Dependency that checks API key and remaining quota.
478
- FAILS CLOSED: if tracker not initialised, raises HTTP 503.
479
  """
480
- # P0 fix: No fallback that allows all requests
481
  if tracker is None:
482
- raise HTTPException(
483
- status_code=503,
484
- detail="Usage tracking service not initialised. Please contact administrator.")
485
 
486
- # Extract API key from header or query
487
  if api_key is None:
488
  auth_header = request.headers.get("Authorization")
489
  if auth_header and auth_header.startswith("Bearer "):
@@ -496,16 +426,19 @@ async def enforce_quota(request: Request, api_key: str = None):
496
 
497
  tier = tracker.get_tier(api_key)
498
  if tier is None:
499
- raise HTTPException(
500
- status_code=403,
501
- detail="Invalid or inactive API key")
502
 
503
  remaining = tracker.get_remaining_quota(api_key, tier)
504
  if remaining is not None and remaining <= 0:
505
- raise HTTPException(status_code=429,
506
- detail="Monthly evaluation quota exceeded")
 
 
 
 
507
 
508
- # Store in request state for later logging (optional)
509
  request.state.api_key = api_key
510
  request.state.tier = tier
511
- return {"api_key": api_key, "tier": tier, "remaining": remaining}
 
 
 
1
  """
2
  Usage Tracker for ARF API – quotas, tiers, and audit logging.
3
  Thread‑safe, atomic quota consumption, idempotent, fail‑closed.
 
4
 
5
+ Extended for multi‑tenancy: each API key is linked to a tenant ID.
6
+ Tenant ID is stored in the `api_keys` table and used for resource isolation.
7
+ """
8
  import json
9
  import sqlite3
10
  import threading
 
12
  from contextlib import contextmanager
13
  from datetime import datetime, timedelta
14
  from dataclasses import dataclass
15
+ from typing import Dict, Any, Optional, List, Tuple, Callable
16
  from enum import Enum
17
  from fastapi import BackgroundTasks, HTTPException, Request
18
 
 
26
 
27
 
28
  class Tier(str, Enum):
29
+ """Pricing tiers with associated quota limits and audit retention."""
30
  FREE = "free"
31
  PRO = "pro"
32
  PREMIUM = "premium"
 
34
 
35
  @property
36
  def monthly_evaluation_limit(self) -> Optional[int]:
37
+ """Monthly evaluation quota. None = unlimited."""
38
  limits = {
39
  Tier.FREE: 1000,
40
  Tier.PRO: 10_000,
41
  Tier.PREMIUM: 50_000,
42
+ Tier.ENTERPRISE: None,
43
  }
44
  return limits[self]
45
 
46
  @property
47
  def audit_log_retention_days(self) -> int:
48
+ """How many days to keep usage and decision audit logs."""
49
  retention = {
50
  Tier.FREE: 7,
51
  Tier.PRO: 30,
 
57
 
58
  @dataclass
59
  class UsageRecord:
60
+ """Single API call usage record (for quota and debugging)."""
61
  api_key: str
62
  tier: Tier
63
  timestamp: float
 
71
  class UsageTracker:
72
  """
73
  Thread‑safe usage tracker with atomic quota consumption and idempotency.
74
+ Extended to support tenant isolation: each API key is linked to a tenant.
75
  """
76
 
77
  def __init__(self, db_path: str = "arf_usage.db",
 
84
  if redis_url and REDIS_AVAILABLE:
85
  self._redis_client = redis.from_url(redis_url)
86
  elif redis_url:
87
+ raise ImportError("Redis client not installed. Run: pip install redis")
 
88
 
89
  @contextmanager
90
  def _get_conn(self):
91
+ """Get a thread‑local SQLite connection with WAL and immediate transactions."""
92
  if not hasattr(self._local, "conn"):
93
  self._local.conn = sqlite3.connect(
94
  self.db_path, check_same_thread=False, isolation_level=None)
 
97
  yield self._local.conn
98
 
99
  def _init_db(self):
100
+ """Initialise SQLite tables with tenant_id support."""
101
  with self._get_conn() as conn:
102
+ # Modified: api_keys now has tenant_id column
103
  conn.execute("""
104
  CREATE TABLE IF NOT EXISTS api_keys (
105
  key TEXT PRIMARY KEY,
106
+ tenant_id TEXT NOT NULL,
107
  tier TEXT NOT NULL,
108
  created_at REAL NOT NULL,
109
  last_used_at REAL,
 
147
  def _get_month_key(self) -> str:
148
  return datetime.now().strftime("%Y-%m")
149
 
150
+ def get_or_create_api_key(self, key: str, tenant_id: str, tier: Tier = Tier.FREE) -> bool:
151
+ """
152
+ Register a new API key for a given tenant.
153
+
154
+ Args:
155
+ key: The API key (plain text, will be hashed in production).
156
+ tenant_id: UUID of the tenant (must already exist in main DB).
157
+ tier: Initial tier for the key.
158
+
159
+ Returns:
160
+ True if key was created (or already exists for the same tenant).
161
+ """
162
  with self._get_conn() as conn:
163
  row = conn.execute(
164
  "SELECT key FROM api_keys WHERE key = ?", (key,)).fetchone()
165
  if row:
166
+ # Key already exists – ensure it belongs to the same tenant
167
+ existing_tenant = conn.execute(
168
+ "SELECT tenant_id FROM api_keys WHERE key = ?", (key,)).fetchone()
169
+ if existing_tenant["tenant_id"] != tenant_id:
170
+ raise ValueError(f"Key {key[:8]}... already belongs to a different tenant.")
171
  return True
172
  conn.execute(
173
+ "INSERT INTO api_keys (key, tenant_id, tier, created_at, is_active) VALUES (?, ?, ?, ?, ?)",
174
+ (key, tenant_id, tier.value, time.time(), 1)
175
  )
176
  conn.commit()
177
  return True
 
187
  return None
188
  return Tier(row["tier"])
189
 
190
+ def get_tenant_id(self, api_key: str) -> Optional[str]:
191
+ """Return the tenant ID associated with the API key, or None if key invalid."""
192
+ with self._get_conn() as conn:
193
+ row = conn.execute(
194
+ "SELECT tenant_id FROM api_keys WHERE key = ? AND is_active = 1",
195
+ (api_key,)
196
+ ).fetchone()
197
+ if not row:
198
+ return None
199
+ return row["tenant_id"]
200
+
201
  def update_api_key_tier(self, api_key: str, new_tier: Tier) -> bool:
202
  """Update the tier of an existing API key. Returns True if successful."""
203
  with self._get_conn() as conn:
 
207
  return False
208
  conn.execute(
209
  "UPDATE api_keys SET tier = ? WHERE key = ?",
210
+ (new_tier.value, api_key))
 
211
  conn.commit()
212
  return True
213
 
214
  # --------------------------------------------------------------------------
215
+ # Atomic quota consumption (unchanged, but uses api_key which links to tenant)
216
  # --------------------------------------------------------------------------
217
+ def _consume_quota_atomic_sqlite(self, api_key: str, tier: Tier, month: str) -> bool:
 
 
 
 
 
 
 
 
218
  limit = tier.monthly_evaluation_limit
219
  if limit is None:
 
220
  with self._get_conn() as conn:
221
  conn.execute(
222
+ "INSERT INTO monthly_counts (api_key, year_month, count) VALUES (?, ?, 1) "
223
+ "ON CONFLICT(api_key, year_month) DO UPDATE SET count = count + 1",
 
224
  (api_key, month)
225
  )
226
  conn.commit()
227
  return True
228
 
 
229
  with self._get_conn() as conn:
230
  conn.execute("BEGIN IMMEDIATE")
231
  try:
 
232
  row = conn.execute(
233
  "SELECT count FROM monthly_counts WHERE api_key = ? AND year_month = ?",
234
  (api_key, month)
 
237
  if current >= limit:
238
  conn.rollback()
239
  return False
 
240
  conn.execute(
241
+ "INSERT INTO monthly_counts (api_key, year_month, count) VALUES (?, ?, 1) "
242
+ "ON CONFLICT(api_key, year_month) DO UPDATE SET count = count + 1",
 
243
  (api_key, month)
244
  )
245
  conn.commit()
 
248
  conn.rollback()
249
  raise
250
 
251
+ def _consume_quota_atomic_redis(self, api_key: str, tier: Tier, month: str) -> bool:
 
 
 
 
 
252
  limit = tier.monthly_evaluation_limit
253
  if limit is None:
 
254
  redis_key = f"arf:quota:{api_key}:{month}"
255
  self._redis_client.incr(redis_key)
256
  self._redis_client.expire(redis_key, timedelta(days=31))
 
264
  return 0
265
  end
266
  local new = redis.call('INCR', key)
267
+ redis.call('EXPIRE', key, 2678400)
268
  return 1
269
  """
270
  redis_key = f"arf:quota:{api_key}:{month}"
 
272
  return result == 1
273
 
274
  # --------------------------------------------------------------------------
275
+ # Idempotency handling (unchanged)
276
  # --------------------------------------------------------------------------
277
  def _is_idempotent_key_used(self, key: str) -> bool:
 
278
  with self._get_conn() as conn:
279
  row = conn.execute(
280
  "SELECT 1 FROM idempotency_keys WHERE key = ?", (key,)).fetchone()
281
  return row is not None
282
 
283
  def _mark_idempotent_key_used(self, key: str, ttl_seconds: int = 86400):
 
284
  with self._get_conn() as conn:
285
  conn.execute(
286
  "INSERT INTO idempotency_keys (key, consumed_at) VALUES (?, ?)",
287
  (key, time.time())
288
  )
289
  conn.commit()
 
 
290
 
291
  # --------------------------------------------------------------------------
292
+ # Core usage recording (atomic + idempotent) – unchanged
293
  # --------------------------------------------------------------------------
294
+ def consume_quota_and_log(self, record: UsageRecord, idempotency_key: Optional[str] = None
295
+ ) -> Tuple[bool, Optional[Dict[str, Any]]]:
296
+ if idempotency_key and self._is_idempotent_key_used(idempotency_key):
297
+ return False, {"idempotent": True, "message": "Already processed"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
298
 
299
  month = self._get_month_key()
 
300
  if self._redis_client:
301
+ quota_ok = self._consume_quota_atomic_redis(record.api_key, record.tier, month)
 
302
  else:
303
+ quota_ok = self._consume_quota_atomic_sqlite(record.api_key, record.tier, month)
 
304
 
305
  if not quota_ok:
306
  return False, None
307
 
 
308
  try:
309
  with self._get_conn() as conn:
310
  conn.execute(
311
  """INSERT INTO usage_log
312
+ (api_key, tier, timestamp, endpoint, request_body, response, error, processing_ms, idempotency_key)
 
 
313
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
314
+ (record.api_key, record.tier.value, record.timestamp, record.endpoint,
315
+ json.dumps(record.request_body) if record.request_body else None,
316
+ json.dumps(record.response) if record.response else None,
317
+ record.error, record.processing_ms, idempotency_key)
318
+ )
 
 
 
 
 
 
 
319
  conn.commit()
320
  except sqlite3.IntegrityError as e:
 
 
321
  if "UNIQUE constraint failed: usage_log.idempotency_key" in str(e):
322
+ return False, {"idempotent": True, "message": "Already processed"}
 
323
  raise
324
 
325
  if idempotency_key:
326
  self._mark_idempotent_key_used(idempotency_key)
 
327
  return True, None
328
 
329
  # --------------------------------------------------------------------------
330
+ # Legacy interface (kept for compatibility)
331
  # --------------------------------------------------------------------------
332
+ def increment_usage_sync(self, record: UsageRecord, idempotency_key: Optional[str] = None) -> bool:
 
 
 
 
 
 
 
 
333
  success, _ = self.consume_quota_and_log(record, idempotency_key)
334
  return success
335
 
336
+ async def increment_usage_async(self, record: UsageRecord, background_tasks: BackgroundTasks,
337
+ idempotency_key: Optional[str] = None) -> bool:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
338
  return self.increment_usage_sync(record, idempotency_key)
339
 
340
  # --------------------------------------------------------------------------
341
+ # Quota inspection
342
  # --------------------------------------------------------------------------
343
  def get_remaining_quota(self, api_key: str, tier: Tier) -> Optional[int]:
 
344
  limit = tier.monthly_evaluation_limit
345
  if limit is None:
346
  return None
 
347
  month = self._get_month_key()
348
  if self._redis_client:
349
  redis_key = f"arf:quota:{api_key}:{month}"
350
  count = int(self._redis_client.get(redis_key) or 0)
351
  return max(0, limit - count)
 
352
  with self._get_conn() as conn:
353
  row = conn.execute(
354
  "SELECT count FROM monthly_counts WHERE api_key = ? AND year_month = ?",
 
358
  return max(0, limit - count)
359
 
360
  # --------------------------------------------------------------------------
361
+ # Audit and maintenance (kept for usage_log)
362
  # --------------------------------------------------------------------------
363
+ def get_audit_logs(self, api_key: str, start_date: Optional[datetime] = None,
364
+ end_date: Optional[datetime] = None, limit: int = 100) -> List[Dict[str, Any]]:
 
 
 
 
 
 
365
  query = "SELECT * FROM usage_log WHERE api_key = ?"
366
  params = [api_key]
367
  if start_date:
 
372
  params.append(end_date.timestamp())
373
  query += " ORDER BY timestamp DESC LIMIT ?"
374
  params.append(limit)
 
375
  with self._get_conn() as conn:
376
  rows = conn.execute(query, params).fetchall()
377
  return [dict(row) for row in rows]
378
 
379
  def clean_old_logs(self):
 
380
  with self._get_conn() as conn:
 
381
  for tier in Tier:
382
  retention_days = tier.audit_log_retention_days
 
 
383
  cutoff = time.time() - retention_days * 86400
384
  conn.execute(
385
  "DELETE FROM usage_log WHERE tier = ? AND timestamp < ?",
386
  (tier.value, cutoff)
387
  )
 
388
  cutoff = time.time() - 7 * 86400
389
+ conn.execute("DELETE FROM idempotency_keys WHERE consumed_at < ?", (cutoff,))
 
390
  conn.commit()
391
 
392
 
393
  # --------------------------------------------------------------------------
394
+ # Global instance and FastAPI dependency
395
  # --------------------------------------------------------------------------
396
  tracker: Optional[UsageTracker] = None
397
 
398
 
399
+ def init_tracker(db_path: str = "arf_usage.db", redis_url: Optional[str] = None):
 
 
 
400
  global tracker
401
  tracker = UsageTracker(db_path, redis_url)
402
 
403
 
404
  def update_key_tier(api_key: str, new_tier: Tier) -> bool:
 
405
  if tracker is None:
406
  return False
407
  return tracker.update_api_key_tier(api_key, new_tier)
 
409
 
410
  async def enforce_quota(request: Request, api_key: str = None):
411
  """
412
+ FastAPI dependency that enforces quota and attaches tenant_id to request state.
 
413
  """
 
414
  if tracker is None:
415
+ raise HTTPException(status_code=503, detail="Usage tracking service not initialised.")
 
 
416
 
 
417
  if api_key is None:
418
  auth_header = request.headers.get("Authorization")
419
  if auth_header and auth_header.startswith("Bearer "):
 
426
 
427
  tier = tracker.get_tier(api_key)
428
  if tier is None:
429
+ raise HTTPException(status_code=403, detail="Invalid or inactive API key")
 
 
430
 
431
  remaining = tracker.get_remaining_quota(api_key, tier)
432
  if remaining is not None and remaining <= 0:
433
+ raise HTTPException(status_code=429, detail="Monthly evaluation quota exceeded")
434
+
435
+ # Retrieve tenant_id
436
+ tenant_id = tracker.get_tenant_id(api_key)
437
+ if not tenant_id:
438
+ raise HTTPException(status_code=403, detail="API key not associated with a tenant")
439
 
 
440
  request.state.api_key = api_key
441
  request.state.tier = tier
442
+ request.state.tenant_id = tenant_id
443
+
444
+ return {"api_key": api_key, "tier": tier, "tenant_id": tenant_id, "remaining": remaining}
app/database/models_intents.py CHANGED
@@ -1,50 +1,149 @@
1
- from sqlalchemy import Column, Integer, String, DateTime, Boolean, Text, JSON, Float, ForeignKey, UniqueConstraint
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  from sqlalchemy.orm import relationship
3
  import datetime
4
  from .base import Base
5
 
6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  class IntentDB(Base):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  __tablename__ = "intents"
 
9
  id = Column(Integer, primary_key=True, index=True)
10
- deterministic_id = Column(
11
- String(64),
12
- unique=True,
13
- index=True,
14
- nullable=False)
15
  intent_type = Column(String(64), nullable=False)
16
  payload = Column(JSON, nullable=False)
17
  oss_payload = Column(JSON, nullable=True)
18
  environment = Column(String(32), nullable=True)
19
- created_at = Column(
20
- DateTime,
21
- default=datetime.datetime.utcnow,
22
- nullable=False)
23
  evaluated_at = Column(DateTime, nullable=True)
24
  risk_score = Column(String(32), nullable=True)
25
- outcomes = relationship(
26
- "OutcomeDB",
27
- back_populates="intent",
28
- cascade="all, delete-orphan")
29
 
30
 
31
  class OutcomeDB(Base):
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  __tablename__ = "intent_outcomes"
 
33
  id = Column(Integer, primary_key=True, index=True)
34
- intent_id = Column(
35
- Integer,
36
- ForeignKey(
37
- "intents.id",
38
- ondelete="CASCADE"),
39
- nullable=False)
40
  success = Column(Boolean, nullable=False)
41
  recorded_by = Column(String(128), nullable=True)
42
  notes = Column(Text, nullable=True)
43
- recorded_at = Column(
44
- DateTime,
45
- default=datetime.datetime.utcnow,
46
- nullable=False)
47
  idempotency_key = Column(String(128), unique=True, nullable=True)
 
48
  intent = relationship("IntentDB", back_populates="outcomes")
49
 
50
  __table_args__ = (
@@ -52,24 +151,83 @@ class OutcomeDB(Base):
52
  )
53
 
54
 
55
- # ---------------------------------------------------------------------------
56
- # NEW: Persistence for the conjugate Bayesian state
57
- # ---------------------------------------------------------------------------
 
58
  class BetaStateDB(Base):
59
  """
60
- Stores the per‑category posterior parameters (α, β) of the BetaStore
61
- so that online learning survives API restarts.
 
62
 
63
- Only one row per ActionCategory is expected; the 'category' column is
64
- unique. Updates are performed via merge / upsert.
 
 
 
 
 
65
  """
66
  __tablename__ = "beta_state"
67
 
68
  id = Column(Integer, primary_key=True, index=True)
69
- category = Column(String(32), unique=True, nullable=False, index=True)
 
70
  alpha = Column(Float, nullable=False)
71
  beta = Column(Float, nullable=False)
72
- updated_at = Column(
73
- DateTime,
74
- default=datetime.datetime.utcnow,
75
- onupdate=datetime.datetime.utcnow)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Database models for the ARF API Control Plane.
3
+
4
+ This module defines the SQLAlchemy ORM models for:
5
+ - Intents (InfrastructureIntent evaluations)
6
+ - Outcomes (recorded results of executed intents)
7
+ - Beta state (conjugate Bayesian posteriors per tenant and category)
8
+ - Audit logs (immutable decision records for compliance)
9
+ - Tenants (multi‑tenant isolation)
10
+
11
+ All tables include a `tenant_id` column to enforce data partitioning.
12
+ The BetaStateDB now stores parameters per (tenant, category) pair.
13
+ """
14
+
15
+ from sqlalchemy import (
16
+ Column, Integer, String, DateTime, Boolean, Text, JSON,
17
+ Float, ForeignKey, UniqueConstraint, Index
18
+ )
19
  from sqlalchemy.orm import relationship
20
  import datetime
21
  from .base import Base
22
 
23
 
24
+ # ============================================================================
25
+ # Tenant table – root of multi‑tenancy
26
+ # ============================================================================
27
+
28
+ class TenantDB(Base):
29
+ """
30
+ Represents a customer tenant (organisation). All other tables
31
+ reference this table via a foreign key `tenant_id`.
32
+
33
+ Attributes:
34
+ id (str): UUID of the tenant (primary key).
35
+ name (str): Human‑readable organisation name.
36
+ created_at (datetime): UTC timestamp of creation.
37
+ created_by (str, optional): Email or user ID of the creator.
38
+ """
39
+ __tablename__ = "tenants"
40
+
41
+ id = Column(String(64), primary_key=True, index=True)
42
+ name = Column(String(256), nullable=False)
43
+ created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
44
+ created_by = Column(String(128), nullable=True)
45
+
46
+ # Relationships
47
+ api_keys = relationship("APIKeyDB", back_populates="tenant", cascade="all, delete-orphan")
48
+ intents = relationship("IntentDB", back_populates="tenant")
49
+ beta_states = relationship("BetaStateDB", back_populates="tenant")
50
+ audit_logs = relationship("DecisionAuditLogDB", back_populates="tenant")
51
+
52
+
53
+ # ============================================================================
54
+ # API keys (extended with tenant_id)
55
+ # ============================================================================
56
+
57
+ class APIKeyDB(Base):
58
+ """
59
+ Stores API keys for authentication and tiered quota. Each key belongs
60
+ to exactly one tenant. The `tier` determines monthly evaluation limits.
61
+
62
+ Attributes:
63
+ key (str): The hashed API key (primary key).
64
+ tenant_id (str): Foreign key to `tenants.id`.
65
+ tier (str): Tier enumeration value (free, pro, premium, enterprise).
66
+ created_at (datetime): UTC creation time.
67
+ last_used_at (datetime, optional): Timestamp of last successful request.
68
+ is_active (bool): Soft‑delete flag.
69
+ """
70
+ __tablename__ = "api_keys"
71
+
72
+ key = Column(String(256), primary_key=True, index=True)
73
+ tenant_id = Column(String(64), ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
74
+ tier = Column(String(32), nullable=False)
75
+ created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
76
+ last_used_at = Column(DateTime, nullable=True)
77
+ is_active = Column(Boolean, default=True, nullable=False)
78
+
79
+ # Relationships
80
+ tenant = relationship("TenantDB", back_populates="api_keys")
81
+ usage_logs = relationship("UsageLogDB", back_populates="api_key")
82
+
83
+
84
+ # ============================================================================
85
+ # Intents (evaluations) – now tenant‑scoped
86
+ # ============================================================================
87
+
88
  class IntentDB(Base):
89
+ """
90
+ Stores each InfrastructureIntent evaluation request and its resulting
91
+ risk score. One‑to‑many with OutcomeDB.
92
+
93
+ Attributes:
94
+ id (int): Auto‑increment primary key.
95
+ deterministic_id (str): Client‑provided idempotency identifier (unique).
96
+ tenant_id (str): Tenant that owns this intent.
97
+ intent_type (str): Type of intent (e.g., "provision_resource").
98
+ payload (JSON): Original API request payload.
99
+ oss_payload (JSON): Canonical OSS intent representation.
100
+ environment (str, optional): Environment label (prod, staging, etc.).
101
+ created_at (datetime): UTC timestamp of evaluation.
102
+ evaluated_at (datetime, optional): When the risk engine processed it.
103
+ risk_score (str, optional): String representation of the risk score.
104
+ """
105
  __tablename__ = "intents"
106
+
107
  id = Column(Integer, primary_key=True, index=True)
108
+ deterministic_id = Column(String(64), unique=True, index=True, nullable=False)
109
+ tenant_id = Column(String(64), ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
 
 
 
110
  intent_type = Column(String(64), nullable=False)
111
  payload = Column(JSON, nullable=False)
112
  oss_payload = Column(JSON, nullable=True)
113
  environment = Column(String(32), nullable=True)
114
+ created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
 
 
 
115
  evaluated_at = Column(DateTime, nullable=True)
116
  risk_score = Column(String(32), nullable=True)
117
+
118
+ # Relationships
119
+ tenant = relationship("TenantDB", back_populates="intents")
120
+ outcomes = relationship("OutcomeDB", back_populates="intent", cascade="all, delete-orphan")
121
 
122
 
123
  class OutcomeDB(Base):
124
+ """
125
+ Records the outcome (success/failure) of a previously evaluated intent.
126
+ Only one outcome per intent is allowed (unique constraint on intent_id).
127
+
128
+ Attributes:
129
+ id (int): Primary key.
130
+ intent_id (int): Foreign key to `intents.id`.
131
+ success (bool): Whether the executed action succeeded.
132
+ recorded_by (str, optional): Identity of the caller (e.g., API key owner).
133
+ notes (str, optional): Free‑text notes.
134
+ recorded_at (datetime): UTC timestamp.
135
+ idempotency_key (str, optional): Unique idempotency key for this outcome.
136
+ """
137
  __tablename__ = "intent_outcomes"
138
+
139
  id = Column(Integer, primary_key=True, index=True)
140
+ intent_id = Column(Integer, ForeignKey("intents.id", ondelete="CASCADE"), nullable=False)
 
 
 
 
 
141
  success = Column(Boolean, nullable=False)
142
  recorded_by = Column(String(128), nullable=True)
143
  notes = Column(Text, nullable=True)
144
+ recorded_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
 
 
 
145
  idempotency_key = Column(String(128), unique=True, nullable=True)
146
+
147
  intent = relationship("IntentDB", back_populates="outcomes")
148
 
149
  __table_args__ = (
 
151
  )
152
 
153
 
154
+ # ============================================================================
155
+ # Bayesian conjugate state now per tenant and per category
156
+ # ============================================================================
157
+
158
  class BetaStateDB(Base):
159
  """
160
+ Stores the posterior parameters (α, β) of the conjugate Beta model
161
+ for each (tenant, category) pair. This allows online learning to be
162
+ isolated per customer.
163
 
164
+ Attributes:
165
+ id (int): Primary key.
166
+ tenant_id (str): Tenant that owns this state.
167
+ category (str): ActionCategory value (e.g., "database", "compute").
168
+ alpha (float): α parameter of the Beta distribution.
169
+ beta (float): β parameter of the Beta distribution.
170
+ updated_at (datetime): Last update timestamp (auto‑set).
171
  """
172
  __tablename__ = "beta_state"
173
 
174
  id = Column(Integer, primary_key=True, index=True)
175
+ tenant_id = Column(String(64), ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
176
+ category = Column(String(32), nullable=False, index=True)
177
  alpha = Column(Float, nullable=False)
178
  beta = Column(Float, nullable=False)
179
+ updated_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
180
+
181
+ # Composite unique constraint: (tenant_id, category)
182
+ __table_args__ = (
183
+ UniqueConstraint("tenant_id", "category", name="uq_beta_state_tenant_category"),
184
+ )
185
+
186
+ # Relationships
187
+ tenant = relationship("TenantDB", back_populates="beta_states")
188
+
189
+
190
+ # ============================================================================
191
+ # NEW: Audit log for compliance (immutable decision records)
192
+ # ============================================================================
193
+
194
+ class DecisionAuditLogDB(Base):
195
+ """
196
+ Immutable, tamper‑evident record of every governance decision.
197
+ Designed for compliance (SOC2, ISO) and forensic analysis.
198
+
199
+ Attributes:
200
+ id (str): UUID primary key.
201
+ tenant_id (str): Tenant that owns the decision.
202
+ deterministic_id (str): Intent identifier (idempotency key).
203
+ timestamp (datetime): UTC decision time.
204
+ risk_score (float): Fused Bayesian risk score (0‑1).
205
+ action (str): Selected action (approve, deny, escalate).
206
+ justification (str): Human‑readable explanation.
207
+ memory_success_rate (float, optional): Memory‑based correction value.
208
+ memory_weight (float, optional): Weight assigned to memory.
209
+ counterfactual (JSON, optional): Structured counterfactual explanation.
210
+ trace_id (str, optional): OpenTelemetry trace ID for debugging.
211
+ signature (str, optional): Ed25519 signature for tamper‑proofing.
212
+ """
213
+ __tablename__ = "decision_audit_log"
214
+
215
+ id = Column(String(64), primary_key=True, default=lambda: str(uuid.uuid4()))
216
+ tenant_id = Column(String(64), ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
217
+ deterministic_id = Column(String(64), nullable=False, index=True)
218
+ timestamp = Column(DateTime, default=datetime.datetime.utcnow, nullable=False, index=True)
219
+ risk_score = Column(Float, nullable=False)
220
+ action = Column(String(32), nullable=False)
221
+ justification = Column(Text, nullable=False)
222
+ memory_success_rate = Column(Float, nullable=True)
223
+ memory_weight = Column(Float, nullable=True)
224
+ counterfactual = Column(JSON, nullable=True)
225
+ trace_id = Column(String(128), nullable=True)
226
+ signature = Column(String(256), nullable=True)
227
+
228
+ # Composite index for fast filtered queries
229
+ __table_args__ = (
230
+ Index("idx_audit_tenant_time", "tenant_id", "timestamp"),
231
+ )
232
+
233
+ tenant = relationship("TenantDB", back_populates="audit_logs")