=== Diff for src/app.py === --- agentmemory/src/app.py +++ agentmemory-python/src/app.py @@ -34,7 +34,7 @@ from db import StateKV import search import functions -from functions import KV +from functions import KV, query_audit app = Flask(__name__) # Enable CORS for all routes (important for client scripts connecting locally) @@ -358,42 +358,6 @@ sessions.sort(key=lambda s: s.get("startedAt", ""), reverse=True) return jsonify({"success": True, "sessions": sessions}), 200 -@app.route("/agentmemory/replay/load", methods=["GET"]) -def api_replay_load(): - auth_err = check_auth() - if auth_err: - return auth_err - - session_id = request.args.get("sessionId") - if not session_id: - return jsonify({"error": "sessionId is required"}), 400 - - session = kv.get(KV.sessions, session_id) - obs = kv.list(KV.observations(session_id)) - - import replay_import - timeline = replay_import.project_timeline(obs) - return jsonify({"success": True, "timeline": timeline, "session": session}), 200 - -@app.route("/agentmemory/replay/import-jsonl", methods=["POST"]) -def api_replay_import_jsonl(): - auth_err = check_auth() - if auth_err: - return auth_err - - try: - body = request.get_json(force=True) or {} - path = body.get("path") - max_files = body.get("maxFiles") - - import replay_import - res = replay_import.import_jsonl_data(kv, path=path, max_files=max_files) - - status_code = 200 if res.get("success", True) else 400 - return jsonify(res), status_code - except Exception as e: - return jsonify({"error": str(e), "success": False}), 500 - @app.route("/agentmemory/session/start", methods=["POST"]) def api_session_start(): auth_err = check_auth() @@ -895,12 +859,292 @@ auth_err = check_auth() if auth_err: return auth_err - + project = request.args.get("project") if not project: - return jsonify({"error": "project query param is required"}), 400 + sessions = kv.list(KV.sessions) + projects = sorted(set(s.get("project", "") for s in sessions if s.get("project"))) + return jsonify({"projects": projects, "success": True}), 200 + res = functions.get_project_profile(kv, project) + + # Stored profile may lack topConcepts/topFiles — compute from observations if empty + if not res.get("topConcepts") and not res.get("topFiles"): + import re as _re, json as _j, os.path as _osp + from collections import Counter + sessions = kv.list(KV.sessions) + project_sessions = [s for s in sessions if s.get("project") == project] + concept_counts = Counter() + file_counts = Counter() + + def _harvest_file(path, fc, cc): + if not isinstance(path, str) or not path: + return + fc[path] += 1 + # dirname components → concepts + parts = _re.split(r'[\\/]', path) + fname = parts[-1] if parts else "" + # dir components (skip drive letters, temp paths) + skip = {"tmp", "temp", "claude", "appdata", "local", "users", "windows"} + for part in parts[:-1]: + p = part.lower().strip() + if p and len(p) > 2 and p not in skip and not _re.match(r'^[a-z]:|^\.|^--', p): + cc[p] += 1 + # file stem → concept + stem = _osp.splitext(fname)[0] + if stem and len(stem) > 2: + cc[stem.lower()] += 1 + # extension → technology concept + ext = _osp.splitext(fname)[1].lstrip(".") + if ext in ("py", "ts", "js", "jsx", "tsx", "go", "rs", "java", "cs", "cpp"): + cc[ext] += 1 + + for s in project_sessions: + sid = s.get("id", "") + if not sid: + continue + for o in kv.list(KV.observations(sid)): + # top-level concepts / files + for c in (o.get("concepts") or []): + if isinstance(c, str) and c: + concept_counts[c] += 1 + for f in (o.get("files") or []): + _harvest_file(f, file_counts, concept_counts) + # toolName → concept + tn = o.get("toolName") + if tn: + concept_counts[tn] += 1 + # toolInput path fields + ti = o.get("toolInput") + if isinstance(ti, str): + try: ti = _j.loads(ti) + except Exception: ti = {} + if isinstance(ti, dict): + for fk in ("path", "file_path", "file", "filename"): + _harvest_file(ti.get(fk, ""), file_counts, concept_counts) + # narrative may be JSON string with path/tool info + narr = o.get("narrative") or o.get("raw") or "" + if isinstance(narr, str) and narr.startswith("{"): + try: + nd = _j.loads(narr) + if isinstance(nd, dict): + tn2 = nd.get("toolName") or nd.get("tool_name") + if tn2: concept_counts[tn2] += 1 + for fk in ("path", "file_path", "file", "filename"): + _harvest_file(nd.get(fk, ""), file_counts, concept_counts) + except Exception: + pass + + # memories for this project + for m in kv.list(KV.memories): + if m.get("project") == project: + for c in (m.get("concepts") or []): + if c: concept_counts[c] += 1 + for f in (m.get("files") or []): + _harvest_file(f, file_counts, concept_counts) + + res["topConcepts"] = [{"concept": c, "frequency": n} for c, n in concept_counts.most_common(20)] + res["topFiles"] = [{"file": f, "frequency": n} for f, n in file_counts.most_common(20)] + res["sessionCount"] = len(project_sessions) + return jsonify(res), 200 + +# ===================================================================== +# Missing endpoints the JS viewer calls +# ===================================================================== + +@app.route("/agentmemory/memories", methods=["GET"]) +def api_memories_list(): + auth_err = check_auth() + if auth_err: + return auth_err + + latest_only = request.args.get("latest", "false").lower() == "true" + limit = int(request.args.get("limit", "500")) + all_mems = kv.list(KV.memories) + if latest_only: + all_mems = [m for m in all_mems if m.get("isLatest") is not False] + all_mems.sort(key=lambda m: m.get("createdAt", ""), reverse=True) + return jsonify({"memories": all_mems[:limit], "total": len(all_mems)}), 200 + +@app.route("/agentmemory/graph/stats", methods=["GET"]) +def api_graph_stats(): + auth_err = check_auth() + if auth_err: + return auth_err + nodes = kv.list(KV.graphNodes) + edges = kv.list(KV.graphEdges) + return jsonify({"nodes": len(nodes), "edges": len(edges), "success": True}), 200 + +@app.route("/agentmemory/semantic", methods=["GET"]) +def api_semantic_list(): + auth_err = check_auth() + if auth_err: + return auth_err + items = kv.list(KV.semantic) + return jsonify({"items": items, "total": len(items)}), 200 + +@app.route("/agentmemory/procedural", methods=["GET"]) +def api_procedural_list(): + auth_err = check_auth() + if auth_err: + return auth_err + items = kv.list(KV.procedural) + return jsonify({"items": items, "total": len(items)}), 200 + +@app.route("/agentmemory/crystals", methods=["GET"]) +def api_crystals_list(): + auth_err = check_auth() + if auth_err: + return auth_err + items = kv.list(KV.crystals) + return jsonify({"crystals": items, "total": len(items)}), 200 + +@app.route("/agentmemory/actions", methods=["GET"]) +def api_actions_list(): + auth_err = check_auth() + if auth_err: + return auth_err + limit = int(request.args.get("limit", "200")) + status = request.args.get("status") + items = kv.list(KV.actions) + if status: + items = [a for a in items if a.get("status") == status] + items.sort(key=lambda a: a.get("updatedAt", ""), reverse=True) + return jsonify({"actions": items[:limit], "total": len(items)}), 200 + +@app.route("/agentmemory/actions", methods=["POST"]) +def api_action_create(): + auth_err = check_auth() + if auth_err: + return auth_err + try: + body = request.get_json(force=True) or {} + action_id = functions.generate_id("act") + from datetime import datetime, timezone + now = datetime.now(timezone.utc).isoformat() + action = { + "id": action_id, + "title": body.get("title", ""), + "description": body.get("description"), + "priority": body.get("priority", 0), + "status": body.get("status", "pending"), + "tags": body.get("tags", []), + "sessionId": body.get("sessionId"), + "createdAt": now, + "updatedAt": now, + } + kv.set(KV.actions, action_id, action) + return jsonify({"action": action, "success": True}), 200 + except Exception as e: + return jsonify({"error": str(e)}), 400 + +@app.route("/agentmemory/actions/", methods=["PATCH"]) +def api_action_update(action_id): + auth_err = check_auth() + if auth_err: + return auth_err + try: + body = request.get_json(force=True) or {} + existing = kv.get(KV.actions, action_id) + if not existing: + return jsonify({"error": "not found"}), 404 + from datetime import datetime, timezone + existing.update({k: v for k, v in body.items() if k not in ("id", "createdAt")}) + existing["updatedAt"] = datetime.now(timezone.utc).isoformat() + kv.set(KV.actions, action_id, existing) + return jsonify({"action": existing, "success": True}), 200 + except Exception as e: + return jsonify({"error": str(e)}), 400 + +@app.route("/agentmemory/frontier", methods=["GET"]) +def api_frontier(): + auth_err = check_auth() + if auth_err: + return auth_err + items = kv.list(KV.actions) + frontier = [a for a in items if a.get("status") in ("pending", "active")] + frontier.sort(key=lambda a: (-(a.get("priority") or 0), a.get("createdAt", ""))) + return jsonify({"frontier": frontier[:50], "total": len(frontier)}), 200 + +@app.route("/agentmemory/insights", methods=["GET"]) +def api_insights_list(): + auth_err = check_auth() + if auth_err: + return auth_err + limit = int(request.args.get("limit", "200")) + items = kv.list(KV.insights) + items.sort(key=lambda x: x.get("createdAt", ""), reverse=True) + return jsonify({"insights": items[:limit], "total": len(items)}), 200 + +@app.route("/agentmemory/replay/load", methods=["GET"]) +def api_replay_load(): + auth_err = check_auth() + if auth_err: + return auth_err + session_id = request.args.get("sessionId") + if not session_id: + return jsonify({"error": "sessionId required"}), 400 + session = kv.get(KV.sessions, session_id) + if not session: + return jsonify({"error": "session not found"}), 404 + obs = kv.list(KV.observations(session_id)) + obs.sort(key=lambda o: o.get("timestamp", "")) + + from dateutil import parser as dtparser + session_start = session.get("startedAt", "") + try: + t0_ms = dtparser.parse(session_start).timestamp() * 1000 if session_start else None + except Exception: + t0_ms = None + + _hook_kind = { + "prompt_submit": "prompt", + "post_tool_use": "tool_call", + "post_tool_failure": "tool_error", + "subagent_stop": "response", + "task_completed": "response", + "notification": "response", + } + + events = [] + for o in obs: + ts = o.get("timestamp", "") + try: + t_ms = dtparser.parse(ts).timestamp() * 1000 if ts else 0 + except Exception: + t_ms = 0 + offset_ms = max(0, t_ms - t0_ms) if t0_ms is not None else t_ms + + hook = o.get("hookType", "") + kind = _hook_kind.get(hook, "prompt") + + # obs fields are top-level (no data wrapper) + tool_name = o.get("toolName") + tool_input = o.get("toolInput") + tool_output = o.get("toolOutput") + + label = (o.get("userPrompt") or + o.get("narrative") or o.get("title") or o.get("summary") or + tool_name or + (o.get("raw") or "")[:80] or hook) + if isinstance(label, str): + label = label[:120] + + events.append({ + "kind": kind, + "label": label, + "offsetMs": offset_ms, + "ts": ts, + "body": o.get("narrative") or o.get("raw") or o.get("summary") or "", + "toolName": tool_name, + "toolInput": tool_input, + "toolOutput": tool_output, + }) + + total_ms = (events[-1]["offsetMs"] + 1000) if events else 0 + timeline = {"events": events, "eventCount": len(events), "totalDurationMs": total_ms} + return jsonify({"session": session, "timeline": timeline, "success": True}), 200 @app.route("/agentmemory/auto-forget", methods=["POST"]) def api_auto_forget(): @@ -1191,6 +1435,311 @@ return jsonify({"error": str(e)}), 500 # ===================================================================== +# Agent Discovery Endpoint (/agent.md and /auth.md) +# ===================================================================== + +AGENT_MD_TEMPLATE = """# AgentMemory — Agent Connection Guide + +> This document is auto-generated. Fetch it fresh each session so you always have current values. + +## 1. Server + +| Field | Value | +|-------|-------| +| Base URL | `http://127.0.0.1:{port}` | +| API prefix | `/agentmemory` | +| Liveness | `GET http://127.0.0.1:{port}/agentmemory/livez` (no auth) | +| Auth required | `{auth_required}` | +| Auth header | `Authorization: Bearer ` | + +{auth_note} + +--- + +## 2. Quick-Start: Minimum Viable Session + +Do these in order at the start of every session: + +``` +1. POST /agentmemory/session/start ← register yourself +2. POST /agentmemory/context ← inject past memory into your system prompt + ... do work, call observe after each tool use ... +3. POST /agentmemory/session/end ← mark session complete +``` + +### 2a. Start a session + +```http +POST /agentmemory/session/start +Content-Type: application/json +{auth_header_example} + +{{ + "sessionId": "", + "project": "", + "cwd": "", + "agentId": "", + "title": "" +}} +``` + +Response includes `context` — paste it into your system prompt before answering. + +### 2b. Compile context (also works mid-session) + +```http +POST /agentmemory/context +Content-Type: application/json +{auth_header_example} + +{{ + "sessionId": "", + "project": "", + "budget": 2000 +}} +``` + +Returns `context` string wrapped in `` tags. Insert verbatim. + +### 2c. Log an observation (call after every significant tool use) + +```http +POST /agentmemory/agent/observe +Content-Type: application/json +{auth_header_example} + +{{ + "agentId": "", + "sessionId": "", + "project": "", + "cwd": "", + "text": "", + "type": "tool | command | thought | error | result | conversation | other", + "title": "" +}} +``` + +### 2d. Save a long-term memory (decisions, patterns, preferences) + +```http +POST /agentmemory/agent/remember +Content-Type: application/json +{auth_header_example} + +{{ + "agentId": "", + "project": "", + "content": "", + "type": "fact | preference | architecture | bug | workflow | pattern", + "concepts": "auth, jwt, middleware", + "files": "src/middleware/auth.ts, tests/auth.test.ts" +}} +``` + +### 2e. End session + +```http +POST /agentmemory/session/end +Content-Type: application/json +{auth_header_example} + +{{ + "sessionId": "" +}} +``` + +--- + +## 3. MCP Tools (JSON-RPC style) + +Call via: `POST /agentmemory/mcp/tools` + +```http +POST /agentmemory/mcp/tools +Content-Type: application/json +{auth_header_example} + +{{ + "name": "", + "arguments": {{ ... }} +}} +``` + +| Tool | Required args | Description | +|------|---------------|-------------| +| `memory_recall` | `query` | BM25+vector hybrid search over all past observations | +| `memory_smart_search` | `query` | Same as recall with progressive disclosure | +| `memory_save` | `content` | Save long-term memory (type, concepts, files optional) | +| `memory_sessions` | — | List recent sessions with observation counts | +| `memory_timeline` | `anchor` | Observations around a date or keyword anchor | +| `memory_profile` | `project` | Top concepts, files, conventions for a project | +| `memory_lesson_save` | `content` | Save a lesson (auto-strengthens duplicates) | +| `memory_lesson_recall` | `query` | Search lessons scored by confidence + recency | +| `agent_observe` | `agentId`, `sessionId`, `project`, `text` | Log an observation (same as REST) | +| `agent_remember` | `agentId`, `content`, `project` | Save memory (same as REST) | + +Get full schemas: `GET /agentmemory/mcp/tools` + +--- + +## 4. Full REST Endpoint Reference + +All endpoints require `Authorization: Bearer ` when auth is enabled. +Prefix every path with `http://127.0.0.1:{port}`. + +### Session Management + +| Method | Path | Key body fields | Notes | +|--------|------|-----------------|-------| +| `POST` | `/agentmemory/session/start` | `sessionId`, `project`, `cwd`, `agentId` | Returns `session` + `context` | +| `POST` | `/agentmemory/session/end` | `sessionId` | Marks status=completed | +| `GET` | `/agentmemory/sessions` | — | List all sessions | +| `GET` | `/agentmemory/observations?sessionId=` | — | Observations for a session | +| `POST` | `/agentmemory/session/commit` | `sha`, `sessionId`, `branch`, `message` | Link a git commit to a session | +| `GET` | `/agentmemory/session/by-commit?sha=` | — | Sessions linked to a commit | +| `GET` | `/agentmemory/commits` | `branch`, `repo`, `limit` | List tracked commits | + +### Observations + +| Method | Path | Key body fields | Notes | +|--------|------|-----------------|-------| +| `POST` | `/agentmemory/observe` | `sessionId`, `hookType`, `timestamp`, `data` | Raw hook payload | +| `POST` | `/agentmemory/agent/observe` | `agentId`, `sessionId`, `project`, `text`, `type` | Simplified agent endpoint | + +`hookType` values: `post_tool_use`, `post_tool_failure`, `prompt_submit`, `subagent_stop`, `task_completed`, `notification` + +### Memory + +| Method | Path | Key body fields | Notes | +|--------|------|-----------------|-------| +| `POST` | `/agentmemory/remember` | `content`, `type`, `concepts[]`, `files[]`, `project` | Supersedes similar memories automatically | +| `POST` | `/agentmemory/agent/remember` | `agentId`, `content`, `project`, `type`, `concepts`, `files` | Simplified agent endpoint | +| `POST` | `/agentmemory/forget` | `memoryId` OR `sessionId` [+ `observationIds[]`] | Delete memory/session/observations | +| `POST` | `/agentmemory/evolve` | `memoryId`, `newContent`, `newTitle` | Version a memory (creates new, marks old superseded) | +| `POST` | `/agentmemory/search` | `query`, `limit` | Hybrid BM25+vector search | +| `POST` | `/agentmemory/context` | `sessionId`, `project`, `budget` | Compile context block for injection | + +Memory `type` values: `fact`, `preference`, `architecture`, `bug`, `workflow`, `pattern` + +### Lessons + +| Method | Path | Key body fields | Notes | +|--------|------|-----------------|-------| +| `POST` | `/agentmemory/lessons` | `content`, `context`, `project`, `confidence`, `tags[]` | Create/strengthen (duplicate = auto-reinforce) | +| `GET` | `/agentmemory/lessons` | `project`, `minConfidence`, `limit` | List lessons | +| `POST` | `/agentmemory/lessons/search` | `query`, `project`, `limit` | Keyword search scored by confidence×recency | +| `POST` | `/agentmemory/lessons/strengthen` | `lessonId` | Manually reinforce a lesson (+0.1 confidence) | + +### Memory Slots (Pinned Context) + +Slots are named text buffers injected into every context compilation. + +| Method | Path | Key body/query fields | Notes | +|--------|------|-----------------------|-------| +| `GET` | `/agentmemory/slots` | — | List all slots | +| `GET` | `/agentmemory/slot?label=` | — | Get single slot | +| `POST` | `/agentmemory/slot` | `label`, `content`, `scope`, `sizeLimit`, `pinned` | Create slot | +| `POST` | `/agentmemory/slot/append` | `label`, `text` | Append text to slot | +| `POST` | `/agentmemory/slot/replace` | `label`, `content` | Replace slot content | +| `DELETE` | `/agentmemory/slot?label=` | — | Delete slot | +| `POST` | `/agentmemory/slot/reflect` | `sessionId`, `maxObservations` | Auto-populate slots from session observations | + +Default slot labels: `persona`, `user_preferences`, `tool_guidelines`, `project_context`, `guidance`, `pending_items`, `session_patterns`, `self_notes` + +### Knowledge Graph & Analytics + +| Method | Path | Key body/query fields | Notes | +|--------|------|-----------------------|-------| +| `GET` | `/agentmemory/relations` | — | List all knowledge graph edges | +| `POST` | `/agentmemory/relations` | `sourceId`, `targetId`, `type` | Add a relation edge | +| `POST` | `/agentmemory/timeline` | `anchor`, `project`, `before`, `after` | Observations around anchor | +| `GET` | `/agentmemory/profile?project=` | — | Project profile (top concepts/files) | +| `GET` | `/agentmemory/audit` | `operation`, `limit` | Audit log | +| `GET` | `/agentmemory/profile` | `project` (optional) | Project profile; omit `project` to list all known projects | +| `GET` | `/agentmemory/actions` | `limit`, `status` | List actions | +| `POST` | `/agentmemory/actions` | `title`, `description`, `priority`, `tags[]`, `status` | Create action | +| `PATCH`| `/agentmemory/actions/` | any action fields | Update action | +| `GET` | `/agentmemory/frontier` | — | Pending+active actions sorted by priority | +| `GET` | `/agentmemory/insights` | `limit` | List insights | +| `GET` | `/agentmemory/replay/sessions` | — | Sessions list for replay | +| `GET` | `/agentmemory/replay/load?sessionId=` | — | Full session + ordered observations for replay | +| `POST` | `/agentmemory/auto-forget` | `dryRun` | Evict stale observations | + +### Health & Config + +| Method | Path | Auth | Notes | +|--------|------|------|-------| +| `GET` | `/agentmemory/livez` | No | Liveness check | +| `GET` | `/agentmemory/health` | Yes | Full health + version | +| `GET` | `/agentmemory/config/flags` | Yes | Feature flags (graph, consolidation, compression) | + +--- + +## 5. WebSocket Live Stream + +``` +ws://127.0.0.1:{port}/stream/mem-live/viewer +``` + +Connect to receive real-time events. Message types: +- `raw_observation` — raw hook payload as received +- `compressed_observation` — after synthetic compression + indexing + +No auth on the WebSocket endpoint. + +--- + +## 6. Agent Identity & Scope Isolation + +Set `AGENT_ID` env var on the server to scope all read operations to a single agent. +Set `AGENTMEMORY_AGENT_SCOPE=isolated` to enforce per-agent isolation. + +Always pass `agentId` in every request body — it's stored on every observation/memory and enables per-agent filtering via `?agentId=` on list endpoints. + +--- + +## 7. Secrets Are Redacted + +Any text containing API keys, Bearer tokens, passwords, or JWTs is automatically redacted to `[REDACTED_SECRET]` before storage. Wrap sensitive blocks in `...` tags to force redaction. + +--- + +*Generated by agentmemory v{version} — `GET /agent.md` to refresh* +""" + +@app.route("/agent.md", methods=["GET"]) +@app.route("/auth.md", methods=["GET"]) +def agent_discovery(): + port = int(os.getenv("III_REST_PORT", os.getenv("PORT", "3111"))) + secret = os.getenv("AGENTMEMORY_SECRET") + auth_required = "yes" if secret else "no" + + if secret: + auth_note = ( + "> **Auth is enabled.** Add `Authorization: Bearer ` to every request.\n" + "> The secret is set via `AGENTMEMORY_SECRET` env var in `~/.agentmemory/.env`." + ) + auth_header_example = 'Authorization: Bearer ' + else: + auth_note = ( + "> **Auth is disabled.** No `Authorization` header needed — all endpoints are open." + ) + auth_header_example = '' + + md = AGENT_MD_TEMPLATE.format( + port=port, + auth_required=auth_required, + auth_note=auth_note, + auth_header_example=auth_header_example, + version="0.9.8", + ) + + response = make_response(md) + response.headers["Content-Type"] = "text/markdown; charset=utf-8" + response.headers["Cache-Control"] = "no-cache" + return response + +# ===================================================================== # Lifecycle Helpers # ===================================================================== === Diff for src/viewer/index.html === --- agentmemory/src/viewer/index.html +++ agentmemory-python/src/viewer/index.html @@ -1039,15 +1039,15 @@ var wsProto = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; var REST, WS_URL, WS_DIRECT_URL, wsPort; if (paramPort) { - var resolvedPort = parseInt(paramPort) === 3111 ? '3113' : paramPort; + var resolvedPort = paramPort; REST = window.location.protocol + '//' + window.location.hostname + ':' + resolvedPort; - wsPort = params.get('wsPort') || String(parseInt(resolvedPort) - 1); + wsPort = params.get('wsPort') || resolvedPort; WS_URL = wsProto + '//' + window.location.hostname + ':' + wsPort; WS_DIRECT_URL = WS_URL + '/stream/mem-live/viewer'; } else if (locPort) { - var resolvedPort = parseInt(locPort) === 3111 ? '3113' : locPort; + var resolvedPort = locPort; REST = window.location.protocol + '//' + window.location.hostname + ':' + resolvedPort; - wsPort = params.get('wsPort') || String(parseInt(resolvedPort) - 1); + wsPort = params.get('wsPort') || resolvedPort; WS_URL = wsProto + '//' + window.location.hostname + ':' + wsPort; WS_DIRECT_URL = WS_URL + '/stream/mem-live/viewer'; } else {