From 999ec6d1bb9d5870455008276c4387b3622d94ea Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Sat, 7 Feb 2026 04:24:06 +0530 Subject: [PATCH] Add gateway template sync for agents --- Makefile | 5 + backend/app/api/gateways.py | 37 ++- backend/app/schemas/gateways.py | 19 +- backend/app/services/agent_provisioning.py | 7 +- backend/app/services/template_sync.py | 309 +++++++++++++++++++++ backend/scripts/sync_gateway_templates.py | 82 ++++++ 6 files changed, 455 insertions(+), 4 deletions(-) create mode 100644 backend/app/services/template_sync.py create mode 100644 backend/scripts/sync_gateway_templates.py diff --git a/Makefile b/Makefile index ca640a4a..35695933 100644 --- a/Makefile +++ b/Makefile @@ -89,5 +89,10 @@ frontend-build: ## Build frontend (next build) api-gen: ## Regenerate TS API client (requires backend running at 127.0.0.1:8000) cd $(FRONTEND_DIR) && npm run api:gen +.PHONY: backend-templates-sync +backend-templates-sync: ## Sync templates to existing gateway agents (usage: make backend-templates-sync GATEWAY_ID= SYNC_ARGS="--reset-sessions") + @if [ -z "$(GATEWAY_ID)" ]; then echo "GATEWAY_ID is required (uuid)"; exit 1; fi + cd $(BACKEND_DIR) && uv run python scripts/sync_gateway_templates.py --gateway-id "$(GATEWAY_ID)" $(SYNC_ARGS) + .PHONY: check check: lint typecheck test build ## Run lint + typecheck + tests + build diff --git a/backend/app/api/gateways.py b/backend/app/api/gateways.py index 9e06c4f5..7808f38d 100644 --- a/backend/app/api/gateways.py +++ b/backend/app/api/gateways.py @@ -2,10 +2,11 @@ from __future__ import annotations from uuid import UUID -from fastapi import APIRouter, Depends, HTTPException, status +from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlmodel import col, select from sqlmodel.ext.asyncio.session import AsyncSession +from app.api.deps import require_admin_auth from app.core.agent_tokens import generate_agent_token, hash_agent_token from app.core.auth import AuthContext, get_auth_context from app.core.time import utcnow @@ -16,9 +17,15 @@ from app.integrations.openclaw_gateway import OpenClawGatewayError, ensure_sessi from app.models.agents import Agent from app.models.gateways import Gateway from app.schemas.common import OkResponse -from app.schemas.gateways import GatewayCreate, GatewayRead, GatewayUpdate +from app.schemas.gateways import ( + GatewayCreate, + GatewayRead, + GatewayTemplatesSyncResult, + GatewayUpdate, +) from app.schemas.pagination import DefaultLimitOffsetPage from app.services.agent_provisioning import DEFAULT_HEARTBEAT_CONFIG, provision_main_agent +from app.services.template_sync import sync_gateway_templates as sync_gateway_templates_service router = APIRouter(prefix="/gateways", tags=["gateways"]) @@ -186,6 +193,32 @@ async def update_gateway( return gateway +@router.post("/{gateway_id}/templates/sync", response_model=GatewayTemplatesSyncResult) +async def sync_gateway_templates( + gateway_id: UUID, + include_main: bool = Query(default=True), + reset_sessions: bool = Query(default=False), + rotate_tokens: bool = Query(default=False), + force_bootstrap: bool = Query(default=False), + board_id: UUID | None = Query(default=None), + session: AsyncSession = Depends(get_session), + auth: AuthContext = Depends(require_admin_auth), +) -> GatewayTemplatesSyncResult: + gateway = await session.get(Gateway, gateway_id) + if gateway is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Gateway not found") + return await sync_gateway_templates_service( + session, + gateway, + user=auth.user, + include_main=include_main, + reset_sessions=reset_sessions, + rotate_tokens=rotate_tokens, + force_bootstrap=force_bootstrap, + board_id=board_id, + ) + + @router.delete("/{gateway_id}", response_model=OkResponse) async def delete_gateway( gateway_id: UUID, diff --git a/backend/app/schemas/gateways.py b/backend/app/schemas/gateways.py index 2140dd95..04142fba 100644 --- a/backend/app/schemas/gateways.py +++ b/backend/app/schemas/gateways.py @@ -5,7 +5,7 @@ from typing import Any from uuid import UUID from pydantic import field_validator -from sqlmodel import SQLModel +from sqlmodel import Field, SQLModel class GatewayBase(SQLModel): @@ -52,3 +52,20 @@ class GatewayRead(GatewayBase): token: str | None = None created_at: datetime updated_at: datetime + + +class GatewayTemplatesSyncError(SQLModel): + agent_id: UUID | None = None + agent_name: str | None = None + board_id: UUID | None = None + message: str + + +class GatewayTemplatesSyncResult(SQLModel): + gateway_id: UUID + include_main: bool + reset_sessions: bool + agents_updated: int + agents_skipped: int + main_updated: bool + errors: list[GatewayTemplatesSyncError] = Field(default_factory=list) diff --git a/backend/app/services/agent_provisioning.py b/backend/app/services/agent_provisioning.py index c3640f9e..1a54a062 100644 --- a/backend/app/services/agent_provisioning.py +++ b/backend/app/services/agent_provisioning.py @@ -55,7 +55,12 @@ DEFAULT_GATEWAY_FILES = frozenset( # These files are intended to evolve within the agent workspace. Provision them if missing, # but avoid overwriting existing content during updates. -PRESERVE_AGENT_EDITABLE_FILES = frozenset({"SELF.md", "AUTONOMY.md"}) +# +# Examples: +# - SELF.md: evolving identity/preferences +# - USER.md: human-provided context + lead intake notes +# - MEMORY.md: curated long-term memory (consolidated) +PRESERVE_AGENT_EDITABLE_FILES = frozenset({"SELF.md", "USER.md", "MEMORY.md"}) HEARTBEAT_LEAD_TEMPLATE = "HEARTBEAT_LEAD.md" HEARTBEAT_AGENT_TEMPLATE = "HEARTBEAT_AGENT.md" diff --git a/backend/app/services/template_sync.py b/backend/app/services/template_sync.py new file mode 100644 index 00000000..825a9f65 --- /dev/null +++ b/backend/app/services/template_sync.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import re +from uuid import UUID, uuid4 + +from sqlmodel import col, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from app.core.agent_tokens import generate_agent_token, hash_agent_token, verify_agent_token +from app.core.time import utcnow +from app.integrations.openclaw_gateway import GatewayConfig as GatewayClientConfig +from app.integrations.openclaw_gateway import OpenClawGatewayError, openclaw_call +from app.models.agents import Agent +from app.models.boards import Board +from app.models.gateways import Gateway +from app.models.users import User +from app.schemas.gateways import GatewayTemplatesSyncError, GatewayTemplatesSyncResult +from app.services.agent_provisioning import provision_agent, provision_main_agent + +_TOOLS_KV_RE = re.compile(r"^(?P[A-Z0-9_]+)=(?P.*)$") + + +def _slugify(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + return slug or uuid4().hex + + +def _gateway_agent_id(agent: Agent) -> str: + session_key = agent.openclaw_session_id or "" + if session_key.startswith("agent:"): + parts = session_key.split(":") + if len(parts) >= 2 and parts[1]: + return parts[1] + return _slugify(agent.name) + + +def _parse_tools_md(content: str) -> dict[str, str]: + values: dict[str, str] = {} + for raw in content.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + match = _TOOLS_KV_RE.match(line) + if not match: + continue + values[match.group("key")] = match.group("value").strip() + return values + + +async def _get_agent_file( + *, + agent_gateway_id: str, + name: str, + config: GatewayClientConfig, +) -> str | None: + try: + payload = await openclaw_call( + "agents.files.get", + {"agentId": agent_gateway_id, "name": name}, + config=config, + ) + except OpenClawGatewayError: + return None + if isinstance(payload, str): + return payload + if isinstance(payload, dict): + # Common shapes: + # - {"name": "...", "content": "..."} + # - {"file": {"name": "...", "content": "..." }} + content = payload.get("content") + if isinstance(content, str): + return content + file_obj = payload.get("file") + if isinstance(file_obj, dict): + nested = file_obj.get("content") + if isinstance(nested, str): + return nested + return None + + +async def _get_existing_auth_token( + *, + agent_gateway_id: str, + config: GatewayClientConfig, +) -> str | None: + tools = await _get_agent_file(agent_gateway_id=agent_gateway_id, name="TOOLS.md", config=config) + if not tools: + return None + values = _parse_tools_md(tools) + token = values.get("AUTH_TOKEN") + if not token: + return None + token = token.strip() + return token or None + + +async def _gateway_default_agent_id(config: GatewayClientConfig) -> str | None: + try: + payload = await openclaw_call("agents.list", config=config) + except OpenClawGatewayError: + return None + if not isinstance(payload, dict): + return None + default_id = payload.get("defaultId") or payload.get("default_id") + if isinstance(default_id, str) and default_id: + return default_id + agents = payload.get("agents") or [] + if isinstance(agents, list) and agents: + first = agents[0] + if isinstance(first, dict): + agent_id = first.get("id") + if isinstance(agent_id, str) and agent_id: + return agent_id + return None + + +async def sync_gateway_templates( + session: AsyncSession, + gateway: Gateway, + *, + user: User | None, + include_main: bool = True, + reset_sessions: bool = False, + rotate_tokens: bool = False, + force_bootstrap: bool = False, + board_id: UUID | None = None, +) -> GatewayTemplatesSyncResult: + result = GatewayTemplatesSyncResult( + gateway_id=gateway.id, + include_main=include_main, + reset_sessions=reset_sessions, + agents_updated=0, + agents_skipped=0, + main_updated=False, + ) + if not gateway.url: + result.errors.append( + GatewayTemplatesSyncError(message="Gateway URL is not configured for this gateway.") + ) + return result + + client_config = GatewayClientConfig(url=gateway.url, token=gateway.token) + + boards = list(await session.exec(select(Board).where(col(Board.gateway_id) == gateway.id))) + boards_by_id = {board.id: board for board in boards} + if board_id is not None: + board = boards_by_id.get(board_id) + if board is None: + result.errors.append( + GatewayTemplatesSyncError( + board_id=board_id, + message="Board does not belong to this gateway.", + ) + ) + return result + boards_by_id = {board_id: board} + + if boards_by_id: + agents = list( + await session.exec( + select(Agent) + .where(col(Agent.board_id).in_(list(boards_by_id.keys()))) + .order_by(col(Agent.created_at).asc()) + ) + ) + else: + agents = [] + + for agent in agents: + board = boards_by_id.get(agent.board_id) if agent.board_id is not None else None + if board is None: + result.agents_skipped += 1 + result.errors.append( + GatewayTemplatesSyncError( + agent_id=agent.id, + agent_name=agent.name, + board_id=agent.board_id, + message="Skipping agent: board not found for agent.", + ) + ) + continue + + agent_gateway_id = _gateway_agent_id(agent) + auth_token = await _get_existing_auth_token( + agent_gateway_id=agent_gateway_id, config=client_config + ) + + if not auth_token: + if not rotate_tokens: + result.agents_skipped += 1 + result.errors.append( + GatewayTemplatesSyncError( + agent_id=agent.id, + agent_name=agent.name, + board_id=board.id, + message="Skipping agent: unable to read AUTH_TOKEN from TOOLS.md (run with rotate_tokens=true to re-key).", + ) + ) + continue + raw_token = generate_agent_token() + agent.agent_token_hash = hash_agent_token(raw_token) + agent.updated_at = utcnow() + session.add(agent) + await session.commit() + await session.refresh(agent) + auth_token = raw_token + + if agent.agent_token_hash and not verify_agent_token(auth_token, agent.agent_token_hash): + # Do not block template sync on token drift; optionally re-key. + if rotate_tokens: + raw_token = generate_agent_token() + agent.agent_token_hash = hash_agent_token(raw_token) + agent.updated_at = utcnow() + session.add(agent) + await session.commit() + await session.refresh(agent) + auth_token = raw_token + else: + result.errors.append( + GatewayTemplatesSyncError( + agent_id=agent.id, + agent_name=agent.name, + board_id=board.id, + message="Warning: AUTH_TOKEN in TOOLS.md does not match backend token hash (agent auth may be broken).", + ) + ) + + try: + await provision_agent( + agent, + board, + gateway, + auth_token, + user, + action="update", + force_bootstrap=force_bootstrap, + reset_session=reset_sessions, + ) + result.agents_updated += 1 + except Exception as exc: # pragma: no cover - gateway/network dependent + result.agents_skipped += 1 + result.errors.append( + GatewayTemplatesSyncError( + agent_id=agent.id, + agent_name=agent.name, + board_id=board.id, + message=f"Failed to sync templates: {exc}", + ) + ) + + if include_main: + main_agent = ( + await session.exec( + select(Agent).where(col(Agent.openclaw_session_id) == gateway.main_session_key) + ) + ).first() + if main_agent is None: + result.errors.append( + GatewayTemplatesSyncError( + message="Gateway main agent record not found; skipping main agent template sync.", + ) + ) + return result + + main_gateway_agent_id = await _gateway_default_agent_id(client_config) + if not main_gateway_agent_id: + result.errors.append( + GatewayTemplatesSyncError( + agent_id=main_agent.id, + agent_name=main_agent.name, + message="Unable to resolve gateway default agent id for main agent.", + ) + ) + return result + + main_token = await _get_existing_auth_token( + agent_gateway_id=main_gateway_agent_id, config=client_config + ) + if not main_token: + result.errors.append( + GatewayTemplatesSyncError( + agent_id=main_agent.id, + agent_name=main_agent.name, + message="Skipping main agent: unable to read AUTH_TOKEN from TOOLS.md.", + ) + ) + return result + + try: + await provision_main_agent( + main_agent, + gateway, + main_token, + user, + action="update", + force_bootstrap=force_bootstrap, + reset_session=reset_sessions, + ) + result.main_updated = True + except Exception as exc: # pragma: no cover - gateway/network dependent + result.errors.append( + GatewayTemplatesSyncError( + agent_id=main_agent.id, + agent_name=main_agent.name, + message=f"Failed to sync main agent templates: {exc}", + ) + ) + + return result diff --git a/backend/scripts/sync_gateway_templates.py b/backend/scripts/sync_gateway_templates.py new file mode 100644 index 00000000..e58f6097 --- /dev/null +++ b/backend/scripts/sync_gateway_templates.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import argparse +import asyncio +from uuid import UUID + +from app.db.session import async_session_maker +from app.models.gateways import Gateway +from app.services.template_sync import sync_gateway_templates + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Sync templates/ to existing OpenClaw gateway agent workspaces." + ) + parser.add_argument("--gateway-id", type=str, required=True, help="Gateway UUID") + parser.add_argument("--board-id", type=str, default=None, help="Optional Board UUID filter") + parser.add_argument( + "--include-main", + action=argparse.BooleanOptionalAction, + default=True, + help="Also sync the gateway main agent (default: true)", + ) + parser.add_argument( + "--reset-sessions", + action="store_true", + help="Reset agent sessions after syncing files (forces agents to re-read workspace)", + ) + parser.add_argument( + "--rotate-tokens", + action="store_true", + help="Rotate agent tokens when TOOLS.md is missing/unreadable or token drift is detected", + ) + parser.add_argument( + "--force-bootstrap", + action="store_true", + help="Force BOOTSTRAP.md to be provisioned during sync", + ) + return parser.parse_args() + + +async def _run() -> int: + args = _parse_args() + gateway_id = UUID(args.gateway_id) + board_id = UUID(args.board_id) if args.board_id else None + + async with async_session_maker() as session: + gateway = await session.get(Gateway, gateway_id) + if gateway is None: + raise SystemExit(f"Gateway not found: {gateway_id}") + + result = await sync_gateway_templates( + session, + gateway, + user=None, + include_main=bool(args.include_main), + reset_sessions=bool(args.reset_sessions), + rotate_tokens=bool(args.rotate_tokens), + force_bootstrap=bool(args.force_bootstrap), + board_id=board_id, + ) + + print(f"gateway_id={result.gateway_id}") + print(f"include_main={result.include_main} reset_sessions={result.reset_sessions}") + print( + f"agents_updated={result.agents_updated} agents_skipped={result.agents_skipped} main_updated={result.main_updated}" + ) + if result.errors: + print("errors:") + for err in result.errors: + agent = f"{err.agent_name} ({err.agent_id})" if err.agent_id else "n/a" + print(f"- agent={agent} board_id={err.board_id} message={err.message}") + return 1 + return 0 + + +def main() -> None: + raise SystemExit(asyncio.run(_run())) + + +if __name__ == "__main__": + main()