Add gateway template sync for agents

Makefile

@@ -89,5 +89,10 @@ frontend-build: ## Build frontend (next build)
 api-gen: ## Regenerate TS API client (requires backend running at 127.0.0.1:8000)
 	cd $(FRONTEND_DIR) && npm run api:gen
 
+.PHONY: backend-templates-sync
+backend-templates-sync: ## Sync templates to existing gateway agents (usage: make backend-templates-sync GATEWAY_ID=<uuid> SYNC_ARGS="--reset-sessions")
+	@if [ -z "$(GATEWAY_ID)" ]; then echo "GATEWAY_ID is required (uuid)"; exit 1; fi
+	cd $(BACKEND_DIR) && uv run python scripts/sync_gateway_templates.py --gateway-id "$(GATEWAY_ID)" $(SYNC_ARGS)
+
 .PHONY: check
 check: lint typecheck test build ## Run lint + typecheck + tests + build

backend/app/api/gateways.py

@@ -2,10 +2,11 @@ from __future__ import annotations
 
 from uuid import UUID
 
-from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi import APIRouter, Depends, HTTPException, Query, status
 from sqlmodel import col, select
 from sqlmodel.ext.asyncio.session import AsyncSession
 
+from app.api.deps import require_admin_auth
 from app.core.agent_tokens import generate_agent_token, hash_agent_token
 from app.core.auth import AuthContext, get_auth_context
 from app.core.time import utcnow
@@ -16,9 +17,15 @@ from app.integrations.openclaw_gateway import OpenClawGatewayError, ensure_sessi
 from app.models.agents import Agent
 from app.models.gateways import Gateway
 from app.schemas.common import OkResponse
-from app.schemas.gateways import GatewayCreate, GatewayRead, GatewayUpdate
+from app.schemas.gateways import (
+    GatewayCreate,
+    GatewayRead,
+    GatewayTemplatesSyncResult,
+    GatewayUpdate,
+)
 from app.schemas.pagination import DefaultLimitOffsetPage
 from app.services.agent_provisioning import DEFAULT_HEARTBEAT_CONFIG, provision_main_agent
+from app.services.template_sync import sync_gateway_templates as sync_gateway_templates_service
 
 router = APIRouter(prefix="/gateways", tags=["gateways"])
 
@@ -186,6 +193,32 @@ async def update_gateway(
     return gateway
 
 
+@router.post("/{gateway_id}/templates/sync", response_model=GatewayTemplatesSyncResult)
+async def sync_gateway_templates(
+    gateway_id: UUID,
+    include_main: bool = Query(default=True),
+    reset_sessions: bool = Query(default=False),
+    rotate_tokens: bool = Query(default=False),
+    force_bootstrap: bool = Query(default=False),
+    board_id: UUID | None = Query(default=None),
+    session: AsyncSession = Depends(get_session),
+    auth: AuthContext = Depends(require_admin_auth),
+) -> GatewayTemplatesSyncResult:
+    gateway = await session.get(Gateway, gateway_id)
+    if gateway is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Gateway not found")
+    return await sync_gateway_templates_service(
+        session,
+        gateway,
+        user=auth.user,
+        include_main=include_main,
+        reset_sessions=reset_sessions,
+        rotate_tokens=rotate_tokens,
+        force_bootstrap=force_bootstrap,
+        board_id=board_id,
+    )
+
+
 @router.delete("/{gateway_id}", response_model=OkResponse)
 async def delete_gateway(
     gateway_id: UUID,

backend/app/schemas/gateways.py

@@ -5,7 +5,7 @@ from typing import Any
 from uuid import UUID
 
 from pydantic import field_validator
-from sqlmodel import SQLModel
+from sqlmodel import Field, SQLModel
 
 
 class GatewayBase(SQLModel):
@@ -52,3 +52,20 @@ class GatewayRead(GatewayBase):
     token: str | None = None
     created_at: datetime
     updated_at: datetime
+
+
+class GatewayTemplatesSyncError(SQLModel):
+    agent_id: UUID | None = None
+    agent_name: str | None = None
+    board_id: UUID | None = None
+    message: str
+
+
+class GatewayTemplatesSyncResult(SQLModel):
+    gateway_id: UUID
+    include_main: bool
+    reset_sessions: bool
+    agents_updated: int
+    agents_skipped: int
+    main_updated: bool
+    errors: list[GatewayTemplatesSyncError] = Field(default_factory=list)

backend/app/services/agent_provisioning.py

@@ -55,7 +55,12 @@ DEFAULT_GATEWAY_FILES = frozenset(
 
 # These files are intended to evolve within the agent workspace. Provision them if missing,
 # but avoid overwriting existing content during updates.
-PRESERVE_AGENT_EDITABLE_FILES = frozenset({"SELF.md", "AUTONOMY.md"})
+#
+# Examples:
+# - SELF.md: evolving identity/preferences
+# - USER.md: human-provided context + lead intake notes
+# - MEMORY.md: curated long-term memory (consolidated)
+PRESERVE_AGENT_EDITABLE_FILES = frozenset({"SELF.md", "USER.md", "MEMORY.md"})
 
 HEARTBEAT_LEAD_TEMPLATE = "HEARTBEAT_LEAD.md"
 HEARTBEAT_AGENT_TEMPLATE = "HEARTBEAT_AGENT.md"

backend/app/services/template_sync.py

@@ -0,0 +1,309 @@
+from __future__ import annotations
+
+import re
+from uuid import UUID, uuid4
+
+from sqlmodel import col, select
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.core.agent_tokens import generate_agent_token, hash_agent_token, verify_agent_token
+from app.core.time import utcnow
+from app.integrations.openclaw_gateway import GatewayConfig as GatewayClientConfig
+from app.integrations.openclaw_gateway import OpenClawGatewayError, openclaw_call
+from app.models.agents import Agent
+from app.models.boards import Board
+from app.models.gateways import Gateway
+from app.models.users import User
+from app.schemas.gateways import GatewayTemplatesSyncError, GatewayTemplatesSyncResult
+from app.services.agent_provisioning import provision_agent, provision_main_agent
+
+_TOOLS_KV_RE = re.compile(r"^(?P<key>[A-Z0-9_]+)=(?P<value>.*)$")
+
+
+def _slugify(value: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
+    return slug or uuid4().hex
+
+
+def _gateway_agent_id(agent: Agent) -> str:
+    session_key = agent.openclaw_session_id or ""
+    if session_key.startswith("agent:"):
+        parts = session_key.split(":")
+        if len(parts) >= 2 and parts[1]:
+            return parts[1]
+    return _slugify(agent.name)
+
+
+def _parse_tools_md(content: str) -> dict[str, str]:
+    values: dict[str, str] = {}
+    for raw in content.splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        match = _TOOLS_KV_RE.match(line)
+        if not match:
+            continue
+        values[match.group("key")] = match.group("value").strip()
+    return values
+
+
+async def _get_agent_file(
+    *,
+    agent_gateway_id: str,
+    name: str,
+    config: GatewayClientConfig,
+) -> str | None:
+    try:
+        payload = await openclaw_call(
+            "agents.files.get",
+            {"agentId": agent_gateway_id, "name": name},
+            config=config,
+        )
+    except OpenClawGatewayError:
+        return None
+    if isinstance(payload, str):
+        return payload
+    if isinstance(payload, dict):
+        # Common shapes:
+        # - {"name": "...", "content": "..."}
+        # - {"file": {"name": "...", "content": "..." }}
+        content = payload.get("content")
+        if isinstance(content, str):
+            return content
+        file_obj = payload.get("file")
+        if isinstance(file_obj, dict):
+            nested = file_obj.get("content")
+            if isinstance(nested, str):
+                return nested
+    return None
+
+
+async def _get_existing_auth_token(
+    *,
+    agent_gateway_id: str,
+    config: GatewayClientConfig,
+) -> str | None:
+    tools = await _get_agent_file(agent_gateway_id=agent_gateway_id, name="TOOLS.md", config=config)
+    if not tools:
+        return None
+    values = _parse_tools_md(tools)
+    token = values.get("AUTH_TOKEN")
+    if not token:
+        return None
+    token = token.strip()
+    return token or None
+
+
+async def _gateway_default_agent_id(config: GatewayClientConfig) -> str | None:
+    try:
+        payload = await openclaw_call("agents.list", config=config)
+    except OpenClawGatewayError:
+        return None
+    if not isinstance(payload, dict):
+        return None
+    default_id = payload.get("defaultId") or payload.get("default_id")
+    if isinstance(default_id, str) and default_id:
+        return default_id
+    agents = payload.get("agents") or []
+    if isinstance(agents, list) and agents:
+        first = agents[0]
+        if isinstance(first, dict):
+            agent_id = first.get("id")
+            if isinstance(agent_id, str) and agent_id:
+                return agent_id
+    return None
+
+
+async def sync_gateway_templates(
+    session: AsyncSession,
+    gateway: Gateway,
+    *,
+    user: User | None,
+    include_main: bool = True,
+    reset_sessions: bool = False,
+    rotate_tokens: bool = False,
+    force_bootstrap: bool = False,
+    board_id: UUID | None = None,
+) -> GatewayTemplatesSyncResult:
+    result = GatewayTemplatesSyncResult(
+        gateway_id=gateway.id,
+        include_main=include_main,
+        reset_sessions=reset_sessions,
+        agents_updated=0,
+        agents_skipped=0,
+        main_updated=False,
+    )
+    if not gateway.url:
+        result.errors.append(
+            GatewayTemplatesSyncError(message="Gateway URL is not configured for this gateway.")
+        )
+        return result
+
+    client_config = GatewayClientConfig(url=gateway.url, token=gateway.token)
+
+    boards = list(await session.exec(select(Board).where(col(Board.gateway_id) == gateway.id)))
+    boards_by_id = {board.id: board for board in boards}
+    if board_id is not None:
+        board = boards_by_id.get(board_id)
+        if board is None:
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    board_id=board_id,
+                    message="Board does not belong to this gateway.",
+                )
+            )
+            return result
+        boards_by_id = {board_id: board}
+
+    if boards_by_id:
+        agents = list(
+            await session.exec(
+                select(Agent)
+                .where(col(Agent.board_id).in_(list(boards_by_id.keys())))
+                .order_by(col(Agent.created_at).asc())
+            )
+        )
+    else:
+        agents = []
+
+    for agent in agents:
+        board = boards_by_id.get(agent.board_id) if agent.board_id is not None else None
+        if board is None:
+            result.agents_skipped += 1
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=agent.id,
+                    agent_name=agent.name,
+                    board_id=agent.board_id,
+                    message="Skipping agent: board not found for agent.",
+                )
+            )
+            continue
+
+        agent_gateway_id = _gateway_agent_id(agent)
+        auth_token = await _get_existing_auth_token(
+            agent_gateway_id=agent_gateway_id, config=client_config
+        )
+
+        if not auth_token:
+            if not rotate_tokens:
+                result.agents_skipped += 1
+                result.errors.append(
+                    GatewayTemplatesSyncError(
+                        agent_id=agent.id,
+                        agent_name=agent.name,
+                        board_id=board.id,
+                        message="Skipping agent: unable to read AUTH_TOKEN from TOOLS.md (run with rotate_tokens=true to re-key).",
+                    )
+                )
+                continue
+            raw_token = generate_agent_token()
+            agent.agent_token_hash = hash_agent_token(raw_token)
+            agent.updated_at = utcnow()
+            session.add(agent)
+            await session.commit()
+            await session.refresh(agent)
+            auth_token = raw_token
+
+        if agent.agent_token_hash and not verify_agent_token(auth_token, agent.agent_token_hash):
+            # Do not block template sync on token drift; optionally re-key.
+            if rotate_tokens:
+                raw_token = generate_agent_token()
+                agent.agent_token_hash = hash_agent_token(raw_token)
+                agent.updated_at = utcnow()
+                session.add(agent)
+                await session.commit()
+                await session.refresh(agent)
+                auth_token = raw_token
+            else:
+                result.errors.append(
+                    GatewayTemplatesSyncError(
+                        agent_id=agent.id,
+                        agent_name=agent.name,
+                        board_id=board.id,
+                        message="Warning: AUTH_TOKEN in TOOLS.md does not match backend token hash (agent auth may be broken).",
+                    )
+                )
+
+        try:
+            await provision_agent(
+                agent,
+                board,
+                gateway,
+                auth_token,
+                user,
+                action="update",
+                force_bootstrap=force_bootstrap,
+                reset_session=reset_sessions,
+            )
+            result.agents_updated += 1
+        except Exception as exc:  # pragma: no cover - gateway/network dependent
+            result.agents_skipped += 1
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=agent.id,
+                    agent_name=agent.name,
+                    board_id=board.id,
+                    message=f"Failed to sync templates: {exc}",
+                )
+            )
+
+    if include_main:
+        main_agent = (
+            await session.exec(
+                select(Agent).where(col(Agent.openclaw_session_id) == gateway.main_session_key)
+            )
+        ).first()
+        if main_agent is None:
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    message="Gateway main agent record not found; skipping main agent template sync.",
+                )
+            )
+            return result
+
+        main_gateway_agent_id = await _gateway_default_agent_id(client_config)
+        if not main_gateway_agent_id:
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=main_agent.id,
+                    agent_name=main_agent.name,
+                    message="Unable to resolve gateway default agent id for main agent.",
+                )
+            )
+            return result
+
+        main_token = await _get_existing_auth_token(
+            agent_gateway_id=main_gateway_agent_id, config=client_config
+        )
+        if not main_token:
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=main_agent.id,
+                    agent_name=main_agent.name,
+                    message="Skipping main agent: unable to read AUTH_TOKEN from TOOLS.md.",
+                )
+            )
+            return result
+
+        try:
+            await provision_main_agent(
+                main_agent,
+                gateway,
+                main_token,
+                user,
+                action="update",
+                force_bootstrap=force_bootstrap,
+                reset_session=reset_sessions,
+            )
+            result.main_updated = True
+        except Exception as exc:  # pragma: no cover - gateway/network dependent
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=main_agent.id,
+                    agent_name=main_agent.name,
+                    message=f"Failed to sync main agent templates: {exc}",
+                )
+            )
+
+    return result

backend/scripts/sync_gateway_templates.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+from uuid import UUID
+
+from app.db.session import async_session_maker
+from app.models.gateways import Gateway
+from app.services.template_sync import sync_gateway_templates
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Sync templates/ to existing OpenClaw gateway agent workspaces."
+    )
+    parser.add_argument("--gateway-id", type=str, required=True, help="Gateway UUID")
+    parser.add_argument("--board-id", type=str, default=None, help="Optional Board UUID filter")
+    parser.add_argument(
+        "--include-main",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Also sync the gateway main agent (default: true)",
+    )
+    parser.add_argument(
+        "--reset-sessions",
+        action="store_true",
+        help="Reset agent sessions after syncing files (forces agents to re-read workspace)",
+    )
+    parser.add_argument(
+        "--rotate-tokens",
+        action="store_true",
+        help="Rotate agent tokens when TOOLS.md is missing/unreadable or token drift is detected",
+    )
+    parser.add_argument(
+        "--force-bootstrap",
+        action="store_true",
+        help="Force BOOTSTRAP.md to be provisioned during sync",
+    )
+    return parser.parse_args()
+
+
+async def _run() -> int:
+    args = _parse_args()
+    gateway_id = UUID(args.gateway_id)
+    board_id = UUID(args.board_id) if args.board_id else None
+
+    async with async_session_maker() as session:
+        gateway = await session.get(Gateway, gateway_id)
+        if gateway is None:
+            raise SystemExit(f"Gateway not found: {gateway_id}")
+
+        result = await sync_gateway_templates(
+            session,
+            gateway,
+            user=None,
+            include_main=bool(args.include_main),
+            reset_sessions=bool(args.reset_sessions),
+            rotate_tokens=bool(args.rotate_tokens),
+            force_bootstrap=bool(args.force_bootstrap),
+            board_id=board_id,
+        )
+
+    print(f"gateway_id={result.gateway_id}")
+    print(f"include_main={result.include_main} reset_sessions={result.reset_sessions}")
+    print(
+        f"agents_updated={result.agents_updated} agents_skipped={result.agents_skipped} main_updated={result.main_updated}"
+    )
+    if result.errors:
+        print("errors:")
+        for err in result.errors:
+            agent = f"{err.agent_name} ({err.agent_id})" if err.agent_id else "n/a"
+            print(f"- agent={agent} board_id={err.board_id} message={err.message}")
+        return 1
+    return 0
+
+
+def main() -> None:
+    raise SystemExit(asyncio.run(_run()))
+
+
+if __name__ == "__main__":
+    main()