Merge remote-tracking branch 'origin/master' into kunal/remove-e2e-auth-bypass

.gitignore

@@ -20,6 +20,6 @@ node_modules/
 
 # Accidental literal "~" directories (e.g. when a configured path contains "~" but isn't expanded)
 backend/~/
-backend/coverage.xml
+backend/coverage.*
 backend/.coverage
 frontend/coverage

Makefile

@@ -6,6 +6,8 @@ SHELL := /usr/bin/env bash
 BACKEND_DIR := backend
 FRONTEND_DIR := frontend
 
+NODE_WRAP := bash scripts/with_node.sh
+
 .PHONY: help
 help: ## Show available targets
 	@grep -E '^[a-zA-Z0-9_.-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*## "}; {printf "  %-26s %s\n", $$1, $$2}'
@@ -13,13 +15,20 @@ help: ## Show available targets
 .PHONY: setup
 setup: backend-sync frontend-sync ## Install/sync backend + frontend deps
 
+.PHONY: all
+all: setup format check ## Run everything (deps + format + CI-equivalent checks)
+
 .PHONY: backend-sync
 backend-sync: ## uv sync backend deps (includes dev extra)
 	cd $(BACKEND_DIR) && uv sync --extra dev
 
+.PHONY: frontend-tooling
+frontend-tooling: ## Verify frontend toolchain (node + npm)
+	@$(NODE_WRAP) --check
+
 .PHONY: frontend-sync
-frontend-sync: ## npm install frontend deps
-	cd $(FRONTEND_DIR) && npm install
+frontend-sync: frontend-tooling ## npm install frontend deps
+	$(NODE_WRAP) --cwd $(FRONTEND_DIR) npm install
 
 .PHONY: format
 format: backend-format frontend-format ## Format backend + frontend
@@ -30,8 +39,8 @@ backend-format: ## Format backend (isort + black)
 	cd $(BACKEND_DIR) && uv run black .
 
 .PHONY: frontend-format
-frontend-format: ## Format frontend (prettier)
-	cd $(FRONTEND_DIR) && npx prettier --write "src/**/*.{ts,tsx,js,jsx,json,css,md}" "*.{ts,js,json,md,mdx}"
+frontend-format: frontend-tooling ## Format frontend (prettier)
+	$(NODE_WRAP) --cwd $(FRONTEND_DIR) npx prettier --write "src/**/*.{ts,tsx,js,jsx,json,css,md}" "*.{ts,js,json,md,mdx}"
 
 .PHONY: format-check
 format-check: backend-format-check frontend-format-check ## Check formatting (no changes)
@@ -42,8 +51,8 @@ backend-format-check: ## Check backend formatting (isort + black)
 	cd $(BACKEND_DIR) && uv run black . --check --diff
 
 .PHONY: frontend-format-check
-frontend-format-check: ## Check frontend formatting (prettier)
-	cd $(FRONTEND_DIR) && npx prettier --check "src/**/*.{ts,tsx,js,jsx,json,css,md}" "*.{ts,js,json,md,mdx}"
+frontend-format-check: frontend-tooling ## Check frontend formatting (prettier)
+	$(NODE_WRAP) --cwd $(FRONTEND_DIR) npx prettier --check "src/**/*.{ts,tsx,js,jsx,json,css,md}" "*.{ts,js,json,md,mdx}"
 
 .PHONY: lint
 lint: backend-lint frontend-lint ## Lint backend + frontend
@@ -53,8 +62,8 @@ backend-lint: ## Lint backend (flake8)
 	cd $(BACKEND_DIR) && uv run flake8 --config .flake8
 
 .PHONY: frontend-lint
-frontend-lint: ## Lint frontend (eslint)
-	cd $(FRONTEND_DIR) && npm run lint
+frontend-lint: frontend-tooling ## Lint frontend (eslint)
+	$(NODE_WRAP) --cwd $(FRONTEND_DIR) npm run lint
 
 .PHONY: typecheck
 typecheck: backend-typecheck frontend-typecheck ## Typecheck backend + frontend
@@ -64,8 +73,8 @@ backend-typecheck: ## Typecheck backend (mypy --strict)
 	cd $(BACKEND_DIR) && uv run mypy
 
 .PHONY: frontend-typecheck
-frontend-typecheck: ## Typecheck frontend (tsc)
-	cd $(FRONTEND_DIR) && npx tsc -p tsconfig.json --noEmit
+frontend-typecheck: frontend-tooling ## Typecheck frontend (tsc)
+	$(NODE_WRAP) --cwd $(FRONTEND_DIR) npx tsc -p tsconfig.json --noEmit
 
 .PHONY: test
 test: backend-test frontend-test ## Run tests
@@ -88,8 +97,8 @@ backend-coverage: ## Backend tests with coverage gate (scoped 100% stmt+branch o
 		--cov-fail-under=100
 
 .PHONY: frontend-test
-frontend-test: ## Frontend tests (vitest)
-	cd $(FRONTEND_DIR) && npm run test
+frontend-test: frontend-tooling ## Frontend tests (vitest)
+	$(NODE_WRAP) --cwd $(FRONTEND_DIR) npm run test
 
 .PHONY: backend-migrate
 backend-migrate: ## Apply backend DB migrations (alembic upgrade head)
@@ -99,12 +108,12 @@ backend-migrate: ## Apply backend DB migrations (alembic upgrade head)
 build: frontend-build ## Build artifacts
 
 .PHONY: frontend-build
-frontend-build: ## Build frontend (next build)
-	cd $(FRONTEND_DIR) && npm run build
+frontend-build: frontend-tooling ## Build frontend (next build)
+	$(NODE_WRAP) --cwd $(FRONTEND_DIR) npm run build
 
 .PHONY: api-gen
-api-gen: ## Regenerate TS API client (requires backend running at 127.0.0.1:8000)
-	cd $(FRONTEND_DIR) && npm run api:gen
+api-gen: frontend-tooling ## Regenerate TS API client (requires backend running at 127.0.0.1:8000)
+	$(NODE_WRAP) --cwd $(FRONTEND_DIR) npm run api:gen
 
 .PHONY: backend-templates-sync
 backend-templates-sync: ## Sync templates to existing gateway agents (usage: make backend-templates-sync GATEWAY_ID=<uuid> SYNC_ARGS="--reset-sessions")

backend/app/api/agent.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
+import re
 from collections.abc import Sequence
 from typing import Any, cast
 from uuid import UUID
 
 from fastapi import APIRouter, Depends, HTTPException, Query, status
-from sqlmodel import col, select
+from sqlmodel import SQLModel, col, select
 from sqlmodel.ext.asyncio.session import AsyncSession
 
 from app.api import agents as agents_api
@@ -16,10 +17,16 @@ from app.api import tasks as tasks_api
 from app.api.deps import ActorContext, get_board_or_404, get_task_or_404
 from app.core.agent_auth import AgentAuthContext, get_agent_auth_context
 from app.core.config import settings
+from app.core.time import utcnow
 from app.db.pagination import paginate
 from app.db.session import get_session
 from app.integrations.openclaw_gateway import GatewayConfig as GatewayClientConfig
-from app.integrations.openclaw_gateway import OpenClawGatewayError, ensure_session, send_message
+from app.integrations.openclaw_gateway import (
+    OpenClawGatewayError,
+    ensure_session,
+    openclaw_call,
+    send_message,
+)
 from app.models.activity_events import ActivityEvent
 from app.models.agents import Agent
 from app.models.approvals import Approval
@@ -62,6 +69,26 @@ from app.services.task_dependencies import (
 
 router = APIRouter(prefix="/agent", tags=["agent"])
 
+_AGENT_SESSION_PREFIX = "agent:"
+
+
+def _gateway_agent_id(agent: Agent) -> str:
+    session_key = agent.openclaw_session_id or ""
+    if session_key.startswith(_AGENT_SESSION_PREFIX):
+        parts = session_key.split(":")
+        if len(parts) >= 2 and parts[1]:
+            return parts[1]
+    # Fall back to a stable slug derived from name (matches provisioning behavior).
+    value = agent.name.lower().strip()
+    value = re.sub(r"[^a-z0-9]+", "-", value).strip("-")
+    return value or str(agent.id)
+
+
+class SoulUpdateRequest(SQLModel):
+    content: str
+    source_url: str | None = None
+    reason: str | None = None
+
 
 def _actor(agent_ctx: AgentAuthContext) -> ActorContext:
     return ActorContext(actor_type="agent", agent=agent_ctx.agent)
@@ -492,6 +519,96 @@ async def agent_heartbeat(
     )
 
 
+@router.get("/boards/{board_id}/agents/{agent_id}/soul", response_model=str)
+async def get_agent_soul(
+    agent_id: str,
+    board: Board = Depends(get_board_or_404),
+    session: AsyncSession = Depends(get_session),
+    agent_ctx: AgentAuthContext = Depends(get_agent_auth_context),
+) -> str:
+    _guard_board_access(agent_ctx, board)
+    if not agent_ctx.agent.is_board_lead and str(agent_ctx.agent.id) != agent_id:
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN)
+    target = await session.get(Agent, agent_id)
+    if target is None or (target.board_id and target.board_id != board.id):
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+    config = await _gateway_config(session, board)
+    gateway_id = _gateway_agent_id(target)
+    try:
+        payload = await openclaw_call(
+            "agents.files.get",
+            {"agentId": gateway_id, "name": "SOUL.md"},
+            config=config,
+        )
+    except OpenClawGatewayError as exc:
+        raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
+    if isinstance(payload, str):
+        return payload
+    if isinstance(payload, dict):
+        content = payload.get("content")
+        if isinstance(content, str):
+            return content
+        file_obj = payload.get("file")
+        if isinstance(file_obj, dict):
+            nested = file_obj.get("content")
+            if isinstance(nested, str):
+                return nested
+    raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail="Invalid gateway response")
+
+
+@router.put("/boards/{board_id}/agents/{agent_id}/soul", response_model=OkResponse)
+async def update_agent_soul(
+    agent_id: str,
+    payload: SoulUpdateRequest,
+    board: Board = Depends(get_board_or_404),
+    session: AsyncSession = Depends(get_session),
+    agent_ctx: AgentAuthContext = Depends(get_agent_auth_context),
+) -> OkResponse:
+    _guard_board_access(agent_ctx, board)
+    if not agent_ctx.agent.is_board_lead:
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN)
+    target = await session.get(Agent, agent_id)
+    if target is None or (target.board_id and target.board_id != board.id):
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+    config = await _gateway_config(session, board)
+    gateway_id = _gateway_agent_id(target)
+    content = payload.content.strip()
+    if not content:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="content is required",
+        )
+
+    # Persist the SOUL in the DB so future reprovision/update doesn't overwrite it.
+    target.soul_template = content
+    target.updated_at = utcnow()
+    session.add(target)
+    await session.commit()
+    try:
+        await openclaw_call(
+            "agents.files.set",
+            {"agentId": gateway_id, "name": "SOUL.md", "content": content},
+            config=config,
+        )
+    except OpenClawGatewayError as exc:
+        raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
+    reason = (payload.reason or "").strip()
+    source_url = (payload.source_url or "").strip()
+    note = f"SOUL.md updated for {target.name}."
+    if reason:
+        note = f"{note} Reason: {reason}"
+    if source_url:
+        note = f"{note} Source: {source_url}"
+    record_activity(
+        session,
+        event_type="agent.soul.updated",
+        message=note,
+        agent_id=agent_ctx.agent.id,
+    )
+    await session.commit()
+    return OkResponse()
+
+
 @router.post(
     "/boards/{board_id}/gateway/main/ask-user",
     response_model=GatewayMainAskUserResponse,

backend/app/api/souls_directory.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import re
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+
+from app.api.deps import ActorContext, require_admin_or_agent
+from app.schemas.souls_directory import (
+    SoulsDirectoryMarkdownResponse,
+    SoulsDirectorySearchResponse,
+    SoulsDirectorySoulRef,
+)
+from app.services import souls_directory
+
+router = APIRouter(prefix="/souls-directory", tags=["souls-directory"])
+
+_SAFE_SEGMENT_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_-]*$")
+_SAFE_SLUG_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_-]*$")
+
+
+def _validate_segment(value: str, *, field: str) -> str:
+    cleaned = value.strip().strip("/")
+    if not cleaned:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"{field} is required",
+        )
+    if field == "handle":
+        ok = bool(_SAFE_SEGMENT_RE.match(cleaned))
+    else:
+        ok = bool(_SAFE_SLUG_RE.match(cleaned))
+    if not ok:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"{field} contains unsupported characters",
+        )
+    return cleaned
+
+
+@router.get("/search", response_model=SoulsDirectorySearchResponse)
+async def search(
+    q: str = Query(default="", min_length=0),
+    limit: int = Query(default=20, ge=1, le=100),
+    _actor: ActorContext = Depends(require_admin_or_agent),
+) -> SoulsDirectorySearchResponse:
+    refs = await souls_directory.list_souls_directory_refs()
+    matches = souls_directory.search_souls(refs, query=q, limit=limit)
+    items = [
+        SoulsDirectorySoulRef(
+            handle=ref.handle,
+            slug=ref.slug,
+            page_url=ref.page_url,
+            raw_md_url=ref.raw_md_url,
+        )
+        for ref in matches
+    ]
+    return SoulsDirectorySearchResponse(items=items)
+
+
+@router.get("/{handle}/{slug}.md", response_model=SoulsDirectoryMarkdownResponse)
+@router.get("/{handle}/{slug}", response_model=SoulsDirectoryMarkdownResponse)
+async def get_markdown(
+    handle: str,
+    slug: str,
+    _actor: ActorContext = Depends(require_admin_or_agent),
+) -> SoulsDirectoryMarkdownResponse:
+    safe_handle = _validate_segment(handle, field="handle")
+    safe_slug = _validate_segment(slug.removesuffix(".md"), field="slug")
+    try:
+        content = await souls_directory.fetch_soul_markdown(handle=safe_handle, slug=safe_slug)
+    except Exception as exc:
+        raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
+    return SoulsDirectoryMarkdownResponse(handle=safe_handle, slug=safe_slug, content=content)

backend/app/main.py

@@ -20,6 +20,7 @@ from app.api.boards import router as boards_router
 from app.api.gateway import router as gateway_router
 from app.api.gateways import router as gateways_router
 from app.api.metrics import router as metrics_router
+from app.api.souls_directory import router as souls_directory_router
 from app.api.tasks import router as tasks_router
 from app.api.users import router as users_router
 from app.core.config import settings
@@ -74,6 +75,7 @@ api_v1.include_router(activity_router)
 api_v1.include_router(gateway_router)
 api_v1.include_router(gateways_router)
 api_v1.include_router(metrics_router)
+api_v1.include_router(souls_directory_router)
 api_v1.include_router(board_groups_router)
 api_v1.include_router(board_group_memory_router)
 api_v1.include_router(boards_router)

backend/app/schemas/__init__.py

@@ -12,6 +12,11 @@ from app.schemas.board_onboarding import (
 from app.schemas.boards import BoardCreate, BoardRead, BoardUpdate
 from app.schemas.gateways import GatewayCreate, GatewayRead, GatewayUpdate
 from app.schemas.metrics import DashboardMetrics
+from app.schemas.souls_directory import (
+    SoulsDirectoryMarkdownResponse,
+    SoulsDirectorySearchResponse,
+    SoulsDirectorySoulRef,
+)
 from app.schemas.tasks import TaskCreate, TaskRead, TaskUpdate
 from app.schemas.users import UserCreate, UserRead, UserUpdate
 
@@ -38,6 +43,9 @@ __all__ = [
     "GatewayRead",
     "GatewayUpdate",
     "DashboardMetrics",
+    "SoulsDirectoryMarkdownResponse",
+    "SoulsDirectorySearchResponse",
+    "SoulsDirectorySoulRef",
     "TaskCreate",
     "TaskRead",
     "TaskUpdate",

backend/app/schemas/souls_directory.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class SoulsDirectorySoulRef(BaseModel):
+    handle: str
+    slug: str
+    page_url: str
+    raw_md_url: str
+
+
+class SoulsDirectorySearchResponse(BaseModel):
+    items: list[SoulsDirectorySoulRef]
+
+
+class SoulsDirectoryMarkdownResponse(BaseModel):
+    handle: str
+    slug: str
+    content: str

backend/app/services/agent_provisioning.py

@@ -34,6 +34,9 @@ EXTRA_IDENTITY_PROFILE_FIELDS = {
     "verbosity": "identity_verbosity",
     "output_format": "identity_output_format",
     "update_cadence": "identity_update_cadence",
+    # Per-agent charter (optional). Used to give agents a "purpose in life" and a distinct vibe.
+    "purpose": "identity_purpose",
+    "personality": "identity_personality",
     "custom_instructions": "identity_custom_instructions",
 }
 

backend/app/services/souls_directory.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from typing import Final
+
+import httpx
+
+SOULS_DIRECTORY_BASE_URL: Final[str] = "https://souls.directory"
+SOULS_DIRECTORY_SITEMAP_URL: Final[str] = f"{SOULS_DIRECTORY_BASE_URL}/sitemap.xml"
+
+_SITEMAP_TTL_SECONDS: Final[int] = 60 * 60
+
+
+@dataclass(frozen=True, slots=True)
+class SoulRef:
+    handle: str
+    slug: str
+
+    @property
+    def page_url(self) -> str:
+        return f"{SOULS_DIRECTORY_BASE_URL}/souls/{self.handle}/{self.slug}"
+
+    @property
+    def raw_md_url(self) -> str:
+        return f"{SOULS_DIRECTORY_BASE_URL}/api/souls/{self.handle}/{self.slug}.md"
+
+
+def _parse_sitemap_soul_refs(sitemap_xml: str) -> list[SoulRef]:
+    try:
+        root = ET.fromstring(sitemap_xml)
+    except ET.ParseError:
+        return []
+
+    # Handle both namespaced and non-namespaced sitemap XML.
+    urls: list[str] = []
+    for loc in root.iter():
+        if loc.tag.endswith("loc") and loc.text:
+            urls.append(loc.text.strip())
+
+    refs: list[SoulRef] = []
+    for url in urls:
+        if not url.startswith(f"{SOULS_DIRECTORY_BASE_URL}/souls/"):
+            continue
+        # Expected: https://souls.directory/souls/{handle}/{slug}
+        parts = url.split("/")
+        if len(parts) < 6:
+            continue
+        handle = parts[4].strip()
+        slug = parts[5].strip()
+        if not handle or not slug:
+            continue
+        refs.append(SoulRef(handle=handle, slug=slug))
+    return refs
+
+
+_sitemap_cache: dict[str, object] = {
+    "loaded_at": 0.0,
+    "refs": [],
+}
+
+
+async def list_souls_directory_refs(*, client: httpx.AsyncClient | None = None) -> list[SoulRef]:
+    now = time.time()
+    loaded_raw = _sitemap_cache.get("loaded_at")
+    loaded_at = loaded_raw if isinstance(loaded_raw, (int, float)) else 0.0
+    cached = _sitemap_cache.get("refs")
+    if cached and isinstance(cached, list) and now - loaded_at < _SITEMAP_TTL_SECONDS:
+        return cached
+
+    owns_client = client is None
+    if client is None:
+        client = httpx.AsyncClient(
+            timeout=httpx.Timeout(10.0, connect=5.0),
+            headers={"User-Agent": "openclaw-mission-control/1.0"},
+        )
+    try:
+        resp = await client.get(SOULS_DIRECTORY_SITEMAP_URL)
+        resp.raise_for_status()
+        refs = _parse_sitemap_soul_refs(resp.text)
+        _sitemap_cache["loaded_at"] = now
+        _sitemap_cache["refs"] = refs
+        return refs
+    finally:
+        if owns_client:
+            await client.aclose()
+
+
+async def fetch_soul_markdown(
+    *,
+    handle: str,
+    slug: str,
+    client: httpx.AsyncClient | None = None,
+) -> str:
+    normalized_handle = handle.strip().strip("/")
+    normalized_slug = slug.strip().strip("/")
+    if normalized_slug.endswith(".md"):
+        normalized_slug = normalized_slug[: -len(".md")]
+    url = f"{SOULS_DIRECTORY_BASE_URL}/api/souls/{normalized_handle}/{normalized_slug}.md"
+
+    owns_client = client is None
+    if client is None:
+        client = httpx.AsyncClient(
+            timeout=httpx.Timeout(15.0, connect=5.0),
+            headers={"User-Agent": "openclaw-mission-control/1.0"},
+        )
+    try:
+        resp = await client.get(url)
+        resp.raise_for_status()
+        return resp.text
+    finally:
+        if owns_client:
+            await client.aclose()
+
+
+def search_souls(refs: list[SoulRef], *, query: str, limit: int = 20) -> list[SoulRef]:
+    q = query.strip().lower()
+    if not q:
+        return refs[: max(0, min(limit, len(refs)))]
+
+    matches: list[SoulRef] = []
+    for ref in refs:
+        hay = f"{ref.handle}/{ref.slug}".lower()
+        if q in hay:
+            matches.append(ref)
+        if len(matches) >= limit:
+            break
+    return matches

backend/tests/test_error_handling.py

@@ -3,10 +3,14 @@ from __future__ import annotations
 from fastapi import FastAPI, HTTPException
 from fastapi.testclient import TestClient
 from pydantic import BaseModel, Field
-
 from starlette.requests import Request
 
-from app.core.error_handling import REQUEST_ID_HEADER, _error_payload, _get_request_id, install_error_handling
+from app.core.error_handling import (
+    REQUEST_ID_HEADER,
+    _error_payload,
+    _get_request_id,
+    install_error_handling,
+)
 
 
 def test_request_validation_error_includes_request_id():

backend/tests/test_request_id_middleware.py

@@ -46,7 +46,9 @@ async def test_request_id_middleware_ignores_blank_client_header_and_generates_o
 
     assert isinstance(captured_request_id, str) and captured_request_id
     # Header should reflect the generated id, not the blank one.
-    values = [v for k, v in response_headers if k.lower() == REQUEST_ID_HEADER.lower().encode("latin-1")]
+    values = [
+        v for k, v in response_headers if k.lower() == REQUEST_ID_HEADER.lower().encode("latin-1")
+    ]
     assert values == [captured_request_id.encode("latin-1")]
 
 
@@ -81,5 +83,7 @@ async def test_request_id_middleware_does_not_duplicate_existing_header() -> Non
     assert start_headers is not None
 
     # Ensure the middleware did not append a second copy.
-    values = [v for k, v in start_headers if k.lower() == REQUEST_ID_HEADER.lower().encode("latin-1")]
+    values = [
+        v for k, v in start_headers if k.lower() == REQUEST_ID_HEADER.lower().encode("latin-1")
+    ]
     assert values == [b"already"]

backend/tests/test_souls_directory.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from app.services.souls_directory import SoulRef, _parse_sitemap_soul_refs, search_souls
+
+
+def test_parse_sitemap_extracts_soul_refs() -> None:
+    xml = """<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://souls.directory</loc></url>
+  <url><loc>https://souls.directory/souls/thedaviddias/code-reviewer</loc></url>
+  <url><loc>https://souls.directory/souls/someone/technical-writer</loc></url>
+</urlset>
+"""
+    refs = _parse_sitemap_soul_refs(xml)
+    assert refs == [
+        SoulRef(handle="thedaviddias", slug="code-reviewer"),
+        SoulRef(handle="someone", slug="technical-writer"),
+    ]
+
+
+def test_search_souls_matches_handle_or_slug() -> None:
+    refs = [
+        SoulRef(handle="thedaviddias", slug="code-reviewer"),
+        SoulRef(handle="thedaviddias", slug="technical-writer"),
+        SoulRef(handle="someone", slug="pirate-captain"),
+    ]
+    assert search_souls(refs, query="writer", limit=20) == [refs[1]]
+    assert search_souls(refs, query="thedaviddias", limit=20) == [refs[0], refs[1]]

frontend/README.md

@@ -4,7 +4,7 @@ This package is the **Next.js** web UI for OpenClaw Mission Control.
 
 - Talks to the Mission Control **backend** over HTTP (typically `http://localhost:8000`).
 - Uses **React Query** for data fetching.
-- Can optionally enable **Clerk** authentication (disabled by default unless you provide a *real* Clerk publishable key).
+- Can optionally enable **Clerk** authentication (disabled by default unless you provide a _real_ Clerk publishable key).
 
 ## Prerequisites
 
@@ -73,7 +73,7 @@ Implementation detail: we gate on a conservative regex (`pk_test_...` / `pk_live
   - `NEXT_PUBLIC_CLERK_SIGN_UP_FALLBACK_REDIRECT_URL`
 
 **Important:** `frontend/.env.example` contains placeholder values like `YOUR_PUBLISHABLE_KEY`.
-Those placeholders are *not* valid keys and are intentionally treated as “Clerk disabled”.
+Those placeholders are _not_ valid keys and are intentionally treated as “Clerk disabled”.
 
 ## How the frontend talks to the backend
 
@@ -157,8 +157,11 @@ Clerk should be **off** unless you set a real `pk_test_...` or `pk_live_...` pub
 
 ### Dev server blocked by origin restrictions
 
-`next.config.ts` sets:
+`next.config.ts` sets `allowedDevOrigins` for dev proxy safety.
 
-- `allowedDevOrigins: ["192.168.1.101"]`
+If you see repeated proxy errors (often `ECONNRESET`), make sure your dev server hostname and browser URL match (e.g. `localhost` vs `127.0.0.1`), and that your origin is included in `allowedDevOrigins`.
 
-If you’re developing from a different hostname/IP, you may need to update `allowedDevOrigins` (or use `npm run dev` on localhost).
+Notes:
+- Local dev should work via `http://localhost:3000` and `http://127.0.0.1:3000`.
+- LAN dev should work via the configured LAN IP (e.g. `http://192.168.1.101:3000`) **only** if you bind the dev server to all interfaces (`npm run dev:lan`).
+- If you bind Next to `127.0.0.1` only, remote LAN clients won’t connect.

frontend/next.config.ts

@@ -1,7 +1,11 @@
 import type { NextConfig } from "next";
 
 const nextConfig: NextConfig = {
-  allowedDevOrigins: ["192.168.1.101"],
+  // In dev, Next may proxy requests based on the request origin/host.
+  // Allow common local origins so `next dev --hostname 127.0.0.1` works
+  // when users access via http://localhost:3000 or http://127.0.0.1:3000.
+  // Keep the LAN IP as well for dev on the local network.
+  allowedDevOrigins: ["192.168.1.101", "localhost", "127.0.0.1"],
   images: {
     remotePatterns: [
       {

frontend/src/app/activity/page.test.tsx

@@ -26,15 +26,23 @@ vi.mock("next/link", () => {
 // wrappers still render <SignedOut/> from @clerk/nextjs (which crashes in real builds).
 vi.mock("@clerk/nextjs", () => {
   return {
-    ClerkProvider: ({ children }: { children: React.ReactNode }) => <>{children}</>,
+    ClerkProvider: ({ children }: { children: React.ReactNode }) => (
+      <>{children}</>
+    ),
     SignedIn: () => {
-      throw new Error("@clerk/nextjs SignedIn rendered (unexpected in secretless mode)");
+      throw new Error(
+        "@clerk/nextjs SignedIn rendered (unexpected in secretless mode)",
+      );
     },
     SignedOut: () => {
       throw new Error("@clerk/nextjs SignedOut rendered without ClerkProvider");
     },
-    SignInButton: ({ children }: { children: React.ReactNode }) => <>{children}</>,
-    SignOutButton: ({ children }: { children: React.ReactNode }) => <>{children}</>,
+    SignInButton: ({ children }: { children: React.ReactNode }) => (
+      <>{children}</>
+    ),
+    SignOutButton: ({ children }: { children: React.ReactNode }) => (
+      <>{children}</>
+    ),
     useAuth: () => ({ isLoaded: true, isSignedIn: false }),
     useUser: () => ({ isLoaded: true, isSignedIn: false, user: null }),
   };

frontend/src/app/agents/[agentId]/edit/page.tsx

@@ -68,16 +68,27 @@ const getBoardOptions = (boards: BoardRead[]): SearchableSelectOption[] =>
     label: board.name,
   }));
 
-const normalizeIdentityProfile = (
-  profile: IdentityProfile,
-): IdentityProfile | null => {
-  const normalized: IdentityProfile = {
-    role: profile.role.trim(),
-    communication_style: profile.communication_style.trim(),
-    emoji: profile.emoji.trim(),
+const mergeIdentityProfile = (
+  existing: unknown,
+  patch: IdentityProfile,
+): Record<string, unknown> | null => {
+  const resolved: Record<string, unknown> =
+    existing && typeof existing === "object"
+      ? { ...(existing as Record<string, unknown>) }
+      : {};
+  const updates: Record<string, string> = {
+    role: patch.role.trim(),
+    communication_style: patch.communication_style.trim(),
+    emoji: patch.emoji.trim(),
   };
-  const hasValue = Object.values(normalized).some((value) => value.length > 0);
-  return hasValue ? normalized : null;
+  for (const [key, value] of Object.entries(updates)) {
+    if (value) {
+      resolved[key] = value;
+    } else {
+      delete resolved[key];
+    }
+  }
+  return Object.keys(resolved).length > 0 ? resolved : null;
 };
 
 const withIdentityDefaults = (
@@ -241,7 +252,8 @@ export default function EditAgentPage() {
         every: resolvedHeartbeatEvery.trim() || "10m",
         target: resolvedHeartbeatTarget,
       } as unknown as Record<string, unknown>,
-      identity_profile: normalizeIdentityProfile(
+      identity_profile: mergeIdentityProfile(
+        loadedAgent.identity_profile,
         resolvedIdentityProfile,
       ) as unknown as Record<string, unknown> | null,
       soul_template: resolvedSoulTemplate.trim() || null,

frontend/src/app/board-groups/[groupId]/page.tsx

@@ -135,7 +135,11 @@ const SSE_RECONNECT_BACKOFF = {
 
 type HeartbeatUnit = "s" | "m" | "h" | "d";
 
-const HEARTBEAT_PRESETS: Array<{ label: string; amount: number; unit: HeartbeatUnit }> = [
+const HEARTBEAT_PRESETS: Array<{
+  label: string;
+  amount: number;
+  unit: HeartbeatUnit;
+}> = [
   { label: "30s", amount: 30, unit: "s" },
   { label: "1m", amount: 1, unit: "m" },
   { label: "2m", amount: 2, unit: "m" },
@@ -781,22 +785,22 @@ export default function BoardGroupDetailPage() {
                     {HEARTBEAT_PRESETS.map((preset) => {
                       const value = `${preset.amount}${preset.unit}`;
                       return (
-                      <button
-                        key={value}
-                        type="button"
-                        className={cn(
-                          "rounded-md px-2.5 py-1 text-xs font-semibold transition-colors",
-                          heartbeatEvery === value
-                            ? "bg-slate-900 text-white"
-                            : "text-slate-600 hover:bg-slate-100 hover:text-slate-900",
-                        )}
-                        onClick={() => {
-                          setHeartbeatAmount(String(preset.amount));
-                          setHeartbeatUnit(preset.unit);
-                        }}
-                      >
-                        {preset.label}
-                      </button>
+                        <button
+                          key={value}
+                          type="button"
+                          className={cn(
+                            "rounded-md px-2.5 py-1 text-xs font-semibold transition-colors",
+                            heartbeatEvery === value
+                              ? "bg-slate-900 text-white"
+                              : "text-slate-600 hover:bg-slate-100 hover:text-slate-900",
+                          )}
+                          onClick={() => {
+                            setHeartbeatAmount(String(preset.amount));
+                            setHeartbeatUnit(preset.unit);
+                          }}
+                        >
+                          {preset.label}
+                        </button>
                       );
                     })}
                   </div>

frontend/src/app/boards/page.tsx

@@ -21,7 +21,11 @@ import {
   useDeleteBoardApiV1BoardsBoardIdDelete,
   useListBoardsApiV1BoardsGet,
 } from "@/api/generated/boards/boards";
-import type { BoardRead } from "@/api/generated/model";
+import {
+  type listBoardGroupsApiV1BoardGroupsGetResponse,
+  useListBoardGroupsApiV1BoardGroupsGet,
+} from "@/api/generated/board-groups/board-groups";
+import type { BoardGroupRead, BoardRead } from "@/api/generated/model";
 import { DashboardSidebar } from "@/components/organisms/DashboardSidebar";
 import { DashboardShell } from "@/components/templates/DashboardShell";
 import { Button, buttonVariants } from "@/components/ui/button";
@@ -46,6 +50,9 @@ const formatTimestamp = (value?: string | null) => {
   });
 };
 
+const compactId = (value: string) =>
+  value.length > 8 ? `${value.slice(0, 8)}…` : value;
+
 export default function BoardsPage() {
   const { isSignedIn } = useAuth();
   const queryClient = useQueryClient();
@@ -63,6 +70,20 @@ export default function BoardsPage() {
     },
   });
 
+  const groupsQuery = useListBoardGroupsApiV1BoardGroupsGet<
+    listBoardGroupsApiV1BoardGroupsGetResponse,
+    ApiError
+  >(
+    { limit: 200 },
+    {
+      query: {
+        enabled: Boolean(isSignedIn),
+        refetchInterval: 30_000,
+        refetchOnMount: "always",
+      },
+    },
+  );
+
   const boards = useMemo(
     () =>
       boardsQuery.data?.status === 200
@@ -71,6 +92,19 @@ export default function BoardsPage() {
     [boardsQuery.data],
   );
 
+  const groups = useMemo<BoardGroupRead[]>(() => {
+    if (groupsQuery.data?.status !== 200) return [];
+    return groupsQuery.data.data.items ?? [];
+  }, [groupsQuery.data]);
+
+  const groupById = useMemo(() => {
+    const map = new Map<string, BoardGroupRead>();
+    for (const group of groups) {
+      map.set(group.id, group);
+    }
+    return map;
+  }, [groups]);
+
   const deleteMutation = useDeleteBoardApiV1BoardsBoardIdDelete<
     ApiError,
     { previous?: listBoardsApiV1BoardsGetResponse }
@@ -136,6 +170,28 @@ export default function BoardsPage() {
           </Link>
         ),
       },
+      {
+        id: "group",
+        header: "Group",
+        cell: ({ row }) => {
+          const groupId = row.original.board_group_id;
+          if (!groupId) {
+            return <span className="text-sm text-slate-400">—</span>;
+          }
+          const group = groupById.get(groupId);
+          const label = group?.name ?? compactId(groupId);
+          const title = group?.name ?? groupId;
+          return (
+            <Link
+              href={`/board-groups/${groupId}`}
+              className="text-sm font-medium text-slate-700 hover:text-blue-600"
+              title={title}
+            >
+              {label}
+            </Link>
+          );
+        },
+      },
       {
         accessorKey: "updated_at",
         header: "Updated",
@@ -167,7 +223,7 @@ export default function BoardsPage() {
         ),
       },
     ],
-    [],
+    [groupById],
   );
 
   // eslint-disable-next-line react-hooks/incompatible-library

frontend/src/auth/clerkKey.ts

@@ -3,7 +3,9 @@
 // IMPORTANT: keep this file dependency-free (no `"use client"`, no React, no Clerk imports)
 // so it can be used from both client and server/edge entrypoints.
 
-export function isLikelyValidClerkPublishableKey(key: string | undefined): key is string {
+export function isLikelyValidClerkPublishableKey(
+  key: string | undefined,
+): key is string {
   if (!key) return false;
 
   // Clerk publishable keys look like: pk_test_... or pk_live_...

frontend/src/components/BoardOnboardingChat.tsx

@@ -447,36 +447,6 @@ export function BoardOnboardingChat({
                   <span className="font-medium text-slate-900">Emoji:</span>{" "}
                   {draft.lead_agent.identity_profile?.emoji || "—"}
                 </p>
-                <p className="text-slate-700">
-                  <span className="font-medium text-slate-900">Autonomy:</span>{" "}
-                  {draft.lead_agent.autonomy_level || "—"}
-                </p>
-                <p className="text-slate-700">
-                  <span className="font-medium text-slate-900">Verbosity:</span>{" "}
-                  {draft.lead_agent.verbosity || "—"}
-                </p>
-                <p className="text-slate-700">
-                  <span className="font-medium text-slate-900">
-                    Output format:
-                  </span>{" "}
-                  {draft.lead_agent.output_format || "—"}
-                </p>
-                <p className="text-slate-700">
-                  <span className="font-medium text-slate-900">
-                    Update cadence:
-                  </span>{" "}
-                  {draft.lead_agent.update_cadence || "—"}
-                </p>
-                {draft.lead_agent.custom_instructions ? (
-                  <>
-                    <p className="mt-3 font-semibold text-slate-900">
-                      Custom instructions
-                    </p>
-                    <pre className="mt-1 whitespace-pre-wrap text-xs text-slate-600">
-                      {draft.lead_agent.custom_instructions}
-                    </pre>
-                  </>
-                ) : null}
               </>
             ) : null}
           </div>

frontend/src/proxy.ts

@@ -4,7 +4,9 @@ import { clerkMiddleware } from "@clerk/nextjs/server";
 import { isLikelyValidClerkPublishableKey } from "@/auth/clerkKey";
 
 const isClerkEnabled = () =>
-  isLikelyValidClerkPublishableKey(process.env.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY);
+  isLikelyValidClerkPublishableKey(
+    process.env.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY,
+  );
 
 export default isClerkEnabled() ? clerkMiddleware() : () => NextResponse.next();
 

scripts/with_node.sh

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  with_node.sh [--check] [--cwd DIR] [--] <command> [args...]
+
+Ensures node/npm/npx are available (optionally via nvm) before running a command.
+
+Options:
+  --check       Only verify node/npm/npx are available (no command required).
+  --cwd DIR     Change to DIR before running.
+  -h, --help    Show help.
+EOF
+}
+
+CHECK_ONLY="false"
+CWD=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --check)
+      CHECK_ONLY="true"
+      shift
+      ;;
+    --cwd)
+      CWD="${2:-}"
+      shift 2
+      ;;
+    --)
+      shift
+      break
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+if [[ -n "$CWD" ]]; then
+  : # handled after we resolve repo root from this script's location
+fi
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd -P)"
+
+if [[ -n "$CWD" ]]; then
+  cd "$CWD"
+fi
+
+read_nvmrc() {
+  local path="$1"
+  if [[ -f "$path" ]]; then
+    command tr -d ' \t\r\n' <"$path" || true
+  fi
+}
+
+version_greater() {
+  # Returns 0 (true) if $1 > $2 for simple semver-ish values like "v22.21.1".
+  local v1="${1#v}"
+  local v2="${2#v}"
+  local a1 b1 c1 a2 b2 c2
+  IFS=. read -r a1 b1 c1 <<<"$v1"
+  IFS=. read -r a2 b2 c2 <<<"$v2"
+  a1="${a1:-0}"; b1="${b1:-0}"; c1="${c1:-0}"
+  a2="${a2:-0}"; b2="${b2:-0}"; c2="${c2:-0}"
+  if ((a1 != a2)); then ((a1 > a2)); return; fi
+  if ((b1 != b2)); then ((b1 > b2)); return; fi
+  ((c1 > c2))
+}
+
+bootstrap_nvm_if_needed() {
+  if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1 && command -v npx >/dev/null 2>&1; then
+    return 0
+  fi
+
+  local nvm_dir="${NVM_DIR:-$HOME/.nvm}"
+  if [[ ! -s "$nvm_dir/nvm.sh" ]]; then
+    return 0
+  fi
+
+  # nvm is not guaranteed to be safe under `set -u`.
+  set +u
+  # shellcheck disable=SC1090
+  . "$nvm_dir/nvm.sh"
+
+  local version=""
+  version="$(read_nvmrc "$REPO_ROOT/.nvmrc")"
+  if [[ -z "$version" ]]; then
+    version="$(read_nvmrc "$REPO_ROOT/frontend/.nvmrc")"
+  fi
+
+  if [[ -n "$version" ]]; then
+    nvm use --silent "$version" >/dev/null 2>&1 || true
+  else
+    # Prefer a user-defined nvm default, otherwise pick the newest installed version.
+    nvm use --silent default >/dev/null 2>&1 || true
+    if ! command -v node >/dev/null 2>&1; then
+      local versions_dir="$nvm_dir/versions/node"
+      if [[ -d "$versions_dir" ]]; then
+        local latest=""
+        local candidate=""
+        for candidate in "$versions_dir"/*; do
+          [[ -d "$candidate" ]] || continue
+          candidate="$(basename "$candidate")"
+          [[ "$candidate" =~ ^v?[0-9]+(\\.[0-9]+){0,2}$ ]] || continue
+          if [[ -z "$latest" ]] || version_greater "$candidate" "$latest"; then
+            latest="$candidate"
+          fi
+        done
+        [[ -n "$latest" ]] && nvm use --silent "$latest" >/dev/null 2>&1 || true
+      fi
+    fi
+  fi
+
+  set -u
+}
+
+bootstrap_nvm_if_needed
+
+if ! command -v node >/dev/null 2>&1; then
+  echo "ERROR: node is required to run frontend targets." >&2
+  echo "Install Node.js or make it available via nvm (set NVM_DIR and ensure \$NVM_DIR/nvm.sh exists)." >&2
+  echo "Tip: add a project .nvmrc or set an nvm default alias (e.g. 'nvm alias default <version>')." >&2
+  exit 127
+fi
+
+if ! command -v npm >/dev/null 2>&1; then
+  echo "ERROR: npm is required to run frontend targets." >&2
+  echo "Install Node.js (includes npm/npx) or ensure your nvm-selected Node provides npm." >&2
+  exit 127
+fi
+
+if ! command -v npx >/dev/null 2>&1; then
+  echo "ERROR: npx is required to run frontend targets (usually installed with npm)." >&2
+  echo "Install Node.js (includes npm/npx) or ensure your npm install includes npx." >&2
+  exit 127
+fi
+
+if [[ "$CHECK_ONLY" == "true" ]]; then
+  exit 0
+fi
+
+if [[ $# -lt 1 ]]; then
+  usage >&2
+  exit 2
+fi
+
+exec "$@"

templates/HEARTBEAT_LEAD.md

@@ -235,6 +235,11 @@ Body: {"depends_on_task_ids":["DEP_TASK_ID_1","DEP_TASK_ID_2"]}
 - When creating a new agent, always set `identity_profile.role` using real-world team roles so humans and other agents can coordinate quickly.
   - Use Title Case role nouns: `Researcher`, `Analyst 1`, `Analyst 2`, `Engineer 1`, `QA`, `Reviewer`, `Scribe`.
   - If you create multiple agents with the same base role, number them sequentially starting at 1 (pick the next unused number by scanning the current agent list).
+- When creating a new agent, always give them a lightweight "charter" so they are not a generic interchangeable worker:
+  - The charter must be derived from the requirements of the work you plan to delegate next (tasks, constraints, success metrics, risks). If you cannot articulate it, do **not** create the agent yet.
+  - Set `identity_profile.purpose` (1-2 sentences): what outcomes they own, what artifacts they should produce, and how it advances the board objective.
+  - Set `identity_profile.personality` (short): a distinct working style that changes decisions and tradeoffs (e.g., speed vs correctness, skeptical vs optimistic, detail vs breadth).
+  - Optional: set `identity_profile.custom_instructions` when you need stronger guardrails (3-8 short bullets). Examples: "always cite sources", "always propose tests", "prefer smallest change", "ask clarifying questions before coding", "do not touch prod configs".
   Agent create (lead‑allowed):
   POST $BASE_URL/api/v1/agent/agents
   Body example:
@@ -243,6 +248,8 @@ Body: {"depends_on_task_ids":["DEP_TASK_ID_1","DEP_TASK_ID_2"]}
     "board_id": "$BOARD_ID",
     "identity_profile": {
       "role": "Researcher",
+      "purpose": "Find authoritative sources on X and write a 10-bullet summary with links + key risks.",
+      "personality": "curious, skeptical, citation-happy, concise",
       "communication_style": "concise, structured",
       "emoji": ":brain:"
     }
@@ -290,6 +297,32 @@ Body: {"depends_on_task_ids":["DEP_TASK_ID_1","DEP_TASK_ID_2"]}
 
 9) Post a brief status update in board memory (1-3 bullets).
 
+## Soul Inspiration (Optional)
+
+Sometimes it's useful to improve your `SOUL.md` (or an agent's `SOUL.md`) to better match the work, constraints, and desired collaboration style.
+
+Rules:
+- Use external SOUL templates (e.g. souls.directory) as inspiration only. Do not copy-paste large sections verbatim.
+- Prefer small, reversible edits. Keep `SOUL.md` stable; put fast-evolving preferences in `SELF.md`.
+- When proposing a change, include:
+  - The source page URL(s) you looked at.
+  - A short summary of the principles you are borrowing.
+  - A minimal diff-like description of what would change.
+  - A rollback note (how to revert).
+- Do not apply changes silently. Create a board approval first if the change is non-trivial.
+
+Tools:
+- Search souls directory:
+  GET $BASE_URL/api/v1/souls-directory/search?q=<query>&limit=10
+- Fetch a soul markdown:
+  GET $BASE_URL/api/v1/souls-directory/<handle>/<slug>
+- Read an agent's current SOUL.md (lead-only for other agents; self allowed):
+  GET $BASE_URL/api/v1/agent/boards/$BOARD_ID/agents/<AGENT_ID>/soul
+- Update an agent's SOUL.md (lead-only):
+  PUT $BASE_URL/api/v1/agent/boards/$BOARD_ID/agents/<AGENT_ID>/soul
+  Body: {"content":"<new SOUL.md>","source_url":"<optional>","reason":"<optional>"}
+  Notes: this persists as the agent's `soul_template` so future reprovision won't overwrite it.
+
 ## Memory Maintenance (every 2-3 days)
 Lightweight consolidation (modeled on human "sleep consolidation"):
 1) Read recent `memory/YYYY-MM-DD.md` files (since last consolidation, or last 2-3 days).

templates/IDENTITY.md

@@ -9,3 +9,11 @@ Creature: {{ identity_role }}
 Vibe: {{ identity_communication_style }}
 
 Emoji: {{ identity_emoji }}
+
+{% if identity_purpose %}
+Purpose: {{ identity_purpose }}
+{% endif %}
+
+{% if identity_personality %}
+Personality: {{ identity_personality }}
+{% endif %}

templates/SELF.md

@@ -15,6 +15,12 @@ every message.
 - Role: {{ identity_role }}
 - Communication: {{ identity_communication_style }}
 - Emoji: {{ identity_emoji }}
+{% if identity_purpose %}
+- Purpose: {{ identity_purpose }}
+{% endif %}
+{% if identity_personality %}
+- Personality: {{ identity_personality }}
+{% endif %}
 
 {% if board_id is defined %}
 - Board: {{ board_name }}