diff --git a/backend/app/api/agent.py b/backend/app/api/agent.py index 0ea12bab..1c4ed53c 100644 --- a/backend/app/api/agent.py +++ b/backend/app/api/agent.py @@ -1,11 +1,12 @@ from __future__ import annotations +import re from collections.abc import Sequence from typing import Any, cast from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, status -from sqlmodel import col, select +from sqlmodel import SQLModel, col, select from sqlmodel.ext.asyncio.session import AsyncSession from app.api import agents as agents_api @@ -19,7 +20,7 @@ from app.core.config import settings from app.db.pagination import paginate from app.db.session import get_session from app.integrations.openclaw_gateway import GatewayConfig as GatewayClientConfig -from app.integrations.openclaw_gateway import OpenClawGatewayError, ensure_session, send_message +from app.integrations.openclaw_gateway import OpenClawGatewayError, ensure_session, openclaw_call, send_message from app.models.activity_events import ActivityEvent from app.models.agents import Agent from app.models.approvals import Approval @@ -62,6 +63,26 @@ from app.services.task_dependencies import ( router = APIRouter(prefix="/agent", tags=["agent"]) +_AGENT_SESSION_PREFIX = "agent:" + + +def _gateway_agent_id(agent: Agent) -> str: + session_key = agent.openclaw_session_id or "" + if session_key.startswith(_AGENT_SESSION_PREFIX): + parts = session_key.split(":") + if len(parts) >= 2 and parts[1]: + return parts[1] + # Fall back to a stable slug derived from name (matches provisioning behavior). + value = agent.name.lower().strip() + value = re.sub(r"[^a-z0-9]+", "-", value).strip("-") + return value or str(agent.id) + + +class SoulUpdateRequest(SQLModel): + content: str + source_url: str | None = None + reason: str | None = None + def _actor(agent_ctx: AgentAuthContext) -> ActorContext: return ActorContext(actor_type="agent", agent=agent_ctx.agent) @@ -492,6 +513,90 @@ async def agent_heartbeat( ) +@router.get("/boards/{board_id}/agents/{agent_id}/soul", response_model=str) +async def get_agent_soul( + agent_id: str, + board: Board = Depends(get_board_or_404), + session: AsyncSession = Depends(get_session), + agent_ctx: AgentAuthContext = Depends(get_agent_auth_context), +) -> str: + _guard_board_access(agent_ctx, board) + if not agent_ctx.agent.is_board_lead and str(agent_ctx.agent.id) != agent_id: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN) + target = await session.get(Agent, agent_id) + if target is None or (target.board_id and target.board_id != board.id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + config = await _gateway_config(session, board) + gateway_id = _gateway_agent_id(target) + try: + payload = await openclaw_call( + "agents.files.get", + {"agentId": gateway_id, "name": "SOUL.md"}, + config=config, + ) + except OpenClawGatewayError as exc: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc + if isinstance(payload, str): + return payload + if isinstance(payload, dict): + content = payload.get("content") + if isinstance(content, str): + return content + file_obj = payload.get("file") + if isinstance(file_obj, dict): + nested = file_obj.get("content") + if isinstance(nested, str): + return nested + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail="Invalid gateway response") + + +@router.put("/boards/{board_id}/agents/{agent_id}/soul", response_model=OkResponse) +async def update_agent_soul( + agent_id: str, + payload: SoulUpdateRequest, + board: Board = Depends(get_board_or_404), + session: AsyncSession = Depends(get_session), + agent_ctx: AgentAuthContext = Depends(get_agent_auth_context), +) -> OkResponse: + _guard_board_access(agent_ctx, board) + if not agent_ctx.agent.is_board_lead: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN) + target = await session.get(Agent, agent_id) + if target is None or (target.board_id and target.board_id != board.id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + config = await _gateway_config(session, board) + gateway_id = _gateway_agent_id(target) + content = payload.content.strip() + if not content: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="content is required", + ) + try: + await openclaw_call( + "agents.files.set", + {"agentId": gateway_id, "name": "SOUL.md", "content": content}, + config=config, + ) + except OpenClawGatewayError as exc: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc + reason = (payload.reason or "").strip() + source_url = (payload.source_url or "").strip() + note = f"SOUL.md updated for {target.name}." + if reason: + note = f"{note} Reason: {reason}" + if source_url: + note = f"{note} Source: {source_url}" + record_activity( + session, + event_type="agent.soul.updated", + message=note, + agent_id=agent_ctx.agent.id, + ) + await session.commit() + return OkResponse() + + @router.post( "/boards/{board_id}/gateway/main/ask-user", response_model=GatewayMainAskUserResponse, diff --git a/backend/app/api/souls_directory.py b/backend/app/api/souls_directory.py new file mode 100644 index 00000000..c66ce8de --- /dev/null +++ b/backend/app/api/souls_directory.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import re + +from fastapi import APIRouter, Depends, HTTPException, Query, status + +from app.api.deps import ActorContext, require_admin_or_agent +from app.schemas.souls_directory import ( + SoulsDirectoryMarkdownResponse, + SoulsDirectorySearchResponse, + SoulsDirectorySoulRef, +) +from app.services import souls_directory + +router = APIRouter(prefix="/souls-directory", tags=["souls-directory"]) + +_SAFE_SEGMENT_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_-]*$") +_SAFE_SLUG_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_-]*$") + + +def _validate_segment(value: str, *, field: str) -> str: + cleaned = value.strip().strip("/") + if not cleaned: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"{field} is required", + ) + if field == "handle": + ok = bool(_SAFE_SEGMENT_RE.match(cleaned)) + else: + ok = bool(_SAFE_SLUG_RE.match(cleaned)) + if not ok: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"{field} contains unsupported characters", + ) + return cleaned + + +@router.get("/search", response_model=SoulsDirectorySearchResponse) +async def search( + q: str = Query(default="", min_length=0), + limit: int = Query(default=20, ge=1, le=100), + _actor: ActorContext = Depends(require_admin_or_agent), +) -> SoulsDirectorySearchResponse: + refs = await souls_directory.list_souls_directory_refs() + matches = souls_directory.search_souls(refs, query=q, limit=limit) + items = [ + SoulsDirectorySoulRef( + handle=ref.handle, + slug=ref.slug, + page_url=ref.page_url, + raw_md_url=ref.raw_md_url, + ) + for ref in matches + ] + return SoulsDirectorySearchResponse(items=items) + + +@router.get("/{handle}/{slug}.md", response_model=SoulsDirectoryMarkdownResponse) +@router.get("/{handle}/{slug}", response_model=SoulsDirectoryMarkdownResponse) +async def get_markdown( + handle: str, + slug: str, + _actor: ActorContext = Depends(require_admin_or_agent), +) -> SoulsDirectoryMarkdownResponse: + safe_handle = _validate_segment(handle, field="handle") + safe_slug = _validate_segment(slug.removesuffix(".md"), field="slug") + try: + content = await souls_directory.fetch_soul_markdown(handle=safe_handle, slug=safe_slug) + except Exception as exc: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc + return SoulsDirectoryMarkdownResponse(handle=safe_handle, slug=safe_slug, content=content) + diff --git a/backend/app/main.py b/backend/app/main.py index 22723710..5808801a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -20,6 +20,7 @@ from app.api.boards import router as boards_router from app.api.gateway import router as gateway_router from app.api.gateways import router as gateways_router from app.api.metrics import router as metrics_router +from app.api.souls_directory import router as souls_directory_router from app.api.tasks import router as tasks_router from app.api.users import router as users_router from app.core.config import settings @@ -74,6 +75,7 @@ api_v1.include_router(activity_router) api_v1.include_router(gateway_router) api_v1.include_router(gateways_router) api_v1.include_router(metrics_router) +api_v1.include_router(souls_directory_router) api_v1.include_router(board_groups_router) api_v1.include_router(board_group_memory_router) api_v1.include_router(boards_router) diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index c5fd12a0..06687fe6 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -12,6 +12,11 @@ from app.schemas.board_onboarding import ( from app.schemas.boards import BoardCreate, BoardRead, BoardUpdate from app.schemas.gateways import GatewayCreate, GatewayRead, GatewayUpdate from app.schemas.metrics import DashboardMetrics +from app.schemas.souls_directory import ( + SoulsDirectoryMarkdownResponse, + SoulsDirectorySearchResponse, + SoulsDirectorySoulRef, +) from app.schemas.tasks import TaskCreate, TaskRead, TaskUpdate from app.schemas.users import UserCreate, UserRead, UserUpdate @@ -38,6 +43,9 @@ __all__ = [ "GatewayRead", "GatewayUpdate", "DashboardMetrics", + "SoulsDirectoryMarkdownResponse", + "SoulsDirectorySearchResponse", + "SoulsDirectorySoulRef", "TaskCreate", "TaskRead", "TaskUpdate", diff --git a/backend/app/schemas/souls_directory.py b/backend/app/schemas/souls_directory.py new file mode 100644 index 00000000..1902e739 --- /dev/null +++ b/backend/app/schemas/souls_directory.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class SoulsDirectorySoulRef(BaseModel): + handle: str + slug: str + page_url: str + raw_md_url: str + + +class SoulsDirectorySearchResponse(BaseModel): + items: list[SoulsDirectorySoulRef] + + +class SoulsDirectoryMarkdownResponse(BaseModel): + handle: str + slug: str + content: str + diff --git a/backend/app/services/souls_directory.py b/backend/app/services/souls_directory.py new file mode 100644 index 00000000..190c8d73 --- /dev/null +++ b/backend/app/services/souls_directory.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from typing import Final + +import httpx + +SOULS_DIRECTORY_BASE_URL: Final[str] = "https://souls.directory" +SOULS_DIRECTORY_SITEMAP_URL: Final[str] = f"{SOULS_DIRECTORY_BASE_URL}/sitemap.xml" + +_SITEMAP_TTL_SECONDS: Final[int] = 60 * 60 + + +@dataclass(frozen=True, slots=True) +class SoulRef: + handle: str + slug: str + + @property + def page_url(self) -> str: + return f"{SOULS_DIRECTORY_BASE_URL}/souls/{self.handle}/{self.slug}" + + @property + def raw_md_url(self) -> str: + return f"{SOULS_DIRECTORY_BASE_URL}/api/souls/{self.handle}/{self.slug}.md" + + +def _parse_sitemap_soul_refs(sitemap_xml: str) -> list[SoulRef]: + try: + root = ET.fromstring(sitemap_xml) + except ET.ParseError: + return [] + + # Handle both namespaced and non-namespaced sitemap XML. + urls: list[str] = [] + for loc in root.iter(): + if loc.tag.endswith("loc") and loc.text: + urls.append(loc.text.strip()) + + refs: list[SoulRef] = [] + for url in urls: + if not url.startswith(f"{SOULS_DIRECTORY_BASE_URL}/souls/"): + continue + # Expected: https://souls.directory/souls/{handle}/{slug} + parts = url.split("/") + if len(parts) < 6: + continue + handle = parts[4].strip() + slug = parts[5].strip() + if not handle or not slug: + continue + refs.append(SoulRef(handle=handle, slug=slug)) + return refs + + +_sitemap_cache: dict[str, object] = { + "loaded_at": 0.0, + "refs": [], +} + + +async def list_souls_directory_refs(*, client: httpx.AsyncClient | None = None) -> list[SoulRef]: + now = time.time() + loaded_raw = _sitemap_cache.get("loaded_at") + loaded_at = loaded_raw if isinstance(loaded_raw, (int, float)) else 0.0 + cached = _sitemap_cache.get("refs") + if cached and isinstance(cached, list) and now - loaded_at < _SITEMAP_TTL_SECONDS: + return cached + + owns_client = client is None + if client is None: + client = httpx.AsyncClient( + timeout=httpx.Timeout(10.0, connect=5.0), + headers={"User-Agent": "openclaw-mission-control/1.0"}, + ) + try: + resp = await client.get(SOULS_DIRECTORY_SITEMAP_URL) + resp.raise_for_status() + refs = _parse_sitemap_soul_refs(resp.text) + _sitemap_cache["loaded_at"] = now + _sitemap_cache["refs"] = refs + return refs + finally: + if owns_client: + await client.aclose() + + +async def fetch_soul_markdown( + *, + handle: str, + slug: str, + client: httpx.AsyncClient | None = None, +) -> str: + normalized_handle = handle.strip().strip("/") + normalized_slug = slug.strip().strip("/") + if normalized_slug.endswith(".md"): + normalized_slug = normalized_slug[: -len(".md")] + url = f"{SOULS_DIRECTORY_BASE_URL}/api/souls/{normalized_handle}/{normalized_slug}.md" + + owns_client = client is None + if client is None: + client = httpx.AsyncClient( + timeout=httpx.Timeout(15.0, connect=5.0), + headers={"User-Agent": "openclaw-mission-control/1.0"}, + ) + try: + resp = await client.get(url) + resp.raise_for_status() + return resp.text + finally: + if owns_client: + await client.aclose() + + +def search_souls(refs: list[SoulRef], *, query: str, limit: int = 20) -> list[SoulRef]: + q = query.strip().lower() + if not q: + return refs[: max(0, min(limit, len(refs)))] + + matches: list[SoulRef] = [] + for ref in refs: + hay = f"{ref.handle}/{ref.slug}".lower() + if q in hay: + matches.append(ref) + if len(matches) >= limit: + break + return matches diff --git a/backend/tests/test_souls_directory.py b/backend/tests/test_souls_directory.py new file mode 100644 index 00000000..cb427285 --- /dev/null +++ b/backend/tests/test_souls_directory.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from app.services.souls_directory import SoulRef, _parse_sitemap_soul_refs, search_souls + + +def test_parse_sitemap_extracts_soul_refs() -> None: + xml = """ + + https://souls.directory + https://souls.directory/souls/thedaviddias/code-reviewer + https://souls.directory/souls/someone/technical-writer + +""" + refs = _parse_sitemap_soul_refs(xml) + assert refs == [ + SoulRef(handle="thedaviddias", slug="code-reviewer"), + SoulRef(handle="someone", slug="technical-writer"), + ] + + +def test_search_souls_matches_handle_or_slug() -> None: + refs = [ + SoulRef(handle="thedaviddias", slug="code-reviewer"), + SoulRef(handle="thedaviddias", slug="technical-writer"), + SoulRef(handle="someone", slug="pirate-captain"), + ] + assert search_souls(refs, query="writer", limit=20) == [refs[1]] + assert search_souls(refs, query="thedaviddias", limit=20) == [refs[0], refs[1]] + diff --git a/templates/HEARTBEAT_LEAD.md b/templates/HEARTBEAT_LEAD.md index f52f1ce3..bb9da903 100644 --- a/templates/HEARTBEAT_LEAD.md +++ b/templates/HEARTBEAT_LEAD.md @@ -297,6 +297,31 @@ Body: {"depends_on_task_ids":["DEP_TASK_ID_1","DEP_TASK_ID_2"]} 9) Post a brief status update in board memory (1-3 bullets). +## Soul Inspiration (Optional) + +Sometimes it's useful to improve your `SOUL.md` (or an agent's `SOUL.md`) to better match the work, constraints, and desired collaboration style. + +Rules: +- Use external SOUL templates (e.g. souls.directory) as inspiration only. Do not copy-paste large sections verbatim. +- Prefer small, reversible edits. Keep `SOUL.md` stable; put fast-evolving preferences in `SELF.md`. +- When proposing a change, include: + - The source page URL(s) you looked at. + - A short summary of the principles you are borrowing. + - A minimal diff-like description of what would change. + - A rollback note (how to revert). +- Do not apply changes silently. Create a board approval first if the change is non-trivial. + +Tools: +- Search souls directory: + GET $BASE_URL/api/v1/souls-directory/search?q=&limit=10 +- Fetch a soul markdown: + GET $BASE_URL/api/v1/souls-directory// +- Read an agent's current SOUL.md (lead-only for other agents; self allowed): + GET $BASE_URL/api/v1/agent/boards/$BOARD_ID/agents//soul +- Update an agent's SOUL.md (lead-only): + PUT $BASE_URL/api/v1/agent/boards/$BOARD_ID/agents//soul + Body: {"content":"","source_url":"","reason":""} + ## Memory Maintenance (every 2-3 days) Lightweight consolidation (modeled on human "sleep consolidation"): 1) Read recent `memory/YYYY-MM-DD.md` files (since last consolidation, or last 2-3 days).