feat(github): add approval-check reconciliation scheduler
This commit is contained in:
@@ -22,6 +22,9 @@ DB_AUTO_MIGRATE=false
|
|||||||
# GitHub integration (for Check Runs / required-check enforcement)
|
# GitHub integration (for Check Runs / required-check enforcement)
|
||||||
# Used by mission-control/approval check updater.
|
# Used by mission-control/approval check updater.
|
||||||
GH_TOKEN=
|
GH_TOKEN=
|
||||||
|
# Periodic reconciliation safety net (rq-scheduler)
|
||||||
|
GITHUB_APPROVAL_CHECK_SCHEDULE_ID=mission-control-approval-check-reconcile
|
||||||
|
GITHUB_APPROVAL_CHECK_SCHEDULE_INTERVAL_SECONDS=900
|
||||||
|
|
||||||
# Webhook queue / worker
|
# Webhook queue / worker
|
||||||
WEBHOOK_REDIS_URL=redis://localhost:6379/0
|
WEBHOOK_REDIS_URL=redis://localhost:6379/0
|
||||||
|
|||||||
@@ -57,6 +57,10 @@ class Settings(BaseSettings):
|
|||||||
validation_alias=AliasChoices("GH_TOKEN", "GITHUB_TOKEN"),
|
validation_alias=AliasChoices("GH_TOKEN", "GITHUB_TOKEN"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Periodic reconciliation safety net for mission-control/approval checks.
|
||||||
|
github_approval_check_schedule_id: str = "mission-control-approval-check-reconcile"
|
||||||
|
github_approval_check_schedule_interval_seconds: int = 900
|
||||||
|
|
||||||
# Database lifecycle
|
# Database lifecycle
|
||||||
db_auto_migrate: bool = False
|
db_auto_migrate: bool = False
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from sqlmodel import col, select
|
|||||||
|
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
from app.core.logging import get_logger
|
from app.core.logging import get_logger
|
||||||
|
from app.db.session import async_session_maker
|
||||||
from app.models.approval_task_links import ApprovalTaskLink
|
from app.models.approval_task_links import ApprovalTaskLink
|
||||||
from app.models.approvals import Approval
|
from app.models.approvals import Approval
|
||||||
from app.models.boards import Board
|
from app.models.boards import Board
|
||||||
@@ -372,5 +373,35 @@ async def reconcile_github_approval_checks_for_board(
|
|||||||
return len(pr_urls)
|
return len(pr_urls)
|
||||||
|
|
||||||
|
|
||||||
|
async def reconcile_mission_control_approval_checks_for_all_boards() -> int:
|
||||||
|
"""Reconcile approval checks for every board.
|
||||||
|
|
||||||
|
Returns total number of distinct PR URLs processed across boards.
|
||||||
|
|
||||||
|
This is intentionally a safety net: the primary, low-latency updates happen on
|
||||||
|
approval create/resolution and task github_pr_url updates.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async with async_session_maker() as session:
|
||||||
|
board_ids = list(
|
||||||
|
await session.exec(
|
||||||
|
select(col(Board.id)).order_by(col(Board.created_at).asc()),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
processed = 0
|
||||||
|
for board_id in board_ids:
|
||||||
|
try:
|
||||||
|
processed += await reconcile_github_approval_checks_for_board(
|
||||||
|
session,
|
||||||
|
board_id=board_id,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"github.approval_check.reconcile.board_failed",
|
||||||
|
extra={"board_id": str(board_id)},
|
||||||
|
)
|
||||||
|
return processed
|
||||||
|
|
||||||
|
|
||||||
def github_approval_check_enabled() -> bool:
|
def github_approval_check_enabled() -> bool:
|
||||||
return bool((settings.github_token or "").strip())
|
return bool((settings.github_token or "").strip())
|
||||||
|
|||||||
84
backend/app/services/github/scheduler.py
Normal file
84
backend/app/services/github/scheduler.py
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
"""Scheduler bootstrap for Mission Control GitHub approval check reconciliation.
|
||||||
|
|
||||||
|
This uses rq-scheduler (same pattern as webhook dispatch scheduler) to periodically
|
||||||
|
reconcile the `mission-control/approval` check run state.
|
||||||
|
|
||||||
|
The periodic job is a safety net; primary updates happen on:
|
||||||
|
- approval create / resolution
|
||||||
|
- task github_pr_url updates
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from redis import Redis
|
||||||
|
from rq_scheduler import Scheduler # type: ignore[import-untyped]
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.core.logging import get_logger
|
||||||
|
from app.services.github.worker import run_reconcile_mission_control_approval_checks
|
||||||
|
|
||||||
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def bootstrap_mission_control_approval_check_schedule(
|
||||||
|
interval_seconds: int | None = None,
|
||||||
|
*,
|
||||||
|
max_attempts: int = 5,
|
||||||
|
retry_sleep_seconds: float = 1.0,
|
||||||
|
) -> None:
|
||||||
|
"""Register a recurring reconciliation job for GitHub approval checks."""
|
||||||
|
|
||||||
|
effective_interval_seconds = (
|
||||||
|
settings.github_approval_check_schedule_interval_seconds
|
||||||
|
if interval_seconds is None
|
||||||
|
else interval_seconds
|
||||||
|
)
|
||||||
|
|
||||||
|
last_exc: Exception | None = None
|
||||||
|
for attempt in range(1, max_attempts + 1):
|
||||||
|
try:
|
||||||
|
connection = Redis.from_url(settings.webhook_redis_url)
|
||||||
|
connection.ping()
|
||||||
|
scheduler = Scheduler(
|
||||||
|
queue_name=settings.webhook_queue_name,
|
||||||
|
connection=connection,
|
||||||
|
)
|
||||||
|
|
||||||
|
for job in scheduler.get_jobs():
|
||||||
|
if job.id == settings.github_approval_check_schedule_id:
|
||||||
|
scheduler.cancel(job)
|
||||||
|
|
||||||
|
scheduler.schedule(
|
||||||
|
datetime.now(tz=timezone.utc) + timedelta(seconds=10),
|
||||||
|
func=run_reconcile_mission_control_approval_checks,
|
||||||
|
interval=effective_interval_seconds,
|
||||||
|
repeat=None,
|
||||||
|
id=settings.github_approval_check_schedule_id,
|
||||||
|
queue_name=settings.webhook_queue_name,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"github.approval_check.scheduler.bootstrapped",
|
||||||
|
extra={
|
||||||
|
"schedule_id": settings.github_approval_check_schedule_id,
|
||||||
|
"queue_name": settings.webhook_queue_name,
|
||||||
|
"interval_seconds": effective_interval_seconds,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return
|
||||||
|
except Exception as exc:
|
||||||
|
last_exc = exc
|
||||||
|
logger.warning(
|
||||||
|
"github.approval_check.scheduler.bootstrap_failed",
|
||||||
|
extra={
|
||||||
|
"attempt": attempt,
|
||||||
|
"max_attempts": max_attempts,
|
||||||
|
"error": str(exc),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if attempt < max_attempts:
|
||||||
|
time.sleep(retry_sleep_seconds * attempt)
|
||||||
|
|
||||||
|
raise RuntimeError("Failed to bootstrap GitHub approval check schedule") from last_exc
|
||||||
30
backend/app/services/github/worker.py
Normal file
30
backend/app/services/github/worker.py
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
"""RQ worker entrypoints for GitHub check reconciliation."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
|
||||||
|
from app.core.logging import get_logger
|
||||||
|
from app.services.github.mission_control_approval_check import (
|
||||||
|
github_approval_check_enabled,
|
||||||
|
reconcile_mission_control_approval_checks_for_all_boards,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def run_reconcile_mission_control_approval_checks() -> None:
|
||||||
|
"""RQ entrypoint for periodically reconciling mission-control/approval checks."""
|
||||||
|
if not github_approval_check_enabled():
|
||||||
|
logger.info("github.approval_check.reconcile.skipped_missing_token")
|
||||||
|
return
|
||||||
|
|
||||||
|
start = time.time()
|
||||||
|
logger.info("github.approval_check.reconcile.started")
|
||||||
|
count = asyncio.run(reconcile_mission_control_approval_checks_for_all_boards())
|
||||||
|
elapsed_ms = int((time.time() - start) * 1000)
|
||||||
|
logger.info(
|
||||||
|
"github.approval_check.reconcile.finished",
|
||||||
|
extra={"duration_ms": elapsed_ms, "pr_urls": count},
|
||||||
|
)
|
||||||
@@ -95,7 +95,7 @@ services:
|
|||||||
- sh
|
- sh
|
||||||
- -c
|
- -c
|
||||||
- |
|
- |
|
||||||
python -c "from app.services.webhooks.scheduler import bootstrap_webhook_dispatch_schedule; bootstrap_webhook_dispatch_schedule()" && \
|
python -c "from app.services.webhooks.scheduler import bootstrap_webhook_dispatch_schedule; from app.services.github.scheduler import bootstrap_mission_control_approval_check_schedule; bootstrap_webhook_dispatch_schedule(); bootstrap_mission_control_approval_check_schedule()" && \
|
||||||
rqscheduler -u "${WEBHOOK_REDIS_URL:-redis://redis:6379/0}" -i 60
|
rqscheduler -u "${WEBHOOK_REDIS_URL:-redis://redis:6379/0}" -i 60
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
- redis
|
||||||
@@ -105,6 +105,8 @@ services:
|
|||||||
WEBHOOK_QUEUE_NAME: webhook-dispatch
|
WEBHOOK_QUEUE_NAME: webhook-dispatch
|
||||||
WEBHOOK_DISPATCH_SCHEDULE_ID: webhook-dispatch-batch
|
WEBHOOK_DISPATCH_SCHEDULE_ID: webhook-dispatch-batch
|
||||||
WEBHOOK_DISPATCH_SCHEDULE_INTERVAL_SECONDS: ${WEBHOOK_DISPATCH_SCHEDULE_INTERVAL_SECONDS:-900}
|
WEBHOOK_DISPATCH_SCHEDULE_INTERVAL_SECONDS: ${WEBHOOK_DISPATCH_SCHEDULE_INTERVAL_SECONDS:-900}
|
||||||
|
GITHUB_APPROVAL_CHECK_SCHEDULE_ID: ${GITHUB_APPROVAL_CHECK_SCHEDULE_ID:-mission-control-approval-check-reconcile}
|
||||||
|
GITHUB_APPROVAL_CHECK_SCHEDULE_INTERVAL_SECONDS: ${GITHUB_APPROVAL_CHECK_SCHEDULE_INTERVAL_SECONDS:-900}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
Reference in New Issue
Block a user