refactor(runtime): decouple scheduler tasks and align worker group names
This commit is contained in:
@@ -2,7 +2,6 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from datetime import timezone
|
||||
from typing import Any, cast
|
||||
from uuid import UUID
|
||||
|
||||
@@ -17,11 +16,6 @@ from core.agentscope.runtime.context_service import AgentContextService
|
||||
from core.agentscope.runtime.orchestrator import AgentScopeRuntimeOrchestrator
|
||||
from core.agentscope.runtime.pipeline_registry import build_default_pipeline_spec
|
||||
from core.agentscope.schemas.agui_input import parse_run_input
|
||||
from core.automation.scheduler import (
|
||||
AutomationSchedulerService,
|
||||
SqlAlchemyAutomationSchedulerRepository,
|
||||
utc_now,
|
||||
)
|
||||
from core.auth.models import CurrentUser
|
||||
from core.config.settings import config
|
||||
from core.db.session import AsyncSessionLocal
|
||||
@@ -46,18 +40,6 @@ logger = get_logger("core.agentscope.runtime.tasks")
|
||||
_MAX_CONTEXT_ATTACHMENTS = 3
|
||||
|
||||
|
||||
class _BulkQueueAdapter:
|
||||
async def enqueue(
|
||||
self,
|
||||
*,
|
||||
command: dict[str, object],
|
||||
dedup_key: str | None,
|
||||
) -> str:
|
||||
del dedup_key
|
||||
result = await run_command_task_bulk.kiq(command)
|
||||
return str(result.task_id)
|
||||
|
||||
|
||||
def _serialize_tool_agent_output(
|
||||
*,
|
||||
metadata: AgentChatMessageMetadata | dict[str, object] | None,
|
||||
@@ -314,35 +296,6 @@ async def run_agentscope_task(command: dict[str, Any]) -> dict[str, object]:
|
||||
}
|
||||
|
||||
|
||||
async def run_automation_scheduler_scan(
|
||||
*,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, int]:
|
||||
now = utc_now()
|
||||
safe_limit = (
|
||||
max(int(limit), 1)
|
||||
if isinstance(limit, int)
|
||||
else int(config.automation_scheduler.batch_limit)
|
||||
)
|
||||
async with AsyncSessionLocal() as session:
|
||||
repository = SqlAlchemyAutomationSchedulerRepository(session=session)
|
||||
service = AutomationSchedulerService(
|
||||
repository=repository,
|
||||
queue=_BulkQueueAdapter(),
|
||||
)
|
||||
result = await service.scan_and_dispatch(now_utc=now, limit=safe_limit)
|
||||
logger.info(
|
||||
"automation scheduler scan completed",
|
||||
scanned=result.scanned,
|
||||
dispatched=result.dispatched,
|
||||
now_utc=now.astimezone(timezone.utc).isoformat(),
|
||||
)
|
||||
return {
|
||||
"scanned": int(result.scanned),
|
||||
"dispatched": int(result.dispatched),
|
||||
}
|
||||
|
||||
|
||||
@default_broker.task(task_name="tasks.agentscope.run_command")
|
||||
async def run_command_task(command: dict[str, Any]) -> dict[str, object]:
|
||||
return await run_agentscope_task(command)
|
||||
@@ -356,8 +309,3 @@ async def run_command_task_critical(command: dict[str, Any]) -> dict[str, object
|
||||
@bulk_broker.task(task_name="tasks.agentscope.run_command.bulk")
|
||||
async def run_command_task_bulk(command: dict[str, Any]) -> dict[str, object]:
|
||||
return await run_agentscope_task(command)
|
||||
|
||||
|
||||
@default_broker.task(task_name="tasks.automation.scan_due_jobs")
|
||||
async def scan_due_automation_jobs_task(limit: int | None = None) -> dict[str, int]:
|
||||
return await run_automation_scheduler_scan(limit=limit)
|
||||
|
||||
@@ -8,6 +8,7 @@ from uuid import UUID, uuid4
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from core.config.settings import config
|
||||
from core.logging import get_logger
|
||||
from models.agent_chat_session import AgentChatSession, SessionType
|
||||
from models.automation_jobs import AutomationJob, ScheduleType
|
||||
@@ -17,6 +18,20 @@ from schemas.automation.scheduler import DueAutomationJob, SchedulerDispatchComm
|
||||
logger = get_logger("core.automation.scheduler")
|
||||
|
||||
|
||||
class _BulkQueueAdapter:
|
||||
async def enqueue(
|
||||
self,
|
||||
*,
|
||||
command: dict[str, object],
|
||||
dedup_key: str | None,
|
||||
) -> str:
|
||||
del dedup_key
|
||||
from core.agentscope.runtime.tasks import run_command_task_bulk
|
||||
|
||||
result = await run_command_task_bulk.kiq(command)
|
||||
return str(result.task_id)
|
||||
|
||||
|
||||
class QueueLike(Protocol):
|
||||
async def enqueue(
|
||||
self,
|
||||
@@ -245,3 +260,34 @@ def _compute_next_run_at(
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
async def run_automation_scheduler_scan(
|
||||
*,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, int]:
|
||||
now = utc_now()
|
||||
safe_limit = (
|
||||
max(int(limit), 1)
|
||||
if isinstance(limit, int)
|
||||
else int(config.automation_scheduler.batch_limit)
|
||||
)
|
||||
from core.db.session import AsyncSessionLocal
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
repository = SqlAlchemyAutomationSchedulerRepository(session=session)
|
||||
service = AutomationSchedulerService(
|
||||
repository=repository,
|
||||
queue=_BulkQueueAdapter(),
|
||||
)
|
||||
result = await service.scan_and_dispatch(now_utc=now, limit=safe_limit)
|
||||
logger.info(
|
||||
"automation scheduler scan completed",
|
||||
scanned=result.scanned,
|
||||
dispatched=result.dispatched,
|
||||
now_utc=now.astimezone(timezone.utc).isoformat(),
|
||||
)
|
||||
return {
|
||||
"scanned": int(result.scanned),
|
||||
"dispatched": int(result.dispatched),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from core.logging import get_logger
|
||||
from core.taskiq.app import bulk_broker
|
||||
|
||||
logger = get_logger("core.automation.tasks")
|
||||
|
||||
|
||||
@bulk_broker.task(task_name="tasks.automation.scan_due_jobs")
|
||||
async def scan_due_automation_jobs_task(limit: int | None = None) -> dict[str, int]:
|
||||
from core.automation.scheduler import run_automation_scheduler_scan
|
||||
|
||||
return await run_automation_scheduler_scan(limit=limit)
|
||||
@@ -5,7 +5,10 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from core.agentscope.runtime.tasks import run_automation_scheduler_scan
|
||||
from apscheduler.schedulers.blocking import BlockingScheduler
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
from core.automation.scheduler import run_automation_scheduler_scan
|
||||
from core.config.settings import config
|
||||
from core.config.initial.init_data import initialize_data
|
||||
from core.logging import get_logger
|
||||
@@ -103,24 +106,34 @@ async def bootstrap() -> bool:
|
||||
return True
|
||||
|
||||
|
||||
async def run_automation_scheduler_forever() -> bool:
|
||||
async def run_automation_scheduler_forever() -> None:
|
||||
if not config.automation_scheduler.enabled:
|
||||
logger.info("Automation scheduler disabled by config")
|
||||
return True
|
||||
return
|
||||
|
||||
interval_seconds = int(config.automation_scheduler.interval_seconds)
|
||||
batch_limit = int(config.automation_scheduler.batch_limit)
|
||||
logger.info(
|
||||
"Starting automation scheduler loop",
|
||||
"Starting automation scheduler",
|
||||
interval_seconds=interval_seconds,
|
||||
batch_limit=batch_limit,
|
||||
)
|
||||
while True:
|
||||
|
||||
def scan_job() -> None:
|
||||
try:
|
||||
await run_automation_scheduler_scan(limit=batch_limit)
|
||||
asyncio.run(run_automation_scheduler_scan(limit=batch_limit))
|
||||
except Exception as exc:
|
||||
logger.exception("Automation scheduler scan failed", error=str(exc))
|
||||
await asyncio.sleep(interval_seconds)
|
||||
|
||||
scheduler = BlockingScheduler()
|
||||
scheduler.add_job(
|
||||
scan_job,
|
||||
trigger=IntervalTrigger(seconds=interval_seconds),
|
||||
id="automation_scheduler_scan",
|
||||
name="Automation scheduler scan",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.start()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -142,7 +155,8 @@ def main() -> int:
|
||||
elif command == "bootstrap":
|
||||
success = asyncio.run(bootstrap())
|
||||
elif command == "automation-scheduler":
|
||||
success = asyncio.run(run_automation_scheduler_forever())
|
||||
asyncio.run(run_automation_scheduler_forever())
|
||||
return 0
|
||||
else:
|
||||
logger.error("Unknown command", command=command)
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user