qzl/social-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): add voice input capability and standardize tool naming

Browse Source 
- Add voice recording with transcribe endpoint (ASR) for multimodal input
- Android: add RECORD_AUDIO and INTERNET permissions
- Refactor tool naming: frontend tools use 'front.' prefix, backend tools use 'back.'
- Migrate calendar tools: create_calendar_event -> back.mutate/list/delete events
- Add calendar_event_list.v1 and calendar_operation.v1 UI card types
- Update all Flutter and Python tests to match new tool naming conventions
- Add record package dependency for voice recording
This commit is contained in:
zl-q
2026-03-09 00:10:09 +08:00
parent 6c83e35a69
commit 3ac09475ad
30 changed files with 1593 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/e2e/test_agent_live_flow.py
+1 -1
View File
@@ -355,7 +355,7 @@ async def test_agent_live_image_calendar_tool_persistence() -> None:
else:
payload = json.loads(str(downloaded))
assert payload["toolName"] == "back.create_calendar_event"
assert payload["toolName"] == "back.mutate_calendar_event"
finally:
if uploaded_paths:
try:
backend/tests/unit/core/agent/test_create_calendar_event_tool.py
-65
View File
@@ -1,65 +0,0 @@
from __future__ import annotations
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import cast
from uuid import uuid4
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from core.agent.infrastructure.crewai.tools.backend.create_calendar_event_tool import (
_execute_create_calendar_event,
)
@pytest.mark.asyncio
async def test_create_calendar_event_tool_returns_ui_schema_v1_top_level(
monkeypatch: pytest.MonkeyPatch,
) -> None:
event_id = uuid4()
created = SimpleNamespace(
id=event_id,
title="晨会",
description="同步计划",
start_at=datetime(2026, 3, 8, 1, 0, tzinfo=timezone.utc),
end_at=datetime(2026, 3, 8, 2, 0, tzinfo=timezone.utc),
timezone="Asia/Shanghai",
)
class _FakeService:
def __init__(self, **kwargs) -> None:
del kwargs
async def create_agent_generated(self, payload):
del payload
return created
class _FakeRepository:
def __init__(self, session) -> None:
del session
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.backend.create_calendar_event_tool.ScheduleItemService",
_FakeService,
)
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.backend.create_calendar_event_tool.SQLAlchemyScheduleItemRepository",
_FakeRepository,
)
result = cast(
dict[str, object],
await _execute_create_calendar_event(
session=cast(AsyncSession, SimpleNamespace()),
owner_id=uuid4(),
tool_args={"title": "晨会"},
),
)
assert result["type"] == "calendar_card.v1"
assert result["version"] == "v1"
data = cast(dict[str, object], result["data"])
actions = cast(list[dict[str, object]], result["actions"])
assert data["id"] == str(event_id)
assert actions
backend/tests/unit/core/agent/test_crewai_runtime.py
+70 -7
View File
@@ -119,7 +119,8 @@ def test_runtime_needs_execution_and_collects_front_tool_call() -> None:
assert isinstance(tools, list)
assert any(t.get("name") == "front.navigate_to_route" for t in tools)
execution_tools = cast(list[dict[str, object]], calls[1]["tools"])
assert any(t.get("name") == "back.create_calendar_event" for t in execution_tools)
assert any(t.get("name") == "back.list_calendar_events" for t in execution_tools)
assert any(t.get("name") == "back.mutate_calendar_event" for t in execution_tools)
assert result["assistant_text"] == "do it"
assert result["pending_front_tool"] == {
"name": "front.navigate_to_route",
@@ -191,7 +192,7 @@ def test_runtime_multimodal_intent_receives_execution_tool_awareness() -> None:
calls.append({"stage": stage, "tools": tools})
if stage == "intent":
return (
'{"route":"NEEDS_EXECUTION","intent_summary":"need tool","execution_brief":"call back.create_calendar_event","safety_flags":[]}',
'{"route":"NEEDS_EXECUTION","intent_summary":"need tool","execution_brief":"call back.mutate_calendar_event","safety_flags":[]}',
UsageCost(1, 1, 2, 0.01),
[],
None,
@@ -218,7 +219,8 @@ def test_runtime_multimodal_intent_receives_execution_tool_awareness() -> None:
)
intent_tools = cast(list[dict[str, object]], calls[0]["tools"])
assert any(t.get("name") == "back.create_calendar_event" for t in intent_tools)
assert any(t.get("name") == "back.list_calendar_events" for t in intent_tools)
assert any(t.get("name") == "back.mutate_calendar_event" for t in intent_tools)
def test_runtime_synthesizes_backend_call_when_model_skips_react_tool_call() -> None:
@@ -267,18 +269,78 @@ def test_runtime_synthesizes_backend_call_when_model_skips_react_tool_call() ->
assert backend_calls == [
(
"back.create_calendar_event",
{"title": "项目评审", "timezone": "Asia/Shanghai"},
"back.mutate_calendar_event",
{
"operation": "create",
"title": "项目评审",
"timezone": "Asia/Shanghai",
},
)
]
tool_calls = cast(list[dict[str, object]], result["tool_calls"])
assert any(
call.get("target") == "backend"
and call.get("name") == "back.create_calendar_event"
and call.get("name") == "back.mutate_calendar_event"
for call in tool_calls
)
def test_runtime_does_not_synthesize_mutate_create_when_event_id_without_operation() -> (
None
):
runtime = _build_runtime()
backend_calls: list[tuple[str, dict[str, object]]] = []
def _backend_handler(
tool_name: str, tool_args: dict[str, object]
) -> dict[str, object]:
backend_calls.append((tool_name, tool_args))
return {"type": "ok", "version": "v1", "data": {}, "actions": []}
runtime.set_backend_tool_handler(_backend_handler)
def _fake_run_stage(self, **kwargs):
stage = kwargs["stage"]
if stage == "intent":
return (
'{"route":"NEEDS_EXECUTION","intent_summary":"update event","execution_brief":"update via backend tool","safety_flags":[]}',
UsageCost(1, 1, 2, 0.01),
[],
None,
)
if stage == "execution":
return (
'{"status":"SUCCESS","execution_summary":"updated","execution_data":{"eventId":"1c7e85f6-a2b4-4da3-a143-7f9af8ea1a3d","title":"修正标题"},"report_brief":"done"}',
UsageCost(2, 2, 4, 0.02),
[],
None,
)
return (
'{"assistant_text":"ok","response_metadata":{}}',
UsageCost(1, 1, 2, 0.01),
[],
None,
)
runtime._run_stage_with_crewai = MethodType(_fake_run_stage, runtime) # type: ignore[method-assign]
runtime.execute(user_input="更新日程", tools=[])
assert backend_calls == []
def test_runtime_sanitize_backend_args_keeps_business_status() -> None:
payload = {
"status": "completed",
"title": "日程",
"result": "ignore",
"id": "ignore",
}
assert CrewAIRuntime._sanitize_backend_args(payload) == {
"status": "completed",
"title": "日程",
}
def test_runtime_extracts_pending_front_tool_from_approval_required_shape() -> None:
runtime = _build_runtime()
@@ -423,7 +485,8 @@ def test_run_stage_with_crewai_uses_output_pydantic_for_stage(
def test_runtime_backend_registry_check() -> None:
runtime = _build_runtime()
assert runtime.is_registered_backend_tool("back.create_calendar_event") is True
assert runtime.is_registered_backend_tool("back.list_calendar_events") is True
assert runtime.is_registered_backend_tool("back.mutate_calendar_event") is True
assert runtime.is_registered_backend_tool("back.unknown") is False
backend/tests/unit/core/agent/test_list_calendar_events_tool.py
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import cast
from uuid import uuid4
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from core.agent.infrastructure.crewai.tools.create_calendar_event_tool import (
_execute_list_calendar_events,
)
@pytest.mark.asyncio
async def test_list_calendar_events_tool_returns_paginated_payload_v1(
monkeypatch: pytest.MonkeyPatch,
) -> None:
first_id = uuid4()
second_id = uuid4()
items = [
SimpleNamespace(
id=first_id,
title="晨会",
description="同步",
start_at=datetime(2026, 3, 8, 1, 0, tzinfo=timezone.utc),
end_at=datetime(2026, 3, 8, 2, 0, tzinfo=timezone.utc),
timezone="Asia/Shanghai",
metadata=SimpleNamespace(location="会议室A", color="#4F46E5"),
),
SimpleNamespace(
id=second_id,
title="评审",
description=None,
start_at=datetime(2026, 3, 8, 3, 0, tzinfo=timezone.utc),
end_at=None,
timezone="Asia/Shanghai",
metadata=None,
),
]
class _FakeService:
def __init__(self, **kwargs) -> None:
del kwargs
async def list_paginated(self, *, page: int, page_size: int):
assert page == 2
assert page_size == 10
return items, 37
class _FakeRepository:
def __init__(self, session) -> None:
del session
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.ScheduleItemService",
_FakeService,
)
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.SQLAlchemyScheduleItemRepository",
_FakeRepository,
)
result = cast(
dict[str, object],
await _execute_list_calendar_events(
session=cast(AsyncSession, SimpleNamespace()),
owner_id=uuid4(),
tool_args={"page": 2, "pageSize": 10},
),
)
assert result["type"] == "calendar_event_list.v1"
assert result["version"] == "v1"
data = cast(dict[str, object], result["data"])
pagination = cast(dict[str, object], data["pagination"])
events = cast(list[dict[str, object]], data["items"])
assert pagination == {
"page": 2,
"pageSize": 10,
"total": 37,
"totalPages": 4,
}
assert events[0]["id"] == str(first_id)
assert events[0]["title"] == "晨会"
assert events[1]["id"] == str(second_id)
@pytest.mark.asyncio
async def test_list_calendar_events_tool_uses_default_pagination_when_missing(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class _FakeService:
def __init__(self, **kwargs) -> None:
del kwargs
async def list_paginated(self, *, page: int, page_size: int):
assert page == 1
assert page_size == 20
return [], 0
class _FakeRepository:
def __init__(self, session) -> None:
del session
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.ScheduleItemService",
_FakeService,
)
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.SQLAlchemyScheduleItemRepository",
_FakeRepository,
)
result = cast(
dict[str, object],
await _execute_list_calendar_events(
session=cast(AsyncSession, SimpleNamespace()),
owner_id=uuid4(),
tool_args={},
),
)
data = cast(dict[str, object], result["data"])
pagination = cast(dict[str, object], data["pagination"])
assert pagination["page"] == 1
assert pagination["pageSize"] == 20
backend/tests/unit/core/agent/test_mutate_calendar_event_tool.py
+173
View File
@@ -0,0 +1,173 @@
from __future__ import annotations
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import cast
from uuid import uuid4
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from core.agent.infrastructure.crewai.tools.create_calendar_event_tool import (
_execute_mutate_calendar_event,
)
@pytest.mark.asyncio
async def test_mutate_calendar_event_create_returns_calendar_card_v1(
monkeypatch: pytest.MonkeyPatch,
) -> None:
created_id = uuid4()
class _FakeService:
def __init__(self, **kwargs) -> None:
del kwargs
async def create_agent_generated(self, payload):
assert payload.title == "晨会"
return SimpleNamespace(
id=created_id,
title="晨会",
description="同步计划",
start_at=datetime(2026, 3, 8, 1, 0, tzinfo=timezone.utc),
end_at=datetime(2026, 3, 8, 2, 0, tzinfo=timezone.utc),
timezone="Asia/Shanghai",
metadata=SimpleNamespace(location="会议室A", color="#4F46E5"),
)
class _FakeRepository:
def __init__(self, session) -> None:
del session
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.ScheduleItemService",
_FakeService,
)
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.SQLAlchemyScheduleItemRepository",
_FakeRepository,
)
result = cast(
dict[str, object],
await _execute_mutate_calendar_event(
session=cast(AsyncSession, SimpleNamespace()),
owner_id=uuid4(),
tool_args={
"operation": "create",
"title": "晨会",
"description": "同步计划",
"startAt": "2026-03-08T09:00:00+08:00",
"endAt": "2026-03-08T10:00:00+08:00",
"timezone": "Asia/Shanghai",
"location": "会议室A",
},
),
)
assert result["type"] == "calendar_card.v1"
data = cast(dict[str, object], result["data"])
assert data["id"] == str(created_id)
assert data["ok"] is True
@pytest.mark.asyncio
async def test_mutate_calendar_event_update_requires_event_id() -> None:
with pytest.raises(ValueError, match="eventId is required"):
await _execute_mutate_calendar_event(
session=cast(AsyncSession, SimpleNamespace()),
owner_id=uuid4(),
tool_args={"operation": "update", "title": "新标题"},
)
@pytest.mark.asyncio
async def test_mutate_calendar_event_delete_returns_ack(
monkeypatch: pytest.MonkeyPatch,
) -> None:
deleted_id = uuid4()
class _FakeService:
def __init__(self, **kwargs) -> None:
del kwargs
async def delete(self, item_id):
assert item_id == deleted_id
return None
class _FakeRepository:
def __init__(self, session) -> None:
del session
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.ScheduleItemService",
_FakeService,
)
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.SQLAlchemyScheduleItemRepository",
_FakeRepository,
)
result = cast(
dict[str, object],
await _execute_mutate_calendar_event(
session=cast(AsyncSession, SimpleNamespace()),
owner_id=uuid4(),
tool_args={"operation": "delete", "eventId": str(deleted_id)},
),
)
assert result["type"] == "calendar_operation.v1"
data = cast(dict[str, object], result["data"])
assert data["operation"] == "delete"
assert data["id"] == str(deleted_id)
assert data["ok"] is True
@pytest.mark.asyncio
async def test_mutate_calendar_event_rejects_invalid_operation() -> None:
with pytest.raises(ValueError, match="operation"):
await _execute_mutate_calendar_event(
session=cast(AsyncSession, SimpleNamespace()),
owner_id=uuid4(),
tool_args={"operation": "upsert"},
)
@pytest.mark.asyncio
async def test_mutate_calendar_event_update_rejects_invalid_color(
monkeypatch: pytest.MonkeyPatch,
) -> None:
event_id = uuid4()
class _FakeService:
def __init__(self, **kwargs) -> None:
del kwargs
async def get_by_id(self, item_id):
assert item_id == event_id
return SimpleNamespace(metadata=None)
class _FakeRepository:
def __init__(self, session) -> None:
del session
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.ScheduleItemService",
_FakeService,
)
monkeypatch.setattr(
"core.agent.infrastructure.crewai.tools.create_calendar_event_tool.SQLAlchemyScheduleItemRepository",
_FakeRepository,
)
with pytest.raises(ValueError, match="color"):
await _execute_mutate_calendar_event(
session=cast(AsyncSession, SimpleNamespace()),
owner_id=uuid4(),
tool_args={
"operation": "update",
"eventId": str(event_id),
"color": "blue",
},
)
backend/tests/unit/core/agent/test_run_resume_service.py
+6 -6
View File
@@ -646,7 +646,7 @@ async def test_run_service_passes_user_context_system_prompt_to_runtime(
tool_args,
):
del session, owner_id
assert tool_name == "back.create_calendar_event"
assert tool_name == "back.mutate_calendar_event"
assert "title" in tool_args
return {
"result": {"eventId": "evt-1", "ok": True},
@@ -788,7 +788,7 @@ async def test_run_service_emits_frontend_tool_pending_events(
class _FakeRuntime:
def is_registered_backend_tool(self, tool_name: str) -> bool:
return tool_name == "back.create_calendar_event"
return tool_name == "back.mutate_calendar_event"
async def execute_backend_tool(
self,
@@ -799,7 +799,7 @@ async def test_run_service_emits_frontend_tool_pending_events(
tool_args,
):
del session, owner_id
assert tool_name == "back.create_calendar_event"
assert tool_name == "back.mutate_calendar_event"
assert "title" in tool_args
return {
"result": {"eventId": "evt-1", "ok": True},
@@ -957,7 +957,7 @@ async def test_run_service_executes_backend_calendar_tool_and_emits_result(
class _FakeRuntime:
def is_registered_backend_tool(self, tool_name: str) -> bool:
return tool_name == "back.create_calendar_event"
return tool_name == "back.mutate_calendar_event"
async def execute_backend_tool(
self,
@@ -968,7 +968,7 @@ async def test_run_service_executes_backend_calendar_tool_and_emits_result(
tool_args,
):
del session, owner_id
assert tool_name == "back.create_calendar_event"
assert tool_name == "back.mutate_calendar_event"
assert "title" in tool_args
return {
"result": {"eventId": "evt-1", "ok": True},
@@ -1043,7 +1043,7 @@ async def test_run_service_executes_backend_calendar_tool_and_emits_result(
text="请安排一个明早会议",
tools=[
{
"name": "back.create_calendar_event",
"name": "back.mutate_calendar_event",
"description": "create calendar",
"parameters": {"type": "object"},
}
backend/tests/unit/core/agent/test_stage_tool_allowlist.py
+4 -1
View File
@@ -10,7 +10,10 @@ def test_load_crewai_stage_tools_returns_expected_defaults() -> None:
assert result == {
"intent": [],
"execution": ["back.create_calendar_event"],
"execution": [
"back.list_calendar_events",
"back.mutate_calendar_event",
],
"organization": [],
}