qzl/social-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 添加 Agent 步骤事件与图片附件功能

Browse Source 
- 新增 stepStarted/stepFinished 事件类型支持
- 前端实现图片附件上传和预览功能
- 后端增强工具结果存储和事件处理
- 完善相关单元测试和集成测试
This commit is contained in:
zl-q
2026-03-12 09:29:57 +08:00
parent 87215f9d41
commit 7b8865e256
45 changed files with 3869 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/unit/v1/agent/test_attachment_storage.py
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
import v1.agent.attachment_storage as attachment_storage_module
class _FakeBucket:
def __init__(self) -> None:
self.upload_calls: list[tuple[str, bytes, dict[str, str]]] = []
self.download_calls: list[str] = []
def upload(self, path: str, content: bytes, options: dict[str, str]) -> object:
self.upload_calls.append((path, content, options))
return {"path": path}
def download(self, path: str) -> object:
self.download_calls.append(path)
return b"ok"
class _FakeStorage:
def __init__(self, bucket: _FakeBucket) -> None:
self._bucket = bucket
def from_(self, bucket: str) -> object:
del bucket
return self._bucket
@pytest.mark.asyncio
async def test_attachment_storage_rejects_unexpected_bucket(
monkeypatch: pytest.MonkeyPatch,
) -> None:
storage = attachment_storage_module.AgentAttachmentStorage()
monkeypatch.setattr(
attachment_storage_module.config.storage,
"bucket",
"allowed-bucket",
)
with pytest.raises(RuntimeError, match="Invalid attachment bucket"):
await storage.upload_bytes(
bucket="other-bucket",
path="agent-inputs/u/t/r/file.png",
content=b"data",
content_type="image/png",
)
@pytest.mark.asyncio
async def test_attachment_storage_accepts_configured_bucket(
monkeypatch: pytest.MonkeyPatch,
) -> None:
storage = attachment_storage_module.AgentAttachmentStorage()
fake_bucket = _FakeBucket()
fake_client = SimpleNamespace(storage=_FakeStorage(fake_bucket))
monkeypatch.setattr(
attachment_storage_module.config.storage,
"bucket",
"allowed-bucket",
)
monkeypatch.setattr(
attachment_storage_module.supabase_service,
"get_admin_client",
lambda: fake_client,
)
path = await storage.upload_bytes(
bucket="allowed-bucket",
path="agent-inputs/u/t/r/file.png",
content=b"data",
content_type="image/png",
)
payload = await storage.download_bytes(
bucket="allowed-bucket",
path=path,
)
assert path == "agent-inputs/u/t/r/file.png"
assert payload == b"ok"
assert len(fake_bucket.upload_calls) == 1
assert fake_bucket.download_calls == ["agent-inputs/u/t/r/file.png"]
backend/tests/unit/v1/agent/test_repository.py
+182 -9
View File
@@ -6,6 +6,7 @@ from uuid import uuid4
import pytest
from core.config.settings import config
from models.agent_chat_message import AgentChatMessageRole
from v1.agent.repository import AgentRepository
@@ -62,7 +63,7 @@ async def test_tool_message_hydrates_content_from_object_storage() -> None:
content='{"offloaded":true}',
metadata_json={
"tool_call_id": "call-1",
"storage_bucket": "private",
"storage_bucket": config.storage.bucket,
"storage_path": "tool-results/run-1/call-1.json",
},
)
@@ -73,6 +74,43 @@ async def test_tool_message_hydrates_content_from_object_storage() -> None:
assert payload["content"] == "已跳转"
@pytest.mark.asyncio
async def test_tool_message_hydrates_ui_from_ui_schema_field() -> None:
repository = AgentRepository(
session=SimpleNamespace(), # type: ignore[arg-type]
tool_result_storage=_FakeToolResultStorage(
{
"toolName": "calendar_write",
"ui_schema": {
"type": "calendar_operation.v1",
"version": "v1",
"data": {"ok": True, "operation": "create"},
"actions": [],
},
}
),
)
message = SimpleNamespace(
id=uuid4(),
role=AgentChatMessageRole.TOOL,
created_at=datetime.now(timezone.utc),
content="已创建日程：项目评审（明天 10:00）",
metadata_json={
"tool_call_id": "call-3",
"storage_bucket": config.storage.bucket,
"storage_path": "tool-results/run-1/call-3.json",
},
)
payload = await repository._to_snapshot_message(message) # type: ignore[arg-type]
assert payload["toolCallId"] == "call-3"
assert payload["content"] == "已创建日程：项目评审（明天 10:00）"
ui = payload.get("ui")
assert isinstance(ui, dict)
assert ui["type"] == "calendar_operation.v1"
@pytest.mark.asyncio
async def test_tool_message_keeps_inline_content_when_storage_payload_missing() -> None:
repository = AgentRepository(
@@ -86,7 +124,7 @@ async def test_tool_message_keeps_inline_content_when_storage_payload_missing()
content="inline-tool-content",
metadata_json={
"tool_call_id": "call-2",
"storage_bucket": "private",
"storage_bucket": config.storage.bucket,
"storage_path": "tool-results/run-1/call-2.json",
},
)
@@ -97,6 +135,111 @@ async def test_tool_message_keeps_inline_content_when_storage_payload_missing()
assert payload["content"] == "inline-tool-content"
@pytest.mark.asyncio
async def test_tool_message_skips_storage_when_path_not_matching_session() -> None:
repository = AgentRepository(
session=SimpleNamespace(), # type: ignore[arg-type]
tool_result_storage=_FakeToolResultStorage(
{
"ui_schema": {
"type": "calendar_operation.v1",
"version": "v1",
"data": {"ok": True},
"actions": [],
}
}
),
)
message = SimpleNamespace(
id=uuid4(),
session_id=uuid4(),
role=AgentChatMessageRole.TOOL,
created_at=datetime.now(timezone.utc),
content="summary",
metadata_json={
"tool_call_id": "call-x",
"storage_bucket": config.storage.bucket,
"storage_path": "tool-results/foreign-session/call-y.json",
},
)
payload = await repository._to_snapshot_message(message) # type: ignore[arg-type]
assert payload["content"] == "summary"
assert "ui" not in payload
@pytest.mark.asyncio
async def test_tool_message_rejects_path_traversal() -> None:
repository = AgentRepository(
session=SimpleNamespace(), # type: ignore[arg-type]
tool_result_storage=_FakeToolResultStorage(
{
"ui_schema": {
"type": "calendar_operation.v1",
"version": "v1",
"data": {"ok": True},
"actions": [],
}
}
),
)
message = SimpleNamespace(
id=uuid4(),
session_id=uuid4(),
role=AgentChatMessageRole.TOOL,
created_at=datetime.now(timezone.utc),
content="summary",
metadata_json={
"tool_call_id": "call-z",
"storage_bucket": config.storage.bucket,
"storage_path": "tool-results/ok/../../evil/call-z.json",
},
)
payload = await repository._to_snapshot_message(message) # type: ignore[arg-type]
assert payload["content"] == "summary"
assert "ui" not in payload
@pytest.mark.asyncio
async def test_tool_message_supports_legacy_storage_path() -> None:
repository = AgentRepository(
session=SimpleNamespace(), # type: ignore[arg-type]
tool_result_storage=_FakeToolResultStorage(
{
"ui_schema": {
"type": "calendar_operation.v1",
"version": "v1",
"data": {"ok": True},
"actions": [],
},
"content": "legacy content",
}
),
)
message = SimpleNamespace(
id=uuid4(),
session_id=uuid4(),
role=AgentChatMessageRole.TOOL,
created_at=datetime.now(timezone.utc),
content='{"offloaded":true}',
metadata_json={
"tool_call_id": "call-legacy",
"storage_bucket": config.storage.bucket,
"storage_path": "tool-results/old-run/call-legacy.json",
},
)
payload = await repository._to_snapshot_message(message) # type: ignore[arg-type]
assert payload["content"] == "legacy content"
ui = payload.get("ui")
assert isinstance(ui, dict)
assert ui["type"] == "calendar_operation.v1"
@pytest.mark.asyncio
async def test_user_message_snapshot_includes_renderable_attachments() -> None:
repository = AgentRepository(
@@ -104,6 +247,7 @@ async def test_user_message_snapshot_includes_renderable_attachments() -> None:
)
message = SimpleNamespace(
id=uuid4(),
session_id=uuid4(),
role=AgentChatMessageRole.USER,
created_at=datetime.now(timezone.utc),
content="请分析这张图",
@@ -122,13 +266,13 @@ async def test_user_message_snapshot_includes_renderable_attachments() -> None:
assert payload["role"] == "user"
assert payload["content"] == "请分析这张图"
assert payload["attachments"] == [
{
"bucket": "agent-chat-attachments",
"path": "agent-inputs/u1/t1/r1/m1/att-1.png",
"mimeType": "image/png",
}
]
attachments = payload.get("attachments")
assert isinstance(attachments, list)
assert len(attachments) == 1
first = attachments[0]
assert isinstance(first, dict)
assert first["mimeType"] == "image/png"
assert isinstance(first.get("previewPath"), str)
@pytest.mark.asyncio
@@ -174,3 +318,32 @@ async def test_persist_user_message_keeps_existing_session_title() -> None:
assert session_row.title == "已有标题"
assert session_row.message_count == 2
@pytest.mark.asyncio
async def test_get_message_attachment_reference_returns_item() -> None:
session_id = str(uuid4())
message_id = str(uuid4())
message = SimpleNamespace(
metadata_json={
"attachments": [
{
"bucket": "bucket-test",
"path": "agent-inputs/u/t/r/a.png",
"mimeType": "image/png",
}
]
}
)
fake_session = _FakeSession(message)
repository = AgentRepository(session=fake_session) # type: ignore[arg-type]
ref = await repository.get_message_attachment_reference(
session_id=session_id,
message_id=message_id,
attachment_index=0,
)
assert ref is not None
assert ref["bucket"] == "bucket-test"
assert ref["mimeType"] == "image/png"
backend/tests/unit/v1/agent/test_router_guards.py
+41
View File
@@ -225,3 +225,44 @@ async def test_stream_events_retries_on_redis_timeout(
merged = "".join(chunks)
assert "event: RUN_FINISHED" in merged
@pytest.mark.asyncio
async def test_get_attachment_preview_rejects_negative_index() -> None:
class _Service:
async def get_attachment_preview(self, **kwargs): # noqa: ANN003
del kwargs
raise AssertionError("get_attachment_preview should not be called")
with pytest.raises(HTTPException) as exc_info:
await agent_router.get_attachment_preview(
thread_id="00000000-0000-0000-0000-000000000001",
message_id="00000000-0000-0000-0000-000000000010",
attachment_index=-1,
service=cast(Any, _Service()),
current_user=CurrentUser(id=uuid4(), email="user@example.com"),
)
assert exc_info.value.status_code == 422
@pytest.mark.asyncio
async def test_get_attachment_preview_returns_streaming_response() -> None:
class _Service:
async def get_attachment_preview(self, **kwargs): # noqa: ANN003
del kwargs
return b"png-bytes", "image/png"
response = await agent_router.get_attachment_preview(
thread_id="00000000-0000-0000-0000-000000000001",
message_id="00000000-0000-0000-0000-000000000010",
attachment_index=0,
service=cast(Any, _Service()),
current_user=CurrentUser(id=uuid4(), email="user@example.com"),
)
chunks: list[bytes] = []
async for chunk in response.body_iterator:
chunks.append(cast(bytes, chunk))
assert response.media_type == "image/png"
assert b"".join(chunks) == b"png-bytes"
backend/tests/unit/v1/agent/test_service.py
+341 -13
View File
@@ -6,8 +6,10 @@ from uuid import UUID
from ag_ui.core import RunAgentInput
from fastapi import HTTPException
import pytest
from sqlalchemy.exc import IntegrityError
from core.auth.models import CurrentUser
from core.config.settings import config
import v1.agent.service as agent_service_module
from v1.agent.service import AgentService, AsrService
@@ -74,12 +76,32 @@ class _FakeRepository:
}
)
async def get_message_attachment_reference(
self,
*,
session_id: str,
message_id: str,
attachment_index: int,
) -> dict[str, str] | None:
del session_id, message_id
if attachment_index != 0:
return None
return {
"bucket": config.storage.bucket,
"path": "agent-inputs/00000000-0000-0000-0000-000000000001/00000000-0000-0000-0000-000000000001/run-1/attachment-0-a.png",
"mimeType": "image/png",
}
class _FakeQueue:
def __init__(self) -> None:
self.commands: list[dict[str, object]] = []
async def enqueue(
self, *, command: dict[str, object], dedup_key: str | None
) -> str:
del command, dedup_key
self.commands.append(command)
del dedup_key
return "task-1"
@@ -123,6 +145,33 @@ class _FakeAttachmentStorage:
)
return path
async def download_bytes(self, *, bucket: str, path: str) -> bytes:
self.calls.append(
{
"bucket": bucket,
"path": path,
"download": True,
}
)
return b"png-bytes"
async def create_signed_url(
self,
*,
bucket: str,
path: str,
expires_in_seconds: int,
) -> str:
self.calls.append(
{
"bucket": bucket,
"path": path,
"signed": True,
"expires_in_seconds": expires_in_seconds,
}
)
return f"https://signed.example/{path}?exp={expires_in_seconds}"
class _AlwaysFailAttachmentStorage:
async def upload_bytes(
@@ -136,6 +185,20 @@ class _AlwaysFailAttachmentStorage:
del bucket, path, content, content_type
raise RuntimeError("upload failed")
async def download_bytes(self, *, bucket: str, path: str) -> bytes:
del bucket, path
raise RuntimeError("download failed")
async def create_signed_url(
self,
*,
bucket: str,
path: str,
expires_in_seconds: int,
) -> str:
del bucket, path, expires_in_seconds
raise RuntimeError("sign failed")
def _user() -> CurrentUser:
return CurrentUser(
@@ -186,9 +249,10 @@ async def test_resume_idempotency_uses_redis_lock_and_task_key() -> None:
async def test_enqueue_run_creates_missing_thread_session() -> None:
repository = _FakeRepository()
queue = _FakeQueue()
service = AgentService(
repository=repository,
queue=_FakeQueue(),
queue=queue,
stream=_FakeStream(),
)
run_input = _build_run_input(
@@ -206,6 +270,30 @@ async def test_enqueue_run_creates_missing_thread_session() -> None:
assert accepted.created is True
assert repository.created_with_session_id == "00000000-0000-0000-0000-000000000999"
assert repository.committed is True
assert queue.commands[0]["user_token"] is None
async def test_enqueue_run_uses_explicit_user_token() -> None:
repository = _FakeRepository()
queue = _FakeQueue()
service = AgentService(
repository=repository,
queue=queue,
stream=_FakeStream(),
)
run_input = _build_run_input(
thread_id="00000000-0000-0000-0000-000000000001",
run_id="run-1",
)
await service.enqueue_run(
run_input=run_input,
current_user=_user(),
user_token="Bearer access-token-1",
)
assert queue.commands
assert queue.commands[0]["user_token"] == "access-token-1"
async def test_enqueue_run_keeps_created_session_when_enqueue_fails() -> None:
@@ -270,7 +358,7 @@ async def test_enqueue_run_handles_session_create_race() -> None:
assert repository.rolled_back is True
async def test_enqueue_run_uploads_user_image_to_supabase_and_injects_metadata(
async def test_enqueue_run_uses_forwarded_attachments_and_injects_metadata(
monkeypatch,
) -> None:
monkeypatch.setattr(
@@ -297,15 +385,23 @@ async def test_enqueue_run_uploads_user_image_to_supabase_and_injects_metadata(
{"type": "text", "text": "帮我看下这张图"},
{
"type": "binary",
"data": "aGVsbG8=",
"mimeType": "image/png",
"url": "https://signed.example/upload.png",
},
],
}
],
"tools": [],
"context": [],
"forwardedProps": {},
"forwardedProps": {
"attachments": [
{
"bucket": "agent-test-bucket",
"path": "agent-inputs/00000000-0000-0000-0000-000000000001/00000000-0000-0000-0000-000000000001/upload.png",
"mimeType": "image/png",
}
]
},
}
)
@@ -313,10 +409,9 @@ async def test_enqueue_run_uploads_user_image_to_supabase_and_injects_metadata(
assert accepted.task_id == "task-1"
assert len(attachment_storage.calls) == 1
upload = attachment_storage.calls[0]
assert upload["bucket"] == "agent-test-bucket"
assert upload["content"] == b"hello"
assert upload["content_type"] == "image/png"
download = attachment_storage.calls[0]
assert download["bucket"] == "agent-test-bucket"
assert download["download"] is True
assert repository.persisted_user_messages
persisted = repository.persisted_user_messages[0]
assert persisted["session_id"] == "00000000-0000-0000-0000-000000000001"
@@ -330,7 +425,7 @@ async def test_enqueue_run_uploads_user_image_to_supabase_and_injects_metadata(
assert isinstance(attachments[0]["path"], str)
async def test_enqueue_run_raises_when_attachment_upload_fails_without_fallback(
async def test_enqueue_run_raises_when_attachment_download_fails_without_fallback(
monkeypatch,
) -> None:
monkeypatch.setattr(
@@ -356,15 +451,23 @@ async def test_enqueue_run_raises_when_attachment_upload_fails_without_fallback(
{"type": "text", "text": "帮我看下这张图"},
{
"type": "binary",
"data": "aGVsbG8=",
"mimeType": "image/png",
"url": "https://signed.example/upload.png",
},
],
}
],
"tools": [],
"context": [],
"forwardedProps": {},
"forwardedProps": {
"attachments": [
{
"bucket": "agent-test-bucket",
"path": "agent-inputs/00000000-0000-0000-0000-000000000001/00000000-0000-0000-0000-000000000001/upload.png",
"mimeType": "image/png",
}
]
},
}
)
@@ -373,11 +476,183 @@ async def test_enqueue_run_raises_when_attachment_upload_fails_without_fallback(
raise AssertionError("expected HTTPException")
except HTTPException as exc:
assert exc.status_code == 502
assert exc.detail == "Failed to upload attachment"
assert exc.detail == "Failed to fetch attachment"
assert repository.persisted_user_messages == []
async def test_enqueue_run_rejects_unsupported_attachment_type(
monkeypatch,
) -> None:
monkeypatch.setattr(
agent_service_module.config.storage, "bucket", "agent-test-bucket"
)
repository = _FakeRepository()
attachment_storage = _FakeAttachmentStorage()
service = AgentService(
repository=repository,
queue=_FakeQueue(),
stream=_FakeStream(),
attachment_storage=attachment_storage,
)
run_input = RunAgentInput.model_validate(
{
"threadId": "00000000-0000-0000-0000-000000000001",
"runId": "run-with-bad-image",
"state": {},
"messages": [
{
"id": "u1",
"role": "user",
"content": [
{"type": "text", "text": "请看附件"},
{
"type": "binary",
"mimeType": "image/gif",
"url": "https://signed.example/upload.gif",
},
],
}
],
"tools": [],
"context": [],
"forwardedProps": {
"attachments": [
{
"bucket": "agent-test-bucket",
"path": "agent-inputs/00000000-0000-0000-0000-000000000001/00000000-0000-0000-0000-000000000001/upload.gif",
"mimeType": "image/gif",
}
]
},
}
)
with pytest.raises(HTTPException) as exc_info:
await service.enqueue_run(run_input=run_input, current_user=_user())
assert exc_info.value.status_code == 422
assert exc_info.value.detail == "Unsupported attachment type"
assert attachment_storage.calls == []
async def test_enqueue_run_rejects_attachment_too_large(
monkeypatch,
) -> None:
monkeypatch.setattr(agent_service_module, "_MAX_ATTACHMENT_BYTES", 4)
monkeypatch.setattr(
agent_service_module.config.storage, "bucket", "agent-test-bucket"
)
repository = _FakeRepository()
attachment_storage = _FakeAttachmentStorage()
service = AgentService(
repository=repository,
queue=_FakeQueue(),
stream=_FakeStream(),
attachment_storage=attachment_storage,
)
run_input = RunAgentInput.model_validate(
{
"threadId": "00000000-0000-0000-0000-000000000001",
"runId": "run-with-big-image",
"state": {},
"messages": [
{
"id": "u1",
"role": "user",
"content": [
{"type": "text", "text": "请看附件"},
{
"type": "binary",
"mimeType": "image/png",
"url": "https://signed.example/upload.png",
},
],
}
],
"tools": [],
"context": [],
"forwardedProps": {
"attachments": [
{
"bucket": "agent-test-bucket",
"path": "agent-inputs/00000000-0000-0000-0000-000000000001/00000000-0000-0000-0000-000000000001/upload.png",
"mimeType": "image/png",
}
]
},
}
)
with pytest.raises(HTTPException) as exc_info:
await service.enqueue_run(run_input=run_input, current_user=_user())
assert exc_info.value.status_code == 413
assert exc_info.value.detail == "Attachment too large"
assert len(attachment_storage.calls) == 1
assert attachment_storage.calls[0]["download"] is True
async def test_enqueue_run_accepts_binary_url_and_persists_metadata() -> None:
repository = _FakeRepository()
queue = _FakeQueue()
attachment_storage = _FakeAttachmentStorage()
service = AgentService(
repository=repository,
queue=queue,
stream=_FakeStream(),
attachment_storage=attachment_storage,
)
run_input = RunAgentInput.model_validate(
{
"threadId": "00000000-0000-0000-0000-000000000001",
"runId": "run-with-binary-url",
"state": {},
"messages": [
{
"id": "u1",
"role": "user",
"content": [
{"type": "text", "text": "请分析"},
{
"type": "binary",
"mimeType": "image/png",
"url": "https://signed.example/upload-1.png",
},
],
}
],
"tools": [],
"context": [],
"forwardedProps": {
"attachments": [
{
"bucket": config.storage.bucket,
"path": "agent-inputs/00000000-0000-0000-0000-000000000001/00000000-0000-0000-0000-000000000001/upload-1.png",
"mimeType": "image/png",
}
]
},
}
)
accepted = await service.enqueue_run(run_input=run_input, current_user=_user())
assert accepted.task_id == "task-1"
persisted = repository.persisted_user_messages[-1]
metadata = persisted["metadata"]
assert isinstance(metadata, dict)
attachments = metadata.get("attachments")
assert isinstance(attachments, list)
assert attachments[0]["path"].endswith("upload-1.png")
queue_input = queue.commands[-1]["run_input"]
assert isinstance(queue_input, dict)
content = queue_input["messages"][0]["content"]
assert isinstance(content, list)
assert content[1]["type"] == "binary"
assert content[1]["url"] == "https://signed.example/upload-1.png"
async def test_get_history_snapshot_wraps_history_day_as_state_snapshot_event() -> None:
service = AgentService(
repository=_FakeRepository(),
@@ -415,6 +690,59 @@ async def test_get_user_history_snapshot_uses_latest_thread_when_absent() -> Non
assert event["threadId"] == "00000000-0000-0000-0000-000000000001"
async def test_get_attachment_preview_returns_payload_and_mime() -> None:
service = AgentService(
repository=_FakeRepository(),
queue=_FakeQueue(),
stream=_FakeStream(),
attachment_storage=_FakeAttachmentStorage(),
)
payload, mime_type = await service.get_attachment_preview(
thread_id="00000000-0000-0000-0000-000000000001",
message_id="00000000-0000-0000-0000-000000000010",
attachment_index=0,
current_user=_user(),
)
assert payload == b"png-bytes"
assert mime_type == "image/png"
async def test_get_attachment_preview_rejects_invalid_path() -> None:
class _BadPathRepository(_FakeRepository):
async def get_message_attachment_reference(
self,
*,
session_id: str,
message_id: str,
attachment_index: int,
) -> dict[str, str] | None:
del session_id, message_id, attachment_index
return {
"bucket": "bucket-test",
"path": "agent-inputs/other-user/other-thread/run-1/a.png",
"mimeType": "image/png",
}
service = AgentService(
repository=_BadPathRepository(),
queue=_FakeQueue(),
stream=_FakeStream(),
attachment_storage=_FakeAttachmentStorage(),
)
with pytest.raises(HTTPException) as exc_info:
await service.get_attachment_preview(
thread_id="00000000-0000-0000-0000-000000000001",
message_id="00000000-0000-0000-0000-000000000010",
attachment_index=0,
current_user=_user(),
)
assert exc_info.value.status_code == 403
async def test_asr_service_parses_dict_output_sentence(monkeypatch) -> None:
result = SimpleNamespace(
status_code=200,