qzl/social-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: unify storage config keys and refresh local dev setup

Browse Source
This commit is contained in:
qzl
2026-03-26 13:25:25 +08:00
parent b765b9e3e1
commit 5900993ee7
61 changed files with 1164 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/alembic/versions/20260326_0001_drop_dedup_backup_tables.py
+24
View File
@@ -0,0 +1,24 @@
"""drop obsolete dedup backup tables from public schema
Revision ID: 202603260001
Revises: 202603250001
Create Date: 2026-03-26 14:20:00
"""
from typing import Sequence, Union
from alembic import op
revision: str = "202603260001"
down_revision: Union[str, Sequence[str], None] = "202603250001"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute("DROP TABLE IF EXISTS public.automation_jobs_dedup_backup_202603230003")
op.execute("DROP TABLE IF EXISTS public.memories_dedup_backup_202603230003")
def downgrade() -> None:
"""Downgrade intentionally unsupported for obsolete backup table removal."""
backend/src/core/config/settings.py
+13 -2
View File
@@ -148,11 +148,22 @@ class SupabaseSettings(BaseModel):
class StorageSettings(BaseModel):
provider: Literal["supabase"] = "supabase"
bucket: str = Field(default="agent-chat-attachments", min_length=3, max_length=63)
signed_url_ttl_seconds: int = Field(default=600, ge=60, le=3600)
max_file_size_mb: int = Field(default=20, ge=1, le=200)
retention_days: int = Field(default=30, ge=1, le=3650)
class AttachmentSettings(BaseModel):
bucket: str = Field(
default="agent-chat-attachments", min_length=3, max_length=63
)
max_size_mb: int = Field(default=20, ge=1, le=200)
class AvatarSettings(BaseModel):
bucket: str = Field(default="avatars", min_length=3, max_length=63)
max_size_mb: int = Field(default=2, ge=1, le=10)
attachment: AttachmentSettings = Field(default_factory=AttachmentSettings)
avatar: AvatarSettings = Field(default_factory=AvatarSettings)
class AgentRuntimeSettings(BaseModel):
redis_stream_prefix: str = "agent:events"
backend/src/core/config/static/database/system_agents.yaml
+3 -1
View File
@@ -18,7 +18,9 @@ agents:
temperature: 0.7
max_tokens: null
timeout_seconds: 30
context_messages: null
context_messages:
mode: number
count: 20
enabled_tools:
- calendar.read
- calendar.write
backend/src/core/runtime/cli.py
+4
View File
@@ -107,6 +107,10 @@ async def bootstrap() -> bool:
async def run_automation_scheduler_forever() -> None:
if config.runtime.environment == "dev":
logger.info("Automation scheduler skipped in dev environment")
return
if not config.automation_scheduler.enabled:
logger.info("Automation scheduler disabled by config")
return
backend/src/services/base/supabase.py
+42 -28
View File
@@ -7,7 +7,6 @@ from supabase import create_client
from storage3.exceptions import StorageApiError
from core.config.settings import SupabaseSettings, config
from core.config.settings import config as app_config
from .service_interface import BaseServiceProvider, register_service_instance
@@ -100,7 +99,6 @@ class SupabaseService(BaseServiceProvider):
)
async def _ensure_storage_bucket(self) -> None:
bucket_name = app_config.storage.bucket
storage = getattr(self._admin_client, "storage", None)
if storage is None:
self.logger.warning("Storage client unavailable, skipping bucket check")
@@ -111,32 +109,45 @@ class SupabaseService(BaseServiceProvider):
self.logger.warning("Storage get_bucket unavailable, skipping bucket check")
return
buckets = [
(config.storage.attachment.bucket, False),
(config.storage.avatar.bucket, True),
]
def _check_and_create() -> None:
try:
get_bucket(bucket_name)
self.logger.debug("Storage bucket already exists", bucket=bucket_name)
except Exception: # noqa: BLE001
create_bucket = getattr(storage, "create_bucket", None)
if not callable(create_bucket):
self.logger.warning(
"Storage create_bucket unavailable, skipping bucket creation"
)
return
for bucket_name, is_public in buckets:
try:
create_bucket(bucket_name, options={"public": False})
self.logger.info("Storage bucket created", bucket=bucket_name)
except Exception as exc: # noqa: BLE001
msg = str(exc).lower()
if "already exists" in msg or "duplicate" in msg:
self.logger.debug(
"Storage bucket already exists (race)", bucket=bucket_name
get_bucket(bucket_name)
self.logger.debug(
"Storage bucket already exists", bucket=bucket_name
)
except Exception: # noqa: BLE001
create_bucket = getattr(storage, "create_bucket", None)
if not callable(create_bucket):
self.logger.warning(
"Storage create_bucket unavailable, skipping bucket creation"
)
return
self.logger.warning(
"Failed to create storage bucket",
bucket=bucket_name,
error=str(exc),
)
try:
create_bucket(bucket_name, options={"public": is_public})
self.logger.info(
"Storage bucket created",
bucket=bucket_name,
public=is_public,
)
except Exception as exc: # noqa: BLE001
msg = str(exc).lower()
if "already exists" in msg or "duplicate" in msg:
self.logger.debug(
"Storage bucket already exists (race)",
bucket=bucket_name,
)
continue
self.logger.warning(
"Failed to create storage bucket",
bucket=bucket_name,
error=str(exc),
)
await asyncio.to_thread(_check_and_create)
@@ -157,10 +168,13 @@ class SupabaseService(BaseServiceProvider):
return from_bucket(bucket)
def _validate_bucket(self, bucket: str) -> None:
"""Validate that the bucket matches the configured bucket."""
expected = app_config.storage.bucket
if bucket != expected:
raise RuntimeError("Invalid attachment bucket")
"""Validate that the bucket matches one of configured storage buckets."""
allowed_buckets = {
config.storage.attachment.bucket,
config.storage.avatar.bucket,
}
if bucket not in allowed_buckets:
raise RuntimeError("Invalid storage bucket")
def _ensure_bucket_client(self, bucket: str) -> Any:
"""Validate bucket and return authenticated bucket client."""
backend/src/v1/agent/service.py
+3 -3
View File
@@ -333,7 +333,7 @@ class AgentService:
f"agent-inputs/{current_user.id}/{thread_id}/uploads/"
f"{filename_hash}-{checksum}.{suffix}"
)
bucket_name = config.storage.bucket
bucket_name = config.storage.attachment.bucket
try:
stored_path = await self._attachment_storage.upload_bytes(
bucket=bucket_name,
@@ -377,7 +377,7 @@ class AgentService:
status_code=503, detail="Attachment storage unavailable"
)
normalized_bucket = bucket.strip()
if normalized_bucket != config.storage.bucket:
if normalized_bucket != config.storage.attachment.bucket:
raise HTTPException(status_code=422, detail="Invalid attachment bucket")
normalized_path = path.strip()
@@ -540,7 +540,7 @@ class AgentService:
status_code=422, detail="Invalid signed image url"
) from exc
if bucket != config.storage.bucket:
if bucket != config.storage.attachment.bucket:
raise HTTPException(status_code=422, detail="INVALID_BINARY_URL_BUCKET")
expected_prefix = f"agent-inputs/{current_user.id}/{thread_id}/uploads/"
backend/src/v1/users/router.py
+19 -2
View File
@@ -3,11 +3,11 @@ from __future__ import annotations
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, File, UploadFile, status
from schemas.shared.user import UserContext
from v1.users.dependencies import get_user_service
from v1.users.schemas import UserSearchRequest, UserUpdateRequest
from v1.users.schemas import AvatarUploadResponse, UserSearchRequest, UserUpdateRequest
from v1.users.service import UserService
@@ -29,6 +29,23 @@ async def update_me(
return await service.update_me(payload)
@router.post(
"/me/avatar",
response_model=AvatarUploadResponse,
status_code=status.HTTP_200_OK,
)
async def upload_avatar(
service: Annotated[UserService, Depends(get_user_service)],
file: UploadFile = File(),
) -> AvatarUploadResponse:
payload = await file.read()
return await service.upload_avatar(
filename=file.filename,
content_type=file.content_type,
payload=payload,
)
@router.post("/search", response_model=list[UserContext])
async def search_users(
payload: UserSearchRequest,
backend/src/v1/users/schemas.py
+4
View File
@@ -38,3 +38,7 @@ class UserUpdateRequest(BaseModel):
if self.username is None and self.avatar_url is None and self.bio is None:
raise ValueError("At least one field must be provided")
return self
class AvatarUploadResponse(BaseModel):
url: str = Field(description="Public URL of the uploaded avatar")
backend/src/v1/users/service.py
+90 -1
View File
@@ -11,11 +11,13 @@ from core.agentscope.caches.user_context_cache import (
create_user_context_cache,
)
from core.auth.models import CurrentUser
from core.config.settings import config
from core.db.base_service import BaseService
from core.logging import get_logger
from schemas.shared.user import UserContext, parse_profile_settings
from services.base.supabase import supabase_service
from v1.users.repository import UserRepository
from v1.users.schemas import UserSearchRequest, UserUpdateRequest
from v1.users.schemas import AvatarUploadResponse, UserSearchRequest, UserUpdateRequest
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
@@ -27,6 +29,16 @@ logger = get_logger("v1.users.service")
_PHONE_QUERY_PATTERN = re.compile(r"^[+()\-\s\d]{4,32}$")
def _mime_to_suffix(mime_type: str) -> str:
"""Convert MIME type to file suffix."""
mapping = {
"image/jpeg": "jpg",
"image/png": "png",
"image/webp": "webp",
}
return mapping.get(mime_type, "bin")
class AuthLookupGateway(Protocol):
async def search_user_ids_by_phone(
self, query: str, limit: int = 20
@@ -164,6 +176,83 @@ class UserService(BaseService):
settings=parse_profile_settings(user.settings),
)
async def upload_avatar(
self,
*,
filename: str | None,
content_type: str | None,
payload: bytes,
) -> AvatarUploadResponse:
user_id = self.require_user_id()
if not isinstance(content_type, str):
raise HTTPException(status_code=422, detail="Unsupported image type")
mime_type = content_type.lower()
allowed_types = {"image/jpeg", "image/png", "image/webp"}
if mime_type not in allowed_types:
raise HTTPException(
status_code=422,
detail="Unsupported image type. Allowed: JPEG, PNG, WebP",
)
max_size_bytes = config.storage.avatar.max_size_mb * 1024 * 1024
if len(payload) > max_size_bytes:
raise HTTPException(
status_code=413,
detail=f"Image too large. Maximum size: {config.storage.avatar.max_size_mb}MB",
)
if not payload:
raise HTTPException(status_code=422, detail="Empty image")
suffix = _mime_to_suffix(mime_type)
path = f"{user_id}/avatar.{suffix}"
bucket_name = config.storage.avatar.bucket
try:
stored_path = await supabase_service.upload_bytes(
bucket=bucket_name,
path=path,
content=payload,
content_type=mime_type,
)
except Exception: # noqa: BLE001
logger.exception(
"Avatar upload failed",
extra={
"bucket": bucket_name,
"path": path,
"mime_type": mime_type,
"user_id": str(user_id),
},
)
raise HTTPException(status_code=502, detail="Failed to upload avatar")
public_url = f"{config.supabase.public_url}/storage/v1/object/public/{bucket_name}/{stored_path}"
update_data: dict[str, str | None] = {"avatar_url": public_url}
try:
user = await self._repository.update_by_user_id(user_id, update_data)
await self._session.commit()
except SQLAlchemyError:
await self._session.rollback()
raise HTTPException(status_code=503, detail="User store unavailable")
if user is None:
raise HTTPException(status_code=404, detail="User not found")
try:
await self._user_context_cache.invalidate_user(user_id=user_id)
except Exception as exc:
logger.warning(
"Failed to invalidate user context cache after avatar upload",
user_id=str(user_id),
error=str(exc),
)
return AvatarUploadResponse(url=public_url)
async def get_by_username(self, username: str) -> UserContext:
try:
user = await self._repository.get_by_username(username)
backend/tests/unit/services/base/test_supabase_service.py
+12
View File
@@ -159,3 +159,15 @@ def test_get_admin_client_lazily_initializes_clients(
assert service.get_client() is anon_client
assert service.is_initialized is True
assert len(create_calls) == 2
def test_validate_bucket_accepts_attachment_and_avatar() -> None:
service = SupabaseService(
settings=SupabaseSettings(public_url="https://test.supabase.co")
)
service._validate_bucket("agent-chat-attachments")
service._validate_bucket("avatars")
with pytest.raises(RuntimeError):
service._validate_bucket("unexpected-bucket")
backend/tests/unit/test_settings_storage_env.py
+8 -4
View File
@@ -11,17 +11,21 @@ def test_social_prefixed_storage_env_populates_settings(
monkeypatch: MonkeyPatch,
) -> None:
monkeypatch.setenv("SOCIAL_STORAGE__PROVIDER", "supabase")
monkeypatch.setenv("SOCIAL_STORAGE__BUCKET", "agent-chat-attachments")
monkeypatch.setenv("SOCIAL_STORAGE__ATTACHMENT__BUCKET", "agent-chat-attachments")
monkeypatch.setenv("SOCIAL_STORAGE__AVATAR__BUCKET", "avatars")
monkeypatch.setenv("SOCIAL_STORAGE__SIGNED_URL_TTL_SECONDS", "900")
monkeypatch.setenv("SOCIAL_STORAGE__MAX_FILE_SIZE_MB", "25")
monkeypatch.setenv("SOCIAL_STORAGE__ATTACHMENT__MAX_SIZE_MB", "25")
monkeypatch.setenv("SOCIAL_STORAGE__AVATAR__MAX_SIZE_MB", "3")
monkeypatch.setenv("SOCIAL_STORAGE__RETENTION_DAYS", "45")
settings = Settings()
assert settings.storage.provider == "supabase"
assert settings.storage.bucket == "agent-chat-attachments"
assert settings.storage.attachment.bucket == "agent-chat-attachments"
assert settings.storage.avatar.bucket == "avatars"
assert settings.storage.signed_url_ttl_seconds == 900
assert settings.storage.max_file_size_mb == 25
assert settings.storage.attachment.max_size_mb == 25
assert settings.storage.avatar.max_size_mb == 3
assert settings.storage.retention_days == 45