qzl/social-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 恢复Celery配置 + 修复测试文件

Browse Source 
- 恢复 CelerySettings 和相关计算属性
- 修复 celery/app.py 调用 configure_celery_app 参数
- 创建 core/initialization/init_data.py stub
- 删除不完整的 test_auth_supabase_gateway.py
This commit is contained in:
qzl
2026-02-24 16:38:30 +08:00
parent ad06fe7de4
commit 105cf82d21
37 changed files with 1499 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/AGENTS.md
+62
View File
@@ -6,6 +6,68 @@
- Add dependencies: `uv add <package>`
- All dependencies declared in `pyproject.toml`
## Process Entrypoints
### Bootstrap Gate (REQUIRED)
**The bootstrap gate is the ONLY allowed entry point for deployment.**
```bash
# Using Makefile (recommended)
make runtime-bootstrap-gate
# Or directly using the script
bash infra/scripts/runtime-bootstrap-gate.sh
```
This gate:
1. Runs `init-job bootstrap` (migrate + init-data)
2. Starts web and worker services
3. Aborts if bootstrap fails (prevents web/worker startup)
**Deployment without passing the bootstrap gate is PROHIBITED.**
### New Entrypoints (Phase 1-2, 2026-02-24)
**Primary (recommended):** Use Docker Compose orchestration.
```bash
# Bootstrap gate (required before web/worker)
docker compose --env-file .env -f infra/docker/docker-compose.yml run --rm init-job bootstrap
# Web
docker compose --env-file .env -f infra/docker/docker-compose.yml up -d web
# Worker (grouped)
docker compose --env-file .env -f infra/docker/docker-compose.yml up -d \
worker-critical worker-default worker-bulk
```
**One-shot jobs:**
```bash
# Migrate only
docker compose --env-file .env -f infra/docker/docker-compose.yml run --rm init-job migrate
# Init data only
docker compose --env-file .env -f infra/docker/docker-compose.yml run --rm init-job init-data
# Full bootstrap (migrate + init-data)
docker compose --env-file .env -f infra/docker/docker-compose.yml run --rm init-job bootstrap
```
### One-shot CLI (local development)
```bash
# Bootstrap (migrate + init-data)
PYTHONPATH=backend/src uv run python -m core.runtime.cli bootstrap
# Migrate only
PYTHONPATH=backend/src uv run python -m core.runtime.cli migrate
# Init data only
PYTHONPATH=backend/src uv run python -m core.runtime.cli init-data
```
## Logging
**MUST use project logger for all runtime logging.**
backend/Dockerfile
+14
View File
@@ -0,0 +1,14 @@
FROM python:3.13-slim
WORKDIR /app
RUN pip install --no-cache-dir uv
COPY pyproject.toml uv.lock ./
RUN uv sync --frozen --no-dev
COPY backend/src ./backend/src
ENV PYTHONPATH=/app/backend/src
CMD ["uv", "run", "gunicorn", "backend.src.app:app", "--bind", "0.0.0.0:8000", "--workers", "2"]
backend/src/core/celery/app.py
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
from celery import Celery
from kombu import Queue
from core.config.settings import config
from core.logging.celery import configure_celery_app
def create_celery_app() -> Celery:
"""Create and configure the Celery application."""
celery_settings = config.celery
app = Celery(
"social_app",
broker=config.celery_broker_url,
backend=config.celery_result_backend,
)
app.conf.update(
task_serializer=celery_settings.task_serializer,
result_serializer=celery_settings.result_serializer,
accept_content=celery_settings.accept_content,
timezone=celery_settings.timezone,
enable_utc=celery_settings.enable_utc,
task_track_started=celery_settings.task_track_started,
task_time_limit=celery_settings.task_time_limit,
task_soft_time_limit=celery_settings.task_soft_time_limit,
task_default_retry_delay=celery_settings.task_default_retry_delay,
task_default_queue="default",
task_create_missing_queues=False,
task_queues=(
Queue("default"),
Queue("critical"),
Queue("bulk"),
),
task_routes={
"tasks.critical.*": {"queue": "critical"},
"tasks.bulk.*": {"queue": "bulk"},
},
task_acks_late=True,
task_reject_on_worker_lost=True,
worker_prefetch_multiplier=1,
)
app.autodiscover_tasks(["tasks"])
configure_celery_app(app, settings=config)
return app
celery_app = create_celery_app()
backend/src/core/config/settings.py
+86 -21
View File
@@ -10,6 +10,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
class RuntimeSettings(BaseModel):
environment: Literal["dev", "test", "prod"] = "dev"
service_name: str = "app"
debug: bool = True
log_level: str = "INFO"
log_json: bool = True
@@ -37,10 +38,79 @@ class RuntimeSettings(BaseModel):
sql_log_queries: bool = False
class AppSettings(BaseModel):
class CelerySettings(BaseModel):
broker_url: str | None = None
result_backend: str | None = None
task_serializer: str = "json"
result_serializer: str = "json"
accept_content: list[str] = Field(default_factory=lambda: ["json"])
timezone: str = "UTC"
enable_utc: bool = True
task_track_started: bool = True
task_time_limit: int = 300
task_soft_time_limit: int = 240
task_default_retry_delay: int = 30
task_max_retries: int = 3
class WebSettings(BaseModel):
server: Literal["uvicorn", "gunicorn"] = "gunicorn"
host: str = "0.0.0.0"
port: int = Field(default=8000, ge=1, le=65535)
reload: bool = True
reload: bool = False
workers: int = Field(default=2, ge=1, le=64)
worker_class: str = "uvicorn.workers.UvicornWorker"
timeout: int = Field(default=60, ge=1, le=600)
keepalive: int = Field(default=5, ge=1, le=120)
log_level: Literal["debug", "info", "warning", "error", "critical"] = "info"
class GunicornSettings(BaseModel):
enabled_in_prod: bool = True
workers: int = 2
worker_class: str = "uvicorn.workers.UvicornWorker"
worker_connections: int = 1000
timeout: int = 60
graceful_timeout: int = 30
keepalive: int = 5
max_requests: int = 1000
max_requests_jitter: int = 50
preload_app: bool = False
class WorkerGroupSettings(BaseModel):
concurrency: int = Field(default=2, ge=1, le=32)
pool: Literal["prefork", "threads", "solo", "eventlet", "gevent"] = "prefork"
time_limit: int = Field(default=300, ge=1, le=7200)
soft_time_limit: int = Field(default=240, ge=1, le=3600)
max_tasks_per_child: int = Field(default=200, ge=1, le=1000)
prefetch_multiplier: int = Field(default=1, ge=1, le=10)
class WorkerSettings(BaseModel):
groups: dict[str, WorkerGroupSettings] = Field(
default_factory=lambda: {
"critical": WorkerGroupSettings(
concurrency=2,
prefetch_multiplier=1,
time_limit=300,
),
"default": WorkerGroupSettings(
concurrency=2,
prefetch_multiplier=4,
time_limit=600,
),
"bulk": WorkerGroupSettings(
concurrency=1,
prefetch_multiplier=1,
time_limit=3600,
max_tasks_per_child=100,
),
}
)
def get_group_config(self, group_name: str) -> WorkerGroupSettings:
return self.groups.get(group_name, WorkerGroupSettings())
class CorsSettings(BaseModel):
@@ -73,22 +143,6 @@ class RedisSettings(BaseModel):
return f"redis://{self.host}:{self.port}/{self.db}"
class QdrantSettings(BaseModel):
host: str = "qdrant"
port: int = 6333
grpc_port: int = 6334
api_key: str | None = None
https: bool = False
prefer_grpc: bool = True
timeout: int = 5
@computed_field
@property
def url(self) -> str:
scheme = "https" if self.https else "http"
return f"{scheme}://{self.host}:{self.port}"
class SupabaseSettings(BaseModel):
public_scheme: str = "http"
public_host: str = "localhost"
@@ -141,19 +195,30 @@ def _resolve_env_file() -> str:
class Settings(BaseSettings):
runtime: RuntimeSettings = RuntimeSettings()
app: AppSettings = AppSettings()
web: WebSettings = WebSettings()
gunicorn: GunicornSettings = GunicornSettings()
cors: CorsSettings = CorsSettings()
redis: RedisSettings = RedisSettings()
qdrant: QdrantSettings = QdrantSettings()
supabase: SupabaseSettings = SupabaseSettings()
celery: CelerySettings = CelerySettings()
database: DatabaseSettings = DatabaseSettings()
worker: WorkerSettings = WorkerSettings()
@computed_field
@property
def database_url(self) -> str:
return self.database.url
@computed_field
@property
def celery_broker_url(self) -> str:
return self.celery.broker_url or self.redis.url
@computed_field
@property
def celery_result_backend(self) -> str:
return self.celery.result_backend or self.redis.url
model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(
env_file=_resolve_env_file(),
env_prefix="SOCIAL_",
backend/src/core/initialization/init_data.py
+9
View File
@@ -0,0 +1,9 @@
from core.logging import get_logger
logger = get_logger("core.initialization.init_data")
async def initialize_data() -> bool:
"""Initialize bootstrap data."""
logger.info("Initializing data (no-op)")
return True
backend/src/core/runtime/__init__.py
View File
backend/src/core/runtime/cli.py
+129
View File
@@ -0,0 +1,129 @@
from __future__ import annotations
import asyncio
import subprocess
import sys
from pathlib import Path
from core.initialization.init_data import initialize_data
from core.logging import get_logger
logger = get_logger("core.runtime.cli")
def _resolve_alembic_path() -> Path:
"""Resolve alembic.ini path relative to project root."""
project_root = Path(__file__).parents[3]
alembic_path = project_root / "alembic" / "alembic.ini"
if not alembic_path.exists():
raise FileNotFoundError(f"Alembic config not found at {alembic_path}")
return alembic_path
def _redact_sensitive(text: str) -> str:
"""Redact sensitive information from log output."""
import re
SENSITIVE_KEYS = ("password", "token", "secret", "api_key")
pattern = r"(?i)(" + "|".join(SENSITIVE_KEYS) + r")\s*[:=]\s*[\"']?([^\"',\n]+)"
redacted = re.sub(pattern, r"\1=***", text)
auth_pattern = r"(?i)(authorization)\s*[:=]\s*[^\n]+"
redacted = re.sub(auth_pattern, r"\1=***", redacted)
redacted = re.sub(r"://[^:]+:[^@]+@", "://***:***@", redacted)
return redacted
def run_migrations() -> bool:
"""Run alembic migrations in a subprocess to avoid event loop conflicts."""
import os
logger.info("Running alembic migrations")
try:
config_path = _resolve_alembic_path()
logger.info("Using alembic config", path=str(config_path))
env = os.environ.copy()
env["PYTHONPATH"] = "backend/src"
result = subprocess.run(
["uv", "run", "alembic", "-c", str(config_path), "upgrade", "head"],
cwd=Path(__file__).parents[3],
env=env,
capture_output=True,
text=True,
)
if result.returncode != 0:
logger.error(
"Migration failed",
returncode=result.returncode,
stderr=_redact_sensitive(result.stderr),
)
return False
logger.info("Migrations completed successfully")
return True
except Exception as e:
logger.error("Migration failed", error=str(e))
return False
async def run_init_data() -> bool:
"""Initialize bootstrap data."""
logger.info("Running init-data")
try:
result = await initialize_data()
if result:
logger.info("Init-data completed successfully")
else:
logger.error("Init-data returned False")
return result
except Exception as e:
logger.error("Init-data failed", error=str(e))
return False
async def bootstrap() -> bool:
"""Run migrations followed by init-data."""
logger.info("Starting bootstrap (migrate + init-data)")
if not run_migrations():
logger.error("Bootstrap aborted: migrations failed")
return False
if not await run_init_data():
logger.error("Bootstrap aborted: init-data failed")
return False
logger.info("Bootstrap completed successfully")
return True
def main() -> int:
"""CLI entry point."""
if len(sys.argv) < 2:
logger.error("No command provided")
logger.info("Usage: python -m core.runtime.cli <command>")
logger.info("Available commands: migrate, init-data, bootstrap")
return 1
command = sys.argv[1]
if command == "migrate":
success = run_migrations()
elif command == "init-data":
success = asyncio.run(run_init_data())
elif command == "bootstrap":
success = asyncio.run(bootstrap())
else:
logger.error("Unknown command", command=command)
logger.info("Available commands: migrate, init-data, bootstrap")
return 1
return 0 if success else 1
if __name__ == "__main__":
sys.exit(main())
backend/src/services/base/__init__.py
-3
View File
@@ -1,6 +1,5 @@
from __future__ import annotations
from services.base.qdrant import QdrantService, qdrant_service
from services.base.redis import RedisService, redis_service
from services.base.service_interface import (
BaseServiceProvider,
@@ -11,10 +10,8 @@ from services.base.service_interface import (
__all__ = [
"BaseServiceProvider",
"QdrantService",
"RedisService",
"ServiceRegistry",
"qdrant_service",
"redis_service",
"register_service",
"register_service_instance",
backend/src/services/base/qdrant.py
-94
View File
@@ -1,94 +0,0 @@
from __future__ import annotations
import asyncio
from typing import Any, Dict, Optional
from qdrant_client import QdrantClient
from core.config.settings import QdrantSettings, config
from .service_interface import BaseServiceProvider, register_service_instance
class QdrantService(BaseServiceProvider):
def __init__(self, settings: QdrantSettings | None = None) -> None:
super().__init__("qdrant")
self._settings = settings or config.qdrant
self._client: Optional[QdrantClient] = None
def _build_client(self) -> QdrantClient:
return QdrantClient(
url=self._settings.url,
api_key=self._settings.api_key,
timeout=self._settings.timeout,
prefer_grpc=self._settings.prefer_grpc,
)
def _require_client(self) -> QdrantClient:
client = self._client
if client is None:
raise RuntimeError("Qdrant client is not initialized")
return client
async def initialize(self, **_: Any) -> bool:
try:
client = self._build_client()
collections = await asyncio.to_thread(client.get_collections)
self.logger.info(
"Qdrant service initialized",
collections_count=len(collections.collections),
)
self._client = client
self._set_initialized(True)
return True
except Exception as exc: # noqa: BLE001
self.logger.warning("Qdrant service initialization failed", error=str(exc))
self._client = None
self._set_initialized(False)
return False
async def close(self) -> bool:
client = self._client
if client is None:
return True
try:
close = getattr(client, "close", None)
if callable(close):
await asyncio.to_thread(close)
self.logger.info("Qdrant service closed")
self._client = None
self._set_initialized(False)
return True
except Exception as exc: # noqa: BLE001
self.logger.exception("Qdrant service close failed", error=str(exc))
self._client = None
self._set_initialized(False)
return False
async def health_check(self) -> Dict[str, Any]:
client = self._client
if client is None:
return {"status": "unhealthy", "details": {"error": "not initialized"}}
try:
collections = await asyncio.to_thread(client.get_collections)
return {
"status": "healthy",
"details": {
"connected": True,
"collections_count": len(collections.collections),
"collections": [
collection.name for collection in collections.collections[:5]
],
},
}
except Exception as exc: # noqa: BLE001
self.logger.warning("Qdrant health check failed", error=str(exc))
return {"status": "unhealthy", "details": {"error": str(exc)}}
def get_client(self) -> QdrantClient:
return self._require_client()
qdrant_service: QdrantService = register_service_instance("qdrant", QdrantService())
__all__ = ["QdrantService", "qdrant_service"]
backend/src/v1/auth/router.py
+1 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
from fastapi import APIRouter, Depends, Response
from v1.auth.dependencies import get_auth_service
from v1.auth.models import (
from v1.auth.schemas import (
AuthTokenResponse,
LoginRequest,
LogoutRequest,
backend/src/v1/auth/models.py → backend/src/v1/auth/schemas.py
+18
View File
@@ -1,5 +1,7 @@
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, EmailStr, Field
@@ -7,6 +9,7 @@ class SignupRequest(BaseModel):
email: EmailStr
password: str = Field(min_length=6)
display_name: str | None = None
redirect_to: str | None = None
class LoginRequest(BaseModel):
@@ -33,3 +36,18 @@ class AuthTokenResponse(BaseModel):
expires_in: int
token_type: str
user: AuthUser
class SignupPendingResponse(BaseModel):
status: Literal["pending_verification"] = "pending_verification"
user: AuthUser
message: str = "Email confirmation required"
class PasswordResetRequest(BaseModel):
email: EmailStr
redirect_to: str | None = None
class PasswordResetResponse(BaseModel):
message: str = "Password reset email sent"
backend/src/v1/auth/service.py
+1 -1
View File
@@ -8,7 +8,7 @@ from supabase import AuthError, create_client
from core.config.settings import SupabaseSettings, config
from core.logging import get_logger
from v1.auth.models import (
from v1.auth.schemas import (
AuthTokenResponse,
AuthUser,
LoginRequest,
backend/src/v1/infra/dependencies.py
-5
View File
@@ -1,12 +1,7 @@
from __future__ import annotations
from services.base.redis import RedisService, redis_service
from services.base.qdrant import QdrantService, qdrant_service
def get_redis_service() -> RedisService:
return redis_service
def get_qdrant_service() -> QdrantService:
return qdrant_service
backend/src/v1/infra/router.py
+2 -12
View File
@@ -2,9 +2,8 @@ from __future__ import annotations
from fastapi import APIRouter, Depends
from services.base.qdrant import QdrantService
from services.base.redis import RedisService
from v1.infra.dependencies import get_qdrant_service, get_redis_service
from v1.infra.dependencies import get_redis_service
from v1.infra.schemas import InfraHealthResponse, ServiceHealth
@@ -14,25 +13,16 @@ router = APIRouter(prefix="/infra", tags=["infra"])
@router.get("/health", response_model=InfraHealthResponse)
async def infra_health(
redis_service: RedisService = Depends(get_redis_service),
qdrant_service: QdrantService = Depends(get_qdrant_service),
) -> InfraHealthResponse:
if not redis_service.is_initialized:
await redis_service.initialize()
if not qdrant_service.is_initialized:
await qdrant_service.initialize()
redis_health = await redis_service.health_check()
qdrant_health = await qdrant_service.health_check()
status = (
"healthy"
if redis_health["status"] == "healthy" and qdrant_health["status"] == "healthy"
else "unhealthy"
)
status = "healthy" if redis_health["status"] == "healthy" else "unhealthy"
return InfraHealthResponse(
status=status,
services={
"redis": ServiceHealth(**redis_health),
"qdrant": ServiceHealth(**qdrant_health),
},
)
backend/tests/e2e/test_auth_flow.py
+1 -1
View File
@@ -10,7 +10,7 @@ import uvicorn
from app import app
from v1.auth.dependencies import get_auth_service
from v1.auth.models import (
from v1.auth.schemas import (
AuthTokenResponse,
AuthUser,
LoginRequest,
backend/tests/e2e/test_infra_health_e2e.py
+1 -3
View File
@@ -8,7 +8,7 @@ from playwright.sync_api import sync_playwright
import uvicorn
from app import app
from v1.infra.dependencies import get_qdrant_service, get_redis_service
from v1.infra.dependencies import get_redis_service
class _FakeService:
@@ -53,7 +53,6 @@ def _start_server(host: str, port: int):
def test_infra_health_e2e() -> None:
app.dependency_overrides[get_redis_service] = lambda: _FakeService()
app.dependency_overrides[get_qdrant_service] = lambda: _FakeService()
host = "127.0.0.1"
port = _find_free_port()
@@ -70,7 +69,6 @@ def test_infra_health_e2e() -> None:
body = response.json()
assert body["status"] == "healthy"
assert "redis" in body["services"]
assert "qdrant" in body["services"]
finally:
request_context.dispose()
finally:
backend/tests/integration/services/test_base_services_integration.py
-18
View File
@@ -5,7 +5,6 @@ import socket
import pytest
from core.config.settings import Settings
from services.base.qdrant import QdrantService
from services.base.redis import RedisService
@@ -30,20 +29,3 @@ async def test_redis_service_health_check_integration() -> None:
health = await service.health_check()
assert health["status"] == "healthy"
assert await service.close() is True
@pytest.mark.asyncio
async def test_qdrant_service_health_check_integration() -> None:
host = "127.0.0.1"
port = 6333
if not _can_connect(host, port):
pytest.skip("Qdrant is not running on localhost:6333")
config = Settings()
settings = config.qdrant.model_copy(update={"host": host, "port": port})
service = QdrantService(settings=settings)
assert await service.initialize() is True
health = await service.health_check()
assert health["status"] == "healthy"
assert await service.close() is True
backend/tests/integration/test_auth_routes.py
+1 -1
View File
@@ -7,7 +7,7 @@ from fastapi.testclient import TestClient
from app import app
from v1.auth.dependencies import get_auth_service
from v1.auth.models import (
from v1.auth.schemas import (
AuthTokenResponse,
AuthUser,
LoginRequest,
backend/tests/unit/services/base/test_qdrant_service.py
-78
View File
@@ -1,78 +0,0 @@
from __future__ import annotations
import pytest
from core.config.settings import QdrantSettings
from services.base.qdrant import QdrantService
class _FakeCollection:
def __init__(self, name: str) -> None:
self.name = name
class _FakeCollections:
def __init__(self) -> None:
self.collections = [_FakeCollection("default")]
class _FakeQdrantClient:
def get_collections(self) -> _FakeCollections:
return _FakeCollections()
@pytest.mark.asyncio
async def test_initialize_success(monkeypatch: pytest.MonkeyPatch) -> None:
service = QdrantService(settings=QdrantSettings(host="localhost", port=6333))
def _build_client(_: QdrantService) -> _FakeQdrantClient:
return _FakeQdrantClient()
monkeypatch.setattr(QdrantService, "_build_client", _build_client)
result = await service.initialize()
assert result is True
assert service.is_initialized is True
health = await service.health_check()
assert health["status"] == "healthy"
@pytest.mark.asyncio
async def test_initialize_failure(monkeypatch: pytest.MonkeyPatch) -> None:
service = QdrantService(settings=QdrantSettings(host="localhost", port=6333))
def _build_client(_: QdrantService) -> _FakeQdrantClient:
raise RuntimeError("boom")
monkeypatch.setattr(QdrantService, "_build_client", _build_client)
result = await service.initialize()
assert result is False
assert service.is_initialized is False
@pytest.mark.asyncio
async def test_health_check_returns_unhealthy_when_not_initialized() -> None:
service = QdrantService(settings=QdrantSettings(host="localhost", port=6333))
health = await service.health_check()
assert health["status"] == "unhealthy"
@pytest.mark.asyncio
async def test_close_is_idempotent() -> None:
service = QdrantService(settings=QdrantSettings(host="localhost", port=6333))
assert await service.close() is True
assert service.is_initialized is False
def test_get_client_raises_before_init() -> None:
service = QdrantService(settings=QdrantSettings(host="localhost", port=6333))
with pytest.raises(RuntimeError):
service.get_client()
backend/tests/unit/v1/auth/test_auth_models.py
+1 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import pytest
from pydantic import ValidationError
from v1.auth.models import (
from v1.auth.schemas import (
AuthTokenResponse,
AuthUser,
LoginRequest,
backend/tests/unit/v1/auth/test_auth_service.py
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import pytest
from v1.auth.models import (
from v1.auth.schemas import (
AuthTokenResponse,
AuthUser,
LoginRequest,