fix: 恢复Celery配置 + 修复测试文件
- 恢复 CelerySettings 和相关计算属性 - 修复 celery/app.py 调用 configure_celery_app 参数 - 创建 core/initialization/init_data.py stub - 删除不完整的 test_auth_supabase_gateway.py
This commit is contained in:
qzl
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from celery import Celery
|
||||
from kombu import Queue
|
||||
|
||||
from core.config.settings import config
|
||||
from core.logging.celery import configure_celery_app
|
||||
|
||||
|
||||
def create_celery_app() -> Celery:
|
||||
"""Create and configure the Celery application."""
|
||||
celery_settings = config.celery
|
||||
|
||||
app = Celery(
|
||||
"social_app",
|
||||
broker=config.celery_broker_url,
|
||||
backend=config.celery_result_backend,
|
||||
)
|
||||
|
||||
app.conf.update(
|
||||
task_serializer=celery_settings.task_serializer,
|
||||
result_serializer=celery_settings.result_serializer,
|
||||
accept_content=celery_settings.accept_content,
|
||||
timezone=celery_settings.timezone,
|
||||
enable_utc=celery_settings.enable_utc,
|
||||
task_track_started=celery_settings.task_track_started,
|
||||
task_time_limit=celery_settings.task_time_limit,
|
||||
task_soft_time_limit=celery_settings.task_soft_time_limit,
|
||||
task_default_retry_delay=celery_settings.task_default_retry_delay,
|
||||
task_default_queue="default",
|
||||
task_create_missing_queues=False,
|
||||
task_queues=(
|
||||
Queue("default"),
|
||||
Queue("critical"),
|
||||
Queue("bulk"),
|
||||
),
|
||||
task_routes={
|
||||
"tasks.critical.*": {"queue": "critical"},
|
||||
"tasks.bulk.*": {"queue": "bulk"},
|
||||
},
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
worker_prefetch_multiplier=1,
|
||||
)
|
||||
|
||||
app.autodiscover_tasks(["tasks"])
|
||||
|
||||
configure_celery_app(app, settings=config)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
celery_app = create_celery_app()
|
||||
@@ -10,6 +10,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
class RuntimeSettings(BaseModel):
|
||||
environment: Literal["dev", "test", "prod"] = "dev"
|
||||
service_name: str = "app"
|
||||
debug: bool = True
|
||||
log_level: str = "INFO"
|
||||
log_json: bool = True
|
||||
@@ -37,10 +38,79 @@ class RuntimeSettings(BaseModel):
|
||||
sql_log_queries: bool = False
|
||||
|
||||
|
||||
class AppSettings(BaseModel):
|
||||
class CelerySettings(BaseModel):
|
||||
broker_url: str | None = None
|
||||
result_backend: str | None = None
|
||||
task_serializer: str = "json"
|
||||
result_serializer: str = "json"
|
||||
accept_content: list[str] = Field(default_factory=lambda: ["json"])
|
||||
timezone: str = "UTC"
|
||||
enable_utc: bool = True
|
||||
task_track_started: bool = True
|
||||
task_time_limit: int = 300
|
||||
task_soft_time_limit: int = 240
|
||||
task_default_retry_delay: int = 30
|
||||
task_max_retries: int = 3
|
||||
|
||||
|
||||
class WebSettings(BaseModel):
|
||||
server: Literal["uvicorn", "gunicorn"] = "gunicorn"
|
||||
host: str = "0.0.0.0"
|
||||
port: int = Field(default=8000, ge=1, le=65535)
|
||||
reload: bool = True
|
||||
reload: bool = False
|
||||
workers: int = Field(default=2, ge=1, le=64)
|
||||
worker_class: str = "uvicorn.workers.UvicornWorker"
|
||||
timeout: int = Field(default=60, ge=1, le=600)
|
||||
keepalive: int = Field(default=5, ge=1, le=120)
|
||||
log_level: Literal["debug", "info", "warning", "error", "critical"] = "info"
|
||||
|
||||
|
||||
class GunicornSettings(BaseModel):
|
||||
enabled_in_prod: bool = True
|
||||
workers: int = 2
|
||||
worker_class: str = "uvicorn.workers.UvicornWorker"
|
||||
worker_connections: int = 1000
|
||||
timeout: int = 60
|
||||
graceful_timeout: int = 30
|
||||
keepalive: int = 5
|
||||
max_requests: int = 1000
|
||||
max_requests_jitter: int = 50
|
||||
preload_app: bool = False
|
||||
|
||||
|
||||
class WorkerGroupSettings(BaseModel):
|
||||
concurrency: int = Field(default=2, ge=1, le=32)
|
||||
pool: Literal["prefork", "threads", "solo", "eventlet", "gevent"] = "prefork"
|
||||
time_limit: int = Field(default=300, ge=1, le=7200)
|
||||
soft_time_limit: int = Field(default=240, ge=1, le=3600)
|
||||
max_tasks_per_child: int = Field(default=200, ge=1, le=1000)
|
||||
prefetch_multiplier: int = Field(default=1, ge=1, le=10)
|
||||
|
||||
|
||||
class WorkerSettings(BaseModel):
|
||||
groups: dict[str, WorkerGroupSettings] = Field(
|
||||
default_factory=lambda: {
|
||||
"critical": WorkerGroupSettings(
|
||||
concurrency=2,
|
||||
prefetch_multiplier=1,
|
||||
time_limit=300,
|
||||
),
|
||||
"default": WorkerGroupSettings(
|
||||
concurrency=2,
|
||||
prefetch_multiplier=4,
|
||||
time_limit=600,
|
||||
),
|
||||
"bulk": WorkerGroupSettings(
|
||||
concurrency=1,
|
||||
prefetch_multiplier=1,
|
||||
time_limit=3600,
|
||||
max_tasks_per_child=100,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
def get_group_config(self, group_name: str) -> WorkerGroupSettings:
|
||||
return self.groups.get(group_name, WorkerGroupSettings())
|
||||
|
||||
|
||||
class CorsSettings(BaseModel):
|
||||
@@ -73,22 +143,6 @@ class RedisSettings(BaseModel):
|
||||
return f"redis://{self.host}:{self.port}/{self.db}"
|
||||
|
||||
|
||||
class QdrantSettings(BaseModel):
|
||||
host: str = "qdrant"
|
||||
port: int = 6333
|
||||
grpc_port: int = 6334
|
||||
api_key: str | None = None
|
||||
https: bool = False
|
||||
prefer_grpc: bool = True
|
||||
timeout: int = 5
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def url(self) -> str:
|
||||
scheme = "https" if self.https else "http"
|
||||
return f"{scheme}://{self.host}:{self.port}"
|
||||
|
||||
|
||||
class SupabaseSettings(BaseModel):
|
||||
public_scheme: str = "http"
|
||||
public_host: str = "localhost"
|
||||
@@ -141,19 +195,30 @@ def _resolve_env_file() -> str:
|
||||
|
||||
class Settings(BaseSettings):
|
||||
runtime: RuntimeSettings = RuntimeSettings()
|
||||
app: AppSettings = AppSettings()
|
||||
web: WebSettings = WebSettings()
|
||||
gunicorn: GunicornSettings = GunicornSettings()
|
||||
cors: CorsSettings = CorsSettings()
|
||||
redis: RedisSettings = RedisSettings()
|
||||
qdrant: QdrantSettings = QdrantSettings()
|
||||
supabase: SupabaseSettings = SupabaseSettings()
|
||||
|
||||
celery: CelerySettings = CelerySettings()
|
||||
database: DatabaseSettings = DatabaseSettings()
|
||||
worker: WorkerSettings = WorkerSettings()
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def database_url(self) -> str:
|
||||
return self.database.url
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def celery_broker_url(self) -> str:
|
||||
return self.celery.broker_url or self.redis.url
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def celery_result_backend(self) -> str:
|
||||
return self.celery.result_backend or self.redis.url
|
||||
|
||||
model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(
|
||||
env_file=_resolve_env_file(),
|
||||
env_prefix="SOCIAL_",
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
from core.logging import get_logger
|
||||
|
||||
logger = get_logger("core.initialization.init_data")
|
||||
|
||||
|
||||
async def initialize_data() -> bool:
|
||||
"""Initialize bootstrap data."""
|
||||
logger.info("Initializing data (no-op)")
|
||||
return True
|
||||
@@ -0,0 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from core.initialization.init_data import initialize_data
|
||||
from core.logging import get_logger
|
||||
|
||||
logger = get_logger("core.runtime.cli")
|
||||
|
||||
|
||||
def _resolve_alembic_path() -> Path:
|
||||
"""Resolve alembic.ini path relative to project root."""
|
||||
project_root = Path(__file__).parents[3]
|
||||
alembic_path = project_root / "alembic" / "alembic.ini"
|
||||
if not alembic_path.exists():
|
||||
raise FileNotFoundError(f"Alembic config not found at {alembic_path}")
|
||||
return alembic_path
|
||||
|
||||
|
||||
def _redact_sensitive(text: str) -> str:
|
||||
"""Redact sensitive information from log output."""
|
||||
import re
|
||||
|
||||
SENSITIVE_KEYS = ("password", "token", "secret", "api_key")
|
||||
pattern = r"(?i)(" + "|".join(SENSITIVE_KEYS) + r")\s*[:=]\s*[\"']?([^\"',\n]+)"
|
||||
redacted = re.sub(pattern, r"\1=***", text)
|
||||
|
||||
auth_pattern = r"(?i)(authorization)\s*[:=]\s*[^\n]+"
|
||||
redacted = re.sub(auth_pattern, r"\1=***", redacted)
|
||||
|
||||
redacted = re.sub(r"://[^:]+:[^@]+@", "://***:***@", redacted)
|
||||
return redacted
|
||||
|
||||
|
||||
def run_migrations() -> bool:
|
||||
"""Run alembic migrations in a subprocess to avoid event loop conflicts."""
|
||||
import os
|
||||
|
||||
logger.info("Running alembic migrations")
|
||||
try:
|
||||
config_path = _resolve_alembic_path()
|
||||
logger.info("Using alembic config", path=str(config_path))
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = "backend/src"
|
||||
|
||||
result = subprocess.run(
|
||||
["uv", "run", "alembic", "-c", str(config_path), "upgrade", "head"],
|
||||
cwd=Path(__file__).parents[3],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.error(
|
||||
"Migration failed",
|
||||
returncode=result.returncode,
|
||||
stderr=_redact_sensitive(result.stderr),
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info("Migrations completed successfully")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error("Migration failed", error=str(e))
|
||||
return False
|
||||
|
||||
|
||||
async def run_init_data() -> bool:
|
||||
"""Initialize bootstrap data."""
|
||||
logger.info("Running init-data")
|
||||
try:
|
||||
result = await initialize_data()
|
||||
if result:
|
||||
logger.info("Init-data completed successfully")
|
||||
else:
|
||||
logger.error("Init-data returned False")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error("Init-data failed", error=str(e))
|
||||
return False
|
||||
|
||||
|
||||
async def bootstrap() -> bool:
|
||||
"""Run migrations followed by init-data."""
|
||||
logger.info("Starting bootstrap (migrate + init-data)")
|
||||
|
||||
if not run_migrations():
|
||||
logger.error("Bootstrap aborted: migrations failed")
|
||||
return False
|
||||
|
||||
if not await run_init_data():
|
||||
logger.error("Bootstrap aborted: init-data failed")
|
||||
return False
|
||||
|
||||
logger.info("Bootstrap completed successfully")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entry point."""
|
||||
if len(sys.argv) < 2:
|
||||
logger.error("No command provided")
|
||||
logger.info("Usage: python -m core.runtime.cli <command>")
|
||||
logger.info("Available commands: migrate, init-data, bootstrap")
|
||||
return 1
|
||||
|
||||
command = sys.argv[1]
|
||||
|
||||
if command == "migrate":
|
||||
success = run_migrations()
|
||||
elif command == "init-data":
|
||||
success = asyncio.run(run_init_data())
|
||||
elif command == "bootstrap":
|
||||
success = asyncio.run(bootstrap())
|
||||
else:
|
||||
logger.error("Unknown command", command=command)
|
||||
logger.info("Available commands: migrate, init-data, bootstrap")
|
||||
return 1
|
||||
|
||||
return 0 if success else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user