diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..284007c --- /dev/null +++ b/.env.example @@ -0,0 +1,162 @@ +# 统一环境变量配置模板（根目录 .env.example） +# 使用方法：复制到 .env 并填写实际值 +# 警告：切勿将包含真实密钥的 .env 提交到代码仓库 +# 命名规则：前缀 SOCIAL_，层级分隔符 __（例如 SOCIAL_INFRA__POSTGRES__PASSWORD） + +############ +# 运行时配置（API 后端使用） +############ +# 运行环境：dev（开发）、test（测试）、prod（生产） +SOCIAL_RUNTIME__ENVIRONMENT=dev +# 调试模式：true 开启详细日志和错误堆栈，false 生产环境建议关闭 +SOCIAL_RUNTIME__DEBUG=true +# 日志级别：DEBUG、INFO、WARNING、ERROR、CRITICAL +SOCIAL_RUNTIME__LOG_LEVEL=INFO +# 是否记录 SQL 查询日志：开发调试时可开启，生产环境建议关闭 +SOCIAL_RUNTIME__SQL_LOG_QUERIES=false + +############ +# 应用配置（API 后端使用） +############ +# API 服务监听地址：0.0.0.0 表示所有网络接口，本地开发可用 127.0.0.1 +SOCIAL_APP__HOST=0.0.0.0 +# API 服务监听端口 +SOCIAL_APP__PORT=8000 +# 是否启用代码热重载：开发环境 true，生产环境 false +SOCIAL_APP__RELOAD=true + +############ +# 基础设施密钥（Docker 服务使用） +############ +# PostgreSQL 数据库超级用户密码：生产环境必须更换强密码 +SOCIAL_INFRA__POSTGRES__PASSWORD=CHANGE_ME +# JWT 签名密钥（Supabase 认证服务使用）：生产环境必须更换 +SOCIAL_INFRA__JWT__SECRET=CHANGE_ME +# Supabase 匿名访问密钥：用于前端匿名访问 API +SOCIAL_INFRA__SUPABASE__ANON_KEY=CHANGE_ME +# Supabase 服务角色密钥：拥有完全权限，仅后端服务使用，切勿泄露 +SOCIAL_INFRA__SUPABASE__SERVICE_ROLE_KEY=CHANGE_ME +# Supabase 管理后台用户名 +SOCIAL_INFRA__DASHBOARD__USERNAME=supabase +# Supabase 管理后台密码 +SOCIAL_INFRA__DASHBOARD__PASSWORD=CHANGE_ME +# Supabase 数据库连接池加密密钥 +SOCIAL_INFRA__SUPAVISOR__SECRET_KEY_BASE=CHANGE_ME +# Supabase Vault 加密密钥 +SOCIAL_INFRA__SUPAVISOR__VAULT_ENC_KEY=CHANGE_ME +# Supabase Postgres Meta 加密密钥 +SOCIAL_INFRA__PG_META__CRYPTO_KEY=CHANGE_ME +# Logflare 公共访问令牌 +SOCIAL_INFRA__LOGFLARE__PUBLIC_ACCESS_TOKEN=CHANGE_ME +# Logflare 私有访问令牌 +SOCIAL_INFRA__LOGFLARE__PRIVATE_ACCESS_TOKEN=CHANGE_ME + +############ +# 基础设施数据库配置（Docker 服务使用） +############ +# PostgreSQL 容器内主机名：Docker 内部使用 +SOCIAL_INFRA__POSTGRES__HOST=db +# 数据库名称：与初始化脚本保持一致 +SOCIAL_INFRA__POSTGRES__DB=linksy +# PostgreSQL 容器内端口 +SOCIAL_INFRA__POSTGRES__PORT=54322 + +############ +# Supavisor 数据库连接池配置（Docker 服务使用） +############ +# 连接池代理端口（事务模式） +SOCIAL_INFRA__POOLER__PROXY_PORT_TRANSACTION=6543 +# 每个数据库连接池的默认大小 +SOCIAL_INFRA__POOLER__DEFAULT_POOL_SIZE=20 +# 连接池最大客户端连接数 +SOCIAL_INFRA__POOLER__MAX_CLIENT_CONN=100 +# 连接池租户 ID +SOCIAL_INFRA__POOLER__TENANT_ID=local-tenant +# 每个数据库的连接池大小 +SOCIAL_INFRA__POOLER__DB_POOL_SIZE=5 + +############ +# API 网关 Kong 配置（Docker 服务使用） +############ +# Kong HTTP 端口映射到宿主机的端口 +SOCIAL_INFRA__KONG__HTTP_PORT=8001 +# Kong HTTPS 端口映射到宿主机的端口 +SOCIAL_INFRA__KONG__HTTPS_PORT=8443 + +############ +# PostgREST API 配置（Docker 服务使用） +############ +# PostgREST 暴露的数据库模式列表，逗号分隔 +SOCIAL_INFRA__PGRST__DB_SCHEMAS=public,storage,graphql_public + +############ +# 认证服务 GoTrue 配置（Docker 服务使用） +############ +# 站点 URL：用于生成回调链接等，通常为前端地址 +SOCIAL_INFRA__SITE__URL=http://localhost:3000 +# 允许的重定向 URL 列表，逗号分隔 +SOCIAL_INFRA__ADDITIONAL_REDIRECT_URLS= +# JWT 过期时间（秒） +SOCIAL_INFRA__JWT__EXPIRY=3600 +# 是否禁用用户注册：true 禁止，false 允许 +SOCIAL_INFRA__AUTH__DISABLE_SIGNUP=false +# API 外部访问 URL：用于 Kong 网关对外暴露的地址 +SOCIAL_INFRA__API_EXTERNAL_URL=http://localhost:8001 +############ +# Supabase 公共访问 URL：用于前端/SDK/Studio 访问（可与 API 外部地址不同） +# 反向代理场景请填代理后的公网地址 +SOCIAL_INFRA__SUPABASE__PUBLIC_URL=http://localhost:8001 +# 邮箱验证链接路径 +SOCIAL_INFRA__MAILER__URLPATHS_CONFIRMATION="/auth/v1/verify" +# 邮箱邀请链接路径 +SOCIAL_INFRA__MAILER__URLPATHS_INVITE="/auth/v1/verify" +# 邮箱找回密码链接路径 +SOCIAL_INFRA__MAILER__URLPATHS_RECOVERY="/auth/v1/verify" +# 邮箱变更确认链接路径 +SOCIAL_INFRA__MAILER__URLPATHS_EMAIL_CHANGE="/auth/v1/verify" +# 是否启用邮箱注册：true 启用，false 禁用 +SOCIAL_INFRA__EMAIL__ENABLE_SIGNUP=true +# 是否自动确认邮箱：true 注册后自动登录，false 需要验证邮箱 +SOCIAL_INFRA__EMAIL__ENABLE_AUTOCONFIRM=false +# 管理员邮箱地址：用于发送系统通知等 +SOCIAL_INFRA__SMTP__ADMIN_EMAIL=admin@example.com +# SMTP 服务器主机地址 +SOCIAL_INFRA__SMTP__HOST=supabase-mail +# SMTP 服务器端口：25（不加密）、465（SSL）、587（TLS） +SOCIAL_INFRA__SMTP__PORT=2500 +# SMTP 用户名 +SOCIAL_INFRA__SMTP__USER=fake_mail_user +# SMTP 密码 +SOCIAL_INFRA__SMTP__PASS=fake_mail_password +# 发件人显示名称 +SOCIAL_INFRA__SMTP__SENDER_NAME=fake_sender +# 是否允许匿名用户访问：true 允许，false 禁止 +SOCIAL_INFRA__AUTH__ENABLE_ANONYMOUS_USERS=false +# 是否启用手机号注册：true 启用，false 禁用 +SOCIAL_INFRA__AUTH__ENABLE_PHONE_SIGNUP=true +# 是否自动确认手机号：true 自动验证，false 需要短信验证码 +SOCIAL_INFRA__AUTH__ENABLE_PHONE_AUTOCONFIRM=true + +############ +# Supabase Studio 配置（Docker 服务使用） +############ +# 默认组织名称 +SOCIAL_INFRA__STUDIO__DEFAULT_ORGANIZATION=Default Organization +# 默认项目名称 +SOCIAL_INFRA__STUDIO__DEFAULT_PROJECT=Default Project +# 是否启用 WebP 图片格式检测：true 启用自动转换，false 禁用 +SOCIAL_INFRA__IMGPROXY__ENABLE_WEBP_DETECTION=true +# OpenAI API 密钥：用于 Supabase AI 功能 +SOCIAL_INFRA__OPENAI__API_KEY= + +############ +# Edge Functions 配置（Docker 服务使用） +############ +# 是否验证 JWT：true 验证，false 不验证 +SOCIAL_INFRA__FUNCTIONS__VERIFY_JWT=false + +############ +# 日志与分析配置（Docker 服务使用） +############ +# Docker Socket 路径：用于容器日志收集 +SOCIAL_INFRA__DOCKER__SOCKET_LOCATION=/var/run/docker.sock diff --git a/.gitignore b/.gitignore index 89374a7..da7edf7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,266 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# Pipfile.lock + +# UV +# uv.lock + +# poetry +# poetry.lock +# poetry.toml + +# pdm +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# pixi.lock +.pixi + +# PEP 582 +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml + +# Flutter/Dart/Pub related +**/doc/api/ +.dart_tool/ +.flutter-plugins +.flutter-plugins-dependencies +**/generated_plugin_registrant.dart +.packages +.pub-preload-cache/ +.pub/ +build/ +flutter_*.png +linked_*.ds +unlinked.ds +unlinked_spec.ds + +# Android related +**/android/**/gradle-wrapper.jar +.gradle/ +**/android/captures/ +**/android/gradlew +**/android/gradlew.bat +**/android/local.properties +**/android/**/GeneratedPluginRegistrant.java +**/android/key.properties +*.jks + +# iOS/XCode related +**/ios/**/*.mode1v3 +**/ios/**/*.mode2v3 +**/ios/**/*.moved-aside +**/ios/**/*.pbxuser +**/ios/**/*.perspectivev3 +**/ios/**/*sync/ +**/ios/**/.sconsign.dblite +**/ios/**/.tags* +**/ios/**/.vagrant/ +**/ios/**/DerivedData/ +**/ios/**/Icon? +**/ios/**/Pods/ +**/ios/**/.symlinks/ +**/ios/**/profile +**/ios/**/xcuserdata +**/ios/.generated/ +**/ios/Flutter/.last_build_id +**/ios/Flutter/App.framework +**/ios/Flutter/Flutter.framework +**/ios/Flutter/Flutter.podspec +**/ios/Flutter/Generated.xcconfig +**/ios/Flutter/ephemeral +**/ios/Flutter/app.flx +**/ios/Flutter/app.zip +**/ios/Flutter/flutter_assets/ +**/ios/Flutter/flutter_export_environment.sh +**/ios/ServiceDefinitions.json +**/ios/Runner/GeneratedPluginRegistrant.* + +# macOS +**/Flutter/ephemeral/ +**/Pods/ +**/macos/Flutter/GeneratedPluginRegistrant.swift +**/macos/Flutter/ephemeral +**/xcuserdata/ + +# Windows +**/windows/flutter/generated_plugin_registrant.cc +**/windows/flutter/generated_plugin_registrant.h +**/windows/flutter/generated_plugins.cmake + +# Linux +**/linux/flutter/generated_plugin_registrant.cc +**/linux/flutter/generated_plugin_registrant.h +**/linux/flutter/generated_plugins.cmake + +# Coverage +coverage/ + +# Symbols +app.*.symbols + +# Exceptions to above rules. +!**/ios/**/default.mode1v3 +!**/ios/**/default.mode2v3 +!**/ios/**/default.pbxuser +!**/ios/**/default.perspectivev3 +!/packages/flutter_tools/test/data/dart_dependencies_test/**/.packages +!/dev/ci/**/Gemfile.lock + # Local environment files infra/local/env/*.env configs/env/*.env @@ -5,8 +268,14 @@ infra/cloud/volcano/env/*.env !infra/local/env/*.env.example !configs/env/*.env.example !infra/cloud/volcano/env/*.env.example -.env .env.local .env.*.local .env.cloud .env.*.cloud + +# Misc +*.class +*.lock +*.swp +.buildlog/ +.history diff --git a/.opencode/opencode.json b/.opencode/opencode.json index 2734f01..3893a93 100644 --- a/.opencode/opencode.json +++ b/.opencode/opencode.json @@ -6,6 +6,18 @@ }, "zai-mcp-server": { "enabled": false + }, + "postgres_dev": { + "type": "local", + "command": [ + "docker", + "run", + "-i", + "--rm", + "mcp/postgres", + "postgresql://supabase:${POSTGRES_PASSWORD}@host.docker.internal:54322/linksy" + ], + "enabled": true } } } diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..fa28f64 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: + - repo: https://github.com/DetachHead/basedpyright-prek-mirror + rev: 1.37.2 + hooks: + - id: basedpyright + args: [--level=error] + files: ^api/ + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.6 + hooks: + - id: ruff + files: ^api/ diff --git a/AGENTS.md b/AGENTS.md index ad26777..4b8ddb6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,57 @@ ## Docker Startup -Must use environment file when starting services: +Always start services with the env file: ```bash -docker compose --env-file infra/env/.env.local -f infra/local/docker-compose.yml up -d +docker compose --env-file .env -f docker/docker-compose.yml up -d ``` + +## Python Environment + +**MUST use uv for dependency management and virtual environment execution.** + +- All Python commands: `uv run ` +- Add dependencies: `uv add ` +- All dependencies declared in `pyproject.toml` + +## Code Quality Checks + +**Git pre-commit hook enforces code quality before commit.** + +Pre-commit hook automatically runs on api/ directory: +- `ruff check` - code style and linting +- `basedpyright` - type checking with error level + +If any error detected, commit is rejected. Fix errors before committing. +Do not bypass or weaken checks (no ignores, disables, or config relaxations). Resolve the underlying issues. + + +## TDD First Policy + +**Principle: tests before implementation.** + +### Coverage Requirements +- Minimum coverage: 80% +- Required test types: + - Unit: isolated functions, utilities, components + - Integration: API endpoints, database operations + - E2E: critical user flows (Playwright) + +### Limited Exceptions +- Docs-only changes (README, comments, formatting) may skip integration/E2E +- Non-runtime config changes may skip E2E if no behavior changes +- Any runtime code change requires unit + integration + E2E +- If an exception is used, record the reason in the PR/test notes + +### Mandatory TDD Workflow +1. Write tests (RED) - they must fail +2. Run tests - confirm failure +3. Implement minimal code (GREEN) - only to pass +4. Run tests - confirm success +5. Refactor (IMPROVE) +6. Verify coverage - must be 80%+ + +### Enforcement +- Must use the `tdd-guide` agent for new features +- Do not write implementation before tests +- Do not lower coverage requirements +- Must include unit, integration, and E2E tests diff --git a/README.md b/README.md index 7b061c6..90ac3a7 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,3 @@ # Social App Monorepo Flutter + FastAPI + Supabase + Redis + Milvus - -## 说明 - -本仓库仅初始化结构，不包含业务实现 - -## 目录结构 - -- `apps/` —— 可运行应用（Flutter / FastAPI / Worker） -- `infra/` —— 基础设施（本地 docker / 云部署 / 迁移） -- `configs/` —— 配置规范与公共配置模板（不含密钥） -- `tools/` —— 脚本与生成器 -- `docs/` —— 文档与规则 - -详见 `docs/rules/repo-structure.md` diff --git a/api/src/__init__.py b/api/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/src/core/__init__.py b/api/src/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/src/core/config/__init__.py b/api/src/core/config/__init__.py new file mode 100644 index 0000000..b1019f5 --- /dev/null +++ b/api/src/core/config/__init__.py @@ -0,0 +1,3 @@ +from .settings import Settings, config + +__all__ = ["Settings", "config"] diff --git a/api/src/core/config/settings.py b/api/src/core/config/settings.py new file mode 100644 index 0000000..b316bbf --- /dev/null +++ b/api/src/core/config/settings.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from pathlib import Path +from typing import ClassVar, Literal + +from pydantic import BaseModel, Field, computed_field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class RuntimeSettings(BaseModel): + environment: Literal["dev", "test", "prod"] = "dev" + debug: bool = True + log_level: str = "INFO" + log_json: bool = True + log_rotation: Literal["time", "size", "none"] = "time" + log_rotation_when: str = "midnight" + log_rotation_interval: int = 1 + log_rotation_backup_count: int = 14 + log_rotation_max_bytes: int = 10_000_000 + log_dir: str = "logs" + log_error_dir: str = "logs/errors" + log_file_name: str = "app.log" + log_error_file_name: str = "error.log" + log_sensitive_fields: list[str] = Field( + default_factory=lambda: [ + "password", + "secret", + "token", + "api_key", + "authorization", + "cookie", + "client_ip", + "user_id", + ] + ) + sql_log_queries: bool = False + + +class AppSettings(BaseModel): + host: str = "0.0.0.0" + port: int = Field(default=8000, ge=1, le=65535) + reload: bool = True + + +class CorsSettings(BaseModel): + allow_origins: list[str] = Field( + default_factory=lambda: [ + "http://localhost", + "http://localhost:3000", + ] + ) + allow_credentials: bool = True + allow_methods: list[str] = Field(default_factory=lambda: ["*"]) + allow_headers: list[str] = Field(default_factory=lambda: ["*"]) + + +class SupabaseSettings(BaseModel): + url: str = "http://localhost:8001" + anon_key: str = "CHANGE_ME" + service_role_key: str = "CHANGE_ME" + jwt_secret: str | None = None + + +class InfraSupabaseSettings(BaseModel): + public_url: str = "http://localhost:8001" + anon_key: str = "CHANGE_ME" + service_role_key: str = "CHANGE_ME" + + +class InfraJwtSettings(BaseModel): + secret: str = "CHANGE_ME" + + +class InfraSettings(BaseModel): + api_external_url: str = "http://localhost:8001" + supabase: InfraSupabaseSettings = Field(default_factory=InfraSupabaseSettings) + jwt: InfraJwtSettings = Field(default_factory=InfraJwtSettings) + + +def _resolve_env_file() -> str: + current = Path(__file__).resolve() + for parent in [current, *current.parents]: + candidate = parent / ".env" + if candidate.is_file(): + return str(candidate) + return ".env" + + +class Settings(BaseSettings): + runtime: RuntimeSettings = RuntimeSettings() + app: AppSettings = AppSettings() + cors: CorsSettings = CorsSettings() + infra: InfraSettings = Field(default_factory=InfraSettings) + + @computed_field + def supabase(self) -> SupabaseSettings: + return SupabaseSettings( + url=self.infra.supabase.public_url or self.infra.api_external_url, + anon_key=self.infra.supabase.anon_key, + service_role_key=self.infra.supabase.service_role_key, + jwt_secret=self.infra.jwt.secret, + ) + + model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict( + env_file=_resolve_env_file(), + env_prefix="SOCIAL_", + env_nested_delimiter="__", + case_sensitive=False, + extra="ignore", + ) + + +config = Settings() diff --git a/api/src/core/logging/__init__.py b/api/src/core/logging/__init__.py new file mode 100644 index 0000000..053f72a --- /dev/null +++ b/api/src/core/logging/__init__.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from core.logging import celery +from core.logging.config import configure_logging +from core.logging.context import bind_context, clear_context, get_context +from core.logging.logger import get_logger + +__all__ = [ + "bind_context", + "celery", + "clear_context", + "configure_logging", + "get_context", + "get_logger", +] diff --git a/api/src/core/logging/celery.py b/api/src/core/logging/celery.py new file mode 100644 index 0000000..1676851 --- /dev/null +++ b/api/src/core/logging/celery.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import cast + +from celery import Celery, signals + +from core.config.settings import Settings +from core.logging.config import configure_logging +from core.logging.context import bind_context, clear_context + + +@dataclass(frozen=True) +class CelerySignalHandlers: + on_setup_logging: Callable[..., None] + on_after_setup_task_logger: Callable[..., None] + on_task_prerun: Callable[..., None] + on_task_postrun: Callable[..., None] + + +def build_celery_signal_handlers( + settings: Settings | None = None, +) -> CelerySignalHandlers: + def on_setup_logging(*_args: object, **_kwargs: object) -> None: + configure_logging(settings) + + def on_after_setup_task_logger(*_args: object, **_kwargs: object) -> None: + configure_logging(settings) + + def on_task_prerun(*_args: object, **kwargs: object) -> None: + task_id = cast(str | None, kwargs.get("task_id")) + task = kwargs.get("task") + task_name = getattr(task, "name", None) + bind_context(task_id=task_id, task_name=task_name) + + def on_task_postrun(*_args: object, **_kwargs: object) -> None: + clear_context() + + return CelerySignalHandlers( + on_setup_logging=on_setup_logging, + on_after_setup_task_logger=on_after_setup_task_logger, + on_task_prerun=on_task_prerun, + on_task_postrun=on_task_postrun, + ) + + +def configure_celery_app(app: Celery, settings: Settings | None = None) -> None: + app.conf.worker_hijack_root_logger = False + + handlers = build_celery_signal_handlers(settings) + signals.setup_logging.connect(handlers.on_setup_logging, weak=False) + signals.after_setup_task_logger.connect( + handlers.on_after_setup_task_logger, weak=False + ) + signals.task_prerun.connect(handlers.on_task_prerun, weak=False) + signals.task_postrun.connect(handlers.on_task_postrun, weak=False) diff --git a/api/src/core/logging/config.py b/api/src/core/logging/config.py new file mode 100644 index 0000000..6d92ba3 --- /dev/null +++ b/api/src/core/logging/config.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import logging +from logging.config import dictConfig +from pathlib import Path + +import structlog + +from core.config.settings import RuntimeSettings, Settings +from core.logging.formatters import ( + build_plain_formatter, + build_processor_formatter, + ensure_message_key, +) +from core.logging.filters import build_sensitive_data_processor +from core.logging.handlers import build_file_handler_config + + +def _ensure_log_dirs(runtime: RuntimeSettings) -> None: + Path(runtime.log_dir).mkdir(parents=True, exist_ok=True) + Path(runtime.log_error_dir).mkdir(parents=True, exist_ok=True) + + +def build_logging_config(runtime: RuntimeSettings) -> dict[str, object]: + log_dir = Path(runtime.log_dir) + error_dir = Path(runtime.log_error_dir) + formatter_name = "json" if runtime.log_json else "plain" + + file_handler = build_file_handler_config( + runtime, + file_path=log_dir / runtime.log_file_name, + level=runtime.log_level, + formatter=formatter_name, + ) + error_handler = build_file_handler_config( + runtime, + file_path=error_dir / runtime.log_error_file_name, + level="ERROR", + formatter=formatter_name, + filters=["error_only"], + ) + + return { + "version": 1, + "disable_existing_loggers": False, + "filters": { + "error_only": { + "()": "core.logging.filters.ErrorLevelFilter", + } + }, + "formatters": { + "json": { + "()": build_processor_formatter, + "sensitive_fields": runtime.log_sensitive_fields, + }, + "plain": { + "()": build_plain_formatter, + "sensitive_fields": runtime.log_sensitive_fields, + }, + }, + "handlers": { + "file": file_handler, + "error": error_handler, + }, + "root": { + "handlers": ["file", "error"], + "level": runtime.log_level, + }, + } + + +def configure_logging(settings: Settings | None = None) -> None: + active_settings = settings or Settings() + runtime = active_settings.runtime + + try: + _ensure_log_dirs(runtime) + dictConfig(build_logging_config(runtime)) + except (OSError, ValueError) as exc: + logging.basicConfig(level=runtime.log_level) + logging.getLogger(__name__).error("Logging setup failed", exc_info=exc) + + structlog.configure( + processors=[ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="iso", utc=True), + structlog.processors.CallsiteParameterAdder( + parameters=[ + structlog.processors.CallsiteParameter.MODULE, + structlog.processors.CallsiteParameter.FUNC_NAME, + structlog.processors.CallsiteParameter.LINENO, + ] + ), + build_sensitive_data_processor(runtime.log_sensitive_fields), + ensure_message_key, + structlog.processors.format_exc_info, + structlog.processors.UnicodeDecoder(), + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], + logger_factory=structlog.stdlib.LoggerFactory(), + cache_logger_on_first_use=True, + ) diff --git a/api/src/core/logging/context.py b/api/src/core/logging/context.py new file mode 100644 index 0000000..d909bf3 --- /dev/null +++ b/api/src/core/logging/context.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from structlog import contextvars + + +def bind_context(**values: object) -> None: + contextvars.bind_contextvars(**values) + + +def clear_context() -> None: + contextvars.clear_contextvars() + + +def get_context() -> dict[str, object]: + return contextvars.get_contextvars() diff --git a/api/src/core/logging/filters.py b/api/src/core/logging/filters.py new file mode 100644 index 0000000..2139c7a --- /dev/null +++ b/api/src/core/logging/filters.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import logging +import re +from collections.abc import Callable +from typing import cast + +from structlog.types import EventDict + + +_NORMALIZE_PATTERN = re.compile(r"[^a-z0-9]") + + +def _normalize_key(value: str) -> str: + return _NORMALIZE_PATTERN.sub("", value.lower()) + + +def _is_sensitive_key(key: object, sensitive_fields: set[str]) -> bool: + normalized_key = _normalize_key(str(key)) + return normalized_key in sensitive_fields or any( + fragment in normalized_key for fragment in sensitive_fields + ) + + +def _redact_value(value: object, sensitive_fields: set[str]) -> object: + if isinstance(value, dict): + typed_value = cast(dict[str, object], value) + return { + key: ( + "[REDACTED]" + if _is_sensitive_key(key, sensitive_fields) + else _redact_value(inner, sensitive_fields) + ) + for key, inner in typed_value.items() + } + if isinstance(value, list): + return [_redact_value(item, sensitive_fields) for item in value] + return value + + +def build_sensitive_data_processor( + sensitive_fields: list[str], +) -> Callable[[object, str, EventDict], EventDict]: + normalized = {_normalize_key(field) for field in sensitive_fields} + + def processor( + _logger: object, _method_name: str, event_dict: EventDict + ) -> EventDict: + return cast(EventDict, _redact_value(event_dict, normalized)) + + return processor + + +class ErrorLevelFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + return record.levelno >= logging.ERROR diff --git a/api/src/core/logging/formatters.py b/api/src/core/logging/formatters.py new file mode 100644 index 0000000..71cba98 --- /dev/null +++ b/api/src/core/logging/formatters.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from structlog.dev import ConsoleRenderer +from structlog.processors import JSONRenderer +from structlog.stdlib import ProcessorFormatter +from structlog.types import EventDict +import structlog + +from core.logging.filters import build_sensitive_data_processor + + +def ensure_message_key( + _logger: object, _method_name: str, event_dict: EventDict +) -> EventDict: + if "message" in event_dict: + return event_dict + if "event" not in event_dict: + return event_dict + + without_event = {key: value for key, value in event_dict.items() if key != "event"} + return {**without_event, "message": event_dict["event"]} + + +def build_processor_formatter( + sensitive_fields: list[str] | None = None, +) -> ProcessorFormatter: + redact = build_sensitive_data_processor(sensitive_fields or []) + return ProcessorFormatter( + foreign_pre_chain=[ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="iso", utc=True), + structlog.processors.CallsiteParameterAdder( + parameters=[ + structlog.processors.CallsiteParameter.MODULE, + structlog.processors.CallsiteParameter.FUNC_NAME, + structlog.processors.CallsiteParameter.LINENO, + ] + ), + structlog.stdlib.ExtraAdder(), + ensure_message_key, + ], + processors=[ + redact, + ensure_message_key, + ProcessorFormatter.remove_processors_meta, + structlog.processors.format_exc_info, + structlog.processors.UnicodeDecoder(), + JSONRenderer(sort_keys=True), + ], + ) + + +def build_plain_formatter( + sensitive_fields: list[str] | None = None, +) -> ProcessorFormatter: + redact = build_sensitive_data_processor(sensitive_fields or []) + return ProcessorFormatter( + foreign_pre_chain=[ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="iso", utc=True), + structlog.processors.CallsiteParameterAdder( + parameters=[ + structlog.processors.CallsiteParameter.MODULE, + structlog.processors.CallsiteParameter.FUNC_NAME, + structlog.processors.CallsiteParameter.LINENO, + ] + ), + structlog.stdlib.ExtraAdder(), + ensure_message_key, + ], + processors=[ + redact, + ensure_message_key, + ProcessorFormatter.remove_processors_meta, + structlog.processors.format_exc_info, + structlog.processors.UnicodeDecoder(), + ConsoleRenderer(colors=False), + ], + ) diff --git a/api/src/core/logging/handlers.py b/api/src/core/logging/handlers.py new file mode 100644 index 0000000..e690f6b --- /dev/null +++ b/api/src/core/logging/handlers.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from pathlib import Path + +from core.config.settings import RuntimeSettings + + +def build_file_handler_config( + runtime: RuntimeSettings, + file_path: Path, + level: str, + formatter: str, + filters: list[str] | None = None, +) -> dict[str, object]: + filter_list = list(filters or []) + base_config: dict[str, object] = { + "level": level, + "formatter": formatter, + "filename": str(file_path), + "encoding": "utf-8", + } + + if filter_list: + base_config = {**base_config, "filters": filter_list} + + if runtime.log_rotation == "time": + return { + **base_config, + "class": "logging.handlers.TimedRotatingFileHandler", + "when": runtime.log_rotation_when, + "interval": runtime.log_rotation_interval, + "backupCount": runtime.log_rotation_backup_count, + } + + if runtime.log_rotation == "size": + return { + **base_config, + "class": "logging.handlers.RotatingFileHandler", + "maxBytes": runtime.log_rotation_max_bytes, + "backupCount": runtime.log_rotation_backup_count, + } + + return { + **base_config, + "class": "logging.FileHandler", + } diff --git a/api/src/core/logging/logger.py b/api/src/core/logging/logger.py new file mode 100644 index 0000000..3f11d3c --- /dev/null +++ b/api/src/core/logging/logger.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import structlog + + +def get_logger(name: str) -> structlog.stdlib.BoundLogger: + return structlog.get_logger(name) diff --git a/api/src/core/logging/middleware.py b/api/src/core/logging/middleware.py new file mode 100644 index 0000000..edf1df6 --- /dev/null +++ b/api/src/core/logging/middleware.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import re +from collections.abc import MutableMapping +from typing import cast +from uuid import uuid4 + +from fastapi import FastAPI, Request +from starlette.requests import Request as StarletteRequest +from starlette.responses import JSONResponse, Response +from starlette.types import ASGIApp, Receive, Scope, Send + +from core.logging.context import bind_context, clear_context +from core.logging.logger import get_logger + + +class RequestContextMiddleware: + app: ASGIApp + _header_name: str + _request_id_pattern: re.Pattern[str] + + def __init__(self, app: ASGIApp, header_name: str = "X-Request-ID") -> None: + self.app = app + self._header_name = header_name + self._request_id_pattern = re.compile(r"^[A-Za-z0-9_-]{8,64}$") + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope.get("type") != "http": + await self.app(scope, receive, send) + return + + request = StarletteRequest(scope, receive=receive) + request_id = self._normalize_request_id(request.headers.get(self._header_name)) + client_ip = request.client.host if request.client else None + user_id = getattr(request.state, "user_id", None) + + request.state.request_id = request_id + + bind_context( + request_id=request_id, + method=request.method, + path=request.url.path, + client_ip=client_ip, + user_id=user_id, + ) + + async def send_wrapper(message: MutableMapping[str, object]) -> None: + if message.get("type") == "http.response.start": + raw_headers = message.get("headers") + headers = list(cast(list[tuple[bytes, bytes]], raw_headers or [])) + header_key = self._header_name.lower().encode() + if not any(item[0].lower() == header_key for item in headers): + headers.append((header_key, request_id.encode())) + message = {**message, "headers": headers} + await send(message) + + try: + await self.app(scope, receive, send_wrapper) + finally: + clear_context() + + def _normalize_request_id(self, request_id: str | None) -> str: + if request_id and self._request_id_pattern.match(request_id): + return request_id + return str(uuid4()) + + +def register_exception_handlers(app: FastAPI) -> None: + logger = get_logger("core.logging.exception") + + @app.exception_handler(Exception) + async def unhandled_exception_handler(request: Request, exc: Exception) -> Response: + request_id = getattr(request.state, "request_id", None) + logger.exception( + "Unhandled exception", + error_type=exc.__class__.__name__, + request_id=request_id, + ) + headers = {"X-Request-ID": request_id} if request_id else None + return JSONResponse( + status_code=500, + content={"detail": "Internal Server Error"}, + headers=headers, + ) diff --git a/api/tests/conftest.py b/api/tests/conftest.py new file mode 100644 index 0000000..1118019 --- /dev/null +++ b/api/tests/conftest.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import sys +from pathlib import Path + + +def pytest_configure() -> None: + root = Path(__file__).resolve().parents[2] + src_path = root / "api" / "src" + if str(src_path) not in sys.path: + sys.path.append(str(src_path)) diff --git a/api/tests/e2e/test_logging_e2e.py b/api/tests/e2e/test_logging_e2e.py new file mode 100644 index 0000000..6fdbaa2 --- /dev/null +++ b/api/tests/e2e/test_logging_e2e.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import json +import socket +import threading +import time +from pathlib import Path + +from fastapi import FastAPI +from playwright.sync_api import sync_playwright +import uvicorn + +from core.config.settings import Settings +from core.logging.config import configure_logging +from core.logging.middleware import ( + RequestContextMiddleware, + register_exception_handlers, +) + + +def _read_json_lines(path: Path) -> list[dict[str, object]]: + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _wait_for_port(host: str, port: int, timeout: float = 5.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + if sock.connect_ex((host, port)) == 0: + return + time.sleep(0.05) + raise RuntimeError("Server did not start in time") + + +def _start_server(app: FastAPI, host: str, port: int): + config = uvicorn.Config(app, host=host, port=port, log_level="info") + server = uvicorn.Server(config) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + _wait_for_port(host, port) + return server, thread + + +def test_e2e_error_logging(tmp_path: Path) -> None: + settings = Settings() + runtime = settings.runtime.model_copy( + update={ + "log_dir": str(tmp_path), + "log_error_dir": str(tmp_path / "errors"), + "log_rotation": "size", + "log_rotation_max_bytes": 2048, + } + ) + configure_logging(settings.model_copy(update={"runtime": runtime})) + + app = FastAPI() + app.add_middleware(RequestContextMiddleware) # type: ignore[arg-type] + register_exception_handlers(app) + + @app.get("/boom") + async def boom() -> dict[str, str]: + raise RuntimeError("boom") + + host = "127.0.0.1" + port = _find_free_port() + server, thread = _start_server(app, host, port) + + try: + with sync_playwright() as playwright: + request_context = playwright.request.new_context( + base_url=f"http://{host}:{port}" + ) + response = request_context.get( + "/boom", + headers={"X-Request-ID": "e2e-5000"}, + ) + assert response.status == 500 + request_context.dispose() + finally: + server.should_exit = True + thread.join(timeout=5) + + error_entries = _read_json_lines(Path(tmp_path) / "errors" / "error.log") + entry = next( + item for item in error_entries if item.get("message") == "Unhandled exception" + ) + + assert entry["request_id"] == "e2e-5000" + exception = str(entry["exception"]) + assert "Traceback" in exception diff --git a/api/tests/integration/test_fastapi_logging_integration.py b/api/tests/integration/test_fastapi_logging_integration.py new file mode 100644 index 0000000..88e3581 --- /dev/null +++ b/api/tests/integration/test_fastapi_logging_integration.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from core.config.settings import Settings +from core.logging.config import configure_logging +from core.logging.logger import get_logger +from core.logging.middleware import ( + RequestContextMiddleware, + register_exception_handlers, +) + + +def _read_json_lines(path: Path) -> list[dict[str, object]]: + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +def _configure_test_logging(tmp_path: Path) -> None: + settings = Settings() + runtime = settings.runtime.model_copy( + update={ + "log_dir": str(tmp_path), + "log_error_dir": str(tmp_path / "errors"), + "log_rotation": "size", + "log_rotation_max_bytes": 2048, + } + ) + test_settings = settings.model_copy(update={"runtime": runtime}) + + configure_logging(test_settings) + + +def test_middleware_binds_request_context(tmp_path: Path) -> None: + _configure_test_logging(tmp_path) + + app = FastAPI() + app.add_middleware(RequestContextMiddleware) # type: ignore[arg-type] + + @app.get("/ok") + async def ok() -> dict[str, str]: + logger = get_logger("tests.ok") + logger.info("request accepted", context_key="context_value") + return {"status": "ok"} + + client = TestClient(app) + response = client.get("/ok", headers={"X-Request-ID": "req-1234"}) + + assert response.status_code == 200 + assert response.headers["X-Request-ID"] == "req-1234" + + log_entries = _read_json_lines(Path(tmp_path) / "app.log") + entry = next( + item for item in log_entries if item.get("message") == "request accepted" + ) + assert entry["message"] == "request accepted" + assert entry["request_id"] == "req-1234" + assert entry["method"] == "GET" + assert entry["path"] == "/ok" + assert entry["context_key"] == "context_value" + + logging.shutdown() + + +def test_exception_handler_logs_stack_and_sends_500(tmp_path: Path) -> None: + _configure_test_logging(tmp_path) + + app = FastAPI() + app.add_middleware(RequestContextMiddleware) + register_exception_handlers(app) + + @app.get("/boom") + async def boom() -> dict[str, str]: + raise RuntimeError("boom") + + client = TestClient(app, raise_server_exceptions=False) + response = client.get("/boom", headers={"X-Request-ID": "req-5000"}) + + assert response.status_code == 500 + assert response.json()["detail"] == "Internal Server Error" + + error_entries = _read_json_lines(Path(tmp_path) / "errors" / "error.log") + assert error_entries + entry = error_entries[-1] + assert entry["level"] == "error" + assert entry["request_id"] == "req-5000" + exception = str(entry["exception"]) + assert "Traceback" in exception + assert "test_fastapi_logging_integration" in exception + + logging.shutdown() + + +def test_invalid_request_id_is_replaced_and_used_in_error_context( + tmp_path: Path, +) -> None: + _configure_test_logging(tmp_path) + + app = FastAPI() + app.add_middleware(RequestContextMiddleware) + register_exception_handlers(app) + + @app.get("/boom") + async def boom() -> dict[str, str]: + raise RuntimeError("boom") + + client = TestClient(app, raise_server_exceptions=False) + response = client.get("/boom", headers={"X-Request-ID": "bad"}) + + assert response.status_code == 500 + + response_request_id = response.headers["X-Request-ID"] + assert response_request_id != "bad" + + error_entries = _read_json_lines(Path(tmp_path) / "errors" / "error.log") + assert error_entries + entry = error_entries[-1] + assert entry["request_id"] == response_request_id + exception = str(entry["exception"]) + assert "Traceback" in exception + + logging.shutdown() diff --git a/api/tests/unit/test_celery_logging.py b/api/tests/unit/test_celery_logging.py new file mode 100644 index 0000000..d58158a --- /dev/null +++ b/api/tests/unit/test_celery_logging.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from celery import Celery +from pytest import MonkeyPatch + +from core.logging import celery as celery_logging +from core.logging.context import clear_context, get_context + + +class DummyTask: + name: str = "tasks.sample" + + +def test_celery_prerun_binds_task_context() -> None: + handlers = celery_logging.build_celery_signal_handlers() + + handlers.on_task_prerun(task_id="task-123", task=DummyTask()) + context = get_context() + + assert context["task_id"] == "task-123" + assert context["task_name"] == "tasks.sample" + + clear_context() + + +def test_celery_setup_logging_calls_configure(monkeypatch: MonkeyPatch) -> None: + called = {"value": False} + + def fake_configure_logging(settings: object | None = None) -> None: + called["value"] = True + + monkeypatch.setattr(celery_logging, "configure_logging", fake_configure_logging) + handlers = celery_logging.build_celery_signal_handlers() + + handlers.on_setup_logging() + + assert called["value"] is True + + +def test_configure_celery_app_disables_hijack() -> None: + app = Celery("test") + + celery_logging.configure_celery_app(app) + + assert app.conf.worker_hijack_root_logger is False diff --git a/api/tests/unit/test_logging_config.py b/api/tests/unit/test_logging_config.py new file mode 100644 index 0000000..e6b15d2 --- /dev/null +++ b/api/tests/unit/test_logging_config.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import json +import logging +from collections.abc import Iterator +from pathlib import Path +from typing import cast + +import pytest +import structlog + +from core.config.settings import Settings +from core.logging.config import build_logging_config, configure_logging + + +def _get_handlers(config: dict[str, object]) -> dict[str, dict[str, object]]: + return cast(dict[str, dict[str, object]], config["handlers"]) + + +def test_build_logging_config_time_rotation(tmp_path: Path) -> None: + settings = Settings() + runtime = settings.runtime.model_copy( + update={ + "log_dir": str(tmp_path), + "log_error_dir": str(tmp_path / "errors"), + "log_rotation": "time", + } + ) + + config = build_logging_config(runtime) + handlers = _get_handlers(config) + + assert handlers["file"]["class"] == "logging.handlers.TimedRotatingFileHandler" + assert handlers["error"]["class"] == "logging.handlers.TimedRotatingFileHandler" + assert handlers["error"]["level"] == "ERROR" + + +def test_build_logging_config_size_rotation(tmp_path: Path) -> None: + settings = Settings() + runtime = settings.runtime.model_copy( + update={ + "log_dir": str(tmp_path), + "log_error_dir": str(tmp_path / "errors"), + "log_rotation": "size", + "log_rotation_max_bytes": 2048, + } + ) + + config = build_logging_config(runtime) + handlers = _get_handlers(config) + + assert handlers["file"]["class"] == "logging.handlers.RotatingFileHandler" + assert handlers["error"]["class"] == "logging.handlers.RotatingFileHandler" + assert handlers["file"]["maxBytes"] == 2048 + + +def test_build_logging_config_plain_formatter_when_disabled(tmp_path: Path) -> None: + settings = Settings() + runtime = settings.runtime.model_copy( + update={ + "log_dir": str(tmp_path), + "log_error_dir": str(tmp_path / "errors"), + "log_json": False, + } + ) + + config = build_logging_config(runtime) + handlers = _get_handlers(config) + + assert handlers["file"]["formatter"] == "plain" + assert handlers["error"]["formatter"] == "plain" + + +def _read_last_log_entry(log_path: Path) -> dict[str, object]: + assert log_path.exists(), f"Expected log file at {log_path}" + entries = [ + json.loads(line) for line in log_path.read_text().splitlines() if line.strip() + ] + assert entries, "Expected at least one log entry in app.log" + return entries[-1] + + +def _flush_root_handlers() -> None: + root_logger = logging.getLogger() + for handler in root_logger.handlers: + if hasattr(handler, "flush"): + handler.flush() + + +@pytest.fixture +def configured_logging(tmp_path: Path) -> Iterator[Path]: + settings = Settings() + runtime = settings.runtime.model_copy( + update={ + "log_dir": str(tmp_path), + "log_error_dir": str(tmp_path / "errors"), + "log_rotation": "size", + "log_rotation_max_bytes": 2048, + "log_json": True, + } + ) + root_logger = logging.getLogger() + original_handlers = root_logger.handlers[:] + original_level = root_logger.level + + configure_logging(settings.model_copy(update={"runtime": runtime})) + + yield tmp_path + + for handler in root_logger.handlers: + handler.close() + root_logger.handlers = original_handlers + root_logger.setLevel(original_level) + structlog.reset_defaults() + + +def test_stdlib_logging_redacts_sensitive_fields(configured_logging: Path) -> None: + logger = logging.getLogger("tests.stdlib") + logger.info("login", extra={"password": "secret", "token": "abc"}) + + _flush_root_handlers() + + log_path = configured_logging / "app.log" + entry = _read_last_log_entry(log_path) + + assert entry["password"] == "[REDACTED]" + assert entry["token"] == "[REDACTED]" + + +def test_structlog_redacts_sensitive_fields(configured_logging: Path) -> None: + logger = structlog.get_logger("tests.structlog") + logger.info("login", password="secret", token="abc") + + _flush_root_handlers() + + log_path = configured_logging / "app.log" + entry = _read_last_log_entry(log_path) + + assert entry["password"] == "[REDACTED]" + assert entry["token"] == "[REDACTED]" diff --git a/api/tests/unit/test_logging_filters.py b/api/tests/unit/test_logging_filters.py new file mode 100644 index 0000000..125c836 --- /dev/null +++ b/api/tests/unit/test_logging_filters.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from core.logging.filters import build_sensitive_data_processor + + +def test_redact_sensitive_fields_masks_values() -> None: + processor = build_sensitive_data_processor( + ["password", "token", "api_key", "cookie"] + ) + + event: dict[str, object] = { + "message": "login", + "password": "secret", + "access_token": "token-123", + "apiKey": "apikey-123", + "set-cookie": "cookie-1", + "nested": {"token": "abc", "safe": "ok"}, + "list": [{"password": "x"}], + } + + redacted = processor(None, "info", event) + + assert redacted["password"] == "[REDACTED]" + assert redacted["access_token"] == "[REDACTED]" + assert redacted["apiKey"] == "[REDACTED]" + assert redacted["set-cookie"] == "[REDACTED]" + assert redacted["nested"]["token"] == "[REDACTED]" + assert redacted["nested"]["safe"] == "ok" + assert redacted["list"][0]["password"] == "[REDACTED]" + assert event["password"] == "secret" diff --git a/api/tests/unit/test_logging_settings.py b/api/tests/unit/test_logging_settings.py new file mode 100644 index 0000000..834c622 --- /dev/null +++ b/api/tests/unit/test_logging_settings.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from pytest import MonkeyPatch + +from core.config.settings import Settings + + +def test_runtime_settings_defaults() -> None: + settings = Settings() + + assert settings.runtime.log_json is True + assert settings.runtime.log_rotation == "time" + assert settings.runtime.log_rotation_when == "midnight" + assert settings.runtime.log_rotation_interval == 1 + assert settings.runtime.log_rotation_backup_count == 14 + assert settings.runtime.log_rotation_max_bytes == 10_000_000 + assert settings.runtime.log_dir == "logs" + assert settings.runtime.log_error_dir == "logs/errors" + assert settings.runtime.log_file_name == "app.log" + assert settings.runtime.log_error_file_name == "error.log" + assert "password" in settings.runtime.log_sensitive_fields + + +def test_runtime_settings_env_override(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setenv("SOCIAL_RUNTIME__LOG_DIR", "var/logs") + monkeypatch.setenv("SOCIAL_RUNTIME__LOG_ERROR_DIR", "var/logs/errors") + monkeypatch.setenv("SOCIAL_RUNTIME__LOG_ROTATION", "size") + monkeypatch.setenv("SOCIAL_RUNTIME__LOG_ROTATION_MAX_BYTES", "2048") + + settings = Settings() + + assert settings.runtime.log_dir == "var/logs" + assert settings.runtime.log_error_dir == "var/logs/errors" + assert settings.runtime.log_rotation == "size" + assert settings.runtime.log_rotation_max_bytes == 2048 diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml deleted file mode 100644 index 29a8bc7..0000000 --- a/apps/api/pyproject.toml +++ /dev/null @@ -1,2 +0,0 @@ -# FastAPI 服务占位文件 -# 后续添加依赖和配置 diff --git a/apps/mobile/pubspec.yaml b/apps/mobile/pubspec.yaml deleted file mode 100644 index d4f369f..0000000 --- a/apps/mobile/pubspec.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# Flutter 应用占位文件 -# 后续添加依赖和配置 diff --git a/apps/worker/pyproject.toml b/apps/worker/pyproject.toml deleted file mode 100644 index 7ed8598..0000000 --- a/apps/worker/pyproject.toml +++ /dev/null @@ -1,2 +0,0 @@ -# 异步任务/队列服务占位文件 -# 预留：后续可能添加 diff --git a/configs/env/.env.example b/configs/env/.env.example deleted file mode 100644 index 2baac69..0000000 --- a/configs/env/.env.example +++ /dev/null @@ -1 +0,0 @@ -# 放后端开发项目需要的环境变量 diff --git a/configs/flutter/dev.json b/configs/flutter/dev.json deleted file mode 100644 index f081a4a..0000000 --- a/configs/flutter/dev.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "apiBaseUrl": "http://localhost:8000", - "environment": "development" -} diff --git a/configs/flutter/prod.json b/configs/flutter/prod.json deleted file mode 100644 index 6cf19bc..0000000 --- a/configs/flutter/prod.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "apiBaseUrl": "https://api.yourdomain.com", - "environment": "production" -} diff --git a/configs/openapi/openapi.yaml b/configs/openapi/openapi.yaml deleted file mode 100644 index e966ffa..0000000 --- a/configs/openapi/openapi.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# OpenAPI 规范占位文件 -# 后续通过 FastAPI 自动生成 diff --git a/infra/local/docker-compose.yml b/docker/docker-compose.yml similarity index 61% rename from infra/local/docker-compose.yml rename to docker/docker-compose.yml index 2dbc627..708a6db 100644 --- a/infra/local/docker-compose.yml +++ b/docker/docker-compose.yml @@ -47,22 +47,22 @@ services: environment: HOSTNAME: "::" STUDIO_PG_META_URL: http://meta:8080 - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - PG_META_CRYPTO_KEY: ${PG_META_CRYPTO_KEY} - DEFAULT_ORGANIZATION_NAME: ${STUDIO_DEFAULT_ORGANIZATION} - DEFAULT_PROJECT_NAME: ${STUDIO_DEFAULT_PROJECT} - OPENAI_API_KEY: ${OPENAI_API_KEY:-} + POSTGRES_PORT: ${SOCIAL_INFRA__POSTGRES__PORT} + POSTGRES_HOST: ${SOCIAL_INFRA__POSTGRES__HOST} + POSTGRES_DB: ${SOCIAL_INFRA__POSTGRES__DB} + POSTGRES_PASSWORD: ${SOCIAL_INFRA__POSTGRES__PASSWORD} + PG_META_CRYPTO_KEY: ${SOCIAL_INFRA__PG_META__CRYPTO_KEY} + DEFAULT_ORGANIZATION_NAME: ${SOCIAL_INFRA__STUDIO__DEFAULT_ORGANIZATION} + DEFAULT_PROJECT_NAME: ${SOCIAL_INFRA__STUDIO__DEFAULT_PROJECT} + OPENAI_API_KEY: ${SOCIAL_INFRA__OPENAI__API_KEY:-} SUPABASE_URL: http://kong:8000 - SUPABASE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} - SUPABASE_ANON_KEY: ${ANON_KEY} - SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} - AUTH_JWT_SECRET: ${JWT_SECRET} - LOGFLARE_API_KEY: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} - LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} - LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} + SUPABASE_PUBLIC_URL: ${SOCIAL_INFRA__SUPABASE__PUBLIC_URL} + SUPABASE_ANON_KEY: ${SOCIAL_INFRA__SUPABASE__ANON_KEY} + SUPABASE_SERVICE_KEY: ${SOCIAL_INFRA__SUPABASE__SERVICE_ROLE_KEY} + AUTH_JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} + LOGFLARE_API_KEY: ${SOCIAL_INFRA__LOGFLARE__PUBLIC_ACCESS_TOKEN} + LOGFLARE_PUBLIC_ACCESS_TOKEN: ${SOCIAL_INFRA__LOGFLARE__PUBLIC_ACCESS_TOKEN} + LOGFLARE_PRIVATE_ACCESS_TOKEN: ${SOCIAL_INFRA__LOGFLARE__PRIVATE_ACCESS_TOKEN} LOGFLARE_URL: http://analytics:4000 NEXT_PUBLIC_ENABLE_LOGS: true NEXT_ANALYTICS_BACKEND_PROVIDER: postgres @@ -75,8 +75,8 @@ services: image: kong:2.8.1 restart: unless-stopped ports: - - ${KONG_HTTP_PORT}:8000/tcp - - ${KONG_HTTPS_PORT}:8443/tcp + - ${SOCIAL_INFRA__KONG__HTTP_PORT}:8000/tcp + - ${SOCIAL_INFRA__KONG__HTTPS_PORT}:8443/tcp volumes: - ./supabase/volumes/api/kong.yml:/home/kong/temp.yml:ro,z depends_on: @@ -89,10 +89,10 @@ services: KONG_PLUGINS: request-transformer,cors,key-auth,acl,basic-auth,request-termination,ip-restriction KONG_NGINX_PROXY_PROXY_BUFFER_SIZE: 160k KONG_NGINX_PROXY_PROXY_BUFFERS: 64 160k - SUPABASE_ANON_KEY: ${ANON_KEY} - SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} - DASHBOARD_USERNAME: ${DASHBOARD_USERNAME} - DASHBOARD_PASSWORD: ${DASHBOARD_PASSWORD} + SUPABASE_ANON_KEY: ${SOCIAL_INFRA__SUPABASE__ANON_KEY} + SUPABASE_SERVICE_KEY: ${SOCIAL_INFRA__SUPABASE__SERVICE_ROLE_KEY} + DASHBOARD_USERNAME: ${SOCIAL_INFRA__DASHBOARD__USERNAME} + DASHBOARD_PASSWORD: ${SOCIAL_INFRA__DASHBOARD__PASSWORD} entrypoint: bash -c 'eval "echo \"$$(cat ~/temp.yml)\"" > ~/kong.yml && /docker-entrypoint.sh kong docker-start' auth: @@ -120,32 +120,32 @@ services: environment: GOTRUE_API_HOST: 0.0.0.0 GOTRUE_API_PORT: 9999 - API_EXTERNAL_URL: ${API_EXTERNAL_URL} + API_EXTERNAL_URL: ${SOCIAL_INFRA__API_EXTERNAL_URL} GOTRUE_DB_DRIVER: postgres - GOTRUE_DB_DATABASE_URL: postgres://supabase_auth_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - GOTRUE_SITE_URL: ${SITE_URL} - GOTRUE_URI_ALLOW_LIST: ${ADDITIONAL_REDIRECT_URLS} - GOTRUE_DISABLE_SIGNUP: ${DISABLE_SIGNUP} + GOTRUE_DB_DATABASE_URL: postgres://supabase_auth_admin:${SOCIAL_INFRA__POSTGRES__PASSWORD}@${SOCIAL_INFRA__POSTGRES__HOST}:${SOCIAL_INFRA__POSTGRES__PORT}/${SOCIAL_INFRA__POSTGRES__DB} + GOTRUE_SITE_URL: ${SOCIAL_INFRA__SITE__URL} + GOTRUE_URI_ALLOW_LIST: ${SOCIAL_INFRA__ADDITIONAL_REDIRECT_URLS} + GOTRUE_DISABLE_SIGNUP: ${SOCIAL_INFRA__AUTH__DISABLE_SIGNUP} GOTRUE_JWT_ADMIN_ROLES: service_role GOTRUE_JWT_AUD: authenticated GOTRUE_JWT_DEFAULT_GROUP_NAME: authenticated - GOTRUE_JWT_EXP: ${JWT_EXPIRY} - GOTRUE_JWT_SECRET: ${JWT_SECRET} - GOTRUE_EXTERNAL_EMAIL_ENABLED: ${ENABLE_EMAIL_SIGNUP} - GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: ${ENABLE_ANONYMOUS_USERS} - GOTRUE_MAILER_AUTOCONFIRM: ${ENABLE_EMAIL_AUTOCONFIRM} - GOTRUE_SMTP_ADMIN_EMAIL: ${SMTP_ADMIN_EMAIL} - GOTRUE_SMTP_HOST: ${SMTP_HOST} - GOTRUE_SMTP_PORT: ${SMTP_PORT} - GOTRUE_SMTP_USER: ${SMTP_USER} - GOTRUE_SMTP_PASS: ${SMTP_PASS} - GOTRUE_SMTP_SENDER_NAME: ${SMTP_SENDER_NAME} - GOTRUE_MAILER_URLPATHS_INVITE: ${MAILER_URLPATHS_INVITE} - GOTRUE_MAILER_URLPATHS_CONFIRMATION: ${MAILER_URLPATHS_CONFIRMATION} - GOTRUE_MAILER_URLPATHS_RECOVERY: ${MAILER_URLPATHS_RECOVERY} - GOTRUE_MAILER_URLPATHS_EMAIL_CHANGE: ${MAILER_URLPATHS_EMAIL_CHANGE} - GOTRUE_EXTERNAL_PHONE_ENABLED: ${ENABLE_PHONE_SIGNUP} - GOTRUE_SMS_AUTOCONFIRM: ${ENABLE_PHONE_AUTOCONFIRM} + GOTRUE_JWT_EXP: ${SOCIAL_INFRA__JWT__EXPIRY} + GOTRUE_JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} + GOTRUE_EXTERNAL_EMAIL_ENABLED: ${SOCIAL_INFRA__EMAIL__ENABLE_SIGNUP} + GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: ${SOCIAL_INFRA__AUTH__ENABLE_ANONYMOUS_USERS} + GOTRUE_MAILER_AUTOCONFIRM: ${SOCIAL_INFRA__EMAIL__ENABLE_AUTOCONFIRM} + GOTRUE_SMTP_ADMIN_EMAIL: ${SOCIAL_INFRA__SMTP__ADMIN_EMAIL} + GOTRUE_SMTP_HOST: ${SOCIAL_INFRA__SMTP__HOST} + GOTRUE_SMTP_PORT: ${SOCIAL_INFRA__SMTP__PORT} + GOTRUE_SMTP_USER: ${SOCIAL_INFRA__SMTP__USER} + GOTRUE_SMTP_PASS: ${SOCIAL_INFRA__SMTP__PASS} + GOTRUE_SMTP_SENDER_NAME: ${SOCIAL_INFRA__SMTP__SENDER_NAME} + GOTRUE_MAILER_URLPATHS_INVITE: ${SOCIAL_INFRA__MAILER__URLPATHS_INVITE} + GOTRUE_MAILER_URLPATHS_CONFIRMATION: ${SOCIAL_INFRA__MAILER__URLPATHS_CONFIRMATION} + GOTRUE_MAILER_URLPATHS_RECOVERY: ${SOCIAL_INFRA__MAILER__URLPATHS_RECOVERY} + GOTRUE_MAILER_URLPATHS_EMAIL_CHANGE: ${SOCIAL_INFRA__MAILER__URLPATHS_EMAIL_CHANGE} + GOTRUE_EXTERNAL_PHONE_ENABLED: ${SOCIAL_INFRA__AUTH__ENABLE_PHONE_SIGNUP} + GOTRUE_SMS_AUTOCONFIRM: ${SOCIAL_INFRA__AUTH__ENABLE_PHONE_AUTOCONFIRM} rest: container_name: supabase-rest @@ -157,13 +157,13 @@ services: analytics: condition: service_healthy environment: - PGRST_DB_URI: postgres://authenticator:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} + PGRST_DB_URI: postgres://authenticator:${SOCIAL_INFRA__POSTGRES__PASSWORD}@${SOCIAL_INFRA__POSTGRES__HOST}:${SOCIAL_INFRA__POSTGRES__PORT}/${SOCIAL_INFRA__POSTGRES__DB} + PGRST_DB_SCHEMAS: ${SOCIAL_INFRA__PGRST__DB_SCHEMAS} PGRST_DB_ANON_ROLE: anon - PGRST_JWT_SECRET: ${JWT_SECRET} + PGRST_JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} PGRST_DB_USE_LEGACY_GUCS: "false" - PGRST_APP_SETTINGS_JWT_SECRET: ${JWT_SECRET} - PGRST_APP_SETTINGS_JWT_EXP: ${JWT_EXPIRY} + PGRST_APP_SETTINGS_JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} + PGRST_APP_SETTINGS_JWT_EXP: ${SOCIAL_INFRA__JWT__EXPIRY} command: ["postgrest"] realtime: @@ -179,7 +179,7 @@ services: test: [ "CMD-SHELL", - 'curl -sSfL --head -o /dev/null -H "Authorization: Bearer ${ANON_KEY}" http://localhost:4000/api/tenants/realtime-dev/health', + 'curl -sSfL --head -o /dev/null -H "Authorization: Bearer ${SOCIAL_INFRA__SUPABASE__ANON_KEY}" http://localhost:4000/api/tenants/realtime-dev/health', ] timeout: 5s interval: 30s @@ -187,16 +187,16 @@ services: start_period: 10s environment: PORT: 4000 - DB_HOST: ${POSTGRES_HOST} - DB_PORT: ${POSTGRES_PORT} + DB_HOST: ${SOCIAL_INFRA__POSTGRES__HOST} + DB_PORT: ${SOCIAL_INFRA__POSTGRES__PORT} DB_USER: supabase_admin - DB_PASSWORD: ${POSTGRES_PASSWORD} - DB_NAME: ${POSTGRES_DB} + DB_PASSWORD: ${SOCIAL_INFRA__POSTGRES__PASSWORD} + DB_NAME: ${SOCIAL_INFRA__POSTGRES__DB} DB_AFTER_CONNECT_QUERY: "SET search_path TO _realtime" DB_ENC_KEY: supabaserealtime - API_JWT_SECRET: ${JWT_SECRET} - ANON_KEY: ${ANON_KEY} - SECRET_KEY_BASE: ${SECRET_KEY_BASE} + API_JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} + ANON_KEY: ${SOCIAL_INFRA__SUPABASE__ANON_KEY} + SECRET_KEY_BASE: ${SOCIAL_INFRA__SUPAVISOR__SECRET_KEY_BASE} ERL_AFLAGS: -proto_dist inet_tcp DNS_NODES: "''" RLIMIT_NOFILE: "10000" @@ -232,11 +232,11 @@ services: imgproxy: condition: service_started environment: - ANON_KEY: ${ANON_KEY} - SERVICE_KEY: ${SERVICE_ROLE_KEY} + ANON_KEY: ${SOCIAL_INFRA__SUPABASE__ANON_KEY} + SERVICE_KEY: ${SOCIAL_INFRA__SUPABASE__SERVICE_ROLE_KEY} POSTGREST_URL: http://rest:3000 - PGRST_JWT_SECRET: ${JWT_SECRET} - DATABASE_URL: postgres://supabase_storage_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + PGRST_JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} + DATABASE_URL: postgres://supabase_storage_admin:${SOCIAL_INFRA__POSTGRES__PASSWORD}@${SOCIAL_INFRA__POSTGRES__HOST}:${SOCIAL_INFRA__POSTGRES__PORT}/${SOCIAL_INFRA__POSTGRES__DB} REQUEST_ALLOW_X_FORWARDED_PATH: "true" FILE_SIZE_LIMIT: 52428800 STORAGE_BACKEND: file @@ -262,7 +262,7 @@ services: IMGPROXY_BIND: ":5001" IMGPROXY_LOCAL_FILESYSTEM_ROOT: / IMGPROXY_USE_ETAG: "true" - IMGPROXY_ENABLE_WEBP_DETECTION: ${IMGPROXY_ENABLE_WEBP_DETECTION} + IMGPROXY_ENABLE_WEBP_DETECTION: ${SOCIAL_INFRA__IMGPROXY__ENABLE_WEBP_DETECTION} IMGPROXY_MAX_SRC_RESOLUTION: 16.8 meta: @@ -276,12 +276,12 @@ services: condition: service_healthy environment: PG_META_PORT: 8080 - PG_META_DB_HOST: ${POSTGRES_HOST} - PG_META_DB_PORT: ${POSTGRES_PORT} - PG_META_DB_NAME: ${POSTGRES_DB} + PG_META_DB_HOST: ${SOCIAL_INFRA__POSTGRES__HOST} + PG_META_DB_PORT: ${SOCIAL_INFRA__POSTGRES__PORT} + PG_META_DB_NAME: ${SOCIAL_INFRA__POSTGRES__DB} PG_META_DB_USER: supabase_admin - PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} - CRYPTO_KEY: ${PG_META_CRYPTO_KEY} + PG_META_DB_PASSWORD: ${SOCIAL_INFRA__POSTGRES__PASSWORD} + CRYPTO_KEY: ${SOCIAL_INFRA__PG_META__CRYPTO_KEY} functions: container_name: supabase-edge-functions @@ -293,12 +293,12 @@ services: analytics: condition: service_healthy environment: - JWT_SECRET: ${JWT_SECRET} + JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} SUPABASE_URL: http://kong:8000 - SUPABASE_ANON_KEY: ${ANON_KEY} - SUPABASE_SERVICE_ROLE_KEY: ${SERVICE_ROLE_KEY} - SUPABASE_DB_URL: postgresql://postgres:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - VERIFY_JWT: "${FUNCTIONS_VERIFY_JWT}" + SUPABASE_ANON_KEY: ${SOCIAL_INFRA__SUPABASE__ANON_KEY} + SUPABASE_SERVICE_ROLE_KEY: ${SOCIAL_INFRA__SUPABASE__SERVICE_ROLE_KEY} + SUPABASE_DB_URL: postgresql://postgres:${SOCIAL_INFRA__POSTGRES__PASSWORD}@${SOCIAL_INFRA__POSTGRES__HOST}:${SOCIAL_INFRA__POSTGRES__PORT}/${SOCIAL_INFRA__POSTGRES__DB} + VERIFY_JWT: "${SOCIAL_INFRA__FUNCTIONS__VERIFY_JWT}" command: ["start", "--main-service", "/home/deno/functions/main"] analytics: @@ -319,15 +319,15 @@ services: LOGFLARE_NODE_HOST: 127.0.0.1 DB_USERNAME: supabase_admin DB_DATABASE: _supabase - DB_HOSTNAME: ${POSTGRES_HOST} - DB_PORT: ${POSTGRES_PORT} - DB_PASSWORD: ${POSTGRES_PASSWORD} + DB_HOSTNAME: ${SOCIAL_INFRA__POSTGRES__HOST} + DB_PORT: ${SOCIAL_INFRA__POSTGRES__PORT} + DB_PASSWORD: ${SOCIAL_INFRA__POSTGRES__PASSWORD} DB_SCHEMA: _analytics - LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} - LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} + LOGFLARE_PUBLIC_ACCESS_TOKEN: ${SOCIAL_INFRA__LOGFLARE__PUBLIC_ACCESS_TOKEN} + LOGFLARE_PRIVATE_ACCESS_TOKEN: ${SOCIAL_INFRA__LOGFLARE__PRIVATE_ACCESS_TOKEN} LOGFLARE_SINGLE_TENANT: true LOGFLARE_SUPABASE_MODE: true - POSTGRES_BACKEND_URL: postgresql://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase + POSTGRES_BACKEND_URL: postgresql://supabase_admin:${SOCIAL_INFRA__POSTGRES__PASSWORD}@${SOCIAL_INFRA__POSTGRES__HOST}:${SOCIAL_INFRA__POSTGRES__PORT}/_supabase POSTGRES_BACKEND_SCHEMA: _analytics LOGFLARE_FEATURE_FLAG_OVERRIDE: multibackend=true @@ -355,14 +355,14 @@ services: condition: service_healthy environment: POSTGRES_HOST: /var/run/postgresql - PGPORT: ${POSTGRES_PORT} - POSTGRES_PORT: ${POSTGRES_PORT} - PGPASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - PGDATABASE: ${POSTGRES_DB} - POSTGRES_DB: ${POSTGRES_DB} - JWT_SECRET: ${JWT_SECRET} - JWT_EXP: ${JWT_EXPIRY} + PGPORT: ${SOCIAL_INFRA__POSTGRES__PORT} + POSTGRES_PORT: ${SOCIAL_INFRA__POSTGRES__PORT} + PGPASSWORD: ${SOCIAL_INFRA__POSTGRES__PASSWORD} + POSTGRES_PASSWORD: ${SOCIAL_INFRA__POSTGRES__PASSWORD} + PGDATABASE: ${SOCIAL_INFRA__POSTGRES__DB} + POSTGRES_DB: ${SOCIAL_INFRA__POSTGRES__DB} + JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} + JWT_EXP: ${SOCIAL_INFRA__JWT__EXPIRY} command: [ "postgres", @@ -378,7 +378,7 @@ services: restart: unless-stopped volumes: - ./supabase/volumes/logs/vector.yml:/etc/vector/vector.yml:ro,z - - ${DOCKER_SOCKET_LOCATION}:/var/run/docker.sock:ro,z + - ${SOCIAL_INFRA__DOCKER__SOCKET_LOCATION}:/var/run/docker.sock:ro,z healthcheck: test: [ @@ -393,7 +393,7 @@ services: interval: 5s retries: 3 environment: - LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} + LOGFLARE_PUBLIC_ACCESS_TOKEN: ${SOCIAL_INFRA__LOGFLARE__PUBLIC_ACCESS_TOKEN} command: ["--config", "/etc/vector/vector.yml"] security_opt: - "label=disable" @@ -403,8 +403,8 @@ services: image: supabase/supavisor:2.7.4 restart: unless-stopped ports: - - ${POSTGRES_PORT}:5432 - - ${POOLER_PROXY_PORT_TRANSACTION}:6543 + - ${SOCIAL_INFRA__POSTGRES__PORT}:5432 + - ${SOCIAL_INFRA__POOLER__PROXY_PORT_TRANSACTION}:6543 volumes: - ./supabase/volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro,z healthcheck: @@ -428,22 +428,22 @@ services: condition: service_healthy environment: PORT: 4000 - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - DATABASE_URL: ecto://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase + POSTGRES_PORT: ${SOCIAL_INFRA__POSTGRES__PORT} + POSTGRES_DB: ${SOCIAL_INFRA__POSTGRES__DB} + POSTGRES_PASSWORD: ${SOCIAL_INFRA__POSTGRES__PASSWORD} + DATABASE_URL: ecto://supabase_admin:${SOCIAL_INFRA__POSTGRES__PASSWORD}@${SOCIAL_INFRA__POSTGRES__HOST}:${SOCIAL_INFRA__POSTGRES__PORT}/_supabase CLUSTER_POSTGRES: true - SECRET_KEY_BASE: ${SECRET_KEY_BASE} - VAULT_ENC_KEY: ${VAULT_ENC_KEY} - API_JWT_SECRET: ${JWT_SECRET} - METRICS_JWT_SECRET: ${JWT_SECRET} + SECRET_KEY_BASE: ${SOCIAL_INFRA__SUPAVISOR__SECRET_KEY_BASE} + VAULT_ENC_KEY: ${SOCIAL_INFRA__SUPAVISOR__VAULT_ENC_KEY} + API_JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} + METRICS_JWT_SECRET: ${SOCIAL_INFRA__JWT__SECRET} REGION: local ERL_AFLAGS: -proto_dist inet_tcp - POOLER_TENANT_ID: ${POOLER_TENANT_ID} - POOLER_DEFAULT_POOL_SIZE: ${POOLER_DEFAULT_POOL_SIZE} - POOLER_MAX_CLIENT_CONN: ${POOLER_MAX_CLIENT_CONN} + POOLER_TENANT_ID: ${SOCIAL_INFRA__POOLER__TENANT_ID} + POOLER_DEFAULT_POOL_SIZE: ${SOCIAL_INFRA__POOLER__DEFAULT_POOL_SIZE} + POOLER_MAX_CLIENT_CONN: ${SOCIAL_INFRA__POOLER__MAX_CLIENT_CONN} POOLER_POOL_MODE: transaction - DB_POOL_SIZE: ${POOLER_DB_POOL_SIZE} + DB_POOL_SIZE: ${SOCIAL_INFRA__POOLER__DB_POOL_SIZE} command: [ "/bin/sh", diff --git a/infra/local/supabase/volumes/api/kong.yml b/docker/supabase/volumes/api/kong.yml similarity index 100% rename from infra/local/supabase/volumes/api/kong.yml rename to docker/supabase/volumes/api/kong.yml diff --git a/infra/local/supabase/volumes/db/_supabase.sql b/docker/supabase/volumes/db/_supabase.sql similarity index 100% rename from infra/local/supabase/volumes/db/_supabase.sql rename to docker/supabase/volumes/db/_supabase.sql diff --git a/infra/local/supabase/volumes/db/jwt.sql b/docker/supabase/volumes/db/jwt.sql similarity index 100% rename from infra/local/supabase/volumes/db/jwt.sql rename to docker/supabase/volumes/db/jwt.sql diff --git a/infra/local/supabase/volumes/db/logs.sql b/docker/supabase/volumes/db/logs.sql similarity index 100% rename from infra/local/supabase/volumes/db/logs.sql rename to docker/supabase/volumes/db/logs.sql diff --git a/infra/local/supabase/volumes/db/pooler.sql b/docker/supabase/volumes/db/pooler.sql similarity index 100% rename from infra/local/supabase/volumes/db/pooler.sql rename to docker/supabase/volumes/db/pooler.sql diff --git a/infra/local/supabase/volumes/db/realtime.sql b/docker/supabase/volumes/db/realtime.sql similarity index 100% rename from infra/local/supabase/volumes/db/realtime.sql rename to docker/supabase/volumes/db/realtime.sql diff --git a/infra/local/supabase/volumes/db/roles.sql b/docker/supabase/volumes/db/roles.sql similarity index 100% rename from infra/local/supabase/volumes/db/roles.sql rename to docker/supabase/volumes/db/roles.sql diff --git a/infra/local/supabase/volumes/db/webhooks.sql b/docker/supabase/volumes/db/webhooks.sql similarity index 100% rename from infra/local/supabase/volumes/db/webhooks.sql rename to docker/supabase/volumes/db/webhooks.sql diff --git a/infra/local/supabase/volumes/functions/main/index.ts b/docker/supabase/volumes/functions/main/index.ts similarity index 100% rename from infra/local/supabase/volumes/functions/main/index.ts rename to docker/supabase/volumes/functions/main/index.ts diff --git a/infra/local/supabase/volumes/logs/vector.yml b/docker/supabase/volumes/logs/vector.yml similarity index 100% rename from infra/local/supabase/volumes/logs/vector.yml rename to docker/supabase/volumes/logs/vector.yml diff --git a/infra/local/supabase/volumes/pooler/pooler.exs b/docker/supabase/volumes/pooler/pooler.exs similarity index 100% rename from infra/local/supabase/volumes/pooler/pooler.exs rename to docker/supabase/volumes/pooler/pooler.exs diff --git a/docs/adr/0001-tech-stack.md b/docs/adr/0001-tech-stack.md deleted file mode 100644 index 8cd55f7..0000000 --- a/docs/adr/0001-tech-stack.md +++ /dev/null @@ -1,38 +0,0 @@ -# 技术栈选择 - -## 背景 - -本项目需要构建一个跨平台社交应用，支持本地开发和云端部署。 - -## 决策 - -1. **前端框架：Flutter** - - 跨平台支持（iOS / Android / Web） - - 高性能原生渲染 - - 丰富的 UI 组件 - -2. **后端框架：FastAPI** - - 高性能异步框架 - - 自动生成 OpenAPI 文档 - - 类型安全 - -3. **数据库：Supabase（PostgreSQL）** - - 开箱即用的 PostgreSQL - - 内置认证和权限管理 - - 实时订阅功能 - -4. **缓存：Redis** - - 高性能键值存储 - - 支持多种数据结构 - -5. **向量数据库：Milvus** - - 高性能向量检索 - - 支持大规模向量存储 - - 适合 RAG 和推荐场景 - -## 后续考虑 - -根据业务发展，可能需要评估： -- CDN 方案 -- 消息队列 -- 监控和日志系统 diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md deleted file mode 100644 index 7fb5f73..0000000 --- a/docs/architecture/overview.md +++ /dev/null @@ -1,17 +0,0 @@ -# 系统架构概述 - -## 技术栈 - -- **前端**：Flutter -- **后端**：FastAPI -- **数据库**：Supabase（PostgreSQL） -- **缓存**：Redis -- **向量数据库**：Milvus -- **部署**：Docker + 火山云（未来） - -## 架构特点 - -- Monorepo 结构 -- 微服务架构（API + Worker） -- 云原生设计 -- 支持本地 Docker 开发和云端部署 diff --git a/docs/plans/PLAN-logging-manager-2026-01-29.md b/docs/plans/PLAN-logging-manager-2026-01-29.md new file mode 100644 index 0000000..e8d7b6c --- /dev/null +++ b/docs/plans/PLAN-logging-manager-2026-01-29.md @@ -0,0 +1,147 @@ +# Plan: FastAPI + Celery 日志管理器系统 + +**Date:** 2026-01-29 +**Author:** AI Assistant +**Status:** Draft + +## Overview + +构建一个统一、可扩展的日志管理器系统，覆盖 FastAPI 与 Celery worker 的运行时日志，提供结构化 JSON 输出、错误分离、日志轮转与上下文追踪。目标是满足生产环境可观测性需求，便于检索、关联与故障排查，并与当前项目配置体系保持一致。 + +## Requirements + +### Functional +- [ ] 统一管理 FastAPI 与 Celery worker 日志 +- [ ] 日志持久化到 `logs/`，错误日志单独输出到 `logs/errors/` +- [ ] 支持按大小或按时间进行日志轮转 +- [ ] 结构化日志（JSON），包含时间戳、级别、模块/函数、消息与上下文 +- [ ] ERROR/CRITICAL 记录完整堆栈与错误上下文 +- [ ] 支持环境差异化配置（dev/test/prod） + +### Non-Functional +- [ ] 性能：日志写入对请求延迟影响可控，支持异步队列化扩展 +- [ ] 安全：避免记录敏感信息，支持字段脱敏 +- [ ] 可维护性：模块化、可测试、与现有配置体系一致 + +## Technical Approach + +### 调研摘要 +- Python 官方建议使用 `logging` + `dictConfig` 管理多 handler、多 formatter 与过滤器，适用于生产环境配置化管理。 +- FastAPI 通常通过中间件注入 request_id 和上下文，并使用结构化日志输出以便集中检索。 +- Celery 官方文档建议在自定义场景下关闭 `worker_hijack_root_logger`，通过信号配置自定义 handler。 +- 结构化日志库中，structlog 更贴近标准 logging，可与 `logging` 生态协同；loguru 简化配置但替换性强、与 Celery 深度集成时可控性较弱。 +- 生产环境推荐 JSON 结构化日志 + 轮转 + 错误分离，并通过外部系统聚合与告警（如 Sentry）。 + +### 方案对比（至少两种） +| 方案 | 描述 | 优点 | 缺点 | 结论 | +|------|------|------|------|------| +| 方案 A：stdlib logging + 自定义 JSON Formatter | 纯标准库实现 JSON formatter + handler/filters | 依赖最少，符合标准库，易与 Celery/FastAPI 集成 | 结构化上下文绑定与 request_id 传递需手写 | 可作为备选最小方案 | +| 方案 B：stdlib logging + structlog | 用 structlog 生成结构化事件，输出到 logging handler | 结构化上下文与 contextvars 支持好，兼容 logging handler | 引入第三方依赖与配置复杂度 | 推荐主方案 | +| 方案 C：loguru | 直接使用 loguru logger | 配置简单、体验好 | 与 Celery/标准 logging 生态整合成本高 | 不推荐作为主方案 | + +### 选型结论 +- 采用方案 B：`logging` 作为底座，structlog 负责结构化事件与上下文绑定；保留可切换到方案 A 的最小实现路径。 +- 通过 `dictConfig` 做环境配置，使用 Rotating/TimedRotating handler 支持按大小或时间轮转。 + +## Implementation Steps + +### Phase 1: 基础日志骨架与配置 (3 hours) +1. 新增日志配置模型（Settings 扩展），支持环境、轮转方式与路径配置。 +2. 创建日志模块骨架：formatter、handler、filter、context。 +3. 集成 `dictConfig` 初始化入口，支持 dev/test/prod 配置切换。 + +### Phase 2: FastAPI 集成与上下文 (4 hours) +1. 实现请求中间件：生成 `request_id`，绑定用户与请求上下文（IP、路径、方法）。 +2. 定义异常处理器：捕获未处理异常并记录堆栈与上下文。 +3. 添加应用启动时日志初始化流程。 + +### Phase 3: Celery 集成 (3 hours) +1. 在 Celery 应用配置中设置 `worker_hijack_root_logger = False`。 +2. 使用 Celery 信号（`setup_logging`、`after_setup_task_logger`）初始化日志并注入 task 上下文。 +3. 统一日志格式、error 处理与 request_id 关联（如 task_id）。 + +### Phase 4: 错误分离与轮转策略 (3 hours) +1. 添加 error handler：仅接受 ERROR/CRITICAL，输出到 `logs/errors/`。 +2. 实现轮转策略配置（按大小、按时间），并提供统一切换配置项。 +3. 增加字段脱敏与敏感字段黑名单过滤器。 + +### Phase 5: 可选增强功能 (4 hours) +1. 日志查询与过滤接口（基础 API + 分页）。 +2. 日志聚合统计（按级别/模块/时间窗口）。 +3. Sentry 集成与异常告警。 + +## Files to Modify + +| File | Changes | +|------|---------| +| api/src/core/config/settings.py | 扩展日志相关配置模型 | + +## Files to Create + +| File | Purpose | +|------|---------| +| api/src/core/logging/__init__.py | 模块导出与初始化入口 | +| api/src/core/logging/config.py | dictConfig 构建与环境配置 | +| api/src/core/logging/formatters.py | JSON formatter 与字段规范 | +| api/src/core/logging/handlers.py | 文件、控制台、错误 handler | +| api/src/core/logging/filters.py | 等级过滤、敏感字段脱敏 | +| api/src/core/logging/context.py | contextvars 绑定与获取 | +| api/src/core/logging/middleware.py | FastAPI 请求中间件 | +| api/src/core/logging/celery.py | Celery 日志信号集成 | +| api/src/core/logging/examples.py | 使用示例（可选） | + +## Dependencies + +- [ ] structlog: 结构化日志与 contextvars 支持 +- [ ] python-json-logger（备选）: 若需要纯 logging JSON formatter +- [ ] sentry-sdk（可选）: 异常告警与追踪 + +## 配置示例 + +```toml +# .env 示例（通过 pydantic settings 读取） +SOCIAL_RUNTIME__LOG_LEVEL=INFO +SOCIAL_RUNTIME__LOG_JSON=true +SOCIAL_RUNTIME__LOG_ROTATION=TIME +SOCIAL_RUNTIME__LOG_ROTATION_WHEN=midnight +SOCIAL_RUNTIME__LOG_ROTATION_BACKUP_COUNT=14 +SOCIAL_RUNTIME__LOG_DIR=logs +SOCIAL_RUNTIME__LOG_ERROR_DIR=logs/errors +``` + +## 使用示例代码 + +```python +from core.logging import configure_logging, get_logger + +configure_logging() +logger = get_logger(__name__) + +logger.info("user login", extra={"user_id": "u_123"}) +``` + +## Testing Strategy + +- **Unit Tests:** formatter 输出结构、filter 脱敏规则、context 绑定行为 +- **Integration Tests:** FastAPI 中间件注入的 request_id 与错误分离写入 +- **E2E Tests:** 关键流程触发错误，验证 error 日志输出与轮转 + +## Risks & Mitigations + +| Risk | Impact | Likelihood | Mitigation | +|------|--------|------------|------------| +| Celery 日志被自动劫持导致重复或丢失 | High | Medium | 设置 `worker_hijack_root_logger=False` 并通过信号统一配置 | +| 结构化字段不一致导致下游解析失败 | Medium | Medium | 统一 schema，增加单元测试与校验 | +| 误记录敏感信息 | High | Medium | 增加脱敏过滤器与字段黑名单 | +| 日志量过大影响性能 | Medium | Medium | 轮转 + 级别控制 + 可选异步队列化 | + +## Estimated Effort + +| Phase | Effort | +|-------|--------| +| Phase 1 | 3 hours | +| Phase 2 | 4 hours | +| Phase 3 | 3 hours | +| Phase 4 | 3 hours | +| Phase 5 | 4 hours | +| **Total** | **17 hours** | diff --git a/docs/rules/config-rules.md b/docs/rules/config-rules.md deleted file mode 100644 index 4a8f285..0000000 --- a/docs/rules/config-rules.md +++ /dev/null @@ -1,203 +0,0 @@ -# 配置规则与使用原则 - -## 配置文件组织 - -### 1. 环境变量规范（真相源） - -**文件位置**：`configs/env/.env.example` - -**作用**： -- 定义应用层环境变量的名称、类型、用途和敏感级别 -- 作为应用配置的"真相源"（source of truth） -- 不可直接用于运行，仅作为参考和规范 -- Docker/Supabase 栈配置不在本文件，使用 `infra/local/env/.env.example` -- 本文件仅包含应用层连接配置（如 `SUPABASE_URL`、`DATABASE_URL`），不包含 Supabase 栈运行变量 -- 不得在 `infra/local/env/.env.example` 中添加应用层变量（如 `PUBLIC_`、`API_`） - -**变量分类**： -- `public`：可公开的信息，如服务地址、端口号 -- `secret`：敏感信息，如密钥、密码、连接串 - -**变量块说明**： -- A. 通用环境（APP_ENV、LOG_LEVEL、TZ） -- B. Flutter 配置（仅 PUBLIC_ 变量） -- C. FastAPI 服务配置 -- D. Supabase / Postgres 连接配置 -- E. Redis 配置 -- F. Milvus 配置 -- G. 对象存储配置（可选） -- H. 其他配置 - -### 2. 本地开发配置 - -**文件位置**：`configs/env/.env` - -**作用**： -- 本地应用层的实际配置文件 -- 从 `configs/env/.env.example` 复制后填入真实值 -- **禁止提交到 Git 仓库** - -**创建方式**： -```bash -cp configs/env/.env.example configs/env/.env -# 编辑 configs/env/.env，填入实际值 -``` - -### 3. 本地 Supabase 栈配置 - -**文件位置**：`infra/local/env/.env` - -**作用**： -- 本地 Supabase + 周边依赖的实际配置文件 -- 从 `infra/local/env/.env.example` 复制后填入真实值 -- **禁止提交到 Git 仓库** - -**创建方式**： -```bash -cp infra/local/env/.env.example infra/local/env/.env -``` - -### 4. 云端部署配置 - -**方式一：环境文件** -- 文件位置：`infra/cloud/volcano/env/.env`（不提交） -- 从 `infra/cloud/volcano/env/.env.example` 复制后填入云端真实值 -- 仅用于本地测试云端连接 - -**方式二：Secret 注入（推荐）** -- 使用火山云或其他云平台的 Secret 管理服务 -- 通过 CI/CD 或容器运行时注入环境变量 -- 代码无需修改，通过环境切换 - -### 5. 应用特定配置 - -**Flutter 配置**： -- `configs/flutter/dev.json` —— 开发环境 -- `configs/flutter/prod.json` —— 生产环境 -- 通过 `--dart-define` 在构建时注入 - -## 安全原则 - -### Public vs Secret - -**Public 变量**： -- 可公开的服务地址和端口号 -- 前端可直接使用的配置 -- 示例：`PUBLIC_API_BASE_URL`、`API_PORT` - -**Secret 变量**： -- 密钥、密码、连接串 -- 仅服务端使用的敏感信息 -- 示例：`DATABASE_URL`、`REDIS_URL`、`JWT_SECRET` - -### Flutter 配置限制 - -**严格规则**： -- Flutter 只能使用以 `PUBLIC_` 开头的变量 -- 严禁将任何 secret 信息注入到 Flutter -- 通过 `--dart-define` 在构建时注入，运行时不可修改 - -**示例**： -```bash -# ✅ 正确：Flutter 使用 PUBLIC_ 变量 -flutter run --dart-define=PUBLIC_API_BASE_URL=http://localhost:8000 - -# ❌ 错误：Flutter 不能使用 secret -flutter run --dart-define=DATABASE_URL=postgresql://user:pass@localhost/db -``` - -### 后端配置读取 - -**严格规则**： -- `apps/api` 和 `apps/worker` 只能通过环境变量读取配置 -- 不得直接读取 `infra/local/*.env` 文件路径 -- 使用 Python 的 `os.getenv()` 或环境变量库 - -**示例**： -```python -# ✅ 正确：通过环境变量读取 -import os - -database_url = os.getenv("DATABASE_URL") - -# ❌ 错误：直接读取本地配置文件 -from dotenv import load_dotenv -load_dotenv("configs/env/.env") # 禁止 -``` - -## 使用方式 - -### 1. 本地开发 - -1. 复制环境变量模板： - ```bash - make env - ``` - -2. 启动依赖服务： - ```bash - make up - ``` - -3. 启动后端服务： - ```bash - make api-dev - ``` - -4. 启动 Flutter 应用： - ```bash - make flutter-dev - ``` - -### 2. 云端部署 - -1. 准备云端环境变量： - - 将 `.env.example` 中的变量映射到云平台 Secret -- 或创建 `infra/cloud/volcano/env/.env`（仅用于本地测试） - -2. 修改配置指向云端： - ```bash - # 示例：修改 .env 指向火山云托管地址 - REDIS_URL=redis://volcano-redis:6379/0 - MILVUS_URI=https://volcano-milvus:19530 - DATABASE_URL=postgresql://user:pass@volcano-postgres/db - ``` - -3. 部署应用： - - 通过 CI/CD 自动注入环境变量 - - 或通过云平台控制台配置 - -## 配置优先级 - -从高到低： -1. 运行时环境变量（最高） -2. 环境文件（`.env`） -3. 配置文件（`configs/flutter/*.json`） - -## 常见问题 - -### Q: 如何切换环境？ - -**本地开发**：使用 `configs/env/.env` -**云端部署**：使用云平台 Secret 注入 - -### Q: 如何确保 secret 不泄露？ - -- 所有 secret 变量使用 `CHANGE_ME` 占位符 -- 本地配置文件（`configs/env/.env`、`infra/local/env/.env`）添加到 `.gitignore` -- 云端使用密钥管理服务注入 - -### Q: 如何测试云端配置？ - -1. 在本地创建 `infra/cloud/volcano/env/.env`（不提交） -2. 修改变量指向云端地址 -3. 导出环境变量测试： - ```bash - export $(cat infra/cloud/volcano/env/.env | xargs) - ``` - -### Q: Compose 如何读取配置？ - -- 使用 `--env-file infra/local/env/.env` 注入本地 Supabase 栈变量 -- Compose 文件为 `infra/local/docker-compose.yml` -- 应用代码通过 `os.getenv()` 读取本地应用变量 diff --git a/docs/rules/repo-structure.md b/docs/rules/repo-structure.md deleted file mode 100644 index 1551fc2..0000000 --- a/docs/rules/repo-structure.md +++ /dev/null @@ -1,21 +0,0 @@ -# 目录结构规则 - -## 顶层目录（必须遵守） - -仓库根目录只能包含以下目录： - -- `apps/` —— 可运行应用（Flutter / FastAPI / Worker） -- `infra/` —— 基础设施（本地 docker / 云部署 / 迁移） -- `configs/` —— 配置规范与公共配置模板（不含密钥） -- `tools/` —— 脚本与生成器 -- `docs/` —— 文档与规则 -- `.github/`（可选，用于 CI/CD） -- `README.md` -- `Makefile`（可选） - -## 禁止事项 - -- 禁止在根目录直接出现：`backend/`、`ui/`、`docker/`、`scripts/` 等非规范目录 -- 所有业务代码必须放在 `apps/` 目录下 -- 所有配置文件必须放在 `configs/` 目录下 -- 所有基础设施相关代码必须放在 `infra/` 目录下 diff --git a/docs/runbooks/cloud-volcano.md b/docs/runbooks/cloud-volcano.md deleted file mode 100644 index 3e127e6..0000000 --- a/docs/runbooks/cloud-volcano.md +++ /dev/null @@ -1,25 +0,0 @@ -# 火山云部署指南 - -## 准备工作 - -1. 火山云账号 -2. 配置云端环境变量 -3. 准备镜像仓库 - -## 环境变量模板 - -云端模板文件位置： - -```bash -cp infra/cloud/volcano/env/.env.example infra/cloud/volcano/env/.env -``` - -## 部署流程 - -待补充详细步骤... - -## 注意事项 - -- 确保所有敏感信息使用环境变量或密钥管理 -- 遵循最小权限原则 -- 配置适当的监控和日志 diff --git a/docs/runbooks/local-dev.md b/docs/runbooks/local-dev.md deleted file mode 100644 index 4698821..0000000 --- a/docs/runbooks/local-dev.md +++ /dev/null @@ -1,272 +0,0 @@ -# 本地开发指南 - -## 前置要求 - -- Docker 和 Docker Compose -- Flutter SDK -- Python 3.11+ - -## 快速开始 - -### 1. 配置环境变量 - -```bash -make env -``` - -按照提示创建并编辑应用配置 `configs/env/.env`，并创建 Supabase 本地栈配置 `infra/local/env/.env`。 - -创建配置文件： - -```bash -cp configs/env/.env.example configs/env/.env -cp infra/local/env/.env.example infra/local/env/.env -``` - -确保以下变量配置正确： -- `DATABASE_URL`（连接到 localhost:54322） -- `REDIS_URL`（连接到 localhost:6379） -- `MILVUS_URI`（连接到 localhost:19530） - -### 2. 启动依赖服务 - -```bash -make up -``` - -这将启动以下服务： -- **Redis**：端口 6379 -- **Milvus**：端口 19530 (gRPC) / 19111 (HTTP) -- **Postgres**：端口 54322 - -### 3. 检查服务状态 - -```bash -make ps -``` - -确保所有服务显示为 `Up` 状态。 - -### 4. 查看服务日志 - -```bash -# 查看所有服务日志 -make logs - -# 查看特定服务日志 -make logs SERVICE=redis -make logs SERVICE=milvus -make logs SERVICE=db -``` - -## 启动应用 - -### 启动 FastAPI 后端 - -```bash -make api-dev -``` - -或手动启动： - -```bash -cd apps/api - -# 创建虚拟环境（首次） -python -m venv .venv -source .venv/bin/activate - -# 安装依赖（首次） -pip install -r requirements.txt - -# 启动服务 -uvicorn src.main:app --host 0.0.0.0 --port 8000 --reload -``` - -后端启动后，访问： -- API 文档：http://localhost:8000/docs -- ReDoc：http://localhost:8000/redoc - -### 启动 Flutter 应用 - -```bash -make flutter-dev -``` - -或手动启动： - -```bash -cd apps/mobile - -# 安装依赖（首次） -flutter pub get - -# 启动开发服务器 -flutter run --dart-define=PUBLIC_API_BASE_URL=http://localhost:8000 - -# 或指定设备 -flutter run -d chrome --dart-define=PUBLIC_API_BASE_URL=http://localhost:8000 -``` - -构建 Android APK： - -```bash -flutter build apk --dart-define=PUBLIC_API_BASE_URL=http://localhost:8000 -``` - -## 初始化 Milvus - -```bash -make milvus-init -``` - -或手动运行初始化脚本： - -```bash -bash tools/scripts/init_milvus.sh -``` - -## 常用命令 - -### 依赖服务管理 - -```bash -# 启动服务 -make up - -# 停止服务 -make down - -# 重启服务 -make down && make up - -# 查看状态 -make ps - -# 查看日志 -make logs - -# 清理数据（警告：会丢失数据） -make clean -``` - -### 应用管理 - -```bash -# 启动后端 -make api-dev - -# 启动前端 -make flutter-dev - -# 配置环境变量 -make env -``` - -## 常见问题 - -### 端口冲突 - -如果启动依赖服务时出现端口冲突： - -1. 检查端口占用： - ```bash - # 检查 6379（Redis） - lsof -i :6379 - - # 检查 54322（Postgres） - lsof -i :54322 - - # 检查 19530（Milvus） - lsof -i :19530 - ``` - -2. 停止占用端口的进程，或修改 `infra/local/docker-compose.yml` 中的端口映射 - -### 容器未健康 - -如果服务状态显示 `Up (health: starting)` 但一直未变成 `Up (healthy)`： - -1. 查看服务日志： - ```bash - make logs SERVICE= - ``` - -2. 检查依赖服务是否正常启动： - ```bash - make ps - ``` - -3. 重启服务： - ```bash - docker compose -f infra/local/docker-compose.yml --env-file infra/local/env/.env restart - ``` - -### 后端无法连接数据库 - -1. 检查 Postgres 是否正常启动： - ```bash - make ps - ``` - -2. 检查环境变量配置： - ```bash - cat configs/env/.env | grep DATABASE_URL - ``` - -3. 确保数据库 URL 格式正确： - ``` - postgresql://postgres:postgres@localhost:54322/postgres - ``` - -### Flutter 无法连接后端 - -1. 确保后端服务已启动并监听在 8000 端口： - ```bash - curl http://localhost:8000/docs - ``` - -2. 检查 Flutter 的 API_BASE_URL 是否正确注入： - ```bash - flutter run --dart-define=PUBLIC_API_BASE_URL=http://localhost:8000 - ``` - -3. 如果使用模拟器，确保能访问 localhost： - - Android 模拟器：使用 `10.0.2.2` 代替 `localhost` - - iOS 模拟器：使用 `localhost` 即可 - -### Milvus 连接失败 - -1. 检查 Milvus 服务是否健康： - ```bash - make ps - ``` - -2. 等待 Milvus 完全启动（可能需要 1-2 分钟）： - ```bash - make logs SERVICE=milvus - ``` - -3. 测试连接： - ```bash - curl http://localhost:19530/healthz - ``` - -## 清理环境 - -```bash -# 停止所有服务 -make down - -# 清理数据卷（警告：会丢失所有数据） -make clean - -# 完全清理（包括未使用的镜像） -docker system prune -a -``` - -## 下一步 - -- 阅读架构文档：`docs/architecture/overview.md` -- 了解配置规则：`docs/rules/config-rules.md` -- 查看技术栈决策：`docs/adr/0001-tech-stack.md` diff --git a/infra/README.md b/infra/README.md deleted file mode 100644 index 443449f..0000000 --- a/infra/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Infra - -Local Docker environment for Supabase stack and supporting services. - -## Quick Start - -1. Copy environment file: - ```bash - cp infra/env/.env.example infra/env/.env.local - ``` - -2. Update `infra/env/.env.local` with your secrets and configurations. - **Important**: Ensure all required fields are filled (no `CHANGE_ME` values). - -3. Start services: - ```bash - docker compose --env-file infra/env/.env.local -f infra/local/docker-compose.yml up -d - ``` - -## Access - -- **Supabase Studio**: http://localhost:8001 - Credentials from `DASHBOARD_USERNAME` and `DASHBOARD_PASSWORD` in `.env.local` -- **Qdrant**: http://localhost:6333 -- **Redis**: localhost:6379 - -## Troubleshooting - -### Port Conflicts - -If `localhost:8001` is unreachable, check for port conflicts: - -**Linux/WSL:** -```bash -ss -ltnp 'sport = :8001' -``` - -**Windows:** -```powershell -Get-NetTCPConnection -LocalPort 8001 | Select-Object LocalAddress, LocalPort, State, OwningProcess -Get-Process -Id (Get-NetTCPConnection -LocalPort 8001).OwningProcess -``` - -If a Windows service occupies the port, either: -- Stop the conflicting service -- Change `KONG_HTTP_PORT`, `API_EXTERNAL_URL`, and `SUPABASE_PUBLIC_URL` in `.env.local` - -### Environment Variables Not Applied - -If Docker reports warnings about missing variables, verify: -- The `--env-file` path is correct -- All required variables are set in `.env.local` (no empty values) - -### Service Health - -Check service status: -```bash -docker compose --env-file infra/env/.env.local -f infra/local/docker-compose.yml ps -``` - -View logs: -```bash -docker compose --env-file infra/env/.env.local -f infra/local/docker-compose.yml logs -``` - -## Services - -| Service | Description | -|------------|---------------------------------------| -| kong | API gateway (8001/8443) | -| studio | Supabase UI (via Kong) | -| auth | Authentication (GoTrue) | -| db | PostgreSQL database | -| rest | PostgREST API | -| realtime | Realtime subscriptions | -| storage | Storage API | -| functions | Edge functions | -| analytics | Logflare logging | -| vector | Log aggregator | -| supavisor | Database connection pooler | -| qdrant | Vector database | -| redis | Cache/message broker | - -## Important Notes - -- Never commit `.env.local` to version control -- Always use `--env-file` when running docker compose -- Port conflicts on Windows (especially 8000) can prevent Kong from starting -- Kong configuration is auto-generated from templates on container start diff --git a/infra/env/.env.example b/infra/env/.env.example deleted file mode 100644 index 2e35f6b..0000000 --- a/infra/env/.env.example +++ /dev/null @@ -1,90 +0,0 @@ -# Local Docker (Supabase stack) environment example -# Copy to infra/env/.env.local and fill values -# Do not commit real secrets - -############ -# Secrets -############ -POSTGRES_PASSWORD=CHANGE_ME -JWT_SECRET=CHANGE_ME -ANON_KEY=CHANGE_ME -SERVICE_ROLE_KEY=CHANGE_ME -DASHBOARD_USERNAME=supabase -DASHBOARD_PASSWORD=CHANGE_ME -SECRET_KEY_BASE=CHANGE_ME -VAULT_ENC_KEY=CHANGE_ME -PG_META_CRYPTO_KEY=CHANGE_ME - -############ -# Database -############ -POSTGRES_HOST=db -POSTGRES_DB=postgres -POSTGRES_PORT=5432 - -############ -# Supavisor -- Database pooler -############ -POOLER_PROXY_PORT_TRANSACTION=6543 -POOLER_DEFAULT_POOL_SIZE=20 -POOLER_MAX_CLIENT_CONN=100 -POOLER_TENANT_ID=local-tenant -POOLER_DB_POOL_SIZE=5 - -############ -# API Proxy - Kong -############ -KONG_HTTP_PORT=8001 -KONG_HTTPS_PORT=8443 - -############ -# API - PostgREST -############ -PGRST_DB_SCHEMAS=public,storage,graphql_public - -############ -# Auth - GoTrue -############ -SITE_URL=http://localhost:3000 -ADDITIONAL_REDIRECT_URLS= -JWT_EXPIRY=3600 -DISABLE_SIGNUP=false -API_EXTERNAL_URL=http://localhost:8001 -MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify" -MAILER_URLPATHS_INVITE="/auth/v1/verify" -MAILER_URLPATHS_RECOVERY="/auth/v1/verify" -MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify" -ENABLE_EMAIL_SIGNUP=true -ENABLE_EMAIL_AUTOCONFIRM=false -SMTP_ADMIN_EMAIL=admin@example.com -SMTP_HOST=supabase-mail -SMTP_PORT=2500 -SMTP_USER=fake_mail_user -SMTP_PASS=fake_mail_password -SMTP_SENDER_NAME=fake_sender -ENABLE_ANONYMOUS_USERS=false -ENABLE_PHONE_SIGNUP=true -ENABLE_PHONE_AUTOCONFIRM=true - -############ -# Studio -############ -STUDIO_DEFAULT_ORGANIZATION=Default Organization -STUDIO_DEFAULT_PROJECT=Default Project -SUPABASE_PUBLIC_URL=http://localhost:8000 -IMGPROXY_ENABLE_WEBP_DETECTION=true -OPENAI_API_KEY= - -############ -# Functions -############ -FUNCTIONS_VERIFY_JWT=false - -############ -# Logs / Analytics -############ -LOGFLARE_PUBLIC_ACCESS_TOKEN=CHANGE_ME -LOGFLARE_PRIVATE_ACCESS_TOKEN=CHANGE_ME -DOCKER_SOCKET_LOCATION=/var/run/docker.sock -GOOGLE_PROJECT_ID=GOOGLE_PROJECT_ID -GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ba1626a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[project] +name = "social-app" +version = "0.1.0" +description = "Social application backend" +requires-python = ">=3.12" +dependencies = [ + "basedpyright>=1.37.2", + "celery>=5.6.2", + "fastapi>=0.128.0", + "pydantic>=2.11.0", + "pydantic-settings>=2.10.0", + "structlog>=24.4.0", + "supabase>=2.27.2", + "uvicorn[standard]>=0.40.0", +] + +[project.optional-dependencies] +dev = [ + "httpx>=0.28.0", + "playwright>=1.49.0", + "pytest>=8.3.0", + "pytest-asyncio>=0.24.0", + "pytest-cov>=5.0.0", +] + +[[tool.uv.index]] +url = "https://pypi.tuna.tsinghua.edu.cn/simple" +default = true + +[tool.pytest.ini_options] +testpaths = ["api/tests"] +addopts = "-q" +asyncio_mode = "auto" + +[dependency-groups] +dev = [ + "pre-commit>=4.5.1", +] diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..cee68aa --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,22 @@ +{ + "include": ["api"], + "exclude": ["**/__pycache__", "**/node_modules", "**/.git"], + "typeCheckingMode": "standard", + "pythonVersion": "3.12", + "pythonPlatform": "Linux", + "stubPath": "", + "extraPaths": [ + "api/src" + ], + "reportAssignmentType": "none", + "reportMissingImports": "error", + "reportMissingTypeStubs": "none", + "reportUnknownMemberType": "information", + "reportUnknownParameterType": "information", + "reportUnknownVariableType": "information", + "reportUntypedFunctionDecorator": "warning", + "reportUnannotatedClassAttribute": "warning", + "reportDeprecated": "warning", + "reportPrivateImportUsage": "none", + "reportImportCycles": "none" +}