refactor: align repo layout and logging safeguards

api/tests/conftest.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+
+def pytest_configure() -> None:
+    root = Path(__file__).resolve().parents[2]
+    src_path = root / "api" / "src"
+    if str(src_path) not in sys.path:
+        sys.path.append(str(src_path))

api/tests/e2e/test_logging_e2e.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import json
+import socket
+import threading
+import time
+from pathlib import Path
+
+from fastapi import FastAPI
+from playwright.sync_api import sync_playwright
+import uvicorn
+
+from core.config.settings import Settings
+from core.logging.config import configure_logging
+from core.logging.middleware import (
+    RequestContextMiddleware,
+    register_exception_handlers,
+)
+
+
+def _read_json_lines(path: Path) -> list[dict[str, object]]:
+    return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
+
+
+def _find_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+def _wait_for_port(host: str, port: int, timeout: float = 5.0) -> None:
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            if sock.connect_ex((host, port)) == 0:
+                return
+        time.sleep(0.05)
+    raise RuntimeError("Server did not start in time")
+
+
+def _start_server(app: FastAPI, host: str, port: int):
+    config = uvicorn.Config(app, host=host, port=port, log_level="info")
+    server = uvicorn.Server(config)
+    thread = threading.Thread(target=server.run, daemon=True)
+    thread.start()
+    _wait_for_port(host, port)
+    return server, thread
+
+
+def test_e2e_error_logging(tmp_path: Path) -> None:
+    settings = Settings()
+    runtime = settings.runtime.model_copy(
+        update={
+            "log_dir": str(tmp_path),
+            "log_error_dir": str(tmp_path / "errors"),
+            "log_rotation": "size",
+            "log_rotation_max_bytes": 2048,
+        }
+    )
+    configure_logging(settings.model_copy(update={"runtime": runtime}))
+
+    app = FastAPI()
+    app.add_middleware(RequestContextMiddleware)  # type: ignore[arg-type]
+    register_exception_handlers(app)
+
+    @app.get("/boom")
+    async def boom() -> dict[str, str]:
+        raise RuntimeError("boom")
+
+    host = "127.0.0.1"
+    port = _find_free_port()
+    server, thread = _start_server(app, host, port)
+
+    try:
+        with sync_playwright() as playwright:
+            request_context = playwright.request.new_context(
+                base_url=f"http://{host}:{port}"
+            )
+            response = request_context.get(
+                "/boom",
+                headers={"X-Request-ID": "e2e-5000"},
+            )
+            assert response.status == 500
+            request_context.dispose()
+    finally:
+        server.should_exit = True
+        thread.join(timeout=5)
+
+    error_entries = _read_json_lines(Path(tmp_path) / "errors" / "error.log")
+    entry = next(
+        item for item in error_entries if item.get("message") == "Unhandled exception"
+    )
+
+    assert entry["request_id"] == "e2e-5000"
+    exception = str(entry["exception"])
+    assert "Traceback" in exception

api/tests/integration/test_fastapi_logging_integration.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from core.config.settings import Settings
+from core.logging.config import configure_logging
+from core.logging.logger import get_logger
+from core.logging.middleware import (
+    RequestContextMiddleware,
+    register_exception_handlers,
+)
+
+
+def _read_json_lines(path: Path) -> list[dict[str, object]]:
+    return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
+
+
+def _configure_test_logging(tmp_path: Path) -> None:
+    settings = Settings()
+    runtime = settings.runtime.model_copy(
+        update={
+            "log_dir": str(tmp_path),
+            "log_error_dir": str(tmp_path / "errors"),
+            "log_rotation": "size",
+            "log_rotation_max_bytes": 2048,
+        }
+    )
+    test_settings = settings.model_copy(update={"runtime": runtime})
+
+    configure_logging(test_settings)
+
+
+def test_middleware_binds_request_context(tmp_path: Path) -> None:
+    _configure_test_logging(tmp_path)
+
+    app = FastAPI()
+    app.add_middleware(RequestContextMiddleware)  # type: ignore[arg-type]
+
+    @app.get("/ok")
+    async def ok() -> dict[str, str]:
+        logger = get_logger("tests.ok")
+        logger.info("request accepted", context_key="context_value")
+        return {"status": "ok"}
+
+    client = TestClient(app)
+    response = client.get("/ok", headers={"X-Request-ID": "req-1234"})
+
+    assert response.status_code == 200
+    assert response.headers["X-Request-ID"] == "req-1234"
+
+    log_entries = _read_json_lines(Path(tmp_path) / "app.log")
+    entry = next(
+        item for item in log_entries if item.get("message") == "request accepted"
+    )
+    assert entry["message"] == "request accepted"
+    assert entry["request_id"] == "req-1234"
+    assert entry["method"] == "GET"
+    assert entry["path"] == "/ok"
+    assert entry["context_key"] == "context_value"
+
+    logging.shutdown()
+
+
+def test_exception_handler_logs_stack_and_sends_500(tmp_path: Path) -> None:
+    _configure_test_logging(tmp_path)
+
+    app = FastAPI()
+    app.add_middleware(RequestContextMiddleware)
+    register_exception_handlers(app)
+
+    @app.get("/boom")
+    async def boom() -> dict[str, str]:
+        raise RuntimeError("boom")
+
+    client = TestClient(app, raise_server_exceptions=False)
+    response = client.get("/boom", headers={"X-Request-ID": "req-5000"})
+
+    assert response.status_code == 500
+    assert response.json()["detail"] == "Internal Server Error"
+
+    error_entries = _read_json_lines(Path(tmp_path) / "errors" / "error.log")
+    assert error_entries
+    entry = error_entries[-1]
+    assert entry["level"] == "error"
+    assert entry["request_id"] == "req-5000"
+    exception = str(entry["exception"])
+    assert "Traceback" in exception
+    assert "test_fastapi_logging_integration" in exception
+
+    logging.shutdown()
+
+
+def test_invalid_request_id_is_replaced_and_used_in_error_context(
+    tmp_path: Path,
+) -> None:
+    _configure_test_logging(tmp_path)
+
+    app = FastAPI()
+    app.add_middleware(RequestContextMiddleware)
+    register_exception_handlers(app)
+
+    @app.get("/boom")
+    async def boom() -> dict[str, str]:
+        raise RuntimeError("boom")
+
+    client = TestClient(app, raise_server_exceptions=False)
+    response = client.get("/boom", headers={"X-Request-ID": "bad"})
+
+    assert response.status_code == 500
+
+    response_request_id = response.headers["X-Request-ID"]
+    assert response_request_id != "bad"
+
+    error_entries = _read_json_lines(Path(tmp_path) / "errors" / "error.log")
+    assert error_entries
+    entry = error_entries[-1]
+    assert entry["request_id"] == response_request_id
+    exception = str(entry["exception"])
+    assert "Traceback" in exception
+
+    logging.shutdown()

api/tests/unit/test_celery_logging.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from celery import Celery
+from pytest import MonkeyPatch
+
+from core.logging import celery as celery_logging
+from core.logging.context import clear_context, get_context
+
+
+class DummyTask:
+    name: str = "tasks.sample"
+
+
+def test_celery_prerun_binds_task_context() -> None:
+    handlers = celery_logging.build_celery_signal_handlers()
+
+    handlers.on_task_prerun(task_id="task-123", task=DummyTask())
+    context = get_context()
+
+    assert context["task_id"] == "task-123"
+    assert context["task_name"] == "tasks.sample"
+
+    clear_context()
+
+
+def test_celery_setup_logging_calls_configure(monkeypatch: MonkeyPatch) -> None:
+    called = {"value": False}
+
+    def fake_configure_logging(settings: object | None = None) -> None:
+        called["value"] = True
+
+    monkeypatch.setattr(celery_logging, "configure_logging", fake_configure_logging)
+    handlers = celery_logging.build_celery_signal_handlers()
+
+    handlers.on_setup_logging()
+
+    assert called["value"] is True
+
+
+def test_configure_celery_app_disables_hijack() -> None:
+    app = Celery("test")
+
+    celery_logging.configure_celery_app(app)
+
+    assert app.conf.worker_hijack_root_logger is False

api/tests/unit/test_logging_config.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Iterator
+from pathlib import Path
+from typing import cast
+
+import pytest
+import structlog
+
+from core.config.settings import Settings
+from core.logging.config import build_logging_config, configure_logging
+
+
+def _get_handlers(config: dict[str, object]) -> dict[str, dict[str, object]]:
+    return cast(dict[str, dict[str, object]], config["handlers"])
+
+
+def test_build_logging_config_time_rotation(tmp_path: Path) -> None:
+    settings = Settings()
+    runtime = settings.runtime.model_copy(
+        update={
+            "log_dir": str(tmp_path),
+            "log_error_dir": str(tmp_path / "errors"),
+            "log_rotation": "time",
+        }
+    )
+
+    config = build_logging_config(runtime)
+    handlers = _get_handlers(config)
+
+    assert handlers["file"]["class"] == "logging.handlers.TimedRotatingFileHandler"
+    assert handlers["error"]["class"] == "logging.handlers.TimedRotatingFileHandler"
+    assert handlers["error"]["level"] == "ERROR"
+
+
+def test_build_logging_config_size_rotation(tmp_path: Path) -> None:
+    settings = Settings()
+    runtime = settings.runtime.model_copy(
+        update={
+            "log_dir": str(tmp_path),
+            "log_error_dir": str(tmp_path / "errors"),
+            "log_rotation": "size",
+            "log_rotation_max_bytes": 2048,
+        }
+    )
+
+    config = build_logging_config(runtime)
+    handlers = _get_handlers(config)
+
+    assert handlers["file"]["class"] == "logging.handlers.RotatingFileHandler"
+    assert handlers["error"]["class"] == "logging.handlers.RotatingFileHandler"
+    assert handlers["file"]["maxBytes"] == 2048
+
+
+def test_build_logging_config_plain_formatter_when_disabled(tmp_path: Path) -> None:
+    settings = Settings()
+    runtime = settings.runtime.model_copy(
+        update={
+            "log_dir": str(tmp_path),
+            "log_error_dir": str(tmp_path / "errors"),
+            "log_json": False,
+        }
+    )
+
+    config = build_logging_config(runtime)
+    handlers = _get_handlers(config)
+
+    assert handlers["file"]["formatter"] == "plain"
+    assert handlers["error"]["formatter"] == "plain"
+
+
+def _read_last_log_entry(log_path: Path) -> dict[str, object]:
+    assert log_path.exists(), f"Expected log file at {log_path}"
+    entries = [
+        json.loads(line) for line in log_path.read_text().splitlines() if line.strip()
+    ]
+    assert entries, "Expected at least one log entry in app.log"
+    return entries[-1]
+
+
+def _flush_root_handlers() -> None:
+    root_logger = logging.getLogger()
+    for handler in root_logger.handlers:
+        if hasattr(handler, "flush"):
+            handler.flush()
+
+
+@pytest.fixture
+def configured_logging(tmp_path: Path) -> Iterator[Path]:
+    settings = Settings()
+    runtime = settings.runtime.model_copy(
+        update={
+            "log_dir": str(tmp_path),
+            "log_error_dir": str(tmp_path / "errors"),
+            "log_rotation": "size",
+            "log_rotation_max_bytes": 2048,
+            "log_json": True,
+        }
+    )
+    root_logger = logging.getLogger()
+    original_handlers = root_logger.handlers[:]
+    original_level = root_logger.level
+
+    configure_logging(settings.model_copy(update={"runtime": runtime}))
+
+    yield tmp_path
+
+    for handler in root_logger.handlers:
+        handler.close()
+    root_logger.handlers = original_handlers
+    root_logger.setLevel(original_level)
+    structlog.reset_defaults()
+
+
+def test_stdlib_logging_redacts_sensitive_fields(configured_logging: Path) -> None:
+    logger = logging.getLogger("tests.stdlib")
+    logger.info("login", extra={"password": "secret", "token": "abc"})
+
+    _flush_root_handlers()
+
+    log_path = configured_logging / "app.log"
+    entry = _read_last_log_entry(log_path)
+
+    assert entry["password"] == "[REDACTED]"
+    assert entry["token"] == "[REDACTED]"
+
+
+def test_structlog_redacts_sensitive_fields(configured_logging: Path) -> None:
+    logger = structlog.get_logger("tests.structlog")
+    logger.info("login", password="secret", token="abc")
+
+    _flush_root_handlers()
+
+    log_path = configured_logging / "app.log"
+    entry = _read_last_log_entry(log_path)
+
+    assert entry["password"] == "[REDACTED]"
+    assert entry["token"] == "[REDACTED]"

api/tests/unit/test_logging_filters.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from core.logging.filters import build_sensitive_data_processor
+
+
+def test_redact_sensitive_fields_masks_values() -> None:
+    processor = build_sensitive_data_processor(
+        ["password", "token", "api_key", "cookie"]
+    )
+
+    event: dict[str, object] = {
+        "message": "login",
+        "password": "secret",
+        "access_token": "token-123",
+        "apiKey": "apikey-123",
+        "set-cookie": "cookie-1",
+        "nested": {"token": "abc", "safe": "ok"},
+        "list": [{"password": "x"}],
+    }
+
+    redacted = processor(None, "info", event)
+
+    assert redacted["password"] == "[REDACTED]"
+    assert redacted["access_token"] == "[REDACTED]"
+    assert redacted["apiKey"] == "[REDACTED]"
+    assert redacted["set-cookie"] == "[REDACTED]"
+    assert redacted["nested"]["token"] == "[REDACTED]"
+    assert redacted["nested"]["safe"] == "ok"
+    assert redacted["list"][0]["password"] == "[REDACTED]"
+    assert event["password"] == "secret"

api/tests/unit/test_logging_settings.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from pytest import MonkeyPatch
+
+from core.config.settings import Settings
+
+
+def test_runtime_settings_defaults() -> None:
+    settings = Settings()
+
+    assert settings.runtime.log_json is True
+    assert settings.runtime.log_rotation == "time"
+    assert settings.runtime.log_rotation_when == "midnight"
+    assert settings.runtime.log_rotation_interval == 1
+    assert settings.runtime.log_rotation_backup_count == 14
+    assert settings.runtime.log_rotation_max_bytes == 10_000_000
+    assert settings.runtime.log_dir == "logs"
+    assert settings.runtime.log_error_dir == "logs/errors"
+    assert settings.runtime.log_file_name == "app.log"
+    assert settings.runtime.log_error_file_name == "error.log"
+    assert "password" in settings.runtime.log_sensitive_fields
+
+
+def test_runtime_settings_env_override(monkeypatch: MonkeyPatch) -> None:
+    monkeypatch.setenv("SOCIAL_RUNTIME__LOG_DIR", "var/logs")
+    monkeypatch.setenv("SOCIAL_RUNTIME__LOG_ERROR_DIR", "var/logs/errors")
+    monkeypatch.setenv("SOCIAL_RUNTIME__LOG_ROTATION", "size")
+    monkeypatch.setenv("SOCIAL_RUNTIME__LOG_ROTATION_MAX_BYTES", "2048")
+
+    settings = Settings()
+
+    assert settings.runtime.log_dir == "var/logs"
+    assert settings.runtime.log_error_dir == "var/logs/errors"
+    assert settings.runtime.log_rotation == "size"
+    assert settings.runtime.log_rotation_max_bytes == 2048