import asyncio import logging import xml.etree.ElementTree as ET from pathlib import PurePosixPath from urllib.parse import unquote import httpx from fastapi import APIRouter, BackgroundTasks, Header, HTTPException, status from pydantic import BaseModel, Field from app.config import get_settings from app.ingest.extractors import SUPPORTED_TYPES from app.ingest.pipeline import process_file from app.webhook.auth import verify_secret from app.webhook.models import EventType logger = logging.getLogger(__name__) router = APIRouter() # Limits parallel pipeline execution during a bulk import so we don't # saturate Ollama/WebDAV with thousands of concurrent requests. BULK_CONCURRENCY = 4 _bulk_semaphore = asyncio.Semaphore(BULK_CONCURRENCY) async def _process_with_semaphore(file_path: str, event_type: EventType) -> None: async with _bulk_semaphore: await process_file(file_path, event_type) class BulkRequest(BaseModel): path: str = Field(min_length=1) PROPFIND_BODY = """ """ DAV_NS = {"d": "DAV:"} async def list_files_recursive(base_url: str, user: str, password: str, path: str) -> list[str]: """PROPFIND with Depth: infinity. Returns relative file paths (no folders).""" base = base_url.rstrip("/") rel = path.strip("/") url = f"{base}/{rel}" async with httpx.AsyncClient(auth=(user, password), timeout=120.0) as client: response = await client.request( "PROPFIND", url, headers={"Depth": "infinity", "Content-Type": "application/xml"}, content=PROPFIND_BODY, ) if response.status_code not in (200, 207): raise RuntimeError(f"PROPFIND failed: status={response.status_code}") root = ET.fromstring(response.text) base_path_segment = PurePosixPath(httpx.URL(base).path).as_posix() # "/remote.php/dav/files/u" out: list[str] = [] for resp in root.findall("d:response", DAV_NS): href = resp.findtext("d:href", default="", namespaces=DAV_NS) decoded = unquote(href) # Strip the WebDAV base prefix → leaves "Documents/.../file.pdf" if decoded.startswith(base_path_segment): decoded = decoded[len(base_path_segment):] decoded = decoded.lstrip("/") if not decoded or decoded.endswith("/"): continue # Skip directory entries (those have resourcetype) rt = resp.find("d:propstat/d:prop/d:resourcetype/d:collection", DAV_NS) if rt is not None: continue out.append(decoded) return out @router.post("/bulk-import", status_code=status.HTTP_202_ACCEPTED) async def bulk_import( body: BulkRequest, background: BackgroundTasks, x_webhook_secret: str | None = Header(default=None), ): settings = get_settings() verify_secret(x_webhook_secret, settings.webhook_secret) try: files = await list_files_recursive( settings.nextcloud_webdav_url, settings.nextcloud_user, settings.nextcloud_app_password, body.path, ) except (RuntimeError, httpx.HTTPError, ET.ParseError) as exc: logger.exception( "bulk listing failed", extra={"event": "bulk_listing_failed", "path": body.path, "error": str(exc)}, ) raise HTTPException(status_code=502, detail="webdav listing failed") from exc dispatched = 0 for f in files: ext = PurePosixPath(f).suffix.lstrip(".").lower() if ext not in SUPPORTED_TYPES: continue background.add_task(_process_with_semaphore, f, EventType.CREATED) dispatched += 1 logger.info( "bulk dispatch", extra={"event": "bulk_dispatch", "path": body.path, "dispatched": dispatched, "total_listed": len(files)}, ) return {"status": "accepted", "dispatched": dispatched}