feat: duration_ms-logging, bulk-semaphore und erweitertes README

- Pipeline-Stages (download/extract/embed/qdrant) loggen jetzt duration_ms
- bulk-import dispatcht mit Semaphore(4) statt unbounded → Backpressure
- README dokumentiert Webhook-Payload-Schema mit curl-Beispiel
- README enthaelt Recovery-Runbook (dim-mismatch, crash-recovery, single-file reindex)
This commit is contained in:
2026-05-04 22:54:58 +02:00
parent 7fe2d853ec
commit ca9ff55587
3 changed files with 92 additions and 3 deletions

View File

@@ -1,3 +1,4 @@
import asyncio
import logging
import xml.etree.ElementTree as ET
from pathlib import PurePosixPath
@@ -18,6 +19,17 @@ logger = logging.getLogger(__name__)
router = APIRouter()
# Limits parallel pipeline execution during a bulk import so we don't
# saturate Ollama/WebDAV with thousands of concurrent requests.
BULK_CONCURRENCY = 4
_bulk_semaphore = asyncio.Semaphore(BULK_CONCURRENCY)
async def _process_with_semaphore(file_path: str, event_type: EventType) -> None:
async with _bulk_semaphore:
await process_file(file_path, event_type)
class BulkRequest(BaseModel):
path: str = Field(min_length=1)
@@ -93,7 +105,7 @@ async def bulk_import(
ext = PurePosixPath(f).suffix.lstrip(".").lower()
if ext not in SUPPORTED_TYPES:
continue
background.add_task(process_file, f, EventType.CREATED)
background.add_task(_process_with_semaphore, f, EventType.CREATED)
dispatched += 1
logger.info(