feat: duration_ms-logging, bulk-semaphore und erweitertes README
- Pipeline-Stages (download/extract/embed/qdrant) loggen jetzt duration_ms - bulk-import dispatcht mit Semaphore(4) statt unbounded → Backpressure - README dokumentiert Webhook-Payload-Schema mit curl-Beispiel - README enthaelt Recovery-Runbook (dim-mismatch, crash-recovery, single-file reindex)
This commit is contained in:
14
app/bulk.py
14
app/bulk.py
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import PurePosixPath
|
||||
@@ -18,6 +19,17 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# Limits parallel pipeline execution during a bulk import so we don't
|
||||
# saturate Ollama/WebDAV with thousands of concurrent requests.
|
||||
BULK_CONCURRENCY = 4
|
||||
_bulk_semaphore = asyncio.Semaphore(BULK_CONCURRENCY)
|
||||
|
||||
|
||||
async def _process_with_semaphore(file_path: str, event_type: EventType) -> None:
|
||||
async with _bulk_semaphore:
|
||||
await process_file(file_path, event_type)
|
||||
|
||||
|
||||
class BulkRequest(BaseModel):
|
||||
path: str = Field(min_length=1)
|
||||
|
||||
@@ -93,7 +105,7 @@ async def bulk_import(
|
||||
ext = PurePosixPath(f).suffix.lstrip(".").lower()
|
||||
if ext not in SUPPORTED_TYPES:
|
||||
continue
|
||||
background.add_task(process_file, f, EventType.CREATED)
|
||||
background.add_task(_process_with_semaphore, f, EventType.CREATED)
|
||||
dispatched += 1
|
||||
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user