sirius/osint-dashboard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Wire ingestion: long-running ingester (sources->NATS->Postgres), fix nats-py API (durable=/LIMITS), drop UUID source_id on RSS
All checks were successful
build-and-deploy / build (push) Successful in 4m18s
Details

Browse source
This commit is contained in:
sirius0xdev 2026-07-07 19:35:28 -04:00
parent 5496023285
commit 67fee36c7c
No known key found for this signature in database
4 changed files with 6 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/ingestor.py
View file

@ -66,7 +66,7 @@ async def start_nats_consumer():
"events.earthquake", "events.disaster", "events.weather", "events.earthquake", "events.disaster", "events.weather",
"events.fire", "events.satellite", "events.new", "events.alert", "events.fire", "events.satellite", "events.new", "events.alert",
], ],
retention=nats.js.api.RetentionPolicy.INTERESTS, retention=nats.js.api.RetentionPolicy.LIMITS,
max_msgs=1_000_000, max_msgs=1_000_000,
) )
logger.info("Created NATS stream %s", NATS_STREAM) logger.info("Created NATS stream %s", NATS_STREAM)
@ -76,7 +76,7 @@ async def start_nats_consumer():
# Create durable consumer # Create durable consumer
sub = await js.pull_subscribe( sub = await js.pull_subscribe(
subject="events.>", subject="events.>",
durable_name=NATS_DURABLE, durable=NATS_DURABLE,
) )
logger.info("NATS consumer started, durable=%s", NATS_DURABLE) logger.info("NATS consumer started, durable=%s", NATS_DURABLE)

2
app/run_ingester.py
View file

@ -45,7 +45,7 @@ async def producer_loop() -> None:
try: try:
for url in RSS_URLS: for url in RSS_URLS:
try: try:
n = await ingest_rss_feed(url, source_id=url) n = await ingest_rss_feed(url)
logger.info("RSS %s -> %d items", url, n) logger.info("RSS %s -> %d items", url, n)
except Exception: # noqa: BLE001 except Exception: # noqa: BLE001
logger.exception("RSS fetch failed: %s", url) logger.exception("RSS fetch failed: %s", url)

4
app/sources.py
View file

@ -40,7 +40,7 @@ async def publish_event(subject: str, event: dict):
# ─── RSS Feed Ingestor ────────────────────────────────────────────────── # ─── RSS Feed Ingestor ──────────────────────────────────────────────────
async def ingest_rss_feed(feed_url: str, source_id: str = None): async def ingest_rss_feed(feed_url: str):
"""Fetch and parse an RSS feed, publish items to NATS.""" """Fetch and parse an RSS feed, publish items to NATS."""
async with httpx.AsyncClient(timeout=30) as client: async with httpx.AsyncClient(timeout=30) as client:
resp = await client.get(feed_url) resp = await client.get(feed_url)
@ -51,7 +51,6 @@ async def ingest_rss_feed(feed_url: str, source_id: str = None):
for entry in feed.entries[:100]: # max 100 per run for entry in feed.entries[:100]: # max 100 per run
event = { event = {
"source_type": "rss", "source_type": "rss",
"source_id": source_id,
"title": entry.get("title"), "title": entry.get("title"),
"body": entry.get("summary") or entry.get("description"), "body": entry.get("summary") or entry.get("description"),
"url": entry.get("link"), "url": entry.get("link"),
@ -59,6 +58,7 @@ async def ingest_rss_feed(feed_url: str, source_id: str = None):
or datetime.now(timezone.utc).isoformat(), or datetime.now(timezone.utc).isoformat(),
"tags": [t.get("term") for t in entry.get("tags", []) if t.get("term")], "tags": [t.get("term") for t in entry.get("tags", []) if t.get("term")],
"raw": { "raw": {
"feed_url": feed_url,
"feed_title": feed.feed.get("title"), "feed_title": feed.feed.get("title"),
"author": entry.get("author"), "author": entry.get("author"),
"categories": [c.get("term") for c in entry.get("categories", [])], "categories": [c.get("term") for c in entry.get("categories", [])],

1
docker-compose.yml
View file

@ -70,6 +70,7 @@ services:
INGEST_INTERVAL: ${INGEST_INTERVAL:-300} INGEST_INTERVAL: ${INGEST_INTERVAL:-300}
INGEST_EARTHQUAKES: ${INGEST_EARTHQUAKES:-1} INGEST_EARTHQUAKES: ${INGEST_EARTHQUAKES:-1}
command: ["python", "app/run_ingester.py"] command: ["python", "app/run_ingester.py"]
entrypoint: ["python", "app/run_ingester.py"]
app: app:
build: build: