Discover feeds behind aggregator-only articles
A best-effort pipeline stage after social enrichment asks Miniflux to discover the feeds behind each aggregator-only article, validates every result by sniffing the body, and records the survivors as feed candidates with a per-host memo. Ranking is a shrunk mean over the linked articles' existing telemetry and ratings. `daily-epub feeds discover` runs the same pass over recent articles for seeding. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QVPagF6jfDv78CC5Jv2wp4
This commit is contained in:
+27
-1
@@ -46,7 +46,7 @@ use crate::types::{
|
||||
Article, ArticleId, Artifact, BehindThePaper, Candidate, Colophon, Edition, Issue, IssueMeta,
|
||||
Lineup, Models, TokenUsage, reading_minutes,
|
||||
};
|
||||
use crate::{comments, dedupe, epub, http, miniflux, publish, social, world};
|
||||
use crate::{comments, dedupe, discovery, epub, http, miniflux, publish, social, world};
|
||||
|
||||
/// One `generate` invocation's inputs — the CLI flags, already parsed (§2).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -449,6 +449,32 @@ async fn run_stages(
|
||||
report.counts.social_hits = enricher.enrich_all(&mut articles).await as i64;
|
||||
report.timings.record("social", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 5b: feed discovery (feed discovery plan §4) — best effort ---
|
||||
// Runs here because it needs the real article ids stage 4 minted and the
|
||||
// subscription map stage 1 already loaded, and because `articles` is moved
|
||||
// into hygiene next.
|
||||
if config.discovery.enabled {
|
||||
let stage = Timestamp::now();
|
||||
match discovery::run(
|
||||
db,
|
||||
&client,
|
||||
&http,
|
||||
&config.discovery,
|
||||
&articles,
|
||||
&feeds,
|
||||
Timestamp::now(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(summary) => {
|
||||
tracing::info!(%summary, "feed discovery");
|
||||
report.counts.feed_candidates_new = summary.candidates_new as i64;
|
||||
}
|
||||
Err(error) => report.warn(format!("feed discovery failed: {error:#}")),
|
||||
}
|
||||
report.timings.record("discovery", elapsed_ms(stage));
|
||||
}
|
||||
|
||||
// --- Stage 6: hygiene, embeddings, and cheap signals (§8.1, §9) ---
|
||||
let stage = Timestamp::now();
|
||||
let mut personalized = admit::hygiene(
|
||||
|
||||
Reference in New Issue
Block a user