Unwrap Scour's /r/rss/ redirector when canonicalizing URLs

Scour interest feeds link every entry through
https://scour.ing/r/rss/<percent-encoded article URL>, so the wrapper
became the article's canonical URL: the same post arriving via Hacker
News and Scour never merged, and the publication fallback showed
"scour.ing" instead of the site's own domain. Resolve the wrapper like
the Google News redirector. `publication_label` re-canonicalizes the
stored URL so the 477 articles saved with the wrapper show the real
domain too.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VXYGBPoHZSDSfE5WcJ9bvj
This commit is contained in:
2026-09-09 18:12:30 +00:00
co-authored by Claude Fable 5.1
parent f2a44507d4
commit c538afa540
4 changed files with 44 additions and 1 deletions
Generated
+1
View File
@@ -905,6 +905,7 @@ dependencies = [
"lettre",
"libc",
"password-auth",
"percent-encoding",
"rand 0.10.2",
"reqwest",
"resvg",
+1
View File
@@ -23,6 +23,7 @@ jiff = { version = "0.2.35", features = ["serde"] }
lettre = { version = "0.11", default-features = false, features = ["builder", "smtp-transport", "tokio1", "tokio1-rustls-tls", "hostname"] }
libc = "0.2.189"
password-auth = "1.0.0"
percent-encoding = "2.3.2"
rand = "0.10.2"
reqwest = { version = "0.13.4", default-features = false, features = [
"rustls",
+26
View File
@@ -6,6 +6,7 @@
use std::collections::HashMap;
use jiff::Timestamp;
use percent_encoding::percent_decode_str;
use url::Url;
use crate::types::{Article, Entry, ExtractMethod, SourceKind, SourceRef};
@@ -109,6 +110,16 @@ fn is_tracking_param(key: &str) -> bool {
/// The real destination behind a known redirector, if any (§3.2).
fn redirect_target(url: &Url) -> Option<String> {
let host = url.host_str()?.to_ascii_lowercase();
if matches!(host.as_str(), "scour.ing" | "www.scour.ing")
&& let Some(encoded) = url.path().strip_prefix("/r/rss/")
{
let target = percent_decode_str(encoded).decode_utf8().ok()?.into_owned();
let target_url = Url::parse(&target).ok()?;
if matches!(target_url.scheme(), "http" | "https") && target_url.host_str().is_some() {
return Some(target);
}
return None;
}
let is_google_news = host == "news.google.com" || host.ends_with(".news.google.com");
let is_google_redirect = host == "news.url.google.com"
|| ((host == "www.google.com" || host == "google.com") && url.path() == "/url");
@@ -490,6 +501,21 @@ mod tests {
"https://www.google.com/url?q=https://example.com/real&sa=D",
Some("https://example.com/real"),
),
// Scour RSS redirectors encode the real URL in the path
(
"https://scour.ing/r/rss/https%3A%2F%2Frmzlb.github.io%2Fnotifyd%2Farticles%2Fpostgres-queue-what-skip-locked-does-not-give-you.html",
Some(
"https://rmzlb.github.io/notifyd/articles/postgres-queue-what-skip-locked-does-not-give-you.html",
),
),
(
"https://www.scour.ing/r/rss/https%3A%2F%2Fexample.com%2Fpost%3Fid%3D7%26utm_source%3Dscour%26utm_medium%3Drss",
Some("https://example.com/post?id=7"),
),
(
"https://scour.ing/@tyler/interests/Kernel%20Development",
Some("https://scour.ing/@tyler/interests/Kernel%20Development"),
),
// non-http schemes and junk
("mailto:tyler@hallada.net", None),
("ftp://example.com/file", None),
+16 -1
View File
@@ -160,7 +160,14 @@ impl Article {
let publication = self
.publication
.clone()
.or_else(|| domain(&self.canonical_url))?;
// Covers rows stored before Scour wrappers were unwrapped.
.or_else(|| {
domain(
crate::dedupe::canonical_url(&self.canonical_url)
.as_deref()
.unwrap_or(&self.canonical_url),
)
})?;
(!publication
.trim()
.eq_ignore_ascii_case(self.feed_title.trim()))
@@ -914,6 +921,14 @@ mod tests {
assert_eq!(article.publication_label().as_deref(), Some("example.com"));
}
#[test]
fn publication_label_unwraps_stored_scour_urls() {
let mut article = publication_article();
article.canonical_url =
"https://scour.ing/r/rss/https%3A%2F%2Fwww.example.com%2Fpost".into();
assert_eq!(article.publication_label().as_deref(), Some("example.com"));
}
#[test]
fn publication_label_omits_a_publication_matching_the_feed() {
let mut article = publication_article();