diff --git a/Cargo.lock b/Cargo.lock index a6110b3..3c9da50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -905,6 +905,7 @@ dependencies = [ "lettre", "libc", "password-auth", + "percent-encoding", "rand 0.10.2", "reqwest", "resvg", diff --git a/Cargo.toml b/Cargo.toml index f4611b2..bb617fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ jiff = { version = "0.2.35", features = ["serde"] } lettre = { version = "0.11", default-features = false, features = ["builder", "smtp-transport", "tokio1", "tokio1-rustls-tls", "hostname"] } libc = "0.2.189" password-auth = "1.0.0" +percent-encoding = "2.3.2" rand = "0.10.2" reqwest = { version = "0.13.4", default-features = false, features = [ "rustls", diff --git a/src/dedupe.rs b/src/dedupe.rs index 31c845f..6d0f0c8 100644 --- a/src/dedupe.rs +++ b/src/dedupe.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use jiff::Timestamp; +use percent_encoding::percent_decode_str; use url::Url; use crate::types::{Article, Entry, ExtractMethod, SourceKind, SourceRef}; @@ -109,6 +110,16 @@ fn is_tracking_param(key: &str) -> bool { /// The real destination behind a known redirector, if any (ยง3.2). fn redirect_target(url: &Url) -> Option { let host = url.host_str()?.to_ascii_lowercase(); + if matches!(host.as_str(), "scour.ing" | "www.scour.ing") + && let Some(encoded) = url.path().strip_prefix("/r/rss/") + { + let target = percent_decode_str(encoded).decode_utf8().ok()?.into_owned(); + let target_url = Url::parse(&target).ok()?; + if matches!(target_url.scheme(), "http" | "https") && target_url.host_str().is_some() { + return Some(target); + } + return None; + } let is_google_news = host == "news.google.com" || host.ends_with(".news.google.com"); let is_google_redirect = host == "news.url.google.com" || ((host == "www.google.com" || host == "google.com") && url.path() == "/url"); @@ -490,6 +501,21 @@ mod tests { "https://www.google.com/url?q=https://example.com/real&sa=D", Some("https://example.com/real"), ), + // Scour RSS redirectors encode the real URL in the path + ( + "https://scour.ing/r/rss/https%3A%2F%2Frmzlb.github.io%2Fnotifyd%2Farticles%2Fpostgres-queue-what-skip-locked-does-not-give-you.html", + Some( + "https://rmzlb.github.io/notifyd/articles/postgres-queue-what-skip-locked-does-not-give-you.html", + ), + ), + ( + "https://www.scour.ing/r/rss/https%3A%2F%2Fexample.com%2Fpost%3Fid%3D7%26utm_source%3Dscour%26utm_medium%3Drss", + Some("https://example.com/post?id=7"), + ), + ( + "https://scour.ing/@tyler/interests/Kernel%20Development", + Some("https://scour.ing/@tyler/interests/Kernel%20Development"), + ), // non-http schemes and junk ("mailto:tyler@hallada.net", None), ("ftp://example.com/file", None), diff --git a/src/types.rs b/src/types.rs index 65d88e0..3a3304f 100644 --- a/src/types.rs +++ b/src/types.rs @@ -160,7 +160,14 @@ impl Article { let publication = self .publication .clone() - .or_else(|| domain(&self.canonical_url))?; + // Covers rows stored before Scour wrappers were unwrapped. + .or_else(|| { + domain( + crate::dedupe::canonical_url(&self.canonical_url) + .as_deref() + .unwrap_or(&self.canonical_url), + ) + })?; (!publication .trim() .eq_ignore_ascii_case(self.feed_title.trim())) @@ -914,6 +921,14 @@ mod tests { assert_eq!(article.publication_label().as_deref(), Some("example.com")); } + #[test] + fn publication_label_unwraps_stored_scour_urls() { + let mut article = publication_article(); + article.canonical_url = + "https://scour.ing/r/rss/https%3A%2F%2Fwww.example.com%2Fpost".into(); + assert_eq!(article.publication_label().as_deref(), Some("example.com")); + } + #[test] fn publication_label_omits_a_publication_matching_the_feed() { let mut article = publication_article();