Unwrap Scour's /r/rss/ redirector when canonicalizing URLs
Scour interest feeds link every entry through https://scour.ing/r/rss/<percent-encoded article URL>, so the wrapper became the article's canonical URL: the same post arriving via Hacker News and Scour never merged, and the publication fallback showed "scour.ing" instead of the site's own domain. Resolve the wrapper like the Google News redirector. `publication_label` re-canonicalizes the stored URL so the 477 articles saved with the wrapper show the real domain too. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VXYGBPoHZSDSfE5WcJ9bvj
This commit is contained in:
Generated
+1
@@ -905,6 +905,7 @@ dependencies = [
|
|||||||
"lettre",
|
"lettre",
|
||||||
"libc",
|
"libc",
|
||||||
"password-auth",
|
"password-auth",
|
||||||
|
"percent-encoding",
|
||||||
"rand 0.10.2",
|
"rand 0.10.2",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"resvg",
|
"resvg",
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ jiff = { version = "0.2.35", features = ["serde"] }
|
|||||||
lettre = { version = "0.11", default-features = false, features = ["builder", "smtp-transport", "tokio1", "tokio1-rustls-tls", "hostname"] }
|
lettre = { version = "0.11", default-features = false, features = ["builder", "smtp-transport", "tokio1", "tokio1-rustls-tls", "hostname"] }
|
||||||
libc = "0.2.189"
|
libc = "0.2.189"
|
||||||
password-auth = "1.0.0"
|
password-auth = "1.0.0"
|
||||||
|
percent-encoding = "2.3.2"
|
||||||
rand = "0.10.2"
|
rand = "0.10.2"
|
||||||
reqwest = { version = "0.13.4", default-features = false, features = [
|
reqwest = { version = "0.13.4", default-features = false, features = [
|
||||||
"rustls",
|
"rustls",
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use jiff::Timestamp;
|
use jiff::Timestamp;
|
||||||
|
use percent_encoding::percent_decode_str;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::types::{Article, Entry, ExtractMethod, SourceKind, SourceRef};
|
use crate::types::{Article, Entry, ExtractMethod, SourceKind, SourceRef};
|
||||||
@@ -109,6 +110,16 @@ fn is_tracking_param(key: &str) -> bool {
|
|||||||
/// The real destination behind a known redirector, if any (§3.2).
|
/// The real destination behind a known redirector, if any (§3.2).
|
||||||
fn redirect_target(url: &Url) -> Option<String> {
|
fn redirect_target(url: &Url) -> Option<String> {
|
||||||
let host = url.host_str()?.to_ascii_lowercase();
|
let host = url.host_str()?.to_ascii_lowercase();
|
||||||
|
if matches!(host.as_str(), "scour.ing" | "www.scour.ing")
|
||||||
|
&& let Some(encoded) = url.path().strip_prefix("/r/rss/")
|
||||||
|
{
|
||||||
|
let target = percent_decode_str(encoded).decode_utf8().ok()?.into_owned();
|
||||||
|
let target_url = Url::parse(&target).ok()?;
|
||||||
|
if matches!(target_url.scheme(), "http" | "https") && target_url.host_str().is_some() {
|
||||||
|
return Some(target);
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
}
|
||||||
let is_google_news = host == "news.google.com" || host.ends_with(".news.google.com");
|
let is_google_news = host == "news.google.com" || host.ends_with(".news.google.com");
|
||||||
let is_google_redirect = host == "news.url.google.com"
|
let is_google_redirect = host == "news.url.google.com"
|
||||||
|| ((host == "www.google.com" || host == "google.com") && url.path() == "/url");
|
|| ((host == "www.google.com" || host == "google.com") && url.path() == "/url");
|
||||||
@@ -490,6 +501,21 @@ mod tests {
|
|||||||
"https://www.google.com/url?q=https://example.com/real&sa=D",
|
"https://www.google.com/url?q=https://example.com/real&sa=D",
|
||||||
Some("https://example.com/real"),
|
Some("https://example.com/real"),
|
||||||
),
|
),
|
||||||
|
// Scour RSS redirectors encode the real URL in the path
|
||||||
|
(
|
||||||
|
"https://scour.ing/r/rss/https%3A%2F%2Frmzlb.github.io%2Fnotifyd%2Farticles%2Fpostgres-queue-what-skip-locked-does-not-give-you.html",
|
||||||
|
Some(
|
||||||
|
"https://rmzlb.github.io/notifyd/articles/postgres-queue-what-skip-locked-does-not-give-you.html",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"https://www.scour.ing/r/rss/https%3A%2F%2Fexample.com%2Fpost%3Fid%3D7%26utm_source%3Dscour%26utm_medium%3Drss",
|
||||||
|
Some("https://example.com/post?id=7"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"https://scour.ing/@tyler/interests/Kernel%20Development",
|
||||||
|
Some("https://scour.ing/@tyler/interests/Kernel%20Development"),
|
||||||
|
),
|
||||||
// non-http schemes and junk
|
// non-http schemes and junk
|
||||||
("mailto:tyler@hallada.net", None),
|
("mailto:tyler@hallada.net", None),
|
||||||
("ftp://example.com/file", None),
|
("ftp://example.com/file", None),
|
||||||
|
|||||||
+16
-1
@@ -160,7 +160,14 @@ impl Article {
|
|||||||
let publication = self
|
let publication = self
|
||||||
.publication
|
.publication
|
||||||
.clone()
|
.clone()
|
||||||
.or_else(|| domain(&self.canonical_url))?;
|
// Covers rows stored before Scour wrappers were unwrapped.
|
||||||
|
.or_else(|| {
|
||||||
|
domain(
|
||||||
|
crate::dedupe::canonical_url(&self.canonical_url)
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or(&self.canonical_url),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
(!publication
|
(!publication
|
||||||
.trim()
|
.trim()
|
||||||
.eq_ignore_ascii_case(self.feed_title.trim()))
|
.eq_ignore_ascii_case(self.feed_title.trim()))
|
||||||
@@ -914,6 +921,14 @@ mod tests {
|
|||||||
assert_eq!(article.publication_label().as_deref(), Some("example.com"));
|
assert_eq!(article.publication_label().as_deref(), Some("example.com"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn publication_label_unwraps_stored_scour_urls() {
|
||||||
|
let mut article = publication_article();
|
||||||
|
article.canonical_url =
|
||||||
|
"https://scour.ing/r/rss/https%3A%2F%2Fwww.example.com%2Fpost".into();
|
||||||
|
assert_eq!(article.publication_label().as_deref(), Some("example.com"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn publication_label_omits_a_publication_matching_the_feed() {
|
fn publication_label_omits_a_publication_matching_the_feed() {
|
||||||
let mut article = publication_article();
|
let mut article = publication_article();
|
||||||
|
|||||||
Reference in New Issue
Block a user