Merge branch 'fix-import': historical rating imports
This commit is contained in:
@@ -219,6 +219,21 @@ scoped `systemd/50-daily-epub.rules` polkit rule, and put the server user in
|
|||||||
`systemd-journal` so the page can show status and its configured journal tail.
|
`systemd-journal` so the page can show status and its configured journal tail.
|
||||||
Set `server.jobs_enabled = false` to make starts unavailable.
|
Set `server.jobs_enabled = false` to make starts unavailable.
|
||||||
|
|
||||||
|
| Job name | Action |
|
||||||
|
|---|---|
|
||||||
|
| `generate` / `generate-YYYY-MM-DD` | Build and publish an issue. |
|
||||||
|
| `dry-run` | Run the pipeline without publishing or recording an issue. |
|
||||||
|
| `profile-rebuild` | Rebuild learned taste adjustments from ratings. |
|
||||||
|
| `features-backfill` | Embed rated and recently published articles and interests. |
|
||||||
|
| `backfill-social` | Refresh recent social scores. |
|
||||||
|
| `features-prune` | Remove stale embeddings and curation telemetry. |
|
||||||
|
| `import-ratings` | Fetch, embed, and rate URLs queued from the Ratings page. |
|
||||||
|
|
||||||
|
The Ratings page accepts up to 500 historical article URLs at a time with one
|
||||||
|
verdict and optional note. Imports run in the background and show per-URL
|
||||||
|
status in the page plus live logs on the job page. If dashboard jobs are
|
||||||
|
disabled, queueing still works; run `daily-epub job run import-ratings` by hand.
|
||||||
|
|
||||||
The Settings page derives its fields from `Config`, rewrites `config.toml` in
|
The Settings page derives its fields from `Config`, rewrites `config.toml` in
|
||||||
place with `toml_edit`, preserves comments/order and file permissions, validates
|
place with `toml_edit`, preserves comments/order and file permissions, validates
|
||||||
before an atomic rename, and records attributed history. It re-reads hand edits
|
before an atomic rename, and records attributed history. It re-reads hand edits
|
||||||
@@ -241,7 +256,8 @@ these writes.
|
|||||||
| `GET /account`, `POST /account/password`, `/account/logout-all` | User or admin | Change the current password or revoke sessions. |
|
| `GET /account`, `POST /account/password`, `/account/logout-all` | User or admin | Change the current password or revoke sessions. |
|
||||||
| `POST /rate` | Admin | Append an attributed dashboard rating event. |
|
| `POST /rate` | Admin | Append an attributed dashboard rating event. |
|
||||||
| `GET /dashboard` | Admin | Run, budget, rating, job, and config overview. |
|
| `GET /dashboard` | Admin | Run, budget, rating, job, and config overview. |
|
||||||
| `GET /dashboard/runs[/{id}]`, `/articles[/{id}]`, `/ratings`, `/stats` | Admin | Pipeline history, article explanations, rating contributions/history, and evaluation stats. |
|
| `GET /dashboard/runs[/{id}]`, `/articles[/{id}]`, `/ratings`, `/stats` | Admin | Pipeline history, article explanations, rating contributions/history, historical URL imports, and evaluation stats. |
|
||||||
|
| `POST /dashboard/ratings/import` | Admin | Queue historical URLs with a verdict and start the background import job. |
|
||||||
| `GET/POST /dashboard/profile`, `POST /dashboard/profile/restore` | Admin | Edit `profile.md`, inspect prompts/adjustments, and restore a version. |
|
| `GET/POST /dashboard/profile`, `POST /dashboard/profile/restore` | Admin | Edit `profile.md`, inspect prompts/adjustments, and restore a version. |
|
||||||
| `GET/POST /dashboard/settings`, `POST /dashboard/settings/providers`, `GET /dashboard/settings/history` | Admin | Edit validated configuration and inspect its audit log. |
|
| `GET/POST /dashboard/settings`, `POST /dashboard/settings/providers`, `GET /dashboard/settings/history` | Admin | Edit validated configuration and inspect its audit log. |
|
||||||
| `GET /dashboard/jobs`, `GET /dashboard/jobs/{id}`, `POST /dashboard/jobs/{name}` | Admin | Start fixed systemd jobs and inspect status and logs. |
|
| `GET /dashboard/jobs`, `GET /dashboard/jobs/{id}`, `POST /dashboard/jobs/{name}` | Admin | Start fixed systemd jobs and inspect status and logs. |
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
CREATE TABLE rating_imports (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
url TEXT NOT NULL,
|
||||||
|
label TEXT NOT NULL CHECK (label IN ('loved', 'good', 'not_for_me')),
|
||||||
|
note TEXT,
|
||||||
|
status TEXT NOT NULL CHECK (status IN ('pending', 'ok', 'failed')),
|
||||||
|
message TEXT,
|
||||||
|
article_id INTEGER REFERENCES articles(id) ON DELETE SET NULL,
|
||||||
|
requested_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||||
|
requested_at TEXT NOT NULL,
|
||||||
|
finished_at TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_rating_imports_status_id ON rating_imports(status, id);
|
||||||
@@ -311,7 +311,7 @@ impl Db {
|
|||||||
)
|
)
|
||||||
.bind(&article.canonical_url)
|
.bind(&article.canonical_url)
|
||||||
.bind(&article.title)
|
.bind(&article.title)
|
||||||
.bind(article.best_entry_id)
|
.bind((article.best_entry_id != 0).then_some(article.best_entry_id))
|
||||||
.bind(&article.content_html)
|
.bind(&article.content_html)
|
||||||
.bind(article.word_count)
|
.bind(article.word_count)
|
||||||
.bind(article.excerpt_only)
|
.bind(article.excerpt_only)
|
||||||
@@ -737,7 +737,9 @@ impl Db {
|
|||||||
)
|
)
|
||||||
SELECT r.article_id, r.user_id, r.issue_date, r.label, r.value, r.note, r.event_at,
|
SELECT r.article_id, r.user_id, r.issue_date, r.label, r.value, r.note, r.event_at,
|
||||||
COALESCE(a.title, '') AS title,
|
COALESCE(a.title, '') AS title,
|
||||||
COALESCE(e.feed_title, '') AS feed_title,
|
COALESCE(e.feed_title,
|
||||||
|
CASE WHEN a.best_entry_id IS NULL THEN 'Imported' ELSE '' END)
|
||||||
|
AS feed_title,
|
||||||
(SELECT ia.summary FROM issue_articles ia
|
(SELECT ia.summary FROM issue_articles ia
|
||||||
WHERE ia.article_id = r.article_id
|
WHERE ia.article_id = r.article_id
|
||||||
ORDER BY ia.issue_date DESC LIMIT 1) AS summary,
|
ORDER BY ia.issue_date DESC LIMIT 1) AS summary,
|
||||||
@@ -918,11 +920,12 @@ fn entry_from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Entry> {
|
|||||||
fn article_from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Article> {
|
fn article_from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Article> {
|
||||||
let sources: Vec<SourceRef> =
|
let sources: Vec<SourceRef> =
|
||||||
serde_json::from_str(&row.get::<String, _>("sources_json")).unwrap_or_default();
|
serde_json::from_str(&row.get::<String, _>("sources_json")).unwrap_or_default();
|
||||||
|
let best_entry_id = row.get::<Option<i64>, _>("best_entry_id").unwrap_or(0);
|
||||||
Ok(Article {
|
Ok(Article {
|
||||||
id: row.get("id"),
|
id: row.get("id"),
|
||||||
canonical_url: row.get("canonical_url"),
|
canonical_url: row.get("canonical_url"),
|
||||||
title: row.get::<Option<String>, _>("title").unwrap_or_default(),
|
title: row.get::<Option<String>, _>("title").unwrap_or_default(),
|
||||||
best_entry_id: row.get::<Option<i64>, _>("best_entry_id").unwrap_or(0),
|
best_entry_id,
|
||||||
content_html: row
|
content_html: row
|
||||||
.get::<Option<String>, _>("content_html")
|
.get::<Option<String>, _>("content_html")
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
@@ -938,6 +941,7 @@ fn article_from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Article> {
|
|||||||
feed_id: row.get::<Option<i64>, _>("feed_id").unwrap_or(0),
|
feed_id: row.get::<Option<i64>, _>("feed_id").unwrap_or(0),
|
||||||
feed_title: row
|
feed_title: row
|
||||||
.get::<Option<String>, _>("feed_title")
|
.get::<Option<String>, _>("feed_title")
|
||||||
|
.or_else(|| (best_entry_id == 0).then(|| "Imported".into()))
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
category: row.get("category"),
|
category: row.get("category"),
|
||||||
published_at: row
|
published_at: row
|
||||||
@@ -1079,6 +1083,7 @@ mod tests {
|
|||||||
"config_changes",
|
"config_changes",
|
||||||
"profile_versions",
|
"profile_versions",
|
||||||
"jobs",
|
"jobs",
|
||||||
|
"rating_imports",
|
||||||
] {
|
] {
|
||||||
let exists: i64 = sqlx::query_scalar(
|
let exists: i64 = sqlx::query_scalar(
|
||||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?",
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?",
|
||||||
@@ -1202,6 +1207,63 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn article_without_an_entry_writes_null_and_loads() {
|
||||||
|
let (_dir, db) = temp_db().await;
|
||||||
|
let article = Article {
|
||||||
|
id: 0,
|
||||||
|
canonical_url: "https://example.com/imported".into(),
|
||||||
|
title: "Imported article".into(),
|
||||||
|
best_entry_id: 0,
|
||||||
|
content_html: "<p>Imported body</p>".into(),
|
||||||
|
word_count: 2,
|
||||||
|
excerpt_only: false,
|
||||||
|
image_count: 0,
|
||||||
|
sources: Vec::new(),
|
||||||
|
first_seen: ts("2026-09-06T00:00:00Z"),
|
||||||
|
url: "https://example.com/imported".into(),
|
||||||
|
author: None,
|
||||||
|
feed_id: 0,
|
||||||
|
feed_title: "Imported".into(),
|
||||||
|
category: None,
|
||||||
|
published_at: None,
|
||||||
|
comments_url: None,
|
||||||
|
image_urls: Vec::new(),
|
||||||
|
social: Vec::new(),
|
||||||
|
extract_method: ExtractMethod::Readability,
|
||||||
|
};
|
||||||
|
let id = db.upsert_article(&article).await.unwrap();
|
||||||
|
let stored_entry: Option<i64> =
|
||||||
|
sqlx::query_scalar("SELECT best_entry_id FROM articles WHERE id = ?")
|
||||||
|
.bind(id)
|
||||||
|
.fetch_one(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stored_entry, None);
|
||||||
|
let loaded = db.get_article(id).await.unwrap().unwrap();
|
||||||
|
assert_eq!(loaded.best_entry_id, 0);
|
||||||
|
assert_eq!(loaded.feed_id, 0);
|
||||||
|
assert_eq!(loaded.feed_title, "Imported");
|
||||||
|
assert!(loaded.sources.is_empty());
|
||||||
|
assert!(crate::curate::signals::direct_feeds(&loaded).is_empty());
|
||||||
|
db.append_rating_event(&RatingEvent {
|
||||||
|
id: 0,
|
||||||
|
user_id: None,
|
||||||
|
article_id: id,
|
||||||
|
issue_date: None,
|
||||||
|
kind: "explicit".into(),
|
||||||
|
source: "import".into(),
|
||||||
|
label: "loved".into(),
|
||||||
|
value: 1.0,
|
||||||
|
note: None,
|
||||||
|
event_at: ts("2026-09-06T00:00:00Z"),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let ratings = db.current_ratings(36_500).await.unwrap();
|
||||||
|
assert_eq!(ratings[0].feed_title, "Imported");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn runs_and_issue_numbers() {
|
async fn runs_and_issue_numbers() {
|
||||||
let (_dir, db) = temp_db().await;
|
let (_dir, db) = temp_db().await;
|
||||||
|
|||||||
+29
-13
@@ -155,17 +155,10 @@ impl Extractor {
|
|||||||
// Relative URLs in the markup belong to the page we ended up
|
// Relative URLs in the markup belong to the page we ended up
|
||||||
// on, not the one we asked for: shortener and syndication
|
// on, not the one we asked for: shortener and syndication
|
||||||
// links land on another host entirely.
|
// links land on another host entirely.
|
||||||
let clean =
|
let extracted = self.finish_readable(&article.url, &page);
|
||||||
sanitize_with_base(&normalize_img_tags(&page.html), &page.final_url);
|
let words = extracted.word_count;
|
||||||
let words = word_count(&clean);
|
|
||||||
if words > feed_words && words > 0 {
|
if words > feed_words && words > 0 {
|
||||||
return self.finish(
|
return extracted;
|
||||||
article,
|
|
||||||
clean,
|
|
||||||
words,
|
|
||||||
ExtractMethod::Readability,
|
|
||||||
&page.final_url,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
tracing::debug!(words, feed_words, "readability was not an improvement");
|
tracing::debug!(words, feed_words, "readability was not an improvement");
|
||||||
}
|
}
|
||||||
@@ -286,16 +279,34 @@ impl Extractor {
|
|||||||
body.extend_from_slice(&chunk);
|
body.extend_from_slice(&chunk);
|
||||||
}
|
}
|
||||||
let html = String::from_utf8_lossy(&body).into_owned();
|
let html = String::from_utf8_lossy(&body).into_owned();
|
||||||
|
let (title, html) = readable_page(&html, &final_url)?;
|
||||||
Ok(Page {
|
Ok(Page {
|
||||||
html: readability(&html, &final_url)?,
|
title,
|
||||||
|
html,
|
||||||
final_url,
|
final_url,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sanitize a fetched readability page and derive its article metadata.
|
||||||
|
pub fn finish_readable(&self, requested_url: &str, page: &Page) -> Extracted {
|
||||||
|
let clean = sanitize_with_base(&normalize_img_tags(&page.html), &page.final_url);
|
||||||
|
let words = word_count(&clean);
|
||||||
|
let image_urls = collect_image_urls(&clean, &page.final_url);
|
||||||
|
Extracted {
|
||||||
|
content_html: clean,
|
||||||
|
word_count: words,
|
||||||
|
excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains),
|
||||||
|
image_urls,
|
||||||
|
method: ExtractMethod::Readability,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// An article page after fetching and readability (§3.3).
|
/// An article page after fetching and readability (§3.3).
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct Page {
|
pub struct Page {
|
||||||
|
/// Readability's title for the page.
|
||||||
|
pub title: String,
|
||||||
/// Readability's main-content markup.
|
/// Readability's main-content markup.
|
||||||
pub html: String,
|
pub html: String,
|
||||||
/// Where the fetch ended up, after any redirects — the base for relative URLs.
|
/// Where the fetch ended up, after any redirects — the base for relative URLs.
|
||||||
@@ -310,6 +321,10 @@ pub struct Page {
|
|||||||
/// their image with them) and its lazy-image heuristic overwrites a perfectly
|
/// their image with them) and its lazy-image heuristic overwrites a perfectly
|
||||||
/// good `src` with whatever other attribute happens to contain `.jpg`.
|
/// good `src` with whatever other attribute happens to contain `.jpg`.
|
||||||
pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> {
|
pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> {
|
||||||
|
readable_page(html, url).map(|(_, content)| content)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn readable_page(html: &str, url: &str) -> Result<(String, String), ExtractError> {
|
||||||
let html = prepare_for_readability(html);
|
let html = prepare_for_readability(html);
|
||||||
let config = dom_smoothie::Config {
|
let config = dom_smoothie::Config {
|
||||||
max_elements_to_parse: 60_000,
|
max_elements_to_parse: 60_000,
|
||||||
@@ -322,7 +337,7 @@ pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> {
|
|||||||
if content.trim().is_empty() {
|
if content.trim().is_empty() {
|
||||||
return Err(ExtractError::NoContent);
|
return Err(ExtractError::NoContent);
|
||||||
}
|
}
|
||||||
Ok(content)
|
Ok((parsed.title.trim().to_string(), content))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Copy an [`Extracted`] onto its [`Article`].
|
/// Copy an [`Extracted`] onto its [`Article`].
|
||||||
@@ -707,7 +722,8 @@ mod tests {
|
|||||||
<article><h1>A Post</h1><p>{paragraph}</p><p>{paragraph}</p></article>\
|
<article><h1>A Post</h1><p>{paragraph}</p><p>{paragraph}</p></article>\
|
||||||
<footer>© 2026</footer></body></html>"
|
<footer>© 2026</footer></body></html>"
|
||||||
);
|
);
|
||||||
let content = readability(&html, "https://blog.dev/p").expect("main content");
|
let (title, content) = readable_page(&html, "https://blog.dev/p").expect("main content");
|
||||||
|
assert_eq!(title, "A Post");
|
||||||
assert!(content.contains("Readability keeps the body copy"));
|
assert!(content.contains("Readability keeps the body copy"));
|
||||||
let clean = sanitize_with_base(&content, "https://blog.dev/p");
|
let clean = sanitize_with_base(&content, "https://blog.dev/p");
|
||||||
assert!(word_count(&clean) > 200);
|
assert!(word_count(&clean) > 200);
|
||||||
|
|||||||
+550
@@ -0,0 +1,550 @@
|
|||||||
|
//! Historical rating imports queued from the dashboard.
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use jiff::Timestamp;
|
||||||
|
use sqlx::Row as _;
|
||||||
|
|
||||||
|
use crate::config::Config;
|
||||||
|
use crate::curate::embedding::EmbeddingService;
|
||||||
|
use crate::db::{Db, fmt_ts};
|
||||||
|
use crate::extract::{Extractor, Page};
|
||||||
|
use crate::types::{Article, ExtractMethod, Extracted, Vote};
|
||||||
|
|
||||||
|
const BATCH_SIZE: i64 = 20;
|
||||||
|
|
||||||
|
/// One row in the historical rating import queue.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ImportRow {
|
||||||
|
pub id: i64,
|
||||||
|
pub url: String,
|
||||||
|
pub label: String,
|
||||||
|
pub note: Option<String>,
|
||||||
|
pub status: String,
|
||||||
|
pub message: Option<String>,
|
||||||
|
pub article_id: Option<i64>,
|
||||||
|
pub requested_by: Option<i64>,
|
||||||
|
pub requested_at: String,
|
||||||
|
pub finished_at: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row_from(row: &sqlx::sqlite::SqliteRow) -> ImportRow {
|
||||||
|
ImportRow {
|
||||||
|
id: row.get("id"),
|
||||||
|
url: row.get("url"),
|
||||||
|
label: row.get("label"),
|
||||||
|
note: row.get("note"),
|
||||||
|
status: row.get("status"),
|
||||||
|
message: row.get("message"),
|
||||||
|
article_id: row.get("article_id"),
|
||||||
|
requested_by: row.get("requested_by"),
|
||||||
|
requested_at: row.get("requested_at"),
|
||||||
|
finished_at: row.get("finished_at"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert one pending import row per URL and return how many were queued.
|
||||||
|
pub async fn queue(
|
||||||
|
db: &Db,
|
||||||
|
urls: &[String],
|
||||||
|
label: &str,
|
||||||
|
note: Option<&str>,
|
||||||
|
requested_by: Option<i64>,
|
||||||
|
now: Timestamp,
|
||||||
|
) -> Result<usize, sqlx::Error> {
|
||||||
|
let mut tx = db.pool().begin().await?;
|
||||||
|
let requested_at = fmt_ts(now);
|
||||||
|
for url in urls {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO rating_imports
|
||||||
|
(url, label, note, status, requested_by, requested_at)
|
||||||
|
VALUES (?, ?, ?, 'pending', ?, ?)",
|
||||||
|
)
|
||||||
|
.bind(url)
|
||||||
|
.bind(label)
|
||||||
|
.bind(note)
|
||||||
|
.bind(requested_by)
|
||||||
|
.bind(&requested_at)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(urls.len())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Newest import rows for the ratings dashboard.
|
||||||
|
pub async fn recent(db: &Db, limit: i64) -> Result<Vec<ImportRow>, sqlx::Error> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT id, url, label, note, status, message, article_id, requested_by,
|
||||||
|
requested_at, finished_at
|
||||||
|
FROM rating_imports ORDER BY id DESC LIMIT ?",
|
||||||
|
)
|
||||||
|
.bind(limit)
|
||||||
|
.fetch_all(db.pool())
|
||||||
|
.await?;
|
||||||
|
Ok(rows.iter().map(row_from).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn pending(db: &Db) -> Result<Vec<ImportRow>, sqlx::Error> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT id, url, label, note, status, message, article_id, requested_by,
|
||||||
|
requested_at, finished_at
|
||||||
|
FROM rating_imports WHERE status = 'pending' ORDER BY id LIMIT ?",
|
||||||
|
)
|
||||||
|
.bind(BATCH_SIZE)
|
||||||
|
.fetch_all(db.pool())
|
||||||
|
.await?;
|
||||||
|
Ok(rows.iter().map(row_from).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn finish(
|
||||||
|
db: &Db,
|
||||||
|
id: i64,
|
||||||
|
status: &str,
|
||||||
|
message: &str,
|
||||||
|
article_id: Option<i64>,
|
||||||
|
) -> Result<(), sqlx::Error> {
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE rating_imports
|
||||||
|
SET status = ?, message = ?, article_id = ?, finished_at = ? WHERE id = ?",
|
||||||
|
)
|
||||||
|
.bind(status)
|
||||||
|
.bind(short_message(message))
|
||||||
|
.bind(article_id)
|
||||||
|
.bind(fmt_ts(Timestamp::now()))
|
||||||
|
.bind(id)
|
||||||
|
.execute(db.pool())
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn short_message(message: &str) -> String {
|
||||||
|
message
|
||||||
|
.split_whitespace()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
|
.chars()
|
||||||
|
.take(500)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn vote(label: &str) -> Result<Vote> {
|
||||||
|
match label {
|
||||||
|
"loved" => Ok(Vote::Loved),
|
||||||
|
"good" => Ok(Vote::Good),
|
||||||
|
"not_for_me" => Ok(Vote::NotForMe),
|
||||||
|
other => anyhow::bail!("invalid rating label {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Processed {
|
||||||
|
article_id: i64,
|
||||||
|
created: bool,
|
||||||
|
embedded: bool,
|
||||||
|
embedding_note: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn imported_article(
|
||||||
|
canonical_url: String,
|
||||||
|
page: Page,
|
||||||
|
extracted: Extracted,
|
||||||
|
first_seen: Timestamp,
|
||||||
|
) -> Article {
|
||||||
|
Article {
|
||||||
|
id: 0,
|
||||||
|
canonical_url,
|
||||||
|
title: if page.title.trim().is_empty() {
|
||||||
|
page.final_url.clone()
|
||||||
|
} else {
|
||||||
|
page.title
|
||||||
|
},
|
||||||
|
best_entry_id: 0,
|
||||||
|
content_html: extracted.content_html,
|
||||||
|
word_count: extracted.word_count,
|
||||||
|
excerpt_only: extracted.excerpt_only,
|
||||||
|
image_count: extracted.image_urls.len() as i64,
|
||||||
|
sources: Vec::new(),
|
||||||
|
first_seen,
|
||||||
|
url: page.final_url,
|
||||||
|
author: None,
|
||||||
|
feed_id: 0,
|
||||||
|
feed_title: "Imported".into(),
|
||||||
|
category: None,
|
||||||
|
published_at: None,
|
||||||
|
comments_url: None,
|
||||||
|
image_urls: extracted.image_urls,
|
||||||
|
social: Vec::new(),
|
||||||
|
extract_method: ExtractMethod::Readability,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn process(
|
||||||
|
config: &Config,
|
||||||
|
db: &Db,
|
||||||
|
extractor: &Extractor,
|
||||||
|
embedding: Option<&EmbeddingService>,
|
||||||
|
embedding_unavailable: Option<&str>,
|
||||||
|
row: &ImportRow,
|
||||||
|
) -> Result<Processed> {
|
||||||
|
let canonical = crate::dedupe::canonical_url(&row.url)
|
||||||
|
.with_context(|| format!("invalid URL {:?}", row.url))?;
|
||||||
|
let (article, created) = match db.article_id_for_url(&canonical).await? {
|
||||||
|
Some(id) => (
|
||||||
|
db.get_article(id)
|
||||||
|
.await?
|
||||||
|
.with_context(|| format!("article {id} disappeared"))?,
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
None => {
|
||||||
|
let page = extractor
|
||||||
|
.fetch_readable(&canonical)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("fetching {canonical}"))?;
|
||||||
|
let extracted = extractor.finish_readable(&canonical, &page);
|
||||||
|
let mut article = imported_article(canonical, page, extracted, Timestamp::now());
|
||||||
|
article.id = db.upsert_article(&article).await?;
|
||||||
|
(article, true)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut embedded = false;
|
||||||
|
let mut embedding_note = embedding_unavailable.map(str::to_string);
|
||||||
|
if let Some(service) = embedding {
|
||||||
|
match service.articles(std::slice::from_ref(&article)).await {
|
||||||
|
Ok(vectors) if vectors.contains_key(&article.id) => embedded = true,
|
||||||
|
Ok(_) => {
|
||||||
|
tracing::warn!(url = %row.url, article_id = article.id, "Voyage returned no embedding for rating import");
|
||||||
|
embedding_note = Some("Voyage returned no embedding".into());
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
tracing::warn!(url = %row.url, article_id = article.id, %error, "could not embed rating import");
|
||||||
|
embedding_note = Some(error.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if !config.voyage.enabled {
|
||||||
|
embedding_note = Some("Voyage is disabled".into());
|
||||||
|
}
|
||||||
|
|
||||||
|
crate::rate::record_explicit(
|
||||||
|
config,
|
||||||
|
db,
|
||||||
|
article.id,
|
||||||
|
Some(vote(&row.label)?),
|
||||||
|
"import",
|
||||||
|
row.requested_by,
|
||||||
|
row.note.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(Processed {
|
||||||
|
article_id: article.id,
|
||||||
|
created,
|
||||||
|
embedded,
|
||||||
|
embedding_note,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch, embed and rate all pending historical imports.
|
||||||
|
pub async fn run(config: &Config, db: &Db) -> Result<String> {
|
||||||
|
let http = crate::http::build_client(crate::http::DEFAULT_TIMEOUT)
|
||||||
|
.context("building article HTTP client")?;
|
||||||
|
let extractor = Extractor::new(http, config.curation.paywall_domains.clone());
|
||||||
|
let (embedding, embedding_unavailable) = if config.voyage.enabled {
|
||||||
|
match EmbeddingService::real(db.clone(), config.voyage.clone()) {
|
||||||
|
Ok(service) => (Some(service), None),
|
||||||
|
Err(error) => {
|
||||||
|
tracing::warn!(%error, "Voyage embeddings unavailable for rating imports");
|
||||||
|
(None, Some(error.to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
(None, None)
|
||||||
|
};
|
||||||
|
run_with(
|
||||||
|
config,
|
||||||
|
db,
|
||||||
|
&extractor,
|
||||||
|
embedding.as_ref(),
|
||||||
|
embedding_unavailable.as_deref(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_with(
|
||||||
|
config: &Config,
|
||||||
|
db: &Db,
|
||||||
|
extractor: &Extractor,
|
||||||
|
embedding: Option<&EmbeddingService>,
|
||||||
|
embedding_unavailable: Option<&str>,
|
||||||
|
) -> Result<String> {
|
||||||
|
let mut imported = 0usize;
|
||||||
|
let mut created = 0usize;
|
||||||
|
let mut existing = 0usize;
|
||||||
|
let mut failed = 0usize;
|
||||||
|
loop {
|
||||||
|
let rows = pending(db).await?;
|
||||||
|
if rows.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
for row in rows {
|
||||||
|
match process(
|
||||||
|
config,
|
||||||
|
db,
|
||||||
|
extractor,
|
||||||
|
embedding,
|
||||||
|
embedding_unavailable,
|
||||||
|
&row,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(done) => {
|
||||||
|
imported += 1;
|
||||||
|
if done.created {
|
||||||
|
created += 1;
|
||||||
|
} else {
|
||||||
|
existing += 1;
|
||||||
|
}
|
||||||
|
let mut message = if done.created {
|
||||||
|
"created".to_string()
|
||||||
|
} else {
|
||||||
|
"existing article".to_string()
|
||||||
|
};
|
||||||
|
if done.embedded {
|
||||||
|
message.push_str(" + embedded");
|
||||||
|
} else if let Some(note) = done.embedding_note {
|
||||||
|
message.push_str(&format!("; no embedding ({note})"));
|
||||||
|
}
|
||||||
|
finish(db, row.id, "ok", &message, Some(done.article_id)).await?;
|
||||||
|
tracing::info!(url = %row.url, article_id = done.article_id, %message, "rating import ok");
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
failed += 1;
|
||||||
|
let message = format!("{error:#}");
|
||||||
|
finish(db, row.id, "failed", &message, None).await?;
|
||||||
|
tracing::info!(url = %row.url, %message, "rating import failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if imported == 0 && failed == 0 {
|
||||||
|
Ok("no pending rating imports".into())
|
||||||
|
} else {
|
||||||
|
Ok(format!(
|
||||||
|
"{imported} imported ({created} new, {existing} existing), {failed} failed"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::types::Entry;
|
||||||
|
|
||||||
|
async fn test_db() -> (tempfile::TempDir, Db) {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
(dir, db)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn existing_article(db: &Db, url: &str) -> i64 {
|
||||||
|
let now: Timestamp = "2026-09-01T00:00:00Z".parse().unwrap();
|
||||||
|
db.upsert_entry(&Entry {
|
||||||
|
id: 42,
|
||||||
|
feed_id: 7,
|
||||||
|
feed_title: Some("Feed".into()),
|
||||||
|
category: None,
|
||||||
|
title: "Existing".into(),
|
||||||
|
url: url.into(),
|
||||||
|
canonical_url: Some(url.into()),
|
||||||
|
author: None,
|
||||||
|
published_at: None,
|
||||||
|
comments_url: None,
|
||||||
|
raw_content: "<p>Existing body</p>".into(),
|
||||||
|
fetched_at: now,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
db.upsert_article(&Article {
|
||||||
|
id: 0,
|
||||||
|
canonical_url: url.into(),
|
||||||
|
title: "Existing".into(),
|
||||||
|
best_entry_id: 42,
|
||||||
|
content_html: "<p>Existing body</p>".into(),
|
||||||
|
word_count: 2,
|
||||||
|
excerpt_only: false,
|
||||||
|
image_count: 0,
|
||||||
|
sources: Vec::new(),
|
||||||
|
first_seen: now,
|
||||||
|
url: url.into(),
|
||||||
|
author: None,
|
||||||
|
feed_id: 7,
|
||||||
|
feed_title: "Feed".into(),
|
||||||
|
category: None,
|
||||||
|
published_at: None,
|
||||||
|
comments_url: None,
|
||||||
|
image_urls: Vec::new(),
|
||||||
|
social: Vec::new(),
|
||||||
|
extract_method: ExtractMethod::Miniflux,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn config_without_voyage() -> Config {
|
||||||
|
let mut config = Config::default();
|
||||||
|
config.voyage.enabled = false;
|
||||||
|
config
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn existing_article_is_reused_and_rated() {
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
let id = existing_article(&db, "https://example.com/post").await;
|
||||||
|
let user = crate::web::users::add(&db, "admin", "correct horse battery", true)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
queue(
|
||||||
|
&db,
|
||||||
|
&["https://EXAMPLE.com/post/?utm_source=test".into()],
|
||||||
|
"good",
|
||||||
|
Some("still useful"),
|
||||||
|
Some(user.id),
|
||||||
|
Timestamp::now(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let summary = run_with(
|
||||||
|
&config_without_voyage(),
|
||||||
|
&db,
|
||||||
|
&Extractor::offline(Vec::new()),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(summary, "1 imported (0 new, 1 existing), 0 failed");
|
||||||
|
assert_eq!(db.count_entries().await.unwrap(), 1);
|
||||||
|
let current = db.current_ratings(36500).await.unwrap();
|
||||||
|
assert_eq!(current.len(), 1);
|
||||||
|
assert_eq!(current[0].article_id, id);
|
||||||
|
assert_eq!(current[0].user_id, Some(user.id));
|
||||||
|
assert_eq!(current[0].label, "good");
|
||||||
|
assert_eq!(current[0].note.as_deref(), Some("still useful"));
|
||||||
|
let source: String =
|
||||||
|
sqlx::query_scalar("SELECT source FROM rating_events ORDER BY id DESC LIMIT 1")
|
||||||
|
.fetch_one(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(source, "import");
|
||||||
|
let row = recent(&db, 1).await.unwrap().remove(0);
|
||||||
|
assert_eq!(row.status, "ok");
|
||||||
|
assert_eq!(row.article_id, Some(id));
|
||||||
|
assert!(row.message.unwrap().contains("existing article"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn invalid_url_fails_and_empty_run_is_benign() {
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
queue(
|
||||||
|
&db,
|
||||||
|
&["not-a-url".into()],
|
||||||
|
"loved",
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Timestamp::now(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let config = config_without_voyage();
|
||||||
|
let extractor = Extractor::offline(Vec::new());
|
||||||
|
assert_eq!(
|
||||||
|
run_with(&config, &db, &extractor, None, None)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
"0 imported (0 new, 0 existing), 1 failed"
|
||||||
|
);
|
||||||
|
let row = recent(&db, 1).await.unwrap().remove(0);
|
||||||
|
assert_eq!(row.status, "failed");
|
||||||
|
assert!(row.message.unwrap().contains("invalid URL"));
|
||||||
|
assert_eq!(
|
||||||
|
run_with(&config, &db, &extractor, None, None)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
"no pending rating imports"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn fetched_page_becomes_an_entryless_sanitized_article() {
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
let extractor = Extractor::offline(Vec::new());
|
||||||
|
let page = Page {
|
||||||
|
title: "A historical essay".into(),
|
||||||
|
html: "<article><p>Useful old writing.</p><script>bad()</script><img src=\"/chart.png\"></article>".into(),
|
||||||
|
final_url: "https://example.com/essays/old".into(),
|
||||||
|
};
|
||||||
|
let extracted = extractor.finish_readable("https://example.com/old", &page);
|
||||||
|
let mut article = imported_article(
|
||||||
|
"https://example.com/old".into(),
|
||||||
|
page,
|
||||||
|
extracted,
|
||||||
|
"2026-09-06T00:00:00Z".parse().unwrap(),
|
||||||
|
);
|
||||||
|
assert_eq!(article.title, "A historical essay");
|
||||||
|
assert_eq!(article.best_entry_id, 0);
|
||||||
|
assert_eq!(article.feed_title, "Imported");
|
||||||
|
assert!(article.sources.is_empty());
|
||||||
|
assert!(!article.content_html.contains("script"));
|
||||||
|
assert_eq!(article.image_urls, ["https://example.com/chart.png"]);
|
||||||
|
article.id = db.upsert_article(&article).await.unwrap();
|
||||||
|
let loaded = db.get_article(article.id).await.unwrap().unwrap();
|
||||||
|
assert_eq!(loaded.title, "A historical essay");
|
||||||
|
assert_eq!(loaded.best_entry_id, 0);
|
||||||
|
assert_eq!(loaded.feed_title, "Imported");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rows_queued_while_running_are_picked_up_in_the_next_batch() {
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
existing_article(&db, "https://example.com/post").await;
|
||||||
|
queue(
|
||||||
|
&db,
|
||||||
|
&["https://example.com/post".into()],
|
||||||
|
"loved",
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Timestamp::now(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query(
|
||||||
|
"CREATE TRIGGER queue_during_import AFTER UPDATE OF status ON rating_imports
|
||||||
|
WHEN OLD.status = 'pending' AND NEW.status = 'ok' AND NEW.id = 1
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO rating_imports
|
||||||
|
(url, label, status, requested_at)
|
||||||
|
VALUES ('https://example.com/post', 'good', 'pending', '2026-09-01T00:00:00Z');
|
||||||
|
END",
|
||||||
|
)
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let summary = run_with(
|
||||||
|
&config_without_voyage(),
|
||||||
|
&db,
|
||||||
|
&Extractor::offline(Vec::new()),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(summary, "2 imported (0 new, 2 existing), 0 failed");
|
||||||
|
assert_eq!(recent(&db, 10).await.unwrap().len(), 2);
|
||||||
|
let ratings: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM rating_events")
|
||||||
|
.fetch_one(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(ratings, 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
+14
-2
@@ -42,17 +42,20 @@ pub enum Job {
|
|||||||
BackfillSocial,
|
BackfillSocial,
|
||||||
/// `features-prune` → `features prune`.
|
/// `features-prune` → `features prune`.
|
||||||
FeaturesPrune,
|
FeaturesPrune,
|
||||||
|
/// `import-ratings` → process ratings-dashboard URL imports.
|
||||||
|
ImportRatings,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Job {
|
impl Job {
|
||||||
/// The catalogue in the order the Jobs page lists it.
|
/// The catalogue in the order the Jobs page lists it.
|
||||||
pub const CATALOGUE: [Job; 6] = [
|
pub const CATALOGUE: [Job; 7] = [
|
||||||
Job::Generate { date: None },
|
Job::Generate { date: None },
|
||||||
Job::DryRun,
|
Job::DryRun,
|
||||||
Job::ProfileRebuild,
|
Job::ProfileRebuild,
|
||||||
Job::FeaturesBackfill,
|
Job::FeaturesBackfill,
|
||||||
Job::BackfillSocial,
|
Job::BackfillSocial,
|
||||||
Job::FeaturesPrune,
|
Job::FeaturesPrune,
|
||||||
|
Job::ImportRatings,
|
||||||
];
|
];
|
||||||
|
|
||||||
/// `^[a-z0-9-]+$`: the only characters a job (and so a unit instance) name
|
/// `^[a-z0-9-]+$`: the only characters a job (and so a unit instance) name
|
||||||
@@ -77,6 +80,7 @@ impl Job {
|
|||||||
"features-backfill" => Some(Job::FeaturesBackfill),
|
"features-backfill" => Some(Job::FeaturesBackfill),
|
||||||
"backfill-social" => Some(Job::BackfillSocial),
|
"backfill-social" => Some(Job::BackfillSocial),
|
||||||
"features-prune" => Some(Job::FeaturesPrune),
|
"features-prune" => Some(Job::FeaturesPrune),
|
||||||
|
"import-ratings" => Some(Job::ImportRatings),
|
||||||
_ => {
|
_ => {
|
||||||
let date = name.strip_prefix("generate-")?;
|
let date = name.strip_prefix("generate-")?;
|
||||||
// Exactly `YYYY-MM-DD`; the round trip rejects `2026-9-3`.
|
// Exactly `YYYY-MM-DD`; the round trip rejects `2026-9-3`.
|
||||||
@@ -95,6 +99,7 @@ impl Job {
|
|||||||
Job::FeaturesBackfill => "features-backfill".into(),
|
Job::FeaturesBackfill => "features-backfill".into(),
|
||||||
Job::BackfillSocial => "backfill-social".into(),
|
Job::BackfillSocial => "backfill-social".into(),
|
||||||
Job::FeaturesPrune => "features-prune".into(),
|
Job::FeaturesPrune => "features-prune".into(),
|
||||||
|
Job::ImportRatings => "import-ratings".into(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -122,6 +127,7 @@ impl Job {
|
|||||||
Job::FeaturesPrune => {
|
Job::FeaturesPrune => {
|
||||||
"Drop stale embeddings, old candidate telemetry and old assessments per the retention config."
|
"Drop stale embeddings, old candidate telemetry and old assessments per the retention config."
|
||||||
}
|
}
|
||||||
|
Job::ImportRatings => "Fetch, embed and rate the URLs queued from the Ratings page.",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,7 +139,7 @@ impl Job {
|
|||||||
Job::ProfileRebuild => Some("profile rebuild"),
|
Job::ProfileRebuild => Some("profile rebuild"),
|
||||||
Job::FeaturesBackfill => Some("features backfill"),
|
Job::FeaturesBackfill => Some("features backfill"),
|
||||||
Job::BackfillSocial => Some("backfill-social"),
|
Job::BackfillSocial => Some("backfill-social"),
|
||||||
Job::FeaturesPrune => None,
|
Job::FeaturesPrune | Job::ImportRatings => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -447,6 +453,12 @@ mod tests {
|
|||||||
Some("generate")
|
Some("generate")
|
||||||
);
|
);
|
||||||
assert_eq!(Job::parse("features-prune").unwrap().takes_lock(), None);
|
assert_eq!(Job::parse("features-prune").unwrap().takes_lock(), None);
|
||||||
|
assert_eq!(Job::parse("import-ratings"), Some(Job::ImportRatings));
|
||||||
|
assert_eq!(Job::ImportRatings.takes_lock(), None);
|
||||||
|
assert_eq!(
|
||||||
|
Job::ImportRatings.description(),
|
||||||
|
"Fetch, embed and rate the URLs queued from the Ratings page."
|
||||||
|
);
|
||||||
assert!(Job::parse("generate-2026-09-03").unwrap().dangerous());
|
assert!(Job::parse("generate-2026-09-03").unwrap().dangerous());
|
||||||
assert!(!Job::parse("dry-run").unwrap().dangerous());
|
assert!(!Job::parse("dry-run").unwrap().dangerous());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,11 +24,13 @@ pub mod extract;
|
|||||||
pub mod html;
|
pub mod html;
|
||||||
pub mod http;
|
pub mod http;
|
||||||
pub mod images;
|
pub mod images;
|
||||||
|
pub mod imports;
|
||||||
pub mod jobs;
|
pub mod jobs;
|
||||||
pub mod lock;
|
pub mod lock;
|
||||||
pub mod miniflux;
|
pub mod miniflux;
|
||||||
pub mod pipeline;
|
pub mod pipeline;
|
||||||
pub mod publish;
|
pub mod publish;
|
||||||
|
pub mod rate;
|
||||||
pub mod report;
|
pub mod report;
|
||||||
pub mod server;
|
pub mod server;
|
||||||
pub mod social;
|
pub mod social;
|
||||||
|
|||||||
+22
-25
@@ -16,8 +16,8 @@ use daily_epub::curate::telemetry;
|
|||||||
use daily_epub::db::Db;
|
use daily_epub::db::Db;
|
||||||
use daily_epub::pipeline::{self, GenerateOptions, GenerateOutcome};
|
use daily_epub::pipeline::{self, GenerateOptions, GenerateOutcome};
|
||||||
use daily_epub::report::{RunReport, VOYAGE_PROVIDER};
|
use daily_epub::report::{RunReport, VOYAGE_PROVIDER};
|
||||||
use daily_epub::types::{ArticleId, RatingEvent, Vote};
|
use daily_epub::types::{ArticleId, Vote};
|
||||||
use daily_epub::{curate, http, jobs, lock, server, social};
|
use daily_epub::{curate, http, imports, jobs, lock, rate, server, social};
|
||||||
|
|
||||||
/// A personalized daily newspaper, delivered as an EPUB.
|
/// A personalized daily newspaper, delivered as an EPUB.
|
||||||
#[derive(Debug, Parser)]
|
#[derive(Debug, Parser)]
|
||||||
@@ -71,7 +71,7 @@ enum JobCommand {
|
|||||||
/// Run one catalogue job in-process and record it in the `jobs` table.
|
/// Run one catalogue job in-process and record it in the `jobs` table.
|
||||||
Run {
|
Run {
|
||||||
/// `generate`, `generate-YYYY-MM-DD`, `dry-run`, `profile-rebuild`,
|
/// `generate`, `generate-YYYY-MM-DD`, `dry-run`, `profile-rebuild`,
|
||||||
/// `features-backfill`, `backfill-social` or `features-prune`.
|
/// `features-backfill`, `backfill-social`, `features-prune` or `import-ratings`.
|
||||||
name: String,
|
name: String,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -726,28 +726,7 @@ async fn append_cli_event(
|
|||||||
vote: Option<Vote>,
|
vote: Option<Vote>,
|
||||||
note: Option<String>,
|
note: Option<String>,
|
||||||
) -> Result<i64> {
|
) -> Result<i64> {
|
||||||
let (label, value) = match vote {
|
Ok(rate::record_explicit(config, db, article_id, vote, "cli", None, note).await?)
|
||||||
Some(Vote::Loved) => ("loved", Vote::Loved.value(&config.curation.feedback)),
|
|
||||||
Some(Vote::Good) => ("good", Vote::Good.value(&config.curation.feedback)),
|
|
||||||
Some(Vote::NotForMe) => (
|
|
||||||
"not_for_me",
|
|
||||||
Vote::NotForMe.value(&config.curation.feedback),
|
|
||||||
),
|
|
||||||
None => ("cleared", 0.0),
|
|
||||||
};
|
|
||||||
let event = RatingEvent {
|
|
||||||
id: 0,
|
|
||||||
user_id: None,
|
|
||||||
article_id,
|
|
||||||
issue_date: db.latest_issue_date_for_article(article_id).await?,
|
|
||||||
kind: "explicit".into(),
|
|
||||||
source: "cli".into(),
|
|
||||||
label: label.into(),
|
|
||||||
value,
|
|
||||||
note,
|
|
||||||
event_at: jiff::Timestamp::now(),
|
|
||||||
};
|
|
||||||
Ok(db.append_rating_event(&event).await?)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn cmd_ratings(config: &Config, db: &Db, command: RatingsCommand) -> Result<()> {
|
async fn cmd_ratings(config: &Config, db: &Db, command: RatingsCommand) -> Result<()> {
|
||||||
@@ -964,6 +943,7 @@ async fn run_job(config: &Config, db: &Db, job: &jobs::Job) -> Result<(String, O
|
|||||||
cmd_features(config, db, FeaturesCommand::Prune).await?,
|
cmd_features(config, db, FeaturesCommand::Prune).await?,
|
||||||
None,
|
None,
|
||||||
)),
|
)),
|
||||||
|
jobs::Job::ImportRatings => Ok((imports::run(config, db).await?, None)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1156,6 +1136,13 @@ mod tests {
|
|||||||
Command::Job(JobCommand::Run { name }) => assert_eq!(name, "features-prune"),
|
Command::Job(JobCommand::Run { name }) => assert_eq!(name, "features-prune"),
|
||||||
other => panic!("expected job run, got {other:?}"),
|
other => panic!("expected job run, got {other:?}"),
|
||||||
}
|
}
|
||||||
|
match Cli::try_parse_from(["daily-epub", "job", "run", "import-ratings"])
|
||||||
|
.unwrap()
|
||||||
|
.command
|
||||||
|
{
|
||||||
|
Command::Job(JobCommand::Run { name }) => assert_eq!(name, "import-ratings"),
|
||||||
|
other => panic!("expected job run, got {other:?}"),
|
||||||
|
}
|
||||||
assert!(Cli::try_parse_from(["daily-epub", "job", "run"]).is_err());
|
assert!(Cli::try_parse_from(["daily-epub", "job", "run"]).is_err());
|
||||||
assert!(Cli::try_parse_from(["daily-epub", "job"]).is_err());
|
assert!(Cli::try_parse_from(["daily-epub", "job"]).is_err());
|
||||||
}
|
}
|
||||||
@@ -1219,6 +1206,16 @@ mod tests {
|
|||||||
.contains("voyage.enabled is false"),
|
.contains("voyage.enabled is false"),
|
||||||
"{failed:?}"
|
"{failed:?}"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let import = jobs::Job::ImportRatings;
|
||||||
|
cmd_job_run(&config, &db, &import).await.unwrap();
|
||||||
|
let imported = jobs::list(&db, 1).await.unwrap().remove(0);
|
||||||
|
assert_eq!(imported.name, "import-ratings");
|
||||||
|
assert_eq!(imported.status, "ok");
|
||||||
|
assert_eq!(
|
||||||
|
imported.message.as_deref(),
|
||||||
|
Some("no pending rating imports")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+39
@@ -0,0 +1,39 @@
|
|||||||
|
//! Shared construction of explicit rating events.
|
||||||
|
|
||||||
|
use crate::config::Config;
|
||||||
|
use crate::db::{Db, DbError};
|
||||||
|
use crate::types::{ArticleId, RatingEvent, Vote};
|
||||||
|
|
||||||
|
/// Append an explicit rating event from a named source.
|
||||||
|
pub async fn record_explicit(
|
||||||
|
config: &Config,
|
||||||
|
db: &Db,
|
||||||
|
article_id: ArticleId,
|
||||||
|
vote: Option<Vote>,
|
||||||
|
source: &str,
|
||||||
|
user_id: Option<i64>,
|
||||||
|
note: Option<String>,
|
||||||
|
) -> Result<i64, DbError> {
|
||||||
|
let (label, value) = match vote {
|
||||||
|
Some(Vote::Loved) => ("loved", Vote::Loved.value(&config.curation.feedback)),
|
||||||
|
Some(Vote::Good) => ("good", Vote::Good.value(&config.curation.feedback)),
|
||||||
|
Some(Vote::NotForMe) => (
|
||||||
|
"not_for_me",
|
||||||
|
Vote::NotForMe.value(&config.curation.feedback),
|
||||||
|
),
|
||||||
|
None => ("cleared", 0.0),
|
||||||
|
};
|
||||||
|
db.append_rating_event(&RatingEvent {
|
||||||
|
id: 0,
|
||||||
|
user_id,
|
||||||
|
article_id,
|
||||||
|
issue_date: db.latest_issue_date_for_article(article_id).await?,
|
||||||
|
kind: "explicit".into(),
|
||||||
|
source: source.into(),
|
||||||
|
label: label.into(),
|
||||||
|
value,
|
||||||
|
note,
|
||||||
|
event_at: jiff::Timestamp::now(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
+62
-37
@@ -196,7 +196,7 @@ fn form_date(body: &[u8]) -> Option<String> {
|
|||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn set_flash(session: &Session, kind: &str, text: String) -> Result<(), WebError> {
|
pub(crate) async fn set_flash(session: &Session, kind: &str, text: String) -> Result<(), WebError> {
|
||||||
session
|
session
|
||||||
.insert(
|
.insert(
|
||||||
"flash",
|
"flash",
|
||||||
@@ -209,6 +209,57 @@ async fn set_flash(session: &Session, kind: &str, text: String) -> Result<(), We
|
|||||||
.map_err(|error| WebError::Internal(error.into()))
|
.map_err(|error| WebError::Internal(error.into()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) enum StartJob {
|
||||||
|
Disabled,
|
||||||
|
Active(i64),
|
||||||
|
Started(i64),
|
||||||
|
Failed(i64, String),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Request one catalogue job through the configured runner.
|
||||||
|
pub(crate) async fn start_job(
|
||||||
|
state: &AppState,
|
||||||
|
viewer: &Viewer,
|
||||||
|
job: Job,
|
||||||
|
) -> Result<StartJob, WebError> {
|
||||||
|
if !state.config().server.jobs_enabled {
|
||||||
|
return Ok(StartJob::Disabled);
|
||||||
|
}
|
||||||
|
let unit = job.unit();
|
||||||
|
if let Some(active) = jobs::active_for_unit(&state.db, &unit)
|
||||||
|
.await
|
||||||
|
.map_err(db_err)?
|
||||||
|
{
|
||||||
|
return Ok(StartJob::Active(active));
|
||||||
|
}
|
||||||
|
if let Err(error) = crate::web::WebState::reload_if_changed(state) {
|
||||||
|
tracing::warn!(%error, "config.toml on disk does not load; keeping the previous config");
|
||||||
|
}
|
||||||
|
let id = jobs::insert_requested(&state.db, &job, Some(viewer.id), Timestamp::now())
|
||||||
|
.await
|
||||||
|
.map_err(db_err)?;
|
||||||
|
match state.web.jobs.start(&unit).await {
|
||||||
|
Ok(()) => {
|
||||||
|
tracing::info!(user = %viewer.username, job = %job.name(), job_id = id, %unit, "job requested");
|
||||||
|
Ok(StartJob::Started(id))
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
tracing::warn!(user = %viewer.username, job = %job.name(), job_id = id, %unit, %error, "job start failed");
|
||||||
|
jobs::finish(
|
||||||
|
&state.db,
|
||||||
|
id,
|
||||||
|
jobs::Outcome::Failed,
|
||||||
|
&format!("could not start {unit}: {error}"),
|
||||||
|
None,
|
||||||
|
Timestamp::now(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(db_err)?;
|
||||||
|
Ok(StartJob::Failed(id, error))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// `POST /dashboard/jobs/{name}` (admin, origin-checked): parse the name,
|
/// `POST /dashboard/jobs/{name}` (admin, origin-checked): parse the name,
|
||||||
/// refuse a duplicate `requested`/`running` unit (409), insert `requested`,
|
/// refuse a duplicate `requested`/`running` unit (409), insert `requested`,
|
||||||
/// start the unit; a failed start marks the row `failed`.
|
/// start the unit; a failed start marks the row `failed`.
|
||||||
@@ -233,21 +284,17 @@ async fn start(
|
|||||||
.map_err(|_| WebError::BadRequest(format!("invalid date {date:?}")))?;
|
.map_err(|_| WebError::BadRequest(format!("invalid date {date:?}")))?;
|
||||||
job = Job::Generate { date: Some(parsed) };
|
job = Job::Generate { date: Some(parsed) };
|
||||||
}
|
}
|
||||||
let config = state.config();
|
match start_job(&state, &viewer, job).await? {
|
||||||
if !config.server.jobs_enabled {
|
StartJob::Disabled => {
|
||||||
set_flash(
|
set_flash(
|
||||||
&session,
|
&session,
|
||||||
"error",
|
"error",
|
||||||
"Jobs are disabled on this server (server.jobs_enabled = false).".into(),
|
"Jobs are disabled on this server (server.jobs_enabled = false).".into(),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
return Ok(Redirect::to("/dashboard/jobs").into_response());
|
Ok(Redirect::to("/dashboard/jobs").into_response())
|
||||||
}
|
}
|
||||||
let unit = job.unit();
|
StartJob::Active(active) => {
|
||||||
if let Some(active) = jobs::active_for_unit(&state.db, &unit)
|
|
||||||
.await
|
|
||||||
.map_err(db_err)?
|
|
||||||
{
|
|
||||||
let flash = Flash {
|
let flash = Flash {
|
||||||
kind: "error".into(),
|
kind: "error".into(),
|
||||||
text: format!(
|
text: format!(
|
||||||
@@ -256,44 +303,22 @@ async fn start(
|
|||||||
),
|
),
|
||||||
};
|
};
|
||||||
let template = jobs_template(&state, Some(viewer), Some(flash)).await?;
|
let template = jobs_template(&state, Some(viewer), Some(flash)).await?;
|
||||||
return Ok((StatusCode::CONFLICT, Html(template)).into_response());
|
Ok((StatusCode::CONFLICT, Html(template)).into_response())
|
||||||
}
|
}
|
||||||
|
StartJob::Started(id) => {
|
||||||
// Pick up a hand-edited config.toml before the unit starts (§4.2); a file
|
|
||||||
// that no longer loads keeps the previous config live and is only logged.
|
|
||||||
if let Err(error) = crate::web::WebState::reload_if_changed(&state) {
|
|
||||||
tracing::warn!(%error, "config.toml on disk does not load; keeping the previous config");
|
|
||||||
}
|
|
||||||
let now = Timestamp::now();
|
|
||||||
let id = jobs::insert_requested(&state.db, &job, Some(viewer.id), now)
|
|
||||||
.await
|
|
||||||
.map_err(db_err)?;
|
|
||||||
match state.web.jobs.start(&unit).await {
|
|
||||||
Ok(()) => {
|
|
||||||
tracing::info!(user = %viewer.username, job = %job.name(), job_id = id, %unit, "job requested");
|
|
||||||
set_flash(&session, "success", format!("Started {}.", job.name())).await?;
|
set_flash(&session, "success", format!("Started {}.", job.name())).await?;
|
||||||
|
Ok(Redirect::to(&format!("/dashboard/jobs/{id}")).into_response())
|
||||||
}
|
}
|
||||||
Err(error) => {
|
StartJob::Failed(id, error) => {
|
||||||
tracing::warn!(user = %viewer.username, job = %job.name(), job_id = id, %unit, %error, "job start failed");
|
|
||||||
jobs::finish(
|
|
||||||
&state.db,
|
|
||||||
id,
|
|
||||||
jobs::Outcome::Failed,
|
|
||||||
&format!("could not start {unit}: {error}"),
|
|
||||||
None,
|
|
||||||
Timestamp::now(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(db_err)?;
|
|
||||||
set_flash(
|
set_flash(
|
||||||
&session,
|
&session,
|
||||||
"error",
|
"error",
|
||||||
format!("Could not start {}: {error}", job.name()),
|
format!("Could not start {}: {error}", job.name()),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(Redirect::to(&format!("/dashboard/jobs/{id}")).into_response())
|
Ok(Redirect::to(&format!("/dashboard/jobs/{id}")).into_response())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `GET /dashboard/jobs/{id}`: the row, the live unit status and the journal
|
/// `GET /dashboard/jobs/{id}`: the row, the live unit status and the journal
|
||||||
|
|||||||
@@ -11,8 +11,9 @@ use std::collections::{BTreeMap, HashMap, HashSet};
|
|||||||
|
|
||||||
use askama::Template;
|
use askama::Template;
|
||||||
use axum::Router;
|
use axum::Router;
|
||||||
|
use axum::body::Bytes;
|
||||||
use axum::extract::{Extension, Query, State};
|
use axum::extract::{Extension, Query, State};
|
||||||
use axum::response::{IntoResponse, Response};
|
use axum::response::{IntoResponse, Redirect, Response};
|
||||||
use axum::routing::get;
|
use axum::routing::get;
|
||||||
use axum_login::tower_sessions::Session;
|
use axum_login::tower_sessions::Session;
|
||||||
use jiff::Timestamp;
|
use jiff::Timestamp;
|
||||||
@@ -24,12 +25,17 @@ use crate::curate::profile::{MAX_RATINGS_IN_REBUILD, REBUILD_INTERVAL_DAYS};
|
|||||||
use crate::curate::signals::{self, PreferenceState};
|
use crate::curate::signals::{self, PreferenceState};
|
||||||
use crate::curate::telemetry::SignalsJson;
|
use crate::curate::telemetry::SignalsJson;
|
||||||
use crate::db::{Db, DbError};
|
use crate::db::{Db, DbError};
|
||||||
|
use crate::imports;
|
||||||
|
use crate::jobs::Job;
|
||||||
use crate::server::AppState;
|
use crate::server::AppState;
|
||||||
use crate::types::{Article, ArticleId, FeedId, RatedArticle};
|
use crate::types::{Article, ArticleId, FeedId, RatedArticle};
|
||||||
use crate::web::rate::RatingWidget;
|
use crate::web::rate::RatingWidget;
|
||||||
use crate::web::session::{AuthSession, Viewer};
|
use crate::web::session::{AuthSession, Viewer};
|
||||||
use crate::web::{Html, Page, Pagination, WebError, encode_component, format_time, take_flash};
|
use crate::web::{Html, Page, Pagination, WebError, encode_component, format_time, take_flash};
|
||||||
|
|
||||||
|
use super::jobs::{StartJob, set_flash, start_job};
|
||||||
|
use super::{db_err, fmt_stored_time};
|
||||||
|
|
||||||
/// The verdict block and the rebuild set are bounded by count, not age, so the
|
/// The verdict block and the rebuild set are bounded by count, not age, so the
|
||||||
/// page lists every current verdict (curation plan §8.3, §8.4).
|
/// page lists every current verdict (curation plan §8.3, §8.4).
|
||||||
const CURRENT_LOOKBACK_DAYS: i64 = 36_500;
|
const CURRENT_LOOKBACK_DAYS: i64 = 36_500;
|
||||||
@@ -37,7 +43,10 @@ const EVENTS_PER_PAGE: u32 = 100;
|
|||||||
|
|
||||||
/// Routes contributed by this page group (merged by `dashboard::router`).
|
/// Routes contributed by this page group (merged by `dashboard::router`).
|
||||||
pub fn routes() -> Router<AppState> {
|
pub fn routes() -> Router<AppState> {
|
||||||
Router::new().route("/dashboard/ratings", get(index))
|
Router::new().route("/dashboard/ratings", get(index)).route(
|
||||||
|
"/dashboard/ratings/import",
|
||||||
|
axum::routing::post(queue_import),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -377,6 +386,16 @@ struct EventRow {
|
|||||||
superseded: bool,
|
superseded: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct ImportLine {
|
||||||
|
url: String,
|
||||||
|
label: String,
|
||||||
|
verdict: &'static str,
|
||||||
|
status: String,
|
||||||
|
message: String,
|
||||||
|
article_id: Option<ArticleId>,
|
||||||
|
requested: String,
|
||||||
|
}
|
||||||
|
|
||||||
struct FeedOption {
|
struct FeedOption {
|
||||||
id: FeedId,
|
id: FeedId,
|
||||||
title: String,
|
title: String,
|
||||||
@@ -434,6 +453,9 @@ struct RatingsTemplate {
|
|||||||
tab: String,
|
tab: String,
|
||||||
summary_line: String,
|
summary_line: String,
|
||||||
no_embedding_count: usize,
|
no_embedding_count: usize,
|
||||||
|
imports: Vec<ImportLine>,
|
||||||
|
imports_open: bool,
|
||||||
|
imports_pending: bool,
|
||||||
how: HowValues,
|
how: HowValues,
|
||||||
filter_label: String,
|
filter_label: String,
|
||||||
filter_source: String,
|
filter_source: String,
|
||||||
@@ -477,6 +499,23 @@ async fn index(
|
|||||||
let filters = Filters::parse(query);
|
let filters = Filters::parse(query);
|
||||||
let now = Timestamp::now();
|
let now = Timestamp::now();
|
||||||
let db = &state.db;
|
let db = &state.db;
|
||||||
|
let import_rows = imports::recent(db, 50).await.map_err(db_err)?;
|
||||||
|
let imports_pending = import_rows.iter().any(|row| row.status == "pending");
|
||||||
|
let import_lines = import_rows
|
||||||
|
.into_iter()
|
||||||
|
.map(|row| {
|
||||||
|
let (label, verdict) = widget_label(&row.label);
|
||||||
|
ImportLine {
|
||||||
|
url: row.url,
|
||||||
|
label: label.to_string(),
|
||||||
|
verdict,
|
||||||
|
status: row.status,
|
||||||
|
message: row.message.unwrap_or_default(),
|
||||||
|
article_id: row.article_id,
|
||||||
|
requested: fmt_stored_time(Some(&row.requested_at), &config),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
let preference = PreferenceState::load(db, &config.voyage, &config.curation.ranking, now)
|
let preference = PreferenceState::load(db, &config.voyage, &config.curation.ranking, now)
|
||||||
.await
|
.await
|
||||||
@@ -517,6 +556,9 @@ async fn index(
|
|||||||
tab: filters.tab.clone(),
|
tab: filters.tab.clone(),
|
||||||
summary_line,
|
summary_line,
|
||||||
no_embedding_count,
|
no_embedding_count,
|
||||||
|
imports_open: !import_lines.is_empty(),
|
||||||
|
imports_pending,
|
||||||
|
imports: import_lines,
|
||||||
how: HowValues::from_config(&config),
|
how: HowValues::from_config(&config),
|
||||||
filter_label: filters.label.clone().unwrap_or_default(),
|
filter_label: filters.label.clone().unwrap_or_default(),
|
||||||
filter_source: filters.source.clone().unwrap_or_default(),
|
filter_source: filters.source.clone().unwrap_or_default(),
|
||||||
@@ -590,6 +632,100 @@ async fn index(
|
|||||||
Ok(Html(template).into_response())
|
Ok(Html(template).into_response())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn import_form(body: &[u8]) -> (String, String, Option<String>) {
|
||||||
|
let mut urls = String::new();
|
||||||
|
let mut label = String::new();
|
||||||
|
let mut note = None;
|
||||||
|
for (key, value) in url::form_urlencoded::parse(body) {
|
||||||
|
match key.as_ref() {
|
||||||
|
"urls" => urls = value.into_owned(),
|
||||||
|
"label" => label = value.into_owned(),
|
||||||
|
"note" => note = (!value.trim().is_empty()).then(|| value.trim().to_string()),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(urls, label, note)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn queue_import(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
auth: AuthSession,
|
||||||
|
Extension(session): Extension<Session>,
|
||||||
|
body: Bytes,
|
||||||
|
) -> Result<Response, WebError> {
|
||||||
|
let viewer = auth
|
||||||
|
.user()
|
||||||
|
.await
|
||||||
|
.map(Viewer::from)
|
||||||
|
.ok_or_else(|| WebError::Unauthenticated {
|
||||||
|
next: "/dashboard/ratings#imports".into(),
|
||||||
|
})?;
|
||||||
|
let (raw_urls, label, note) = import_form(&body);
|
||||||
|
let urls = raw_urls
|
||||||
|
.split(|ch: char| ch.is_whitespace() || ch == ',')
|
||||||
|
.filter(|url| !url.is_empty())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if urls.is_empty() || urls.len() > 500 {
|
||||||
|
let message = if urls.is_empty() {
|
||||||
|
"Enter at least one URL.".into()
|
||||||
|
} else {
|
||||||
|
"At most 500 URLs can be queued at once.".into()
|
||||||
|
};
|
||||||
|
set_flash(&session, "error", message).await?;
|
||||||
|
return Ok(Redirect::to("/dashboard/ratings#imports").into_response());
|
||||||
|
}
|
||||||
|
if !matches!(label.as_str(), "loved" | "good" | "not_for_me") {
|
||||||
|
set_flash(&session, "error", "Choose a valid verdict.".into()).await?;
|
||||||
|
return Ok(Redirect::to("/dashboard/ratings#imports").into_response());
|
||||||
|
}
|
||||||
|
let count = imports::queue(
|
||||||
|
&state.db,
|
||||||
|
&urls,
|
||||||
|
&label,
|
||||||
|
note.as_deref(),
|
||||||
|
Some(viewer.id),
|
||||||
|
Timestamp::now(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(db_err)?;
|
||||||
|
match start_job(&state, &viewer, Job::ImportRatings).await? {
|
||||||
|
StartJob::Disabled => {
|
||||||
|
set_flash(
|
||||||
|
&session,
|
||||||
|
"success",
|
||||||
|
format!("Queued {count} URLs; run daily-epub job run import-ratings by hand."),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(Redirect::to("/dashboard/ratings#imports").into_response())
|
||||||
|
}
|
||||||
|
StartJob::Active(_) => {
|
||||||
|
set_flash(
|
||||||
|
&session,
|
||||||
|
"success",
|
||||||
|
format!("Queued {count} URLs; the running import will pick them up."),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(Redirect::to("/dashboard/ratings#imports").into_response())
|
||||||
|
}
|
||||||
|
StartJob::Started(id) => {
|
||||||
|
set_flash(&session, "success", format!("Queued {count} URLs.")).await?;
|
||||||
|
Ok(Redirect::to(&format!("/dashboard/jobs/{id}")).into_response())
|
||||||
|
}
|
||||||
|
StartJob::Failed(id, error) => {
|
||||||
|
set_flash(
|
||||||
|
&session,
|
||||||
|
"error",
|
||||||
|
format!(
|
||||||
|
"Queued {count} URLs, but the import job could not start: {error}. Run daily-epub job run import-ratings by hand."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(Redirect::to(&format!("/dashboard/jobs/{id}")).into_response())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn embedded_article_ids(db: &Db, config: &Config) -> Result<HashSet<ArticleId>, WebError> {
|
async fn embedded_article_ids(db: &Db, config: &Config) -> Result<HashSet<ArticleId>, WebError> {
|
||||||
let rows =
|
let rows =
|
||||||
sqlx::query("SELECT article_id FROM article_embeddings WHERE model = ? AND dimension = ?")
|
sqlx::query("SELECT article_id FROM article_embeddings WHERE model = ? AND dimension = ?")
|
||||||
@@ -909,13 +1045,15 @@ async fn load_events(
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
use axum::body::{Body, to_bytes};
|
use axum::body::{Body, to_bytes};
|
||||||
use axum::http::{Method, Request, StatusCode, header};
|
use axum::http::{Method, Request, StatusCode, header};
|
||||||
use tower::ServiceExt;
|
use tower::ServiceExt;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::types::{Entry, RatingEvent, SourceKind, SourceRef};
|
use crate::types::{Entry, RatingEvent, SourceKind, SourceRef};
|
||||||
use crate::web::users;
|
use crate::web::{MockRunner, users};
|
||||||
|
|
||||||
fn rated(article_id: ArticleId, label: &str, value: f64, event_at: &str) -> RatedArticle {
|
fn rated(article_id: ArticleId, label: &str, value: f64, event_at: &str) -> RatedArticle {
|
||||||
RatedArticle {
|
RatedArticle {
|
||||||
@@ -1190,6 +1328,22 @@ mod tests {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn post(app: &axum::Router, uri: &str, body: &str, cookie: &str) -> Response {
|
||||||
|
app.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::POST)
|
||||||
|
.uri(uri)
|
||||||
|
.header(header::COOKIE, cookie)
|
||||||
|
.header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
||||||
|
.header("sec-fetch-site", "same-origin")
|
||||||
|
.body(Body::from(body.to_string()))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
async fn text(response: Response) -> String {
|
async fn text(response: Response) -> String {
|
||||||
String::from_utf8(
|
String::from_utf8(
|
||||||
to_bytes(response.into_body(), 2 * 1024 * 1024)
|
to_bytes(response.into_body(), 2 * 1024 * 1024)
|
||||||
@@ -1401,4 +1555,97 @@ mod tests {
|
|||||||
let forbidden = get(&app, "/dashboard/ratings?tab=events", Some(&reader)).await;
|
let forbidden = get(&app, "/dashboard/ratings?tab=events", Some(&reader)).await;
|
||||||
assert_eq!(forbidden.status(), StatusCode::FORBIDDEN);
|
assert_eq!(forbidden.status(), StatusCode::FORBIDDEN);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn import_post_queues_rows_starts_job_and_get_lists_them() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
users::add(&db, "admin", "correct horse battery", true)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let runner = Arc::new(MockRunner::default());
|
||||||
|
let app = crate::server::router(AppState::with_jobs(
|
||||||
|
db.clone(),
|
||||||
|
Config::default(),
|
||||||
|
None,
|
||||||
|
runner.clone(),
|
||||||
|
));
|
||||||
|
let cookie = login_cookie(&app, "admin", "correct horse battery").await;
|
||||||
|
let response = post(
|
||||||
|
&app,
|
||||||
|
"/dashboard/ratings/import",
|
||||||
|
"urls=https%3A%2F%2Fone.example%2Fpost%0Ahttps%3A%2F%2Ftwo.example%2Fstory%2Chttps%3A%2F%2Fthree.example%2Fx&label=not_for_me¬e=historical+miss",
|
||||||
|
&cookie,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::SEE_OTHER);
|
||||||
|
let location = response.headers().get(header::LOCATION).unwrap();
|
||||||
|
assert!(
|
||||||
|
location.to_str().unwrap().starts_with("/dashboard/jobs/"),
|
||||||
|
"{location:?}"
|
||||||
|
);
|
||||||
|
let rows = imports::recent(&db, 50).await.unwrap();
|
||||||
|
assert_eq!(rows.len(), 3);
|
||||||
|
assert!(rows.iter().all(|row| row.status == "pending"));
|
||||||
|
assert!(rows.iter().all(|row| row.label == "not_for_me"));
|
||||||
|
assert!(
|
||||||
|
rows.iter()
|
||||||
|
.all(|row| row.note.as_deref() == Some("historical miss"))
|
||||||
|
);
|
||||||
|
assert!(rows.iter().all(|row| row.requested_by.is_some()));
|
||||||
|
assert_eq!(
|
||||||
|
runner.calls(),
|
||||||
|
vec!["start daily-epub-job@import-ratings.service"]
|
||||||
|
);
|
||||||
|
|
||||||
|
let body = text(get(&app, "/dashboard/ratings", Some(&cookie)).await).await;
|
||||||
|
assert!(body.contains("Import ratings"), "{body}");
|
||||||
|
assert!(body.contains("https://one.example/post"), "{body}");
|
||||||
|
assert!(!body.contains("historical miss"), "notes stay on ratings");
|
||||||
|
assert!(body.contains("badge down\">Not for me"), "{body}");
|
||||||
|
assert!(body.contains("badge pending\">pending"), "{body}");
|
||||||
|
assert!(body.contains("data-refresh=\"5\""), "{body}");
|
||||||
|
assert!(body.contains("<details id=\"imports\" class=\"disclosure\" open"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn disabled_jobs_leave_imports_pending_with_manual_command_flash() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
users::add(&db, "admin", "correct horse battery", true)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut config = Config::default();
|
||||||
|
config.server.jobs_enabled = false;
|
||||||
|
let app = crate::server::router(AppState::with_jobs(
|
||||||
|
db.clone(),
|
||||||
|
config,
|
||||||
|
None,
|
||||||
|
Arc::new(crate::web::DisabledRunner),
|
||||||
|
));
|
||||||
|
let cookie = login_cookie(&app, "admin", "correct horse battery").await;
|
||||||
|
let response = post(
|
||||||
|
&app,
|
||||||
|
"/dashboard/ratings/import",
|
||||||
|
"urls=https%3A%2F%2Fexample.com%2Fpost&label=loved¬e=",
|
||||||
|
&cookie,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::SEE_OTHER);
|
||||||
|
assert_eq!(
|
||||||
|
response.headers().get(header::LOCATION).unwrap(),
|
||||||
|
"/dashboard/ratings#imports"
|
||||||
|
);
|
||||||
|
assert_eq!(imports::recent(&db, 10).await.unwrap().len(), 1);
|
||||||
|
assert!(crate::jobs::list(&db, 10).await.unwrap().is_empty());
|
||||||
|
let body = text(get(&app, "/dashboard/ratings", Some(&cookie)).await).await;
|
||||||
|
assert!(
|
||||||
|
body.contains("run daily-epub job run import-ratings by hand"),
|
||||||
|
"{body}"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -73,7 +73,8 @@ document.addEventListener("submit", (event) => {
|
|||||||
document.querySelectorAll("details[id]").forEach((details) => {
|
document.querySelectorAll("details[id]").forEach((details) => {
|
||||||
try {
|
try {
|
||||||
const key = "details:" + details.id;
|
const key = "details:" + details.id;
|
||||||
details.open = localStorage.getItem(key) === "open";
|
const saved = localStorage.getItem(key);
|
||||||
|
if (saved === "open" || saved === "closed") details.open = saved === "open";
|
||||||
details.addEventListener("toggle", () => localStorage.setItem(key, details.open ? "open" : "closed"));
|
details.addEventListener("toggle", () => localStorage.setItem(key, details.open ? "open" : "closed"));
|
||||||
} catch (_) {}
|
} catch (_) {}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -4,6 +4,30 @@
|
|||||||
<p class="page-desc">{{ summary_line }}</p>
|
<p class="page-desc">{{ summary_line }}</p>
|
||||||
</div><div class="page-actions"><a class="btn" href="/dashboard/profile">Profile</a><a class="btn" href="/dashboard/articles?rated=unrated">Unrated articles</a></div></header>
|
</div><div class="page-actions"><a class="btn" href="/dashboard/profile">Profile</a><a class="btn" href="/dashboard/articles?rated=unrated">Unrated articles</a></div></header>
|
||||||
{% if no_embedding_count > 0 %}<p class="notice">{{ no_embedding_count }} rated articles have no embedding and cannot act as neighbours — run the <code>features-backfill</code> job (<code>features backfill --rated-only</code>).</p>{% endif %}
|
{% if no_embedding_count > 0 %}<p class="notice">{{ no_embedding_count }} rated articles have no embedding and cannot act as neighbours — run the <code>features-backfill</code> job (<code>features backfill --rated-only</code>).</p>{% endif %}
|
||||||
|
<details id="imports" class="disclosure"{% if imports_open %} open{% endif %}{% if imports_pending %} data-refresh="5"{% endif %}>
|
||||||
|
<summary>Import ratings</summary>
|
||||||
|
<div class="disclosure-body">
|
||||||
|
<form method="post" action="/dashboard/ratings/import" class="filters">
|
||||||
|
<label class="w-full">URLs <textarea name="urls" rows="5" required class="w-full" placeholder="One URL per line (commas and spaces also work)" spellcheck="false"></textarea></label>
|
||||||
|
<label>Verdict <select name="label"><option value="loved" selected>Loved it</option><option value="good">Good</option><option value="not_for_me">Not for me</option></select></label>
|
||||||
|
<label>Note <input name="note" type="text" placeholder="Optional note"></label>
|
||||||
|
<div class="filter-actions"><button class="btn btn-primary" type="submit">Queue import</button></div>
|
||||||
|
</form>
|
||||||
|
<p class="meta text-sm">The background job fetches new articles, adds Voyage embeddings when available, and records the verdict. Latest 50 requests:</p>
|
||||||
|
<div class="scroll-x"><table>
|
||||||
|
<thead><tr><th>When</th><th>URL</th><th>Verdict</th><th>Status</th><th>Message</th><th>Article</th></tr></thead>
|
||||||
|
<tbody>{% for row in imports %}<tr>
|
||||||
|
<td class="cell-tight text-muted">{{ row.requested }}</td>
|
||||||
|
<td class="cell-wrap"><a href="{{ row.url }}" rel="noopener" target="_blank">{{ row.url }}</a></td>
|
||||||
|
<td class="cell-tight"><span class="badge {{ row.label }}">{{ row.verdict }}</span></td>
|
||||||
|
<td class="cell-tight"><span class="badge {{ row.status }}">{{ row.status }}</span></td>
|
||||||
|
<td class="cell-wrap text-muted">{{ row.message }}</td>
|
||||||
|
<td class="num">{% if let Some(article_id) = row.article_id %}<a href="/dashboard/articles/{{ article_id }}">{{ article_id }}</a>{% endif %}</td>
|
||||||
|
</tr>{% endfor %}
|
||||||
|
{% if imports.is_empty() %}<tr><td colspan="6" class="text-muted">No rating imports yet.</td></tr>{% endif %}</tbody>
|
||||||
|
</table></div>
|
||||||
|
</div>
|
||||||
|
</details>
|
||||||
<details id="ratings-how" class="how disclosure">
|
<details id="ratings-how" class="how disclosure">
|
||||||
<summary>How ratings enter the algorithm</summary>
|
<summary>How ratings enter the algorithm</summary>
|
||||||
<div class="disclosure-body">
|
<div class="disclosure-body">
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# The Daily EPUB — one operator job (dashboard plan §14): an instance of this
|
# The Daily EPUB — one operator job (dashboard plan §14): an instance of this
|
||||||
# template runs `daily-epub job run <name>` for a catalogue name such as
|
# template runs `daily-epub job run <name>` for a catalogue name such as
|
||||||
# `generate`, `generate-2026-09-03`, `dry-run`, `profile-rebuild`,
|
# `generate`, `generate-2026-09-03`, `dry-run`, `profile-rebuild`,
|
||||||
# `features-backfill`, `backfill-social` or `features-prune`. The Jobs page
|
# `features-backfill`, `backfill-social`, `features-prune` or `import-ratings`. The Jobs page
|
||||||
# starts instances through systemd + polkit (systemd/50-daily-epub.rules); by
|
# starts instances through systemd + polkit (systemd/50-daily-epub.rules); by
|
||||||
# hand: `sudo systemctl start daily-epub-job@features-prune`.
|
# hand: `sudo systemctl start daily-epub-job@features-prune`.
|
||||||
#
|
#
|
||||||
|
|||||||
Reference in New Issue
Block a user