Merge branch 'interests'
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01K9PrjtUS16PAQve8D4bHgc
This commit is contained in:
Claude Fable 5.1
@@ -99,8 +99,7 @@ enabled on every request to an `anthropic`-kind provider.
|
|||||||
| **Node.js 18+** and a clone of [`epub-to-xtc-converter`](https://github.com/bigbag/epub-to-xtc-converter) | XTC/XTCH output for the Xteink X4 | Optional (`xtc.enabled = false` turns it off). Needs `npm install` **inside `cli/`**, and a settings JSON naming a real TTF/OTF — see below. It has **no global npm bin** — it is invoked as `node <repo>/cli/index.js convert …`, which is why `xtc.command`/`xtc.args` are fully general. |
|
| **Node.js 18+** and a clone of [`epub-to-xtc-converter`](https://github.com/bigbag/epub-to-xtc-converter) | XTC/XTCH output for the Xteink X4 | Optional (`xtc.enabled = false` turns it off). Needs `npm install` **inside `cli/`**, and a settings JSON naming a real TTF/OTF — see below. It has **no global npm bin** — it is invoked as `node <repo>/cli/index.js convert …`, which is why `xtc.command`/`xtc.args` are fully general. |
|
||||||
| A reverse proxy for `daily.hallada.net` → `127.0.0.1:3499` | rating links must be reachable from e-readers on the internet | TLS via your existing setup. |
|
| A reverse proxy for `daily.hallada.net` → `127.0.0.1:3499` | rating links must be reachable from e-readers on the internet | TLS via your existing setup. |
|
||||||
|
|
||||||
`data/profile.md` is the hand-maintained reader profile; its optional interests are merged
|
`data/profile.md` is the hand-maintained reader profile; its path is configurable.
|
||||||
with `data/scour-interests.opml`. Both paths are configurable.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -136,6 +135,8 @@ daily-epub features backfill [--days 30] [--rated-only] [--all] [--yes]
|
|||||||
daily-epub features prune # stale embeddings, old telemetry and assessments
|
daily-epub features prune # stale embeddings, old telemetry and assessments
|
||||||
daily-epub backfill-social [--days 7] # re-poll social scores for recent articles
|
daily-epub backfill-social [--days 7] # re-poll social scores for recent articles
|
||||||
daily-epub feeds discover [--days 14] [--limit 50] # seed feed candidates from recent aggregator articles
|
daily-epub feeds discover [--days 14] [--limit 50] # seed feed candidates from recent aggregator articles
|
||||||
|
daily-epub interests import [--opml PATH] [--profile PATH] # one-time standing-interest import
|
||||||
|
daily-epub interests backfill # match compatible cached article embeddings to interests
|
||||||
daily-epub db migrate # run migrations (also automatic on every start)
|
daily-epub db migrate # run migrations (also automatic on every start)
|
||||||
daily-epub config check # validate the config, print the resolved roles, keys and paths
|
daily-epub config check # validate the config, print the resolved roles, keys and paths
|
||||||
daily-epub users add USER [--admin] [--password-stdin]
|
daily-epub users add USER [--admin] [--password-stdin]
|
||||||
@@ -213,7 +214,10 @@ sees complete issues and article chapters and can download available formats
|
|||||||
from a single download menu. An
|
from a single download menu. An
|
||||||
`admin` can additionally follow direct dashboard links from issue entries and
|
`admin` can additionally follow direct dashboard links from issue entries and
|
||||||
article chapters, rate articles, and use every `/dashboard/*` page, including
|
article chapters, rate articles, and use every `/dashboard/*` page, including
|
||||||
settings and jobs. Personalization is shared across accounts for now.
|
settings and jobs. `/dashboard/interests` lists the standing interests with
|
||||||
|
their LLM-assigned category and a rating-derived weight, adds or deletes them,
|
||||||
|
and links every interest name in the UI to the articles it matched.
|
||||||
|
Personalization is shared across accounts for now.
|
||||||
|
|
||||||
Visitors can request an account with their preferred username at
|
Visitors can request an account with their preferred username at
|
||||||
`/request-access`; admins review open requests on `/dashboard/users`. Approving
|
`/request-access`; admins review open requests on `/dashboard/users`. Approving
|
||||||
@@ -249,6 +253,7 @@ Set `server.jobs_enabled = false` to make starts unavailable.
|
|||||||
| `backfill-social` | Refresh recent social scores. |
|
| `backfill-social` | Refresh recent social scores. |
|
||||||
| `features-prune` | Remove stale embeddings and curation telemetry. |
|
| `features-prune` | Remove stale embeddings and curation telemetry. |
|
||||||
| `import-ratings` | Fetch, embed, and rate URLs queued from the Ratings page. |
|
| `import-ratings` | Fetch, embed, and rate URLs queued from the Ratings page. |
|
||||||
|
| `interests-categorize` | File uncategorized interests under categories with the bulk model. |
|
||||||
|
|
||||||
The Ratings page accepts up to 500 historical article URLs at a time with one
|
The Ratings page accepts up to 500 historical article URLs at a time with one
|
||||||
verdict and optional note. Imports run in the background and show per-URL
|
verdict and optional note. Imports run in the background and show per-URL
|
||||||
@@ -366,7 +371,6 @@ prints what resolved.
|
|||||||
| `database_path` | `/var/lib/daily-epub/daily-epub.db` | SQLite file; parent dirs are created. |
|
| `database_path` | `/var/lib/daily-epub/daily-epub.db` | SQLite file; parent dirs are created. |
|
||||||
| `out_dir` | `/var/lib/daily-epub/out` | Where `generate` writes artifacts before publishing. |
|
| `out_dir` | `/var/lib/daily-epub/out` | Where `generate` writes artifacts before publishing. |
|
||||||
| `profile_path` | `data/profile.md` | Hand-maintained reader profile, loaded every run. |
|
| `profile_path` | `data/profile.md` | Hand-maintained reader profile, loaded every run. |
|
||||||
| `interests_opml` | `data/scour-interests.opml` | Scour interests merged with the profile interests. |
|
|
||||||
| `miniflux.base_url` | `http://127.0.0.1:8082` | Miniflux root (no `/v1`). |
|
| `miniflux.base_url` | `http://127.0.0.1:8082` | Miniflux root (no `/v1`). |
|
||||||
| `miniflux.api_key` | — | **`DAILY_EPUB_MINIFLUX__API_KEY`**. Required. |
|
| `miniflux.api_key` | — | **`DAILY_EPUB_MINIFLUX__API_KEY`**. Required. |
|
||||||
| `miniflux.page_limit` | `250` | Entries per page; Miniflux caps this at 250. |
|
| `miniflux.page_limit` | `250` | Entries per page; Miniflux caps this at 250. |
|
||||||
@@ -409,6 +413,12 @@ prints what resolved.
|
|||||||
| `curation.recent_rejection_days` | `7` | Churn window for recent low triage/deep assessments. |
|
| `curation.recent_rejection_days` | `7` | Churn window for recent low triage/deep assessments. |
|
||||||
| `curation.recent_rejection_floor` | `3.0` | Scores below this floor are excluded during the churn window (except auto-includes). |
|
| `curation.recent_rejection_floor` | `3.0` | Scores below this floor are excluded during the churn window (except auto-includes). |
|
||||||
| `curation.ranking.*` | see below | Every weight, quota, gate and threshold of the personalized ranker. |
|
| `curation.ranking.*` | see below | Every weight, quota, gate and threshold of the personalized ranker. |
|
||||||
|
| `curation.ranking.affinity_floor` / `affinity_full` | `15` / `40` | Interest-attributable ratings where affinity starts and reaches full weight. |
|
||||||
|
| `curation.ranking.weights.preliminary.affinity` | `0.10` | Rating-derived interest affinity in the preliminary blend. |
|
||||||
|
| `curation.ranking.weights.preliminary.interest` | `0.30` | Interest similarity in the preliminary blend. |
|
||||||
|
| `curation.ranking.weights.preliminary.social` | `0.05` | Social signal in the preliminary blend. |
|
||||||
|
| `curation.ranking.weights.utility.affinity` | `0.05` | Rating-derived interest affinity in the utility score. |
|
||||||
|
| `curation.ranking.weights.utility.knn` | `0.10` | Rated-neighbour preference in the utility score. |
|
||||||
| `editorial.summary_model` | `editor` | Which `[llm]` role writes the per-article summaries: `editor` (with per-article bulk fallback) or `bulk`. |
|
| `editorial.summary_model` | `editor` | Which `[llm]` role writes the per-article summaries: `editor` (with per-article bulk fallback) or `bulk`. |
|
||||||
| `editorial.summary_input_tokens` | `3000` | Article text offered to the summary prompt. |
|
| `editorial.summary_input_tokens` | `3000` | Article text offered to the summary prompt. |
|
||||||
| `publish.epub_dir` | `/srv/bookorbit/libraries/daily-epub` | Both EPUB editions land here by atomic copy, and this is the directory the OPDS feed lists. The editions are distinguished by a `(X4)` tag in **both** the filename and `dc:title` — libraries and OPDS clients list books by title, so the filename alone would make them look identical. Point a BookOrbit watched folder at it if you want its UI too. **Renamed from `bookorbit_dir`**; the old key is a hard config error. |
|
| `publish.epub_dir` | `/srv/bookorbit/libraries/daily-epub` | Both EPUB editions land here by atomic copy, and this is the directory the OPDS feed lists. The editions are distinguished by a `(X4)` tag in **both** the filename and `dc:title` — libraries and OPDS clients list books by title, so the filename alone would make them look identical. Point a BookOrbit watched folder at it if you want its UI too. **Renamed from `bookorbit_dir`**; the old key is a hard config error. |
|
||||||
@@ -448,7 +458,9 @@ prints what resolved.
|
|||||||
gated: `knn` (rated-neighbour preference) ramps from `knn_floor` (8) to
|
gated: `knn` (rated-neighbour preference) ramps from `knn_floor` (8) to
|
||||||
`knn_full` (25) rated articles with embeddings, `feed` (feed affinity) from
|
`knn_full` (25) rated articles with embeddings, `feed` (feed affinity) from
|
||||||
`feed_floor` (15) to `feed_full` (40) attributable ratings; below the floor the
|
`feed_floor` (15) to `feed_full` (40) attributable ratings; below the floor the
|
||||||
signal is absent. Ratings decay with `rating_half_life_days` (60) over
|
signal is absent. `affinity` (rating-derived interest affinity) likewise ramps
|
||||||
|
from `affinity_floor` (15) to `affinity_full` (40) interest-attributable ratings.
|
||||||
|
Ratings decay with `rating_half_life_days` (60) over
|
||||||
`rating_lookback_days` (180); `neighbour_k` (5) neighbours per side and
|
`rating_lookback_days` (180); `neighbour_k` (5) neighbours per side and
|
||||||
`negative_coefficient` (0.75) shape the signal. `slop_author_penalty` (0.75)
|
`negative_coefficient` (0.75) shape the signal. `slop_author_penalty` (0.75)
|
||||||
is the fraction of the blend and utility removed from every candidate whose
|
is the fraction of the blend and utility removed from every candidate whose
|
||||||
@@ -457,13 +469,13 @@ limit. `triage_max` (800),
|
|||||||
`deep_keep` (120), `shortlist_keep` (60), `assessment_reuse_days` (3),
|
`deep_keep` (120), `shortlist_keep` (60), `assessment_reuse_days` (3),
|
||||||
`semantic_min_words` (300), `exploration_slots` (5), `[curation.ranking.quotas]`
|
`semantic_min_words` (300), `exploration_slots` (5), `[curation.ranking.quotas]`
|
||||||
(`triage` 60 · `interest` 20 · `knn` 20), `[curation.ranking.weights.utility]`
|
(`triage` 60 · `interest` 20 · `knn` 20), `[curation.ranking.weights.utility]`
|
||||||
(`quality` 0.40 · `fit` 0.20 · `knn` 0.15 · `interest` 0.10 · `feed` 0.05 ·
|
(`quality` 0.40 · `fit` 0.20 · `knn` 0.10 · `affinity` 0.05 · `interest` 0.10 · `feed` 0.05 ·
|
||||||
`triage` 0.05 · `social` 0.03 · `heuristic` 0.02, over the signals present for
|
`triage` 0.05 · `social` 0.03 · `heuristic` 0.02, over the signals present for
|
||||||
each article of the deep set) and `[curation.ranking.diversity]`
|
each article of the deep set) and `[curation.ranking.diversity]`
|
||||||
(`cluster_threshold` 0.85, `per_cluster_cap` 2, `utility_protected` 10) drive
|
(`cluster_threshold` 0.85, `per_cluster_cap` 2, `utility_protected` 10) drive
|
||||||
the LLM triage, deep assessment, utility ranking and diversification stages.
|
the LLM triage, deep assessment, utility ranking and diversification stages.
|
||||||
`[curation.ranking.weights.preliminary]` (`interest` 0.35 · `knn` 0.25 ·
|
`[curation.ranking.weights.preliminary]` (`interest` 0.30 · `knn` 0.25 ·
|
||||||
`heuristic` 0.20 · `feed` 0.10 · `social` 0.10) blends the cheap signals; weights
|
`affinity` 0.10 · `heuristic` 0.20 · `feed` 0.10 · `social` 0.05) blends the cheap signals; weights
|
||||||
are renormalized over the signals present for each article, so they need not sum
|
are renormalized over the signals present for each article, so they need not sum
|
||||||
to 1. `embedding_retention_days` (120) and `telemetry_retention_days` (180) are
|
to 1. `embedding_retention_days` (120) and `telemetry_retention_days` (180) are
|
||||||
what `features prune` enforces. Validation: weights non-negative; `deep_keep ≥
|
what `features prune` enforces. Validation: weights non-negative; `deep_keep ≥
|
||||||
@@ -923,6 +935,7 @@ images/ article images comments.rs discussion chapters
|
|||||||
encode re-encode publish.rs BookOrbit + XTC
|
encode re-encode publish.rs BookOrbit + XTC
|
||||||
embed into the page server.rs ratings, OPDS
|
embed into the page server.rs ratings, OPDS
|
||||||
html.rs markup helpers db.rs SQLite
|
html.rs markup helpers db.rs SQLite
|
||||||
|
interests.rs standing interests and stored article matches
|
||||||
lock.rs one writer at a time (flock on <database_path>.lock)
|
lock.rs one writer at a time (flock on <database_path>.lock)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -992,11 +1005,12 @@ From spec §7, plus what implementation turned up:
|
|||||||
ceiling; a protocol that is neither means another impl. Voyage AI embeddings
|
ceiling; a protocol that is neither means another impl. Voyage AI embeddings
|
||||||
sit behind the analogous `EmbeddingBackend` trait in `curate/embedding.rs`.
|
sit behind the analogous `EmbeddingBackend` trait in `curate/embedding.rs`.
|
||||||
- **Triage and union admission replace the heuristic gate.** Every eligible
|
- **Triage and union admission replace the heuristic gate.** Every eligible
|
||||||
article gets interest, rated-neighbour, feed-affinity, social and heuristic
|
article gets interest, rated-neighbour, feed-affinity, interest-affinity,
|
||||||
signals, then DeepSeek reads its opening (up to `triage_max`). The deep set is
|
social and heuristic signals, then DeepSeek reads its opening (up to
|
||||||
|
`triage_max`). The deep set is
|
||||||
the union of triage, interest, neighbour, exploration, blend and auto-include
|
the union of triage, interest, neighbour, exploration, blend and auto-include
|
||||||
retrievers. `explain` shows the assessment and `admitted_by`. Learned signals
|
retrievers. `explain` shows the assessment and `admitted_by`. Learned signals
|
||||||
stay absent until their gates open (8 and 15 ratings respectively).
|
stay absent until their gates open (8, 15, and 15 ratings respectively).
|
||||||
- **Deep assessment and diversity are live.** DeepSeek reads a representative
|
- **Deep assessment and diversity are live.** DeepSeek reads a representative
|
||||||
beginning/middle/end sample, separates editorial quality from reader fit, and
|
beginning/middle/end sample, separates editorial quality from reader fit, and
|
||||||
records descriptive facets. Utility is normalized over the deep set; embedding
|
records descriptive facets. Utility is normalized over the deep set; embedding
|
||||||
|
|||||||
+9
-6
@@ -24,9 +24,8 @@ database_path = "/var/lib/daily-epub/daily-epub.db"
|
|||||||
# Default output directory for generated artifacts (overridden by `--out`).
|
# Default output directory for generated artifacts (overridden by `--out`).
|
||||||
out_dir = "/var/lib/daily-epub/out"
|
out_dir = "/var/lib/daily-epub/out"
|
||||||
|
|
||||||
# Hand-maintained reader profile and Scour interests merged into the system prompt.
|
# Hand-maintained reader profile loaded into the system prompt.
|
||||||
profile_path = "data/profile.md"
|
profile_path = "data/profile.md"
|
||||||
interests_opml = "data/scour-interests.opml"
|
|
||||||
|
|
||||||
[miniflux]
|
[miniflux]
|
||||||
base_url = "http://127.0.0.1:8082"
|
base_url = "http://127.0.0.1:8082"
|
||||||
@@ -138,7 +137,7 @@ slop_value = -1.0 # AI slop: a full negative; the author pena
|
|||||||
verdicts_in_prompt = 60
|
verdicts_in_prompt = 60
|
||||||
|
|
||||||
# Every weight, quota, gate and threshold of the personalized ranker. The
|
# Every weight, quota, gate and threshold of the personalized ranker. The
|
||||||
# learned signals (`knn`, `feed`) contribute nothing until their gates open:
|
# learned signals (`knn`, `feed`, `affinity`) contribute nothing until their gates open:
|
||||||
# the weight ramps linearly from `*_floor` to `*_full` rated articles.
|
# the weight ramps linearly from `*_floor` to `*_full` rated articles.
|
||||||
[curation.ranking]
|
[curation.ranking]
|
||||||
triage_max = 800 # eligible articles the triage LLM reads
|
triage_max = 800 # eligible articles the triage LLM reads
|
||||||
@@ -153,6 +152,8 @@ knn_floor = 8
|
|||||||
knn_full = 25
|
knn_full = 25
|
||||||
feed_floor = 15
|
feed_floor = 15
|
||||||
feed_full = 40
|
feed_full = 40
|
||||||
|
affinity_floor = 15
|
||||||
|
affinity_full = 40
|
||||||
slop_author_penalty = 0.75 # blend and utility × 0.25 for authors with an AI slop verdict
|
slop_author_penalty = 0.75 # blend and utility × 0.25 for authors with an AI slop verdict
|
||||||
semantic_min_words = 300
|
semantic_min_words = 300
|
||||||
exploration_slots = 5
|
exploration_slots = 5
|
||||||
@@ -166,16 +167,18 @@ knn = 20
|
|||||||
|
|
||||||
# Weights need not sum to 1; they are renormalized over the present signals.
|
# Weights need not sum to 1; they are renormalized over the present signals.
|
||||||
[curation.ranking.weights.preliminary]
|
[curation.ranking.weights.preliminary]
|
||||||
interest = 0.35
|
interest = 0.30
|
||||||
knn = 0.25
|
knn = 0.25
|
||||||
|
affinity = 0.10
|
||||||
heuristic = 0.20
|
heuristic = 0.20
|
||||||
feed = 0.10
|
feed = 0.10
|
||||||
social = 0.10
|
social = 0.05
|
||||||
|
|
||||||
[curation.ranking.weights.utility]
|
[curation.ranking.weights.utility]
|
||||||
quality = 0.40
|
quality = 0.40
|
||||||
fit = 0.20
|
fit = 0.20
|
||||||
knn = 0.15
|
knn = 0.10
|
||||||
|
affinity = 0.05
|
||||||
interest = 0.10
|
interest = 0.10
|
||||||
feed = 0.05
|
feed = 0.05
|
||||||
triage = 0.05
|
triage = 0.05
|
||||||
|
|||||||
@@ -0,0 +1,247 @@
|
|||||||
|
# First-class interests: a table, a page, categories, links, and a rating-driven weight
|
||||||
|
|
||||||
|
**Date:** 2026-09-12
|
||||||
|
**Repository:** `thallada/the-daily-epub`
|
||||||
|
**Status:** implementation plan, ready to execute
|
||||||
|
**Builds on:** `docs/plans/2026-09-02-personalized-curation-v2.md` (§8 profile and interests, §9 cheap signals, §12 blend and utility), `docs/plans/2026-09-03-web-dashboard.md` (page conventions), `docs/plans/2026-09-07-feed-discovery.md` (the most recent page + job + CLI addition; copy its shapes)
|
||||||
|
|
||||||
|
Written for a fresh implementation agent. Facts about this repo were checked against `main` (`e82f7b0`) on 2026-09-12. Nothing here depends on an external service beyond the LLM providers and Voyage that the pipeline already uses.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Goal
|
||||||
|
|
||||||
|
Today the reader's ~230 standing interests live in a Scour OPML export (`data/scour-interests.opml`) plus an optional `## Interests` section of `profile.md`. They are parsed on every run, embedded, matched against each day's articles (`signals::interest_matches`), grouped for the system prompt by a hand-written keyword table (`profile/themes.rs`), and — since the last commit — shown to the reader as the **Matches:** line under every article header. They cannot be added from the dashboard, ratings never touch them, and nothing links from an interest to the articles that matched it.
|
||||||
|
|
||||||
|
This plan makes interests a first-class record:
|
||||||
|
|
||||||
|
1. **A database table is the only source of interests.** The OPML and the profile's `## Interests` section become one-time import inputs.
|
||||||
|
2. **An Interests dashboard page** lists them with category, weight, and match counts; adds new ones; filters by category; changes a category by hand; deletes.
|
||||||
|
3. **Categories** are kept (the grouped view the operator likes) and assigned by an LLM in one daily batch, only when uncategorized interests exist, with a manual **Categorize now** button.
|
||||||
|
4. **Every interest name in the web UI is a link** to the Articles page filtered (and sorted) to the articles that matched it, with a cut-off so unrelated articles never appear. The Articles page gains an interest filter and an interests column.
|
||||||
|
5. **Ratings nudge interests.** Each explicit rating credits the interests the article matched, scaled by how strongly it matched; the resulting per-interest weight is shown on the Interests page (default sort: highest first) and feeds curation as one bounded signal, so diversity is preserved.
|
||||||
|
6. **Backfill** is two CLI commands the operator runs once.
|
||||||
|
|
||||||
|
## 2. Verified facts
|
||||||
|
|
||||||
|
### Interests today
|
||||||
|
|
||||||
|
- `profile::parse_interests(opml)` reads `<outline text="…">` names; `parse_profile_str` strips `## Interests` from `profile.md` and returns its lines; `union_interests` dedupes case-insensitively (`src/curate/profile/mod.rs:36–150`). `load_standing_interests(opml, profile)` is called from `pipeline::prepare_features` (`src/pipeline.rs:946`), `embedding::plan_backfill` (`src/curate/embedding.rs:901`), and `profile::load_or_build`/`rebuild` (through `prompt_inputs`). `config.interests_opml` is referenced in 20 places in `src/` (mostly tests that write a temp OPML), `config.example.toml:29`, README (lines 103 and 369), and `docs/runbooks/curation-v2-migration.md`.
|
||||||
|
- The system prompt's "Standing interests" section groups names with `profile::group_into_themes` (`src/curate/profile/themes.rs`): twelve keyword themes plus "Other standing interests", deterministic, sorted. The profile dashboard page renders the same grouping (`src/web/dashboard/profile.rs:249`, template `dashboard/profile.html:42`).
|
||||||
|
- `interest_embeddings` (migration 0002) is keyed by the interest **name**; `EmbeddingService::interests(&[String])` returns cached vectors and fetches misses (`src/curate/embedding.rs:743`). A new name is embedded automatically the next time it is passed.
|
||||||
|
- `signals::interest_matches(article_embeddings, interest_embeddings)` (`src/curate/signals.rs:492`) computes cosine per interest × article, z-scores each interest across the day's embedded articles (std floored at 1e-3; raw top-1 cosine fallback under 30 articles), keeps the top `RECORDED_TOP = 3` per article as `TopInterest { name, z, cos }`, and scores `0.7·z₁ + 0.3·mean(top-3 z)`. `signals.top_interests` is serialized into `candidate_runs.signals_json` (`telemetry::serialize_signals`), copied into `Pick.top_interests` as bare names by the editor (`src/curate/editor.rs:598`), and rendered as the Matches line by `chapters::understanding` → `_understanding.html` (web) and `chapter.xhtml` / `in_this_issue.xhtml` (EPUB). Prompts list interests with `z ≥ 1.5` as "matches" (`triage.rs:135`, `assess.rs:192`, `editor.rs:159`).
|
||||||
|
- Interest names also appear on: the dashboard article detail (`dashboard/article.html:61`, "Top interests" table), `_signals_table.html:5` (runs and article history), and the Feeds page "Why" badges (`dashboard/feeds.html:14`, from `discovery::why`).
|
||||||
|
- Daily volume: ~365 entries → a few hundred articles per run; 512-dim `voyage-4-lite` vectors; `embedding_retention_days = 120`, so the embedding cache holds on the order of 30–50 k vectors (≈2 KB each). Scanning the cache per web request is not an option; a per-run cosine pass is milliseconds (§9.1 of the curation plan).
|
||||||
|
|
||||||
|
### Ratings and the learned signals
|
||||||
|
|
||||||
|
- `rating_events` is append-only; `Db::current_ratings(lookback_days) -> Vec<RatedArticle>` returns the latest explicit verdict per article with `value` (`loved` 1.0, `good` 0.35, `not_for_me` −1.0, `slop` per `feedback.slop_value`) and `event_at` (`src/db.rs:713`). Ratings are written by `web::rate::post` (dashboard and issue pages), `rate::record_explicit` (CLI), and `imports::run`.
|
||||||
|
- `signals::PreferenceState::load` (`src/curate/signals.rs:238`) builds the run's learned state from current ratings: decayed weights `value × 0.5^(age/half_life)`, kNN over rated embeddings, and **feed/author affinity** as Beta-smoothed rates `(up+1)/(up+down+2)` with a gate `gate(n, feed_floor=15, feed_full=40)`. Every cheap signal is `Option<f64>`, percentile-normalized over the eligible set (`normalize`, `PERCENTILE_SIGNALS`), blended with renormalized weights (`preliminary_blend`; `rank::calculate_utility_for` for the deep set). Absent is never zero. The slop-author factor scales both blends. Diversity is enforced downstream by `rank::shortlist` (cluster threshold 0.85, `per_cluster_cap = 2`, `utility_protected = 10`) and by the editor prompt.
|
||||||
|
- Signal names are enumerated in: `signals::PERCENTILE_SIGNALS`, `Signals::raw`, `preliminary_blend`'s candidate list, `rank::calculate_utility_for`'s weighted list, `telemetry::serialize_signals` (raw list) and `RENDERED_SIGNALS`, `dashboard::SIGNAL_NAMES`, `config::{PreliminaryWeights, UtilityWeights}`, and `settings::SETTINGS_HELP`. A new signal touches all of them.
|
||||||
|
- The Ratings dashboard page computes each verdict's feed credit on the fly with a pure function (`ratings::contribution`) rather than storing it. Follow that precedent.
|
||||||
|
|
||||||
|
### Dashboard, jobs, CLI, config
|
||||||
|
|
||||||
|
- One submodule per page group under `src/web/dashboard/`, each with `routes()`, merged in `dashboard::router()`; admin gating is applied by the caller. Nav tabs are hard-coded in `src/web/templates/layout.html` (keyed on `page.active_nav`); overview tiles in `dashboard/overview.html`. POST → flash → redirect via `jobs::set_flash`. `Page::is_admin()` is available in every template. Pager partial `dashboard/_pager.html`; client-side row filter `data-table-filter`.
|
||||||
|
- The Articles list (`src/web/dashboard/articles.rs`) builds `ARTICLE_INNER` (articles ⨝ best entry ⨝ latest `candidate_runs` row ⨝ assessments, with correlated subqueries for rating and publication) and wraps it in `SELECT * FROM (…) x WHERE 1=1 {clauses} ORDER BY {sort}`. Filters are allow-listed; sorts come from `ARTICLE_SORTS`; `Pager::new(pagination, path, &filters.params())` round-trips them.
|
||||||
|
- Jobs are a fixed catalogue (`jobs::Job`, `CATALOGUE`, `parse`, `name`, `description`, `takes_lock`, `dangerous`) run as `daily-epub-job@<name>.service` and dispatched in `main::run_job`. The profile page starts `profile-rebuild` with a plain form posting to `/dashboard/jobs/profile-rebuild`. The only daily entry point is the `daily-epub-generate.timer` (05:30 America/New_York).
|
||||||
|
- `pipeline::build_llms` (`src/pipeline.rs:1133`) builds the taste-profile prompt, the clients, and runs the weekly learned-adjustments rebuild, then rebuilds the clients with the new prompt. An LLM step that must precede the prompt goes here.
|
||||||
|
- Config sections are `#[serde(deny_unknown_fields, default)]`; an unknown top-level key makes `Config` fail to load. The settings page has a hard-coded `GROUP_ORDER`, `PATH_KEYS`, and `SETTINGS_HELP`, with a test over the section list. Migrations: `sqlx::migrate!("./migrations")`; latest is `0012_article_publication.sql`, so the new file is `0013_interests.sql`. `db.rs` has a migration test asserting a table list (`src/db.rs:1720`).
|
||||||
|
|
||||||
|
## 3. Options considered
|
||||||
|
|
||||||
|
### 3.1 Where "articles matching interest X" comes from
|
||||||
|
|
||||||
|
| Option | How | Verdict |
|
||||||
|
|---|---|---|
|
||||||
|
| A. Compute on request | Load the interest vector and every cached article embedding, dot, sort. | **Rejected.** 30–50 k × 2 KB per request; the dashboard is 1–7 ms today and should stay there. |
|
||||||
|
| B. Query `candidate_runs.signals_json` with `json_each` | The top-3 names are already persisted per run. | **Rejected.** A JSON scan over every telemetry row (hundreds of thousands, pruned at 180 days) per request, no index, and rows vanish with telemetry retention. |
|
||||||
|
| C. **A junction table written by the signals stage** | `article_interests(article_id, interest_id, cos, z)`: the same top-3 the run already computes, one row each, indexed by interest. The filter is an indexed join. | **Chosen.** ~1 k rows/day, zero extra computation, and the backfill is one pass over cached embeddings. |
|
||||||
|
|
||||||
|
### 3.2 How an interest weight is computed and stored
|
||||||
|
|
||||||
|
| Option | How | Verdict |
|
||||||
|
|---|---|---|
|
||||||
|
| A. Stored counters updated on every rating event | Add `up/down` columns to `interests`; the rating handlers, CLI, and importer bump them; write a backfill migration script. | **Rejected.** Four write paths to keep in sync, decay cannot be stored (it is a function of *now*), and it duplicates the ratings history that already exists. |
|
||||||
|
| B. **Derived on the fly from current ratings × matches** | One pure function over `current_ratings` and their `article_interests` rows, exactly like feed affinity. The run computes it in `PreferenceState::load`; the Interests page computes it on render. | **Chosen.** No new write path, always current, decay and lookback for free, backfill is "make sure rated articles have match rows". ≤ a few hundred ratings × 3 rows: microseconds. |
|
||||||
|
| C. Ask the weekly learned-adjustments rebuild to write per-interest weights | An LLM judges the rating history per interest. | **Rejected.** Non-deterministic, weekly, and the numbers would not be explainable. The existing prose rebuild already sees the ratings. |
|
||||||
|
|
||||||
|
### 3.3 How the weight enters curation
|
||||||
|
|
||||||
|
| Option | How | Verdict |
|
||||||
|
|---|---|---|
|
||||||
|
| A. **A new bounded cheap signal, `affinity`** | Per article: match-strength-weighted mean of its matched interests' weights; percentile-normalized; gated on rating count; small configured weight in the preliminary blend and the utility. | **Chosen.** Fits the existing design (absent ≠ zero, renormalized weights, `explain` shows it), and its influence is capped at its weight share, so one runaway interest cannot dominate. Diversity machinery downstream is untouched. |
|
||||||
|
| B. Multiply each interest's z by its weight before the top-3 is taken | Changes which interests appear as matches. | **Rejected.** Entangles "what does this article match" with "what does he like", and the Matches line would drift with ratings. |
|
||||||
|
| C. Annotate the prompt's Standing interests with ↑/↓ | Cheap and the LLM would use it. | **Deferred** (§9). Worth adding once the weights have a few weeks of ratings behind them; it is a five-line change on top of this plan. |
|
||||||
|
| D. A new deep-set admission retriever by affinity | Like the `interest` and `knn` quotas. | **Rejected.** More slots for the same signal; the blend fill already admits high-affinity articles. |
|
||||||
|
|
||||||
|
### 3.4 Categories
|
||||||
|
|
||||||
|
| Option | Verdict |
|
||||||
|
|---|---|
|
||||||
|
| A. **`interests.category TEXT NULL`; the category set is the distinct values** | **Chosen.** No FK, no second page, renaming is an `UPDATE`. |
|
||||||
|
| B. A separate `interest_categories` table with FK | Rejected: a table with one meaningful column. |
|
||||||
|
| C. Keep the keyword table in `themes.rs` | Rejected as the source of truth (the user wants DB-tracked interests and LLM categorization), but **kept for the one-time import** so the current grouping survives unchanged. |
|
||||||
|
|
||||||
|
### 3.5 When the categorizer runs
|
||||||
|
|
||||||
|
| Option | Verdict |
|
||||||
|
|---|---|
|
||||||
|
| A. **Inside `generate`, before the prompt is built, only when uncategorized interests exist; plus a catalogue job for the button** | **Chosen.** The morning timer is the only daily trigger that exists; a run without new interests spends nothing. |
|
||||||
|
| B. Its own systemd timer | Rejected: another unit to install for a call that takes seconds. |
|
||||||
|
| C. Synchronously in the Add handler | Rejected: an LLM call in a request path, and the user asked for a daily batch. |
|
||||||
|
|
||||||
|
## 4. Design decisions (settled)
|
||||||
|
|
||||||
|
| Topic | Decision |
|
||||||
|
|---|---|
|
||||||
|
| Match rule (one definition everywhere) | An interest **matches** an article when it is among the article's top three interests by z **and** `z ≥ MATCH_MIN_Z = 1.0`. `interest_matches` applies this when it truncates, so `signals.top_interests`, the Matches line, the stored rows, the Articles filter, and the interest weights all agree. The score formula is unchanged (computed before the cut). The prompts keep their stricter `z ≥ 1.5` for "matches interests". |
|
||||||
|
| Stored rows | `article_interests(article_id, interest_id, cos, z, run_id)`, upserted per eligible article per run (the same article can be eligible on consecutive days; the latest run wins; cosine is stable, z is that day's). Rows are never pruned (≈40 bytes each, ~1 k/day). |
|
||||||
|
| Link target and filter key | `/dashboard/articles?interest=<name>`: names are unique (case-insensitive), human-readable, and `Pick.top_interests` already carries names, so no id has to travel through `issue_json`. The handler resolves the name to an id; an unknown name yields an empty list, never an error. When `interest` is set and `sort` is absent, the sort defaults to `match` (cosine descending). |
|
||||||
|
| Who sees links | The dashboard is admin-only, so the Matches line links only for admin viewers (`page.is_admin()`); readers and anonymous visitors see plain text as today. The EPUB never links. |
|
||||||
|
| Interest weight | Beta-smoothed rate over decayed, strength-scaled credits (§5.2): `(up + 1) / (up + down + 2)`, in (0, 1), 0.5 = no information. Shown with two decimals plus `up`, `down`, and the number of rated matches. |
|
||||||
|
| Match strength | `s = clamp(z / 3, 0, 1)`: a z of 3 credits the full rating, a bare match (z = 1) a third. |
|
||||||
|
| Curation signal | `affinity` (§5.3): signed, centred on zero, gated on the number of ratings that credited at least one interest (`affinity_floor = 15`, `affinity_full = 40`, same shape as the feed gate). Default weights: preliminary 0.10 (taken from `interest` 0.35→0.30 and `social` 0.10→0.05), utility 0.05 (taken from `knn` 0.15→0.10). |
|
||||||
|
| Diversity | Guaranteed by construction: the signal's share of the blend is its configured weight (≤ 10 % / 5 %), percentile normalization caps a favourite interest's articles at percentile 1.0 of that one signal, Beta smoothing means a single *loved* moves a weight from 0.50 to at most 0.67, ratings decay with the 60-day half-life, and `rank::shortlist`'s per-cluster cap and the editor's diversity instructions are unchanged. No new cap is needed. |
|
||||||
|
| Source of truth | The `interests` table. `interests_opml` is **removed** from config; `profile.md`'s `## Interests` section is still stripped by the parser but no longer read (the importer consumes it once; the profile page says so). |
|
||||||
|
| Prompt grouping | "Standing interests" groups by `interests.category` (sorted by category name, members sorted case-insensitively); `NULL` renders under the existing label "Other standing interests". `themes.rs` is used only by the importer. |
|
||||||
|
| Categorizer | One JSON call on the bulk provider over every uncategorized interest, given the existing category list; may create a category only when none fits. Unassigned names stay `NULL` and are retried the next day. Also the catalogue job `interests-categorize` behind the **Categorize now** button. |
|
||||||
|
| Interests page actions | Add (name + optional category), change category (per-row select + Save), Delete (confirm; cascades match rows, deletes the cached embedding). No rename (delete + add). |
|
||||||
|
| Backfill | `daily-epub interests import` (OPML + profile section → rows, categorized by the keyword table) and `daily-epub interests backfill` (match rows for every cached embedding). Weights need no backfill: they are derived. |
|
||||||
|
| Config | No new section. `[curation.ranking]` gains `affinity_floor`, `affinity_full`; the two weight tables gain `affinity`. |
|
||||||
|
|
||||||
|
## 5. The numbers
|
||||||
|
|
||||||
|
### 5.1 Match rows
|
||||||
|
|
||||||
|
In `prepare_features` (`src/pipeline.rs`), after `signals::compute_all` and before the candidates are handed on: for every candidate with non-empty `signals.top_interests`, upsert `(article_id, interest_id, cos, z, run_id)`. One transaction, `INSERT … ON CONFLICT(article_id, interest_id) DO UPDATE SET cos, z, run_id`. Names map to ids through the `interests` rows loaded at the top of the stage (the same rows whose names go to `service.interests`). Best effort: a failure is a report warning, never a failed run.
|
||||||
|
|
||||||
|
### 5.2 Interest weight (pure, `interests::rates`)
|
||||||
|
|
||||||
|
Inputs: `current_ratings(rating_lookback_days)` and the `article_interests` rows of those articles.
|
||||||
|
|
||||||
|
```text
|
||||||
|
for each current rating r on article a (value v_r, decay d_r = 0.5^(age_days / half_life_days))
|
||||||
|
for each match row (a, i, z):
|
||||||
|
s = clamp(z / 3, 0, 1)
|
||||||
|
credit = v_r × d_r × s
|
||||||
|
up_i += max(credit, 0)
|
||||||
|
down_i += max(−credit, 0)
|
||||||
|
n_i += 1
|
||||||
|
weight_i = (up_i + 1) / (up_i + down_i + 2) # 0.5 when n_i = 0
|
||||||
|
```
|
||||||
|
|
||||||
|
`cleared` verdicts are already excluded by `current_ratings`; `slop` carries `feedback.slop_value` like everywhere else. A rating is *attributable to interests* when it credits at least one interest; that count drives the gate.
|
||||||
|
|
||||||
|
### 5.3 The `affinity` signal (pure, in `PreferenceState`)
|
||||||
|
|
||||||
|
```text
|
||||||
|
matched = the article's top_interests with n_i > 0
|
||||||
|
affinity = Σ s_i × (weight_i − 0.5) / Σ s_i absent when matched is empty or the gate is 0
|
||||||
|
gate = gate(attributable_interest_ratings, affinity_floor, affinity_full)
|
||||||
|
```
|
||||||
|
|
||||||
|
The raw value lives in [−0.5, 0.5]; it is percentile-normalized with the other cheap signals, weighted by `weights.preliminary.affinity × gate` in the blend and `weights.utility.affinity × gate` in the utility. `explain` and `_signals_table.html` show it like any other signal. The once-per-run preference log line gains `affinity gate … (n=…)`.
|
||||||
|
|
||||||
|
## 6. Data model — `migrations/0013_interests.sql`
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE interests (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
name TEXT NOT NULL COLLATE NOCASE UNIQUE,
|
||||||
|
category TEXT, -- NULL until categorized
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
categorized_at TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX idx_interests_category ON interests(category);
|
||||||
|
|
||||||
|
CREATE TABLE article_interests (
|
||||||
|
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||||
|
interest_id INTEGER NOT NULL REFERENCES interests(id) ON DELETE CASCADE,
|
||||||
|
cos REAL NOT NULL,
|
||||||
|
z REAL NOT NULL,
|
||||||
|
run_id INTEGER, -- NULL for backfilled rows
|
||||||
|
PRIMARY KEY (article_id, interest_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX idx_article_interests_interest ON article_interests(interest_id, cos DESC);
|
||||||
|
```
|
||||||
|
|
||||||
|
`interest_embeddings` stays keyed by name (no migration): a deleted interest also deletes its embedding row; a renamed one is a new interest.
|
||||||
|
|
||||||
|
## 7. Implementation steps
|
||||||
|
|
||||||
|
Sizes are rough line counts including tests. Steps 1–3 must be in order; 4–8 can proceed in parallel after 3; 9 last.
|
||||||
|
|
||||||
|
1. **Migration + `src/interests.rs`** (~350). The table above; the db-list test in `db.rs` gains both tables. Module (following `discovery.rs`: data access, pure functions, and the LLM step in one file):
|
||||||
|
- `Interest { id, name, category, created_at, categorized_at }`; `list(db) -> Vec<Interest>` (ordered by name), `add(db, name, category, now) -> Result<id, Duplicate>` (trimmed, 1–80 chars, unique case-insensitive), `set_category(db, id, Option<category>, now)`, `delete(db, id)` (also `DELETE FROM interest_embeddings WHERE interest = name`), `names(db) -> Vec<String>`, `grouped(db) -> Vec<(String, Vec<String>)>` (for the prompt), `uncategorized(db)`.
|
||||||
|
- `replace_matches(db, run_id, &[(ArticleId, &[TopInterest])], names→ids)`; `matches_for_articles(db, ids) -> Vec<MatchRow { article_id, interest_id, name, cos, z }>`; `match_counts(db) -> HashMap<interest_id, i64>`.
|
||||||
|
- `pub fn rates(ratings: &[(ArticleId, value, decay)], rows: &[(ArticleId, interest_id, z)]) -> (HashMap<interest_id, Rate { up, down, n }>, attributable)` per §5.2, and `Rate::weight()`.
|
||||||
|
- Tests: uniqueness is case-insensitive; `rates` matches hand-checked numbers (a loved z=3 → up 1.0; a good z=1.5 → up 0.175; a not-for-me z=0.9 → nothing, below the cut); delete cascades; `grouped` puts `NULL` last under "Other standing interests".
|
||||||
|
|
||||||
|
2. **Signals** (~250). In `signals.rs`: `MATCH_MIN_Z = 1.0` applied inside `interest_matches` after truncation; `Signals.affinity: Option<f64>`; `"affinity"` in `PERCENTILE_SIGNALS`, `Signals::raw`, `preliminary_blend` (gate `affinity_gate`); `PreferenceState` gains `interest_rates: HashMap<String, Rate>` (keyed by name — `TopInterest` carries names), `affinity_gate`, `attributable_interest_ratings`, `fn affinity(&self, top: &[TopInterest]) -> Option<f64>`, loaded in `load` from `interests::matches_for_articles(rated ids)` and exposed on `PreferenceSummary` and in `log`. `rank::calculate_utility_for` adds `("affinity", configured.affinity, candidate.signals.affinity_gate)`. `telemetry`: `serialize_signals` raw list and `RENDERED_SIGNALS`; `render_explain` prints it with the others. `dashboard::SIGNAL_NAMES` and `_signals_table.html` pick it up automatically once the name is in the list. Config: `RankingConfig { affinity_floor: 15, affinity_full: 40 }`, `PreliminaryWeights { affinity: 0.10, interest: 0.30, social: 0.05 }`, `UtilityWeights { affinity: 0.05, knn: 0.10 }`, `SETTINGS_HELP` lines, `config.example.toml`, README table.
|
||||||
|
Tests: the top-3 cut drops a z=0.4 third interest; affinity is absent under the gate and with no rated interests; a candidate matching a 0.8-weight interest outranks an otherwise identical one matching a 0.3-weight interest in the blend; blend weights still renormalize to 1.
|
||||||
|
|
||||||
|
3. **Replace the OPML/profile plumbing** (~300, mostly deletions and test edits). `profile::load_standing_interests`, `parse_interests`, `union_interests`, and `prompt_inputs`' OPML argument go; `build`, `load_or_build`, `rebuild`, and `weekly_rebuild_if_due` take `grouped: Vec<(String, Vec<String>)>` from `interests::grouped(db)` instead of `interests: &[String]` + `group_into_themes`. `pipeline::prepare_features` and `embedding::plan_backfill` take names from `interests::names(db)`. `themes.rs` stays but is only referenced by the importer (step 8); `pub use themes::group_into_themes` is removed from the profile module. Remove `Config.interests_opml` (struct, `Default`, `config check` line, the `deny_unknown_fields` implication is a rollout note in §10), `settings::PATH_KEYS`/`SETTINGS_HELP`, `config.example.toml`, README (lines 103, 369), and the runbook mention. Every test that writes a temp OPML instead inserts rows with `interests::add`. The profile page loses the OPML card and the "Extracted `## Interests` lines" preview; in their place one line: "N standing interests in M categories — manage them on the Interests page", and the editor note says the `## Interests` section is ignored.
|
||||||
|
|
||||||
|
4. **Match rows in the run** (~80). `prepare_features` writes them per §5.1 (timing folded into the existing `signals` timing; count `interest_matches` on the report counts is optional — skip unless free).
|
||||||
|
|
||||||
|
5. **Articles page** (~200). `ArticlesQuery.interest: Option<String>` → `ArticleFilters.interest: Option<(i64, String)>` resolved by name (case-insensitive) in `from_query`'s caller (it needs the db; resolve in `list` before building filters, or make `from_query` async — pick the former). When set: `ARTICLE_INNER` gains `JOIN article_interests ai ON ai.article_id = a.id AND ai.interest_id = ?` (inside the inner query so `idx_article_interests_interest` drives it; the join is a `{interest_join}` placeholder that is empty otherwise) and exposes `ai.cos AS match_cos`; `ARTICLE_SORTS` gains `("match", "x.match_cos DESC, x.id DESC")`, which is the default when `interest` is set and `sort` is absent, and is ignored (falls back to `first_seen`) when it is not. Filter UI: a `<select name="interest">` over all interest names (blank = any), populated from `interests::list`. New column **interests**: for the page's ≤ 50 rows, one query through `matches_for_articles`, grouped per article, rendered as badges linking to `?interest=<name>` (ordered by z desc). `params()` round-trips `interest` and `sort=match`. Test: a seeded match row is found by the filter, an unknown name yields an empty table (200), and the column links.
|
||||||
|
|
||||||
|
6. **Interests page** (~450). `src/web/dashboard/interests.rs` + `dashboard/interests.html`; routes `GET /dashboard/interests`, `POST /dashboard/interests` (add), `POST /dashboard/interests/{id}/category`, `POST /dashboard/interests/{id}/delete`. Nav tab **Interests** after Ratings; overview tile "N interests · M uncategorized". Page: add form (name, category select with a blank "let the categorizer decide" option), category filter tabs or select (`?category=<name>` and `?category=uncategorized`), sort select (`weight` default, `name`, `matches`, `added`), the `data-table-filter` row filter, no pager (≤ a few hundred rows). Columns: interest (link → `/dashboard/articles?interest=<name>`), category (link → `?category=`), weight (2 decimals, tabular; `—` when n = 0), up, down, rated matches n, matched articles (count), added (`format_time`), actions (category select + Save; Delete with `confirm`). Weights come from `interests::rates` over `current_ratings(rating_lookback_days)` computed on render; the header says which lookback and half-life it uses and shows the affinity gate state ("affinity gate 0.35 · 22 of 40 attributable ratings"), mirroring the Ratings page. **Categorize now** button posts to `/dashboard/jobs/interests-categorize` (the existing generic job route), shown enabled when `jobs_enabled` and disabled with the manual command otherwise, with the uncategorized count next to it. Flash messages for add/duplicate/deleted. Tests: page renders with weights sorted descending; add rejects a duplicate differing only in case; delete removes match rows and the embedding row; routes are admin-only.
|
||||||
|
|
||||||
|
7. **Links everywhere** (~120). `chapters::Understanding.interests` becomes `Vec<InterestRef { name, href }>` (`href = "/dashboard/articles?interest=" + encode_component(name)`) with `interests_line()` for the EPUB templates; `_understanding.html` renders `{% for %}…{% if page.is_admin() %}<a href>…` joined by ` · `. Dashboard article detail "Top interests" table, `_signals_table.html`, and the Feeds page "Why" badges link the same way. Existing tests asserting `Matches: Filesystems · Rust` keep passing for non-admin views; add one admin assertion.
|
||||||
|
|
||||||
|
8. **Categorizer + job + CLI** (~350).
|
||||||
|
- `interests::categorize(config, db) -> Result<String>`: load uncategorized; if none, return "nothing to categorize"; build `Llms::from_config` with the current taste prompt (as `cmd_profile_rebuild` does) and take `bulk` (fall back to `editor_or_bulk`); one `complete_json` at temperature 0.2 with:
|
||||||
|
```text
|
||||||
|
TASK: file each new standing interest under one of the reader's interest categories.
|
||||||
|
Existing categories (reuse these names verbatim): <list>
|
||||||
|
Create a new category only when none of the existing ones fits; a new category must be broad enough to hold several interests and named like the existing ones (two to five words, sentence case). Every interest gets exactly one category.
|
||||||
|
New interests: <one per line>
|
||||||
|
Return JSON exactly: {"assignments":[{"interest":"…","category":"…"}]}
|
||||||
|
```
|
||||||
|
Validate: interest must be one of the batch (case-insensitive), category trimmed, 1–60 chars; unknown names are ignored; anything unassigned stays `NULL`. Log "categorized N (M new categories: …)". Return that as the job message.
|
||||||
|
- `pipeline::build_llms`: after the clients exist, if `interests::uncategorized(db)` is non-empty and a client is available, run `categorize` (warning on error), then rebuild the prompt and the clients exactly as the weekly rebuild path does (the grouping changed). Both the categorizer and the weekly rebuild share that "prompt changed, remake clients" tail.
|
||||||
|
- `jobs::Job::InterestsCategorize` (`interests-categorize`, description "File uncategorized interests under categories with the bulk model, creating new ones only when needed.", `takes_lock` None, not dangerous); `main::run_job` dispatch; catalogue tests.
|
||||||
|
- CLI `daily-epub interests <import|backfill>` in `main.rs`:
|
||||||
|
- `import [--opml PATH] [--profile PATH]` (defaults `data/scour-interests.opml` and `config.profile_path`): parse the OPML with the old `parse_interests` (moved into `interests.rs` as a private helper) and the profile's `## Interests` lines, insert with `add` skipping duplicates, assign each new row's category with `themes::group_into_themes` mapped back per name ("Other standing interests" → `NULL` so the LLM gets a go at them). Print "imported N, skipped M existing, K left for the categorizer".
|
||||||
|
- `backfill`: load every `article_embeddings` row compatible with the configured model/dimension in chunks, load interest vectors from `interest_embeddings` (fetching misses through the service if Voyage is enabled), run `interest_matches` over the whole set (z over the whole cached set stands in for the per-day cohort; say so in the docstring), and `INSERT OR IGNORE` the rows with `run_id NULL` so real-run rows win. Print the row count. Takes the run lock (it writes what the run writes).
|
||||||
|
- Tests: the categorizer with `MockBackend` assigns two interests, creates one new category, ignores a name not in the batch; `import` is idempotent; `backfill` on a two-article fixture writes the expected rows and leaves an existing run-written row alone.
|
||||||
|
|
||||||
|
9. **Docs** (~40). README: the "Web site and dashboard" paragraph gets a sentence on Interests; the Commands section gets `interests import` / `interests backfill`; the configuration table gains the three keys and loses `interests_opml`; the Layout section lists `src/interests.rs`. `docs/ideas.md`: strike the "what interests most match" item.
|
||||||
|
|
||||||
|
Expect roughly 2 100 lines including tests.
|
||||||
|
|
||||||
|
## 8. Verification
|
||||||
|
|
||||||
|
- `cargo test`, `cargo clippy`, `cargo fmt` green.
|
||||||
|
- Dev seed (`dev/`): `interests import --opml data/scour-interests.opml`, then `interests backfill` (no embeddings in the seed → "0 rows", exits 0), open `/dashboard/interests`: 230 rows, every one categorized except a handful under Uncategorized, weight `—` everywhere; add "Rust macros" → appears uncategorized; **Categorize now** with jobs disabled shows the manual command.
|
||||||
|
- After deploy, on the server: run the two commands (§10), open the page, click **Categorize now**, watch the job page, refresh: no Uncategorized left, new categories (if any) look sane. Rate one article from the issue page; its matched interests' weights move on the next refresh with the expected magnitude (`0.5 → 0.67` for a full-strength *loved*).
|
||||||
|
- Next morning's run: `explain` on a selected article shows `affinity` present with a weight, or absent with the gate at 0 if fewer than 15 ratings credited an interest (expected at first; the log line says which). The Articles page filtered by an interest the reader knows well returns articles that obviously belong, sorted by cosine; nothing unrelated appears at the bottom of the first page. Dashboard timing for that page stays single-digit ms (`Server-Timing` header).
|
||||||
|
- The system prompt on the profile page groups Standing interests by the DB categories with the same members as before the change (byte-stable apart from category names that the LLM added).
|
||||||
|
|
||||||
|
## 9. Deferred, with triggers
|
||||||
|
|
||||||
|
- **Prompt annotation** of Standing interests with the strongest and weakest weights (e.g. `Rust ↑`, `Coffee ↓`) — once the affinity gate is open and the weights look right for a couple of weeks.
|
||||||
|
- **Renaming** an interest in place, and merging duplicates — when the operator actually needs it (delete + add loses nothing but the match rows, which the next runs rebuild).
|
||||||
|
- **Match-row retention** — if `article_interests` ever exceeds a few million rows; today it grows ~1 k/day.
|
||||||
|
- **Implicit ratings** feeding the weights — already designed for in the curation plan §6.4; nothing here blocks it.
|
||||||
|
- **Scour** — the Scour feeds in Miniflux keep working as sources; nothing here syncs interests back to Scour.
|
||||||
|
|
||||||
|
## 10. Rollout notes
|
||||||
|
|
||||||
|
1. Deploy the binary. **Before restarting the service, delete `interests_opml` from `/etc/daily-epub/config.toml`** (the key is gone and `Config` rejects unknown keys). Migrations run on the first command.
|
||||||
|
2. `daily-epub interests import --opml /var/lib/daily-epub/data/scour-interests.opml` (the profile's `## Interests` section is read from `profile_path`; it has been empty so far). Then delete that section from `profile.md`, or leave it — it is ignored either way.
|
||||||
|
3. `daily-epub features backfill --rated-only` if any rated article lacks an embedding (it prints the count), then `daily-epub interests backfill`.
|
||||||
|
4. Open `/dashboard/interests`, press **Categorize now** for the leftovers, and read the weights: they are live from that moment because they are derived from the existing ratings.
|
||||||
|
5. Optional: `daily-epub generate --dry-run` to see `affinity` in a run before the morning.
|
||||||
|
|
||||||
|
## 11. Non-goals (do not add)
|
||||||
|
|
||||||
|
- A category editor page, category colours, or ordering beyond alphabetical.
|
||||||
|
- Per-interest notes, pinning, or manual weight overrides — the weight is derived from ratings only, so the operator changes it by rating.
|
||||||
|
- Interest suggestions from articles ("you seem to like X") — a separate feature with its own plan if wanted.
|
||||||
|
- Syncing interests to or from Scour, or OPML export.
|
||||||
|
- A stored weight column, incremental counters, or a scheduled recompute.
|
||||||
|
- Configurable `MATCH_MIN_Z` or match strength curve — constants until there is a reason.
|
||||||
|
- Public (anonymous or reader-role) links from the Matches line; the dashboard is admin-only.
|
||||||
@@ -9,7 +9,7 @@ What changes for the operator, in one paragraph: the binary is replaced; the SQL
|
|||||||
tables and drops `ratings`, `feed_priors` and `scores` (the migration copies your ratings first);
|
tables and drops `ratings`, `feed_priors` and `scores` (the migration copies your ratings first);
|
||||||
`config.toml` loses a few keys and gains the `[llm]` / `[providers.*]` registry plus a
|
`config.toml` loses a few keys and gains the `[llm]` / `[providers.*]` registry plus a
|
||||||
`profile_path`; the env file gains two API keys and renames the DeepSeek one; a hand-maintained
|
`profile_path`; the env file gains two API keys and renames the DeepSeek one; a hand-maintained
|
||||||
`profile.md` is installed next to the OPML; the systemd units are unchanged.
|
`profile.md` is installed for the hand-maintained reader profile; the systemd units are unchanged.
|
||||||
|
|
||||||
## 0. Before touching the server
|
## 0. Before touching the server
|
||||||
|
|
||||||
@@ -70,12 +70,13 @@ Everything you do not mention keeps its documented default, so the edit is small
|
|||||||
|
|
||||||
| Old key | Why |
|
| Old key | Why |
|
||||||
|---|---|
|
|---|---|
|
||||||
|
| `interests_opml = …` (top level) | standing interests now live in SQLite; remove this before rollout because unknown keys fail startup |
|
||||||
| `prefilter_keep = …` (top level) | replaced by `curation.ranking.deep_keep` (default 120) |
|
| `prefilter_keep = …` (top level) | replaced by `curation.ranking.deep_keep` (default 120) |
|
||||||
| `max_daily_usd = …` (top level) | now per provider: `providers.deepseek.max_daily_usd` |
|
| `max_daily_usd = …` (top level) | now per provider: `providers.deepseek.max_daily_usd` |
|
||||||
| the whole `[deepseek]` table | becomes `[providers.deepseek]` + `[llm]` (see below) |
|
| the whole `[deepseek]` table | becomes `[providers.deepseek]` + `[llm]` (see below) |
|
||||||
| any `[anthropic]` table (only if you added one from an interim build) | becomes `[providers.anthropic]` |
|
| any `[anthropic]` table (only if you added one from an interim build) | becomes `[providers.anthropic]` |
|
||||||
|
|
||||||
**Add** near the top, next to `interests_opml`:
|
**Add** near the top:
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
profile_path = "/var/lib/daily-epub/data/profile.md"
|
profile_path = "/var/lib/daily-epub/data/profile.md"
|
||||||
@@ -83,8 +84,7 @@ profile_path = "/var/lib/daily-epub/data/profile.md"
|
|||||||
|
|
||||||
Use an absolute path. The default is `data/profile.md` *relative to the working directory*, which
|
Use an absolute path. The default is `data/profile.md` *relative to the working directory*, which
|
||||||
under the unit is `/var/lib/daily-epub`, so the default would resolve to the same place, but an
|
under the unit is `/var/lib/daily-epub`, so the default would resolve to the same place, but an
|
||||||
explicit path survives running one-off commands from another directory. Point
|
explicit path survives running one-off commands from another directory.
|
||||||
`interests_opml` at an absolute path too if it is still relative.
|
|
||||||
|
|
||||||
**Add** the LLM registry. Carry over the `base_url`, `model` and `price_*` values from your old
|
**Add** the LLM registry. Carry over the `base_url`, `model` and `price_*` values from your old
|
||||||
`[deepseek]` table if you had changed them; the values shown are the defaults.
|
`[deepseek]` table if you had changed them; the values shown are the defaults.
|
||||||
@@ -165,11 +165,9 @@ Voyage dashboards: the in-app `max_daily_usd` meters are runaway guards, not acc
|
|||||||
```sh
|
```sh
|
||||||
sudo install -d -m0750 -o daily-epub -g daily-epub /var/lib/daily-epub/data
|
sudo install -d -m0750 -o daily-epub -g daily-epub /var/lib/daily-epub/data
|
||||||
sudo install -m0640 -o daily-epub -g daily-epub data/profile.md /var/lib/daily-epub/data/profile.md
|
sudo install -m0640 -o daily-epub -g daily-epub data/profile.md /var/lib/daily-epub/data/profile.md
|
||||||
# if the OPML is not already there:
|
|
||||||
sudo install -m0640 -o daily-epub -g daily-epub data/scour-interests.opml /var/lib/daily-epub/data/
|
|
||||||
```
|
```
|
||||||
|
|
||||||
If the file is missing the run does not fail; it logs a warning and uses the OPML interests only,
|
If the file is missing the run does not fail; it logs a warning and uses empty profile prose,
|
||||||
which is a much worse prompt. `config check` in the next step tells you whether it was found.
|
which is a much worse prompt. `config check` in the next step tells you whether it was found.
|
||||||
|
|
||||||
## 6. Check the config as the service user
|
## 6. Check the config as the service user
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
CREATE TABLE interests (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
name TEXT NOT NULL COLLATE NOCASE UNIQUE,
|
||||||
|
category TEXT, -- NULL until categorized
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
categorized_at TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX idx_interests_category ON interests(category);
|
||||||
|
|
||||||
|
CREATE TABLE article_interests (
|
||||||
|
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||||
|
interest_id INTEGER NOT NULL REFERENCES interests(id) ON DELETE CASCADE,
|
||||||
|
cos REAL NOT NULL,
|
||||||
|
z REAL NOT NULL,
|
||||||
|
run_id INTEGER, -- NULL for backfilled rows
|
||||||
|
PRIMARY KEY (article_id, interest_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX idx_article_interests_interest ON article_interests(interest_id, cos DESC);
|
||||||
+34
-16
@@ -37,12 +37,9 @@ impl From<figment::Error> for ConfigError {
|
|||||||
/// Legacy/alternate env var for the rating-link HMAC key (spec §1).
|
/// Legacy/alternate env var for the rating-link HMAC key (spec §1).
|
||||||
pub const ENV_SECRET_ALIAS: &str = "DAILY_EPUB_SECRET";
|
pub const ENV_SECRET_ALIAS: &str = "DAILY_EPUB_SECRET";
|
||||||
|
|
||||||
/// Root configuration document (§3.14).
|
/// Root configuration document; unknown keys fail so retired settings stay visible.
|
||||||
///
|
|
||||||
/// Unknown *top-level* keys are ignored on purpose: the prefix `DAILY_EPUB_` is
|
|
||||||
/// shared with plain operator env vars such as [`ENV_SECRET_ALIAS`].
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
#[serde(default)]
|
#[serde(deny_unknown_fields, default)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
/// IANA tz used for day boundaries and `--date` (§3.14, notes §2).
|
/// IANA tz used for day boundaries and `--date` (§3.14, notes §2).
|
||||||
pub timezone: String,
|
pub timezone: String,
|
||||||
@@ -65,8 +62,6 @@ pub struct Config {
|
|||||||
pub database_path: PathBuf,
|
pub database_path: PathBuf,
|
||||||
/// Default artifact output directory (overridden by `generate --out`).
|
/// Default artifact output directory (overridden by `generate --out`).
|
||||||
pub out_dir: PathBuf,
|
pub out_dir: PathBuf,
|
||||||
/// Scour interests OPML used to seed the taste profile (§3.6).
|
|
||||||
pub interests_opml: PathBuf,
|
|
||||||
/// Hand-maintained reader profile loaded for every curation run (§8.2).
|
/// Hand-maintained reader profile loaded for every curation run (§8.2).
|
||||||
pub profile_path: PathBuf,
|
pub profile_path: PathBuf,
|
||||||
|
|
||||||
@@ -99,7 +94,6 @@ impl Default for Config {
|
|||||||
world_briefing: true,
|
world_briefing: true,
|
||||||
database_path: PathBuf::from("/var/lib/daily-epub/daily-epub.db"),
|
database_path: PathBuf::from("/var/lib/daily-epub/daily-epub.db"),
|
||||||
out_dir: PathBuf::from("/var/lib/daily-epub/out"),
|
out_dir: PathBuf::from("/var/lib/daily-epub/out"),
|
||||||
interests_opml: PathBuf::from("data/scour-interests.opml"),
|
|
||||||
profile_path: PathBuf::from("data/profile.md"),
|
profile_path: PathBuf::from("data/profile.md"),
|
||||||
miniflux: MinifluxConfig::default(),
|
miniflux: MinifluxConfig::default(),
|
||||||
llm: LlmConfig::default(),
|
llm: LlmConfig::default(),
|
||||||
@@ -488,6 +482,8 @@ pub struct RankingConfig {
|
|||||||
pub knn_full: usize,
|
pub knn_full: usize,
|
||||||
pub feed_floor: usize,
|
pub feed_floor: usize,
|
||||||
pub feed_full: usize,
|
pub feed_full: usize,
|
||||||
|
pub affinity_floor: usize,
|
||||||
|
pub affinity_full: usize,
|
||||||
/// Fraction of the preliminary blend and the utility removed from any
|
/// Fraction of the preliminary blend and the utility removed from any
|
||||||
/// candidate whose author has a current *AI slop* verdict (§9.3). `1.0`
|
/// candidate whose author has a current *AI slop* verdict (§9.3). `1.0`
|
||||||
/// zeroes such candidates; `0.0` disables the penalty.
|
/// zeroes such candidates; `0.0` disables the penalty.
|
||||||
@@ -516,6 +512,8 @@ impl Default for RankingConfig {
|
|||||||
knn_full: 25,
|
knn_full: 25,
|
||||||
feed_floor: 15,
|
feed_floor: 15,
|
||||||
feed_full: 40,
|
feed_full: 40,
|
||||||
|
affinity_floor: 15,
|
||||||
|
affinity_full: 40,
|
||||||
slop_author_penalty: 0.75,
|
slop_author_penalty: 0.75,
|
||||||
semantic_min_words: 300,
|
semantic_min_words: 300,
|
||||||
exploration_slots: 5,
|
exploration_slots: 5,
|
||||||
@@ -558,6 +556,7 @@ pub struct RankingWeights {
|
|||||||
pub struct PreliminaryWeights {
|
pub struct PreliminaryWeights {
|
||||||
pub interest: f64,
|
pub interest: f64,
|
||||||
pub knn: f64,
|
pub knn: f64,
|
||||||
|
pub affinity: f64,
|
||||||
pub heuristic: f64,
|
pub heuristic: f64,
|
||||||
pub feed: f64,
|
pub feed: f64,
|
||||||
pub social: f64,
|
pub social: f64,
|
||||||
@@ -566,11 +565,12 @@ pub struct PreliminaryWeights {
|
|||||||
impl Default for PreliminaryWeights {
|
impl Default for PreliminaryWeights {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
interest: 0.35,
|
interest: 0.30,
|
||||||
knn: 0.25,
|
knn: 0.25,
|
||||||
|
affinity: 0.10,
|
||||||
heuristic: 0.20,
|
heuristic: 0.20,
|
||||||
feed: 0.10,
|
feed: 0.10,
|
||||||
social: 0.10,
|
social: 0.05,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -581,6 +581,7 @@ pub struct UtilityWeights {
|
|||||||
pub quality: f64,
|
pub quality: f64,
|
||||||
pub fit: f64,
|
pub fit: f64,
|
||||||
pub knn: f64,
|
pub knn: f64,
|
||||||
|
pub affinity: f64,
|
||||||
pub interest: f64,
|
pub interest: f64,
|
||||||
pub feed: f64,
|
pub feed: f64,
|
||||||
pub triage: f64,
|
pub triage: f64,
|
||||||
@@ -593,7 +594,8 @@ impl Default for UtilityWeights {
|
|||||||
Self {
|
Self {
|
||||||
quality: 0.40,
|
quality: 0.40,
|
||||||
fit: 0.20,
|
fit: 0.20,
|
||||||
knn: 0.15,
|
knn: 0.10,
|
||||||
|
affinity: 0.05,
|
||||||
interest: 0.10,
|
interest: 0.10,
|
||||||
feed: 0.05,
|
feed: 0.05,
|
||||||
triage: 0.05,
|
triage: 0.05,
|
||||||
@@ -934,7 +936,11 @@ impl Config {
|
|||||||
fig = fig.merge(Toml::file(p));
|
fig = fig.merge(Toml::file(p));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(fig.merge(Env::prefixed(ENV_PREFIX).split(ENV_SPLIT)))
|
Ok(fig.merge(
|
||||||
|
Env::prefixed(ENV_PREFIX)
|
||||||
|
.ignore(&["secret"])
|
||||||
|
.split(ENV_SPLIT),
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The file `load` reads: the explicit `--config` path, else `./config.toml`
|
/// The file `load` reads: the explicit `--config` path, else `./config.toml`
|
||||||
@@ -1059,7 +1065,6 @@ impl Config {
|
|||||||
});
|
});
|
||||||
lines.push(file_line("database_path", &self.database_path));
|
lines.push(file_line("database_path", &self.database_path));
|
||||||
lines.push(file_line("profile_path", &self.profile_path));
|
lines.push(file_line("profile_path", &self.profile_path));
|
||||||
lines.push(file_line("interests_opml", &self.interests_opml));
|
|
||||||
for (role, name) in self.llm.roles() {
|
for (role, name) in self.llm.roles() {
|
||||||
match self.providers.get(name) {
|
match self.providers.get(name) {
|
||||||
Some(provider) => lines.push(provider_line(&format!("llm.{role}"), name, provider)),
|
Some(provider) => lines.push(provider_line(&format!("llm.{role}"), name, provider)),
|
||||||
@@ -1204,7 +1209,10 @@ impl Config {
|
|||||||
.into(),
|
.into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if ranking.knn_full <= ranking.knn_floor || ranking.feed_full <= ranking.feed_floor {
|
if ranking.knn_full <= ranking.knn_floor
|
||||||
|
|| ranking.feed_full <= ranking.feed_floor
|
||||||
|
|| ranking.affinity_full <= ranking.affinity_floor
|
||||||
|
{
|
||||||
return Err(ConfigError::Invalid(
|
return Err(ConfigError::Invalid(
|
||||||
"curation.ranking *_full must be > *_floor >= 0".into(),
|
"curation.ranking *_full must be > *_floor >= 0".into(),
|
||||||
));
|
));
|
||||||
@@ -1229,12 +1237,14 @@ impl Config {
|
|||||||
let weights = [
|
let weights = [
|
||||||
preliminary.interest,
|
preliminary.interest,
|
||||||
preliminary.knn,
|
preliminary.knn,
|
||||||
|
preliminary.affinity,
|
||||||
preliminary.heuristic,
|
preliminary.heuristic,
|
||||||
preliminary.feed,
|
preliminary.feed,
|
||||||
preliminary.social,
|
preliminary.social,
|
||||||
utility.quality,
|
utility.quality,
|
||||||
utility.fit,
|
utility.fit,
|
||||||
utility.knn,
|
utility.knn,
|
||||||
|
utility.affinity,
|
||||||
utility.interest,
|
utility.interest,
|
||||||
utility.feed,
|
utility.feed,
|
||||||
utility.triage,
|
utility.triage,
|
||||||
@@ -2018,7 +2028,7 @@ mod tests {
|
|||||||
for (key, default) in defaults.as_object().expect("config is a table") {
|
for (key, default) in defaults.as_object().expect("config is a table") {
|
||||||
let section = match key.as_str() {
|
let section = match key.as_str() {
|
||||||
"curation" | "llm" | "providers" | "voyage" | "editorial" => key,
|
"curation" | "llm" | "providers" | "voyage" | "editorial" => key,
|
||||||
"target_article_count" | "profile_path" | "interests_opml" => key,
|
"target_article_count" | "profile_path" => key,
|
||||||
_ => continue,
|
_ => continue,
|
||||||
};
|
};
|
||||||
let documented = documented
|
let documented = documented
|
||||||
@@ -2084,10 +2094,15 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!((ranking.knn_floor, ranking.knn_full), (8, 25));
|
assert_eq!((ranking.knn_floor, ranking.knn_full), (8, 25));
|
||||||
assert_eq!((ranking.feed_floor, ranking.feed_full), (15, 40));
|
assert_eq!((ranking.feed_floor, ranking.feed_full), (15, 40));
|
||||||
|
assert_eq!((ranking.affinity_floor, ranking.affinity_full), (15, 40));
|
||||||
assert_eq!(ranking.rating_half_life_days, 60.0);
|
assert_eq!(ranking.rating_half_life_days, 60.0);
|
||||||
assert_eq!(ranking.negative_coefficient, 0.75);
|
assert_eq!(ranking.negative_coefficient, 0.75);
|
||||||
assert_eq!(ranking.weights.preliminary.interest, 0.35);
|
assert_eq!(ranking.weights.preliminary.interest, 0.30);
|
||||||
|
assert_eq!(ranking.weights.preliminary.affinity, 0.10);
|
||||||
|
assert_eq!(ranking.weights.preliminary.social, 0.05);
|
||||||
assert_eq!(ranking.weights.utility.quality, 0.40);
|
assert_eq!(ranking.weights.utility.quality, 0.40);
|
||||||
|
assert_eq!(ranking.weights.utility.knn, 0.10);
|
||||||
|
assert_eq!(ranking.weights.utility.affinity, 0.05);
|
||||||
assert_eq!(ranking.diversity.per_cluster_cap, 2);
|
assert_eq!(ranking.diversity.per_cluster_cap, 2);
|
||||||
assert_eq!(ranking.embedding_retention_days, 120);
|
assert_eq!(ranking.embedding_retention_days, 120);
|
||||||
assert_eq!(ranking.telemetry_retention_days, 180);
|
assert_eq!(ranking.telemetry_retention_days, 180);
|
||||||
@@ -2106,6 +2121,9 @@ mod tests {
|
|||||||
bad.curation.ranking.knn_full = bad.curation.ranking.knn_floor;
|
bad.curation.ranking.knn_full = bad.curation.ranking.knn_floor;
|
||||||
assert!(bad.validate().is_err(), "*_full must exceed *_floor");
|
assert!(bad.validate().is_err(), "*_full must exceed *_floor");
|
||||||
let mut bad = Config::default();
|
let mut bad = Config::default();
|
||||||
|
bad.curation.ranking.affinity_full = bad.curation.ranking.affinity_floor;
|
||||||
|
assert!(bad.validate().is_err(), "*_full must exceed *_floor");
|
||||||
|
let mut bad = Config::default();
|
||||||
bad.curation.ranking.shortlist_keep = bad.curation.ranking.deep_keep + 1;
|
bad.curation.ranking.shortlist_keep = bad.curation.ranking.deep_keep + 1;
|
||||||
assert!(bad.validate().is_err(), "deep_keep >= shortlist_keep");
|
assert!(bad.validate().is_err(), "deep_keep >= shortlist_keep");
|
||||||
let mut bad = Config::default();
|
let mut bad = Config::default();
|
||||||
|
|||||||
+9
-20
@@ -19,9 +19,10 @@ use sha2::{Digest as _, Sha256};
|
|||||||
use sqlx::Row as _;
|
use sqlx::Row as _;
|
||||||
|
|
||||||
use crate::config::{Config, VoyageConfig};
|
use crate::config::{Config, VoyageConfig};
|
||||||
use crate::curate::{approx_tokens, profile, prompt_text};
|
use crate::curate::{approx_tokens, prompt_text};
|
||||||
use crate::db::{Db, fmt_ts};
|
use crate::db::{Db, fmt_ts};
|
||||||
use crate::http::RetryPolicy;
|
use crate::http::RetryPolicy;
|
||||||
|
use crate::interests;
|
||||||
use crate::types::{Article, ArticleId};
|
use crate::types::{Article, ArticleId};
|
||||||
|
|
||||||
/// The only place the Voyage key comes from (§4.3).
|
/// The only place the Voyage key comes from (§4.3).
|
||||||
@@ -898,14 +899,7 @@ pub async fn plan_backfill(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let interests =
|
let interest_names = interests::names(db).await?;
|
||||||
match profile::load_standing_interests(&config.interests_opml, &config.profile_path) {
|
|
||||||
Ok(interests) => interests,
|
|
||||||
Err(error) => {
|
|
||||||
tracing::warn!(%error, "could not load standing interests; skipping them");
|
|
||||||
Vec::new()
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut plan = BackfillPlan::default();
|
let mut plan = BackfillPlan::default();
|
||||||
let mut keep = |articles: Vec<Article>, misses: Vec<(ArticleId, i64)>| -> Vec<Article> {
|
let mut keep = |articles: Vec<Article>, misses: Vec<(ArticleId, i64)>| -> Vec<Article> {
|
||||||
@@ -922,8 +916,8 @@ pub async fn plan_backfill(
|
|||||||
let other_misses = service.uncached_articles(&others).await?;
|
let other_misses = service.uncached_articles(&others).await?;
|
||||||
plan.others = keep(others, other_misses);
|
plan.others = keep(others, other_misses);
|
||||||
|
|
||||||
let interest_misses = service.uncached_interests(&interests).await?;
|
let interest_misses = service.uncached_interests(&interest_names).await?;
|
||||||
plan.cached += interests.len() - interest_misses.len();
|
plan.cached += interest_names.len() - interest_misses.len();
|
||||||
plan.estimated_tokens += interest_misses
|
plan.estimated_tokens += interest_misses
|
||||||
.iter()
|
.iter()
|
||||||
.map(|interest| approx_tokens(interest) as i64)
|
.map(|interest| approx_tokens(interest) as i64)
|
||||||
@@ -1391,7 +1385,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn backfill_prioritizes_the_learned_set_and_is_idempotent() {
|
async fn backfill_prioritizes_the_learned_set_and_is_idempotent() {
|
||||||
let (dir, db) = db_with_articles(&[1, 2, 3]).await;
|
let (_dir, db) = db_with_articles(&[1, 2, 3]).await;
|
||||||
// Article 1 is rated, article 2 is published, article 3 is neither.
|
// Article 1 is rated, article 2 is published, article 3 is neither.
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"INSERT INTO rating_events (article_id, kind, source, label, value, event_at)
|
"INSERT INTO rating_events (article_id, kind, source, label, value, event_at)
|
||||||
@@ -1418,16 +1412,11 @@ mod tests {
|
|||||||
|
|
||||||
let config = Config {
|
let config = Config {
|
||||||
voyage: small_config(),
|
voyage: small_config(),
|
||||||
interests_opml: dir.path().join("interests.opml"),
|
|
||||||
profile_path: dir.path().join("profile.md"),
|
|
||||||
..Config::default()
|
..Config::default()
|
||||||
};
|
};
|
||||||
std::fs::write(
|
interests::add(&db, "Writerdeck", Some("Publishing"), Timestamp::now())
|
||||||
&config.interests_opml,
|
.await
|
||||||
"<opml><body><outline text=\"Writerdeck\"/></body></opml>",
|
.unwrap();
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
std::fs::write(&config.profile_path, "# Reader profile\n").unwrap();
|
|
||||||
|
|
||||||
let backend = Arc::new(MockBackend::auto(4));
|
let backend = Arc::new(MockBackend::auto(4));
|
||||||
let svc = service(db.clone(), config.voyage.clone(), backend.clone());
|
let svc = service(db.clone(), config.voyage.clone(), backend.clone());
|
||||||
|
|||||||
+50
-117
@@ -3,7 +3,6 @@
|
|||||||
//! Every run rebuilds one byte-stable prompt from the hand-maintained profile,
|
//! Every run rebuilds one byte-stable prompt from the hand-maintained profile,
|
||||||
//! standing interests, stored weekly adjustments, and current explicit verdicts.
|
//! standing interests, stored weekly adjustments, and current explicit verdicts.
|
||||||
|
|
||||||
use std::collections::BTreeSet;
|
|
||||||
use std::fmt::Write as _;
|
use std::fmt::Write as _;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
@@ -13,6 +12,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
use super::llm::LlmClient;
|
use super::llm::LlmClient;
|
||||||
use crate::db::{Db, KV_PROFILE_VERSION, KV_TASTE_PROFILE};
|
use crate::db::{Db, KV_PROFILE_VERSION, KV_TASTE_PROFILE};
|
||||||
|
use crate::interests;
|
||||||
use crate::types::{Facets, RatedArticle, TasteProfile};
|
use crate::types::{Facets, RatedArticle, TasteProfile};
|
||||||
|
|
||||||
pub const REBUILD_INTERVAL_DAYS: i64 = 7;
|
pub const REBUILD_INTERVAL_DAYS: i64 = 7;
|
||||||
@@ -29,50 +29,9 @@ pub const NO_LEARNED_ADJUSTMENTS: &str = "No reader ratings have been collected
|
|||||||
const EDITOR_IN_CHIEF_FRAMING: &str = "You are the editor-in-chief of *The Daily EPUB*, a personal morning newspaper assembled every day for exactly one reader. Everything you are asked to do — score, select, place, summarize, introduce — serves his taste, not a general audience's. When a judgement call is close, re-read this profile and decide the way he would.";
|
const EDITOR_IN_CHIEF_FRAMING: &str = "You are the editor-in-chief of *The Daily EPUB*, a personal morning newspaper assembled every day for exactly one reader. Everything you are asked to do — score, select, place, summarize, introduce — serves his taste, not a general audience's. When a judgement call is close, re-read this profile and decide the way he would.";
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Interest and profile-file parsing
|
// Profile-file parsing
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
pub fn parse_interests(opml_path: &Path) -> anyhow::Result<Vec<String>> {
|
|
||||||
let raw = std::fs::read_to_string(opml_path)
|
|
||||||
.with_context(|| format!("reading the interests OPML at {}", opml_path.display()))?;
|
|
||||||
let interests = parse_interests_str(&raw);
|
|
||||||
if interests.is_empty() {
|
|
||||||
anyhow::bail!(
|
|
||||||
"no <outline text=\"…\"> interests found in {}",
|
|
||||||
opml_path.display()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
tracing::debug!(count = interests.len(), "parsed scour interests");
|
|
||||||
Ok(interests)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn parse_interests_str(raw: &str) -> Vec<String> {
|
|
||||||
let mut seen = BTreeSet::new();
|
|
||||||
let mut out = Vec::new();
|
|
||||||
for chunk in raw.split("text=\"").skip(1) {
|
|
||||||
let Some((value, _)) = chunk.split_once('"') else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let name = xml_unescape(value).trim().to_string();
|
|
||||||
if !name.is_empty() && seen.insert(name.to_lowercase()) {
|
|
||||||
out.push(name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out
|
|
||||||
}
|
|
||||||
|
|
||||||
fn xml_unescape(s: &str) -> String {
|
|
||||||
if !s.contains('&') {
|
|
||||||
return s.to_string();
|
|
||||||
}
|
|
||||||
s.replace("<", "<")
|
|
||||||
.replace(">", ">")
|
|
||||||
.replace(""", "\"")
|
|
||||||
.replace("'", "'")
|
|
||||||
.replace("'", "'")
|
|
||||||
.replace("&", "&")
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct ProfileFile {
|
pub struct ProfileFile {
|
||||||
/// Original Markdown with every `## Interests` section removed.
|
/// Original Markdown with every `## Interests` section removed.
|
||||||
@@ -116,7 +75,7 @@ pub fn load_profile(path: &Path) -> anyhow::Result<ProfileFile> {
|
|||||||
match std::fs::read_to_string(path) {
|
match std::fs::read_to_string(path) {
|
||||||
Ok(raw) => Ok(parse_profile_str(&raw)),
|
Ok(raw) => Ok(parse_profile_str(&raw)),
|
||||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||||
tracing::warn!(path = %path.display(), "profile file is missing; using OPML interests only");
|
tracing::warn!(path = %path.display(), "profile file is missing; using empty profile prose");
|
||||||
Ok(ProfileFile {
|
Ok(ProfileFile {
|
||||||
body: String::new(),
|
body: String::new(),
|
||||||
interests: Vec::new(),
|
interests: Vec::new(),
|
||||||
@@ -128,30 +87,7 @@ pub fn load_profile(path: &Path) -> anyhow::Result<ProfileFile> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load the exact standing-interest union used in the system prompt.
|
|
||||||
pub fn load_standing_interests(
|
|
||||||
opml_path: &Path,
|
|
||||||
profile_path: &Path,
|
|
||||||
) -> anyhow::Result<Vec<String>> {
|
|
||||||
let opml = parse_interests(opml_path)?;
|
|
||||||
let profile = load_profile(profile_path)?;
|
|
||||||
Ok(union_interests(opml, profile.interests))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn union_interests(opml: Vec<String>, profile: Vec<String>) -> Vec<String> {
|
|
||||||
let mut seen = BTreeSet::new();
|
|
||||||
let mut out = Vec::new();
|
|
||||||
for interest in opml.into_iter().chain(profile) {
|
|
||||||
let interest = interest.trim();
|
|
||||||
if !interest.is_empty() && seen.insert(interest.to_lowercase()) {
|
|
||||||
out.push(interest.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out
|
|
||||||
}
|
|
||||||
|
|
||||||
pub mod themes;
|
pub mod themes;
|
||||||
pub use themes::group_into_themes;
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Prompt assembly
|
// Prompt assembly
|
||||||
@@ -174,7 +110,7 @@ fn one_line(text: &str) -> String {
|
|||||||
/// Assemble sections in the exact cache-friendly order required by §8.4.
|
/// Assemble sections in the exact cache-friendly order required by §8.4.
|
||||||
pub fn build(
|
pub fn build(
|
||||||
profile_body: &str,
|
profile_body: &str,
|
||||||
interests: &[String],
|
grouped: &[(String, Vec<String>)],
|
||||||
learned_adjustments: &str,
|
learned_adjustments: &str,
|
||||||
ratings: &[RatedArticle],
|
ratings: &[RatedArticle],
|
||||||
verdict_limit: usize,
|
verdict_limit: usize,
|
||||||
@@ -193,8 +129,8 @@ pub fn build(
|
|||||||
|
|
||||||
doc.push_str("## Standing interests\n\n");
|
doc.push_str("## Standing interests\n\n");
|
||||||
doc.push_str("These are his subscribed interest topics, grouped. They raise the floor for a match, but never cap the paper: an outstanding article on none of these still belongs.\n\n");
|
doc.push_str("These are his subscribed interest topics, grouped. They raise the floor for a match, but never cap the paper: an outstanding article on none of these still belongs.\n\n");
|
||||||
for (theme, members) in group_into_themes(interests) {
|
for (category, members) in grouped {
|
||||||
let _ = writeln!(doc, "- **{}**: {}", theme, members.join(", "));
|
let _ = writeln!(doc, "- **{}**: {}", category, members.join(", "));
|
||||||
}
|
}
|
||||||
|
|
||||||
doc.push_str("\n## Learned adjustments (rebuilt weekly from ratings)\n\n");
|
doc.push_str("\n## Learned adjustments (rebuilt weekly from ratings)\n\n");
|
||||||
@@ -272,26 +208,27 @@ async fn store_version(db: &Db, version: i64, built_at: Timestamp) -> anyhow::Re
|
|||||||
|
|
||||||
async fn prompt_inputs(
|
async fn prompt_inputs(
|
||||||
db: &Db,
|
db: &Db,
|
||||||
opml_path: &Path,
|
|
||||||
profile_path: &Path,
|
profile_path: &Path,
|
||||||
) -> anyhow::Result<(ProfileFile, Vec<String>, Vec<RatedArticle>, String)> {
|
) -> anyhow::Result<(
|
||||||
let opml = parse_interests(opml_path)?;
|
ProfileFile,
|
||||||
|
Vec<(String, Vec<String>)>,
|
||||||
|
Vec<RatedArticle>,
|
||||||
|
String,
|
||||||
|
)> {
|
||||||
let profile = load_profile(profile_path)?;
|
let profile = load_profile(profile_path)?;
|
||||||
let interests = union_interests(opml, profile.interests.clone());
|
let grouped = interests::grouped(db).await?;
|
||||||
let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?;
|
let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?;
|
||||||
let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default();
|
let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default();
|
||||||
Ok((profile, interests, ratings, learned))
|
Ok((profile, grouped, ratings, learned))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rebuild the complete system prompt from its live inputs on every run.
|
/// Rebuild the complete system prompt from its live inputs on every run.
|
||||||
pub async fn load_or_build(
|
pub async fn load_or_build(
|
||||||
db: &Db,
|
db: &Db,
|
||||||
opml_path: &Path,
|
|
||||||
profile_path: &Path,
|
profile_path: &Path,
|
||||||
verdict_limit: usize,
|
verdict_limit: usize,
|
||||||
) -> anyhow::Result<TasteProfile> {
|
) -> anyhow::Result<TasteProfile> {
|
||||||
let (profile_file, interests, ratings, learned) =
|
let (profile_file, grouped, ratings, learned) = prompt_inputs(db, profile_path).await?;
|
||||||
prompt_inputs(db, opml_path, profile_path).await?;
|
|
||||||
let (version, built_at) = match stored_version(db).await? {
|
let (version, built_at) = match stored_version(db).await? {
|
||||||
Some(stored) => stored,
|
Some(stored) => stored,
|
||||||
None => {
|
None => {
|
||||||
@@ -303,7 +240,7 @@ pub async fn load_or_build(
|
|||||||
let profile = TasteProfile {
|
let profile = TasteProfile {
|
||||||
text: build(
|
text: build(
|
||||||
&profile_file.body,
|
&profile_file.body,
|
||||||
&interests,
|
&grouped,
|
||||||
&learned,
|
&learned,
|
||||||
&ratings,
|
&ratings,
|
||||||
verdict_limit,
|
verdict_limit,
|
||||||
@@ -315,7 +252,10 @@ pub async fn load_or_build(
|
|||||||
db.kv_set(KV_TASTE_PROFILE, &profile.text).await?;
|
db.kv_set(KV_TASTE_PROFILE, &profile.text).await?;
|
||||||
tracing::debug!(
|
tracing::debug!(
|
||||||
version,
|
version,
|
||||||
interests = interests.len(),
|
interests = grouped
|
||||||
|
.iter()
|
||||||
|
.map(|(_, members)| members.len())
|
||||||
|
.sum::<usize>(),
|
||||||
verdicts = ratings.len().min(verdict_limit),
|
verdicts = ratings.len().min(verdict_limit),
|
||||||
chars = profile.text.len(),
|
chars = profile.text.len(),
|
||||||
"rebuilt the taste profile prompt"
|
"rebuilt the taste profile prompt"
|
||||||
@@ -334,7 +274,6 @@ pub async fn is_stale(db: &Db) -> anyhow::Result<bool> {
|
|||||||
pub async fn weekly_rebuild_if_due(
|
pub async fn weekly_rebuild_if_due(
|
||||||
db: &Db,
|
db: &Db,
|
||||||
llm: &LlmClient,
|
llm: &LlmClient,
|
||||||
opml_path: &Path,
|
|
||||||
profile_path: &Path,
|
profile_path: &Path,
|
||||||
verdict_limit: usize,
|
verdict_limit: usize,
|
||||||
) -> anyhow::Result<Option<TasteProfile>> {
|
) -> anyhow::Result<Option<TasteProfile>> {
|
||||||
@@ -346,9 +285,7 @@ pub async fn weekly_rebuild_if_due(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
tracing::info!("taste profile is over a week old; rebuilding learned adjustments");
|
tracing::info!("taste profile is over a week old; rebuilding learned adjustments");
|
||||||
Ok(Some(
|
Ok(Some(rebuild(db, llm, profile_path, verdict_limit).await?))
|
||||||
rebuild(db, llm, opml_path, profile_path, verdict_limit).await?,
|
|
||||||
))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -429,15 +366,12 @@ pub fn build_rebuild_prompt(ratings: &[RatedArticle]) -> String {
|
|||||||
pub async fn rebuild(
|
pub async fn rebuild(
|
||||||
db: &Db,
|
db: &Db,
|
||||||
llm: &LlmClient,
|
llm: &LlmClient,
|
||||||
opml_path: &Path,
|
|
||||||
profile_path: &Path,
|
profile_path: &Path,
|
||||||
verdict_limit: usize,
|
verdict_limit: usize,
|
||||||
) -> anyhow::Result<TasteProfile> {
|
) -> anyhow::Result<TasteProfile> {
|
||||||
// Read the prompt inputs first: a rebuild that dies on a missing OPML must
|
// Read the prompt inputs before spending a model call.
|
||||||
// stay due and must not have spent a model call getting there.
|
|
||||||
let opml = parse_interests(opml_path)?;
|
|
||||||
let profile_file = load_profile(profile_path)?;
|
let profile_file = load_profile(profile_path)?;
|
||||||
let interests = union_interests(opml, profile_file.interests.clone());
|
let grouped = interests::grouped(db).await?;
|
||||||
let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?;
|
let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?;
|
||||||
let previous = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default();
|
let previous = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default();
|
||||||
let learned = if ratings.is_empty() {
|
let learned = if ratings.is_empty() {
|
||||||
@@ -473,7 +407,7 @@ pub async fn rebuild(
|
|||||||
let profile = TasteProfile {
|
let profile = TasteProfile {
|
||||||
text: build(
|
text: build(
|
||||||
&profile_file.body,
|
&profile_file.body,
|
||||||
&interests,
|
&grouped,
|
||||||
&learned,
|
&learned,
|
||||||
&ratings,
|
&ratings,
|
||||||
verdict_limit,
|
verdict_limit,
|
||||||
@@ -495,18 +429,15 @@ pub async fn rebuild(
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
const OPML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data/scour-interests.opml");
|
|
||||||
const PROFILE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data/profile.md");
|
const PROFILE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data/profile.md");
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn profile_interests_are_removed_and_union_case_insensitively() {
|
fn profile_interests_are_removed_for_the_importer() {
|
||||||
let parsed = parse_profile_str(
|
let parsed = parse_profile_str(
|
||||||
"# P\n\n## Interests\n- Rust\nBoston Tech\n- rust\n\n## Notes\nKeep this.\n",
|
"# P\n\n## Interests\n- Rust\nBoston Tech\n- rust\n\n## Notes\nKeep this.\n",
|
||||||
);
|
);
|
||||||
assert_eq!(parsed.body, "# P\n\n## Notes\nKeep this.\n");
|
assert_eq!(parsed.body, "# P\n\n## Notes\nKeep this.\n");
|
||||||
assert_eq!(parsed.interests, ["Rust", "Boston Tech", "rust"]);
|
assert_eq!(parsed.interests, ["Rust", "Boston Tech", "rust"]);
|
||||||
let union = union_interests(vec!["rust".into(), "E-Ink".into()], parsed.interests);
|
|
||||||
assert_eq!(union, ["rust", "E-Ink", "Boston Tech"]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -526,7 +457,7 @@ mod tests {
|
|||||||
};
|
};
|
||||||
let prompt = build(
|
let prompt = build(
|
||||||
"# Reader profile\n\nProfile prose.",
|
"# Reader profile\n\nProfile prose.",
|
||||||
&["Rust".into()],
|
&[("Software".into(), vec!["Rust".into(), "SQLite".into()])],
|
||||||
"- Adjust.",
|
"- Adjust.",
|
||||||
&[rating],
|
&[rating],
|
||||||
60,
|
60,
|
||||||
@@ -539,17 +470,16 @@ mod tests {
|
|||||||
assert!(
|
assert!(
|
||||||
framing < profile && profile < interests && interests < learned && learned < verdicts
|
framing < profile && profile < interests && interests < learned && learned < verdicts
|
||||||
);
|
);
|
||||||
|
assert!(prompt.contains("- **Software**: Rust, SQLite"));
|
||||||
assert!(prompt.contains("NOT FOR ME | A title | A feed | A summary with whitespace."));
|
assert!(prompt.contains("NOT FOR ME | A title | A feed | A summary with whitespace."));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn shipped_profile_and_opml_parse() {
|
fn shipped_profile_parses() {
|
||||||
let profile = load_profile(Path::new(PROFILE_PATH)).unwrap();
|
let profile = load_profile(Path::new(PROFILE_PATH)).unwrap();
|
||||||
assert!(profile.body.contains("## Who he is"));
|
assert!(profile.body.contains("## Who he is"));
|
||||||
assert!(!profile.body.contains("## Interests"));
|
assert!(!profile.body.contains("## Interests"));
|
||||||
assert!(profile.interests.is_empty());
|
assert!(profile.interests.is_empty());
|
||||||
let interests = parse_interests(Path::new(OPML_PATH)).unwrap();
|
|
||||||
assert!(interests.iter().any(|interest| interest == "Rust"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -594,19 +524,21 @@ mod tests {
|
|||||||
let db = Db::open_and_migrate(&dir.path().join("profile.db"))
|
let db = Db::open_and_migrate(&dir.path().join("profile.db"))
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let opml = dir.path().join("interests.opml");
|
|
||||||
let profile_path = dir.path().join("profile.md");
|
let profile_path = dir.path().join("profile.md");
|
||||||
std::fs::write(&opml, r#"<outline text="Rust"/>"#).unwrap();
|
interests::add(&db, "Rust", Some("Software"), Timestamp::now())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
std::fs::write(
|
std::fs::write(
|
||||||
&profile_path,
|
&profile_path,
|
||||||
"# Reader profile\n\nOriginal prose.\n\n## Interests\n- Custom Topic\n",
|
"# Reader profile\n\nOriginal prose.\n\n## Interests\n- Custom Topic\n",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let first = load_or_build(&db, &opml, &profile_path, 60).await.unwrap();
|
let first = load_or_build(&db, &profile_path, 60).await.unwrap();
|
||||||
assert_eq!(first.version, 1);
|
assert_eq!(first.version, 1);
|
||||||
assert!(first.text.contains("Original prose."));
|
assert!(first.text.contains("Original prose."));
|
||||||
assert!(first.text.contains("Custom Topic"));
|
assert!(first.text.contains("- **Software**: Rust"));
|
||||||
|
assert!(!first.text.contains("Custom Topic"));
|
||||||
assert!(!first.text.contains("## Interests"));
|
assert!(!first.text.contains("## Interests"));
|
||||||
|
|
||||||
std::fs::write(
|
std::fs::write(
|
||||||
@@ -614,11 +546,11 @@ mod tests {
|
|||||||
"# Reader profile\n\nChanged prose.\n\n## Interests\n- Another Topic\n",
|
"# Reader profile\n\nChanged prose.\n\n## Interests\n- Another Topic\n",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let second = load_or_build(&db, &opml, &profile_path, 60).await.unwrap();
|
let second = load_or_build(&db, &profile_path, 60).await.unwrap();
|
||||||
assert_eq!(second.version, first.version);
|
assert_eq!(second.version, first.version);
|
||||||
assert_eq!(second.built_at, first.built_at);
|
assert_eq!(second.built_at, first.built_at);
|
||||||
assert!(second.text.contains("Changed prose."));
|
assert!(second.text.contains("Changed prose."));
|
||||||
assert!(second.text.contains("Another Topic"));
|
assert!(!second.text.contains("Another Topic"));
|
||||||
assert!(!second.text.contains("Original prose."));
|
assert!(!second.text.contains("Original prose."));
|
||||||
|
|
||||||
let missing = load_profile(&dir.path().join("missing.md")).unwrap();
|
let missing = load_profile(&dir.path().join("missing.md")).unwrap();
|
||||||
@@ -637,11 +569,12 @@ mod tests {
|
|||||||
let db = Db::open_and_migrate(&dir.path().join("profile.db"))
|
let db = Db::open_and_migrate(&dir.path().join("profile.db"))
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let opml = dir.path().join("interests.opml");
|
|
||||||
let profile_path = dir.path().join("profile.md");
|
let profile_path = dir.path().join("profile.md");
|
||||||
std::fs::write(&opml, r#"<outline text="Rust"/>"#).unwrap();
|
interests::add(&db, "Rust", Some("Software"), Timestamp::now())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap();
|
std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap();
|
||||||
let initial = load_or_build(&db, &opml, &profile_path, 60).await.unwrap();
|
let initial = load_or_build(&db, &profile_path, 60).await.unwrap();
|
||||||
assert_eq!(initial.version, 1);
|
assert_eq!(initial.version, 1);
|
||||||
|
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
@@ -677,7 +610,7 @@ mod tests {
|
|||||||
UsageMeter::for_provider(&ProviderConfig::deepseek()),
|
UsageMeter::for_provider(&ProviderConfig::deepseek()),
|
||||||
backend.clone(),
|
backend.clone(),
|
||||||
);
|
);
|
||||||
let rebuilt = rebuild(&db, &llm, &opml, &profile_path, 60).await.unwrap();
|
let rebuilt = rebuild(&db, &llm, &profile_path, 60).await.unwrap();
|
||||||
assert_eq!(rebuilt.version, 2);
|
assert_eq!(rebuilt.version, 2);
|
||||||
assert!(rebuilt.text.contains("Rank first-hand reports higher."));
|
assert!(rebuilt.text.contains("Rank first-hand reports higher."));
|
||||||
assert!(
|
assert!(
|
||||||
@@ -700,16 +633,16 @@ mod tests {
|
|||||||
let db = Db::open_and_migrate(&dir.path().join("profile.db"))
|
let db = Db::open_and_migrate(&dir.path().join("profile.db"))
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let opml = dir.path().join("interests.opml");
|
|
||||||
let profile_path = dir.path().join("profile.md");
|
let profile_path = dir.path().join("profile.md");
|
||||||
std::fs::write(&opml, r#"<outline text="Rust"/>"#).unwrap();
|
interests::add(&db, "Rust", Some("Software"), Timestamp::now())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap();
|
std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap();
|
||||||
let initial = load_or_build(&db, &opml, &profile_path, 60).await.unwrap();
|
let initial = load_or_build(&db, &profile_path, 60).await.unwrap();
|
||||||
assert_eq!(initial.version, 1);
|
assert_eq!(initial.version, 1);
|
||||||
|
|
||||||
// The OPML goes missing the way a relative path does under a service
|
std::fs::remove_file(&profile_path).unwrap();
|
||||||
// whose working directory is not the checkout.
|
std::fs::create_dir(&profile_path).unwrap();
|
||||||
std::fs::remove_file(&opml).unwrap();
|
|
||||||
|
|
||||||
let backend = Arc::new(MockBackend::new());
|
let backend = Arc::new(MockBackend::new());
|
||||||
let llm = LlmClient::with_backend(
|
let llm = LlmClient::with_backend(
|
||||||
@@ -718,10 +651,10 @@ mod tests {
|
|||||||
UsageMeter::for_provider(&ProviderConfig::deepseek()),
|
UsageMeter::for_provider(&ProviderConfig::deepseek()),
|
||||||
backend.clone(),
|
backend.clone(),
|
||||||
);
|
);
|
||||||
let error = rebuild(&db, &llm, &opml, &profile_path, 60)
|
let error = rebuild(&db, &llm, &profile_path, 60)
|
||||||
.await
|
.await
|
||||||
.expect_err("a missing OPML fails the rebuild");
|
.expect_err("an unreadable profile fails the rebuild");
|
||||||
assert!(format!("{error:#}").contains("reading the interests OPML"));
|
assert!(format!("{error:#}").contains("reading the reader profile"));
|
||||||
|
|
||||||
// Still version 1, so the profile stays stale and the rebuild is retried.
|
// Still version 1, so the profile stays stale and the rebuild is retried.
|
||||||
assert_eq!(stored_version(&db).await.unwrap().unwrap().0, 1);
|
assert_eq!(stored_version(&db).await.unwrap().unwrap().0, 1);
|
||||||
|
|||||||
+6
-1
@@ -62,6 +62,11 @@ fn calculate_utility_for(
|
|||||||
("quality", configured.quality, 1.0),
|
("quality", configured.quality, 1.0),
|
||||||
("fit", configured.fit, 1.0),
|
("fit", configured.fit, 1.0),
|
||||||
("knn", configured.knn, candidate.signals.knn_gate),
|
("knn", configured.knn, candidate.signals.knn_gate),
|
||||||
|
(
|
||||||
|
"affinity",
|
||||||
|
configured.affinity,
|
||||||
|
candidate.signals.affinity_gate,
|
||||||
|
),
|
||||||
("interest", configured.interest, 1.0),
|
("interest", configured.interest, 1.0),
|
||||||
("feed", configured.feed, candidate.signals.feed_gate),
|
("feed", configured.feed, candidate.signals.feed_gate),
|
||||||
("triage", configured.triage, 1.0),
|
("triage", configured.triage, 1.0),
|
||||||
@@ -342,7 +347,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
assert!(!b.signals.weights.contains_key("interest"));
|
assert!(!b.signals.weights.contains_key("interest"));
|
||||||
assert!(
|
assert!(
|
||||||
(a.signals.weights["knn"] / a.signals.weights["quality"] - (0.15 * 0.5) / 0.40).abs()
|
(a.signals.weights["knn"] / a.signals.weights["quality"] - (0.10 * 0.5) / 0.40).abs()
|
||||||
< 1e-9
|
< 1e-9
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
+304
-10
@@ -14,11 +14,14 @@ use crate::config::{PreliminaryWeights, RankingConfig, VoyageConfig};
|
|||||||
use crate::curate::embedding::{dot, load_article_embeddings};
|
use crate::curate::embedding::{dot, load_article_embeddings};
|
||||||
use crate::curate::prefilter;
|
use crate::curate::prefilter;
|
||||||
use crate::db::Db;
|
use crate::db::Db;
|
||||||
|
use crate::interests::{self, Rate};
|
||||||
use crate::types::{Article, ArticleId, FeedId, SourceKind};
|
use crate::types::{Article, ArticleId, FeedId, SourceKind};
|
||||||
|
|
||||||
/// Below this many embedded eligible articles the z-score is too noisy, so the
|
/// Below this many embedded eligible articles the z-score is too noisy, so the
|
||||||
/// interest signal falls back to the raw top-1 cosine (§9.1).
|
/// interest signal falls back to the raw top-1 cosine (§9.1).
|
||||||
pub const INTEREST_ZSCORE_MIN_ARTICLES: usize = 30;
|
pub const INTEREST_ZSCORE_MIN_ARTICLES: usize = 30;
|
||||||
|
/// Weak top-three matches are omitted everywhere they are presented or credited.
|
||||||
|
pub const MATCH_MIN_Z: f64 = 1.0;
|
||||||
/// Standard-deviation floor for the per-interest z-score (§9.1).
|
/// Standard-deviation floor for the per-interest z-score (§9.1).
|
||||||
const ZSCORE_STD_FLOOR: f64 = 1e-3;
|
const ZSCORE_STD_FLOOR: f64 = 1e-3;
|
||||||
/// How many interests and rated neighbours `signals_json` records (§7.5).
|
/// How many interests and rated neighbours `signals_json` records (§7.5).
|
||||||
@@ -30,7 +33,8 @@ pub const AGGREGATOR_FEED_SHARE: f64 = 0.25;
|
|||||||
/// The signal names that go through the percentile normalizer, in the order
|
/// The signal names that go through the percentile normalizer, in the order
|
||||||
/// they are rendered (§12.2). LLM scores (`triage`, `quality`, `fit`) are
|
/// they are rendered (§12.2). LLM scores (`triage`, `quality`, `fit`) are
|
||||||
/// absolute and arrive in steps 4–5.
|
/// absolute and arrive in steps 4–5.
|
||||||
pub const PERCENTILE_SIGNALS: [&str; 5] = ["interest", "knn", "feed", "social", "heuristic"];
|
pub const PERCENTILE_SIGNALS: [&str; 6] =
|
||||||
|
["interest", "knn", "feed", "affinity", "social", "heuristic"];
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct TopInterest {
|
pub struct TopInterest {
|
||||||
@@ -56,6 +60,7 @@ pub struct Signals {
|
|||||||
pub interest_top1_cos: Option<f64>,
|
pub interest_top1_cos: Option<f64>,
|
||||||
pub knn: Option<f64>,
|
pub knn: Option<f64>,
|
||||||
pub feed: Option<f64>,
|
pub feed: Option<f64>,
|
||||||
|
pub affinity: Option<f64>,
|
||||||
pub social: Option<f64>,
|
pub social: Option<f64>,
|
||||||
pub heuristic: Option<f64>,
|
pub heuristic: Option<f64>,
|
||||||
/// Mid-rank percentiles of the present signals (§12.2).
|
/// Mid-rank percentiles of the present signals (§12.2).
|
||||||
@@ -80,6 +85,8 @@ pub struct Signals {
|
|||||||
pub knn_gate: f64,
|
pub knn_gate: f64,
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
pub feed_gate: f64,
|
pub feed_gate: f64,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub affinity_gate: f64,
|
||||||
/// `ranking.slop_author_penalty`, applied when `slop_author` is set.
|
/// `ranking.slop_author_penalty`, applied when `slop_author` is set.
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
pub slop_penalty: f64,
|
pub slop_penalty: f64,
|
||||||
@@ -105,6 +112,7 @@ impl Signals {
|
|||||||
"interest_top1_cos" => self.interest_top1_cos,
|
"interest_top1_cos" => self.interest_top1_cos,
|
||||||
"knn" => self.knn,
|
"knn" => self.knn,
|
||||||
"feed" => self.feed,
|
"feed" => self.feed,
|
||||||
|
"affinity" => self.affinity,
|
||||||
"social" => self.social,
|
"social" => self.social,
|
||||||
"heuristic" => self.heuristic,
|
"heuristic" => self.heuristic,
|
||||||
_ => None,
|
_ => None,
|
||||||
@@ -130,8 +138,10 @@ impl Signals {
|
|||||||
pub struct PreferenceSummary {
|
pub struct PreferenceSummary {
|
||||||
pub rated_with_embeddings: usize,
|
pub rated_with_embeddings: usize,
|
||||||
pub attributable_feed_ratings: usize,
|
pub attributable_feed_ratings: usize,
|
||||||
|
pub attributable_interest_ratings: usize,
|
||||||
pub knn_gate: f64,
|
pub knn_gate: f64,
|
||||||
pub feed_gate: f64,
|
pub feed_gate: f64,
|
||||||
|
pub affinity_gate: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One rated article with an embedding: the unit of the preference state (§9.2).
|
/// One rated article with an embedding: the unit of the preference state (§9.2).
|
||||||
@@ -179,12 +189,15 @@ pub struct PreferenceState {
|
|||||||
pub examples: Vec<RatedExample>,
|
pub examples: Vec<RatedExample>,
|
||||||
feed_rates: HashMap<FeedId, FeedRate>,
|
feed_rates: HashMap<FeedId, FeedRate>,
|
||||||
author_rates: HashMap<String, FeedRate>,
|
author_rates: HashMap<String, FeedRate>,
|
||||||
|
interest_rates: HashMap<String, Rate>,
|
||||||
/// Normalized keys of authors with a current *AI slop* verdict (§9.3).
|
/// Normalized keys of authors with a current *AI slop* verdict (§9.3).
|
||||||
slop_authors: HashSet<String>,
|
slop_authors: HashSet<String>,
|
||||||
pub slop_author_penalty: f64,
|
pub slop_author_penalty: f64,
|
||||||
pub attributable_feed_ratings: usize,
|
pub attributable_feed_ratings: usize,
|
||||||
|
pub attributable_interest_ratings: usize,
|
||||||
pub knn_gate: f64,
|
pub knn_gate: f64,
|
||||||
pub feed_gate: f64,
|
pub feed_gate: f64,
|
||||||
|
pub affinity_gate: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PreferenceState {
|
impl PreferenceState {
|
||||||
@@ -204,12 +217,28 @@ impl PreferenceState {
|
|||||||
examples,
|
examples,
|
||||||
feed_rates,
|
feed_rates,
|
||||||
author_rates,
|
author_rates,
|
||||||
|
interest_rates: HashMap::new(),
|
||||||
slop_authors: HashSet::new(),
|
slop_authors: HashSet::new(),
|
||||||
slop_author_penalty: ranking.slop_author_penalty,
|
slop_author_penalty: ranking.slop_author_penalty,
|
||||||
attributable_feed_ratings,
|
attributable_feed_ratings,
|
||||||
|
attributable_interest_ratings: 0,
|
||||||
|
affinity_gate: 0.0,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Register rating-derived interest rates after the embedding examples are built.
|
||||||
|
pub fn with_interest_rates(
|
||||||
|
mut self,
|
||||||
|
rates_by_name: HashMap<String, Rate>,
|
||||||
|
attributable: usize,
|
||||||
|
ranking: &RankingConfig,
|
||||||
|
) -> Self {
|
||||||
|
self.interest_rates = rates_by_name;
|
||||||
|
self.attributable_interest_ratings = attributable;
|
||||||
|
self.affinity_gate = gate(attributable, ranking.affinity_floor, ranking.affinity_full);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Register the authors whose current verdict is *AI slop*; keys are
|
/// Register the authors whose current verdict is *AI slop*; keys are
|
||||||
/// normalized like [`normalize_author`] and empty ones are dropped.
|
/// normalized like [`normalize_author`] and empty ones are dropped.
|
||||||
pub fn with_slop_authors<I, S>(mut self, authors: I) -> Self
|
pub fn with_slop_authors<I, S>(mut self, authors: I) -> Self
|
||||||
@@ -234,8 +263,7 @@ impl PreferenceState {
|
|||||||
self.slop_authors.len()
|
self.slop_authors.len()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load `db::current_ratings(rating_lookback_days)` joined to
|
/// Load current ratings; only kNN/feed examples require an embedding.
|
||||||
/// `article_embeddings`; ratings without an embedding are skipped (§9.2).
|
|
||||||
pub async fn load(
|
pub async fn load(
|
||||||
db: &Db,
|
db: &Db,
|
||||||
voyage: &VoyageConfig,
|
voyage: &VoyageConfig,
|
||||||
@@ -249,7 +277,7 @@ impl PreferenceState {
|
|||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let embeddings = load_article_embeddings(db, voyage, &ids).await?;
|
let embeddings = load_article_embeddings(db, voyage, &ids).await?;
|
||||||
let mut examples = Vec::new();
|
let mut examples = Vec::new();
|
||||||
for rating in ratings {
|
for rating in &ratings {
|
||||||
let Some(embedding) = embeddings.get(&rating.article_id).cloned() else {
|
let Some(embedding) = embeddings.get(&rating.article_id).cloned() else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
@@ -264,8 +292,8 @@ impl PreferenceState {
|
|||||||
let age_days = (now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0;
|
let age_days = (now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0;
|
||||||
examples.push(RatedExample {
|
examples.push(RatedExample {
|
||||||
article_id: rating.article_id,
|
article_id: rating.article_id,
|
||||||
label: rating.label,
|
label: rating.label.clone(),
|
||||||
title: rating.title,
|
title: rating.title.clone(),
|
||||||
value: rating.value,
|
value: rating.value,
|
||||||
decay: decay(age_days, ranking.rating_half_life_days),
|
decay: decay(age_days, ranking.rating_half_life_days),
|
||||||
embedding,
|
embedding,
|
||||||
@@ -274,16 +302,51 @@ impl PreferenceState {
|
|||||||
aggregator_only,
|
aggregator_only,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
let match_rows = interests::matches_for_articles(db, &ids).await?;
|
||||||
|
let rated = ratings
|
||||||
|
.iter()
|
||||||
|
.map(|rating| {
|
||||||
|
let age_days =
|
||||||
|
(now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0;
|
||||||
|
(
|
||||||
|
rating.article_id,
|
||||||
|
rating.value,
|
||||||
|
decay(age_days, ranking.rating_half_life_days),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let matched = match_rows
|
||||||
|
.iter()
|
||||||
|
.map(|row| (row.article_id, row.interest_id, row.z))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let rates = interests::rates(&rated, &matched);
|
||||||
|
let names = match_rows
|
||||||
|
.iter()
|
||||||
|
.map(|row| (row.interest_id, row.name.as_str()))
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
let rates_by_name = rates
|
||||||
|
.by_interest
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(interest_id, rate)| {
|
||||||
|
names
|
||||||
|
.get(&interest_id)
|
||||||
|
.map(|name| ((*name).to_string(), rate))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
let slop_authors = db.slop_authors().await?;
|
let slop_authors = db.slop_authors().await?;
|
||||||
Ok(Self::build(examples, ranking).with_slop_authors(slop_authors))
|
Ok(Self::build(examples, ranking)
|
||||||
|
.with_interest_rates(rates_by_name, rates.attributable, ranking)
|
||||||
|
.with_slop_authors(slop_authors))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn summary(&self) -> PreferenceSummary {
|
pub fn summary(&self) -> PreferenceSummary {
|
||||||
PreferenceSummary {
|
PreferenceSummary {
|
||||||
rated_with_embeddings: self.examples.len(),
|
rated_with_embeddings: self.examples.len(),
|
||||||
attributable_feed_ratings: self.attributable_feed_ratings,
|
attributable_feed_ratings: self.attributable_feed_ratings,
|
||||||
|
attributable_interest_ratings: self.attributable_interest_ratings,
|
||||||
knn_gate: self.knn_gate,
|
knn_gate: self.knn_gate,
|
||||||
feed_gate: self.feed_gate,
|
feed_gate: self.feed_gate,
|
||||||
|
affinity_gate: self.affinity_gate,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -297,21 +360,54 @@ impl PreferenceState {
|
|||||||
self.attributable_feed_ratings, ranking.feed_floor
|
self.attributable_feed_ratings, ranking.feed_floor
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
let affinity_detail = if self.affinity_gate > 0.0 {
|
||||||
|
format!("(n={})", self.attributable_interest_ratings)
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"(n={} < {})",
|
||||||
|
self.attributable_interest_ratings, ranking.affinity_floor
|
||||||
|
)
|
||||||
|
};
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
rated_with_embeddings = self.examples.len(),
|
rated_with_embeddings = self.examples.len(),
|
||||||
knn_gate = self.knn_gate,
|
knn_gate = self.knn_gate,
|
||||||
feed_gate = self.feed_gate,
|
feed_gate = self.feed_gate,
|
||||||
|
affinity_gate = self.affinity_gate,
|
||||||
slop_authors = self.slop_authors.len(),
|
slop_authors = self.slop_authors.len(),
|
||||||
"preference: {} rated articles with embeddings → knn gate {:.2}; feed gate {:.1} {}; {} slop authors (penalty {:.2})",
|
"preference: {} rated articles with embeddings → knn gate {:.2}; feed gate {:.1} {}; affinity gate {:.1} {}; {} slop authors (penalty {:.2})",
|
||||||
self.examples.len(),
|
self.examples.len(),
|
||||||
self.knn_gate,
|
self.knn_gate,
|
||||||
self.feed_gate,
|
self.feed_gate,
|
||||||
feed_detail,
|
feed_detail,
|
||||||
|
self.affinity_gate,
|
||||||
|
affinity_detail,
|
||||||
self.slop_authors.len(),
|
self.slop_authors.len(),
|
||||||
self.slop_author_penalty
|
self.slop_author_penalty
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Match-strength-weighted preference for the article's rated interests.
|
||||||
|
fn affinity(&self, top: &[TopInterest]) -> Option<f64> {
|
||||||
|
if self.affinity_gate <= 0.0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut weighted = 0.0;
|
||||||
|
let mut strength_sum = 0.0;
|
||||||
|
for interest in top {
|
||||||
|
let Some(rate) = self
|
||||||
|
.interest_rates
|
||||||
|
.get(&interest.name)
|
||||||
|
.filter(|rate| rate.n > 0)
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let strength = (interest.z / 3.0).clamp(0.0, 1.0);
|
||||||
|
weighted += strength * (rate.weight() - 0.5);
|
||||||
|
strength_sum += strength;
|
||||||
|
}
|
||||||
|
(strength_sum > 0.0).then_some(weighted / strength_sum)
|
||||||
|
}
|
||||||
|
|
||||||
/// Signed rated-neighbour preference and the three nearest rated articles
|
/// Signed rated-neighbour preference and the three nearest rated articles
|
||||||
/// (§9.2). Absent when the gate is closed or there are no examples.
|
/// (§9.2). Absent when the gate is closed or there are no examples.
|
||||||
pub fn knn(&self, candidate: &[f32], ranking: &RankingConfig) -> (Option<f64>, Vec<Neighbour>) {
|
pub fn knn(&self, candidate: &[f32], ranking: &RankingConfig) -> (Option<f64>, Vec<Neighbour>) {
|
||||||
@@ -552,6 +648,7 @@ pub fn interest_matches(
|
|||||||
let top_mean = all.iter().map(|item| item.z).sum::<f64>() / all.len() as f64;
|
let top_mean = all.iter().map(|item| item.z).sum::<f64>() / all.len() as f64;
|
||||||
0.7 * all[0].z + 0.3 * top_mean
|
0.7 * all[0].z + 0.3 * top_mean
|
||||||
};
|
};
|
||||||
|
all.retain(|interest| interest.z >= MATCH_MIN_Z);
|
||||||
(
|
(
|
||||||
article_id,
|
article_id,
|
||||||
InterestMatch {
|
InterestMatch {
|
||||||
@@ -580,10 +677,12 @@ pub fn compute(
|
|||||||
let mut signals = Signals::baseline(article);
|
let mut signals = Signals::baseline(article);
|
||||||
signals.knn_gate = preference.knn_gate;
|
signals.knn_gate = preference.knn_gate;
|
||||||
signals.feed_gate = preference.feed_gate;
|
signals.feed_gate = preference.feed_gate;
|
||||||
|
signals.affinity_gate = preference.affinity_gate;
|
||||||
if let Some(matched) = interests.get(&article.id) {
|
if let Some(matched) = interests.get(&article.id) {
|
||||||
signals.interest = Some(matched.score);
|
signals.interest = Some(matched.score);
|
||||||
signals.interest_top1_cos = Some(matched.top1_cos);
|
signals.interest_top1_cos = Some(matched.top1_cos);
|
||||||
signals.top_interests = matched.top_interests.clone();
|
signals.top_interests = matched.top_interests.clone();
|
||||||
|
signals.affinity = preference.affinity(&matched.top_interests);
|
||||||
}
|
}
|
||||||
if let Some(embedding) = article_embeddings.get(&article.id) {
|
if let Some(embedding) = article_embeddings.get(&article.id) {
|
||||||
let (knn, neighbours) = preference.knn(embedding, ranking);
|
let (knn, neighbours) = preference.knn(embedding, ranking);
|
||||||
@@ -680,6 +779,7 @@ pub fn preliminary_blend(signals: &mut Signals, configured: &PreliminaryWeights)
|
|||||||
let candidates = [
|
let candidates = [
|
||||||
("interest", configured.interest, 1.0),
|
("interest", configured.interest, 1.0),
|
||||||
("knn", configured.knn, signals.knn_gate),
|
("knn", configured.knn, signals.knn_gate),
|
||||||
|
("affinity", configured.affinity, signals.affinity_gate),
|
||||||
("heuristic", configured.heuristic, 1.0),
|
("heuristic", configured.heuristic, 1.0),
|
||||||
("feed", configured.feed, signals.feed_gate),
|
("feed", configured.feed, signals.feed_gate),
|
||||||
("social", configured.social, 1.0),
|
("social", configured.social, 1.0),
|
||||||
@@ -817,6 +917,47 @@ mod tests {
|
|||||||
assert!((matched[&2].top1_cos - 0.707).abs() < 0.01);
|
assert!((matched[&2].top1_cos - 0.707).abs() < 0.01);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn match_cut_keeps_the_score_from_the_uncut_top_three() {
|
||||||
|
let mut articles = HashMap::new();
|
||||||
|
for id in 1..=30 {
|
||||||
|
let mut vector = vec![0.0; 30];
|
||||||
|
vector[id - 1] = 1.0;
|
||||||
|
articles.insert(id as ArticleId, vector);
|
||||||
|
}
|
||||||
|
let interest_at_z = |target: f64| {
|
||||||
|
let mean = -target / 29.0;
|
||||||
|
let spread = ((30.0 - target * target - target * target / 29.0) / 812.0).sqrt();
|
||||||
|
let mut vector = vec![mean + spread; 30];
|
||||||
|
vector[0] = target;
|
||||||
|
vector[29] = mean - 28.0 * spread;
|
||||||
|
unit(
|
||||||
|
&vector
|
||||||
|
.into_iter()
|
||||||
|
.map(|value| value as f32)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let interests = HashMap::from([
|
||||||
|
("first".to_string(), interest_at_z(2.0)),
|
||||||
|
("second".to_string(), interest_at_z(1.5)),
|
||||||
|
("weak third".to_string(), interest_at_z(0.4)),
|
||||||
|
]);
|
||||||
|
|
||||||
|
let matched = interest_matches(&articles, &interests);
|
||||||
|
let first = &matched[&1];
|
||||||
|
assert_eq!(
|
||||||
|
first
|
||||||
|
.top_interests
|
||||||
|
.iter()
|
||||||
|
.map(|interest| interest.name.as_str())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec!["first", "second"]
|
||||||
|
);
|
||||||
|
let uncut_score = 0.7 * 2.0 + 0.3 * ((2.0 + 1.5 + 0.4) / 3.0);
|
||||||
|
assert!((first.score - uncut_score).abs() < 1e-5, "{}", first.score);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn interest_falls_back_to_raw_cosine_under_thirty_articles() {
|
fn interest_falls_back_to_raw_cosine_under_thirty_articles() {
|
||||||
let (articles, interests) = interest_fixture(10);
|
let (articles, interests) = interest_fixture(10);
|
||||||
@@ -826,6 +967,11 @@ mod tests {
|
|||||||
(m.score - m.top1_cos).abs() < 1e-9,
|
(m.score - m.top1_cos).abs() < 1e-9,
|
||||||
"article {id} should use raw top-1"
|
"article {id} should use raw top-1"
|
||||||
);
|
);
|
||||||
|
assert!(
|
||||||
|
m.top_interests
|
||||||
|
.iter()
|
||||||
|
.all(|interest| interest.z >= MATCH_MIN_Z)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
assert!((matched[&2].score - 0.707).abs() < 0.01);
|
assert!((matched[&2].score - 0.707).abs() < 0.01);
|
||||||
}
|
}
|
||||||
@@ -930,6 +1076,99 @@ mod tests {
|
|||||||
assert_eq!(state.knn(&unit(&[1.0, 0.0]), &ranking), (None, Vec::new()));
|
assert_eq!(state.knn(&unit(&[1.0, 0.0]), &ranking), (None, Vec::new()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn affinity_is_absent_under_the_gate_and_without_rated_interests() {
|
||||||
|
let top = [TopInterest {
|
||||||
|
name: "Rust".into(),
|
||||||
|
z: 3.0,
|
||||||
|
cos: 0.8,
|
||||||
|
}];
|
||||||
|
let rates = HashMap::from([(
|
||||||
|
"Rust".to_string(),
|
||||||
|
Rate {
|
||||||
|
up: 3.0,
|
||||||
|
down: 0.0,
|
||||||
|
n: 1,
|
||||||
|
},
|
||||||
|
)]);
|
||||||
|
let closed = PreferenceState::build(Vec::new(), &ranking()).with_interest_rates(
|
||||||
|
rates,
|
||||||
|
1,
|
||||||
|
&ranking(),
|
||||||
|
);
|
||||||
|
assert_eq!(closed.affinity(&top), None);
|
||||||
|
|
||||||
|
let mut open_ranking = ranking();
|
||||||
|
open_ranking.affinity_floor = 0;
|
||||||
|
open_ranking.affinity_full = 1;
|
||||||
|
let empty = PreferenceState::build(Vec::new(), &open_ranking).with_interest_rates(
|
||||||
|
HashMap::new(),
|
||||||
|
1,
|
||||||
|
&open_ranking,
|
||||||
|
);
|
||||||
|
assert_eq!(empty.affinity(&top), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn affinity_load_counts_ratings_without_embeddings() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let db = Db::open_and_migrate(&dir.path().join("signals.db"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO articles (id, canonical_url, title, first_seen)
|
||||||
|
VALUES (1, 'https://example.com/1', 'Rated', '2026-09-13T00:00:00Z')",
|
||||||
|
)
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let now = Timestamp::now();
|
||||||
|
let interests::AddOutcome::Added(interest_id) =
|
||||||
|
interests::add(&db, "Rust", None, now).await.unwrap()
|
||||||
|
else {
|
||||||
|
unreachable!();
|
||||||
|
};
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO article_interests (article_id, interest_id, cos, z)
|
||||||
|
VALUES (1, ?, 0.8, 3.0)",
|
||||||
|
)
|
||||||
|
.bind(interest_id)
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO rating_events
|
||||||
|
(article_id, kind, source, label, value, event_at)
|
||||||
|
VALUES (1, 'explicit', 'test', 'loved', 1.0, ?)",
|
||||||
|
)
|
||||||
|
.bind(now.to_string())
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut ranking = ranking();
|
||||||
|
ranking.affinity_floor = 0;
|
||||||
|
ranking.affinity_full = 1;
|
||||||
|
let voyage = VoyageConfig {
|
||||||
|
enabled: false,
|
||||||
|
..VoyageConfig::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let state = PreferenceState::load(&db, &voyage, &ranking, now)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(state.examples.is_empty());
|
||||||
|
assert_eq!(state.attributable_interest_ratings, 1);
|
||||||
|
assert_eq!(state.affinity_gate, 1.0);
|
||||||
|
let affinity = state
|
||||||
|
.affinity(&[TopInterest {
|
||||||
|
name: "Rust".into(),
|
||||||
|
z: 3.0,
|
||||||
|
cos: 0.8,
|
||||||
|
}])
|
||||||
|
.unwrap();
|
||||||
|
assert!((affinity - 1.0 / 6.0).abs() < 1e-9);
|
||||||
|
}
|
||||||
|
|
||||||
// --- §9.3 feed affinity ---
|
// --- §9.3 feed affinity ---
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1165,8 +1404,8 @@ mod tests {
|
|||||||
assert!((signals.weights.values().sum::<f64>() - 1.0).abs() < 1e-9);
|
assert!((signals.weights.values().sum::<f64>() - 1.0).abs() < 1e-9);
|
||||||
assert!(!signals.weights.contains_key("knn"));
|
assert!(!signals.weights.contains_key("knn"));
|
||||||
assert!(!signals.weights.contains_key("social"));
|
assert!(!signals.weights.contains_key("social"));
|
||||||
// 0.35/0.55 × 0.8 + 0.20/0.55 × 0.4 = 0.6545…
|
// 0.30/0.50 × 0.8 + 0.20/0.50 × 0.4 = 0.64.
|
||||||
assert!((blend - 65.4545).abs() < 0.01, "{blend}");
|
assert!((blend - 64.0).abs() < 1e-9, "{blend}");
|
||||||
|
|
||||||
let mut only_heuristic = Signals {
|
let mut only_heuristic = Signals {
|
||||||
heuristic: Some(2.0),
|
heuristic: Some(2.0),
|
||||||
@@ -1192,6 +1431,61 @@ mod tests {
|
|||||||
assert!((signals.weights["knn"] - 0.125 / 0.325).abs() < 1e-9);
|
assert!((signals.weights["knn"] - 0.125 / 0.325).abs() < 1e-9);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn liked_interest_outranks_disliked_interest_with_affinity_in_the_blend() {
|
||||||
|
let mut ranking = ranking();
|
||||||
|
ranking.affinity_floor = 0;
|
||||||
|
ranking.affinity_full = 1;
|
||||||
|
let rates = HashMap::from([
|
||||||
|
(
|
||||||
|
"liked".to_string(),
|
||||||
|
Rate {
|
||||||
|
up: 3.0,
|
||||||
|
down: 0.0,
|
||||||
|
n: 1,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"disliked".to_string(),
|
||||||
|
Rate {
|
||||||
|
up: 0.0,
|
||||||
|
down: 4.0 / 3.0,
|
||||||
|
n: 1,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
let state =
|
||||||
|
PreferenceState::build(Vec::new(), &ranking).with_interest_rates(rates, 1, &ranking);
|
||||||
|
let top = |name: &str| {
|
||||||
|
vec![TopInterest {
|
||||||
|
name: name.into(),
|
||||||
|
z: 3.0,
|
||||||
|
cos: 0.8,
|
||||||
|
}]
|
||||||
|
};
|
||||||
|
let mut signals = [
|
||||||
|
Signals {
|
||||||
|
affinity: state.affinity(&top("liked")),
|
||||||
|
heuristic: Some(1.0),
|
||||||
|
affinity_gate: state.affinity_gate,
|
||||||
|
..Signals::default()
|
||||||
|
},
|
||||||
|
Signals {
|
||||||
|
affinity: state.affinity(&top("disliked")),
|
||||||
|
heuristic: Some(1.0),
|
||||||
|
affinity_gate: state.affinity_gate,
|
||||||
|
..Signals::default()
|
||||||
|
},
|
||||||
|
];
|
||||||
|
normalize(&mut signals.iter_mut().collect::<Vec<_>>());
|
||||||
|
for signal in &mut signals {
|
||||||
|
preliminary_blend(signal, &ranking.weights.preliminary);
|
||||||
|
assert!((signal.weights.values().sum::<f64>() - 1.0).abs() < 1e-9);
|
||||||
|
assert!(signal.weights.contains_key("affinity"));
|
||||||
|
}
|
||||||
|
assert!(signals[0].preliminary > signals[1].preliminary);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn compute_scores_every_article_and_leaves_ungated_signals_absent() {
|
fn compute_scores_every_article_and_leaves_ungated_signals_absent() {
|
||||||
let articles = vec![article(1, &[10]), article(2, &[20]), article(3, &[30])];
|
let articles = vec![article(1, &[10]), article(2, &[20]), article(3, &[30])];
|
||||||
|
|||||||
+18
-5
@@ -19,10 +19,11 @@ use crate::report::RunReport;
|
|||||||
use crate::types::{ArticleId, Candidate, NearMiss};
|
use crate::types::{ArticleId, Candidate, NearMiss};
|
||||||
|
|
||||||
/// Signal names rendered by `explain`, in the order of §7.5.
|
/// Signal names rendered by `explain`, in the order of §7.5.
|
||||||
const RENDERED_SIGNALS: [&str; 8] = [
|
const RENDERED_SIGNALS: [&str; 9] = [
|
||||||
"interest",
|
"interest",
|
||||||
"knn",
|
"knn",
|
||||||
"feed",
|
"feed",
|
||||||
|
"affinity",
|
||||||
"social",
|
"social",
|
||||||
"heuristic",
|
"heuristic",
|
||||||
"triage",
|
"triage",
|
||||||
@@ -159,6 +160,7 @@ pub fn serialize_signals(signals: &Signals, auto_include: bool) -> String {
|
|||||||
"interest_top1_cos",
|
"interest_top1_cos",
|
||||||
"knn",
|
"knn",
|
||||||
"feed",
|
"feed",
|
||||||
|
"affinity",
|
||||||
"social",
|
"social",
|
||||||
"heuristic",
|
"heuristic",
|
||||||
] {
|
] {
|
||||||
@@ -1157,11 +1159,17 @@ mod tests {
|
|||||||
Signals {
|
Signals {
|
||||||
interest: Some(1.2),
|
interest: Some(1.2),
|
||||||
interest_top1_cos: Some(0.61),
|
interest_top1_cos: Some(0.61),
|
||||||
|
affinity: Some(0.2),
|
||||||
heuristic: Some(heuristic),
|
heuristic: Some(heuristic),
|
||||||
norm: BTreeMap::from([("heuristic".into(), norm), ("interest".into(), 0.9)]),
|
norm: BTreeMap::from([
|
||||||
|
("affinity".into(), 0.7),
|
||||||
|
("heuristic".into(), norm),
|
||||||
|
("interest".into(), 0.9),
|
||||||
|
]),
|
||||||
weights: BTreeMap::from([
|
weights: BTreeMap::from([
|
||||||
("heuristic".into(), 0.2 / 0.55),
|
("affinity".into(), 0.1 / 0.6),
|
||||||
("interest".into(), 0.35 / 0.55),
|
("heuristic".into(), 0.2 / 0.6),
|
||||||
|
("interest".into(), 0.3 / 0.6),
|
||||||
]),
|
]),
|
||||||
top_interests: vec![TopInterest {
|
top_interests: vec![TopInterest {
|
||||||
name: "Gaussian Splatting".into(),
|
name: "Gaussian Splatting".into(),
|
||||||
@@ -1186,6 +1194,7 @@ mod tests {
|
|||||||
assert_eq!(parsed["v"], 1);
|
assert_eq!(parsed["v"], 1);
|
||||||
assert_eq!(parsed["raw"]["heuristic"], 41.0);
|
assert_eq!(parsed["raw"]["heuristic"], 41.0);
|
||||||
assert_eq!(parsed["raw"]["interest_top1_cos"], 0.61);
|
assert_eq!(parsed["raw"]["interest_top1_cos"], 0.61);
|
||||||
|
assert_eq!(parsed["raw"]["affinity"], 0.2);
|
||||||
assert_eq!(parsed["present"]["heuristic"], true);
|
assert_eq!(parsed["present"]["heuristic"], true);
|
||||||
assert_eq!(parsed["present"]["knn"], false);
|
assert_eq!(parsed["present"]["knn"], false);
|
||||||
assert_eq!(parsed["present"]["quality"], false);
|
assert_eq!(parsed["present"]["quality"], false);
|
||||||
@@ -1385,7 +1394,11 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let squashed = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
let squashed = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||||
assert!(
|
assert!(
|
||||||
squashed.contains("heuristic 41.000 · 0.550 · 0.364"),
|
squashed.contains("heuristic 41.000 · 0.550 · 0.333"),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
squashed.contains("affinity 0.200 · 0.700 · 0.167"),
|
||||||
"{text}"
|
"{text}"
|
||||||
);
|
);
|
||||||
assert!(squashed.contains("knn absent"), "{text}");
|
assert!(squashed.contains("knn absent"), "{text}");
|
||||||
|
|||||||
@@ -1689,6 +1689,10 @@ mod tests {
|
|||||||
.execute(&pool)
|
.execute(&pool)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
sqlx::raw_sql(include_str!("../migrations/0013_interests.sql"))
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT article_id, issue_date, kind, source, label, value, event_at, user_id
|
"SELECT article_id, issue_date, kind, source, label, value, event_at, user_id
|
||||||
@@ -1724,6 +1728,8 @@ mod tests {
|
|||||||
"interest_embeddings",
|
"interest_embeddings",
|
||||||
"article_assessments",
|
"article_assessments",
|
||||||
"candidate_runs",
|
"candidate_runs",
|
||||||
|
"interests",
|
||||||
|
"article_interests",
|
||||||
"users",
|
"users",
|
||||||
"sessions",
|
"sessions",
|
||||||
"config_changes",
|
"config_changes",
|
||||||
|
|||||||
+40
-4
@@ -232,7 +232,24 @@ fn facet_label(token: &str) -> Option<String> {
|
|||||||
pub struct Understanding {
|
pub struct Understanding {
|
||||||
pub kicker: Option<String>,
|
pub kicker: Option<String>,
|
||||||
pub topics: Option<String>,
|
pub topics: Option<String>,
|
||||||
pub interests: Option<String>,
|
pub interests: Vec<InterestRef>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
pub struct InterestRef {
|
||||||
|
pub name: String,
|
||||||
|
pub href: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Understanding {
|
||||||
|
/// EPUBs keep interest names as plain text, without dashboard links.
|
||||||
|
pub fn interests_line(&self) -> String {
|
||||||
|
self.interests
|
||||||
|
.iter()
|
||||||
|
.map(|interest| interest.name.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" \u{00b7} ")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reader-facing assessment details, split so templates can give each part the
|
/// Reader-facing assessment details, split so templates can give each part the
|
||||||
@@ -257,7 +274,14 @@ pub fn understanding(pick: &Pick) -> Understanding {
|
|||||||
});
|
});
|
||||||
(kicker, topics)
|
(kicker, topics)
|
||||||
});
|
});
|
||||||
let interests = (!pick.top_interests.is_empty()).then(|| pick.top_interests.join(" \u{00b7} "));
|
let interests = pick
|
||||||
|
.top_interests
|
||||||
|
.iter()
|
||||||
|
.map(|name| InterestRef {
|
||||||
|
name: name.clone(),
|
||||||
|
href: crate::interests::articles_href(name),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
Understanding {
|
Understanding {
|
||||||
kicker,
|
kicker,
|
||||||
@@ -811,7 +835,16 @@ mod tests {
|
|||||||
Understanding {
|
Understanding {
|
||||||
kicker: Some("Software engineering · Analysis".into()),
|
kicker: Some("Software engineering · Analysis".into()),
|
||||||
topics: Some("copy-on-write · ZFS".into()),
|
topics: Some("copy-on-write · ZFS".into()),
|
||||||
interests: Some("Filesystems · Rust".into()),
|
interests: vec![
|
||||||
|
InterestRef {
|
||||||
|
name: "Filesystems".into(),
|
||||||
|
href: "/dashboard/articles?interest=Filesystems".into(),
|
||||||
|
},
|
||||||
|
InterestRef {
|
||||||
|
name: "Rust".into(),
|
||||||
|
href: "/dashboard/articles?interest=Rust".into(),
|
||||||
|
},
|
||||||
|
],
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -833,7 +866,10 @@ mod tests {
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
understanding(&pick),
|
understanding(&pick),
|
||||||
Understanding {
|
Understanding {
|
||||||
interests: Some("Rust".into()),
|
interests: vec![InterestRef {
|
||||||
|
name: "Rust".into(),
|
||||||
|
href: "/dashboard/articles?interest=Rust".into(),
|
||||||
|
}],
|
||||||
..Understanding::default()
|
..Understanding::default()
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -10,13 +10,13 @@
|
|||||||
{% if understanding.kicker.is_some() || understanding.topics.is_some() %}
|
{% if understanding.kicker.is_some() || understanding.topics.is_some() %}
|
||||||
<p class="rubric">{% if let Some(kicker) = understanding.kicker %}<span class="kicker">{{ kicker }}</span>{% if let Some(topics) = understanding.topics %}   {{ topics }}{% endif %}{% else %}{% if let Some(topics) = understanding.topics %}{{ topics }}{% endif %}{% endif %}</p>
|
<p class="rubric">{% if let Some(kicker) = understanding.kicker %}<span class="kicker">{{ kicker }}</span>{% if let Some(topics) = understanding.topics %}   {{ topics }}{% endif %}{% else %}{% if let Some(topics) = understanding.topics %}{{ topics }}{% endif %}{% endif %}</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if why.is_some() || understanding.interests.is_some() %}
|
{% if why.is_some() || !understanding.interests.is_empty() %}
|
||||||
<div class="why">
|
<div class="why">
|
||||||
{% if let Some(text) = why %}
|
{% if let Some(text) = why %}
|
||||||
<p class="why-line"><em>Why it's here: {{ text }}</em></p>
|
<p class="why-line"><em>Why it's here: {{ text }}</em></p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if let Some(interests) = understanding.interests %}
|
{% if !understanding.interests.is_empty() %}
|
||||||
<p class="why-matches">Matches: {{ interests }}</p>
|
<p class="why-matches">Matches: {{ understanding.interests_line() }}</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|||||||
@@ -13,13 +13,13 @@
|
|||||||
{% if entry.understanding.kicker.is_some() || entry.understanding.topics.is_some() %}
|
{% if entry.understanding.kicker.is_some() || entry.understanding.topics.is_some() %}
|
||||||
<p class="rubric">{% if let Some(kicker) = entry.understanding.kicker %}<span class="kicker">{{ kicker }}</span>{% if let Some(topics) = entry.understanding.topics %}   {{ topics }}{% endif %}{% else %}{% if let Some(topics) = entry.understanding.topics %}{{ topics }}{% endif %}{% endif %}</p>
|
<p class="rubric">{% if let Some(kicker) = entry.understanding.kicker %}<span class="kicker">{{ kicker }}</span>{% if let Some(topics) = entry.understanding.topics %}   {{ topics }}{% endif %}{% else %}{% if let Some(topics) = entry.understanding.topics %}{{ topics }}{% endif %}{% endif %}</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if entry.why.is_some() || entry.understanding.interests.is_some() %}
|
{% if entry.why.is_some() || !entry.understanding.interests.is_empty() %}
|
||||||
<div class="index-why">
|
<div class="index-why">
|
||||||
{% if let Some(text) = entry.why %}
|
{% if let Some(text) = entry.why %}
|
||||||
<p class="why-line"><em>Why it's here: {{ text }}</em></p>
|
<p class="why-line"><em>Why it's here: {{ text }}</em></p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if let Some(interests) = entry.understanding.interests %}
|
{% if !entry.understanding.interests.is_empty() %}
|
||||||
<p class="why-matches">Matches: {{ interests }}</p>
|
<p class="why-matches">Matches: {{ entry.understanding.interests_line() }}</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|||||||
@@ -0,0 +1,812 @@
|
|||||||
|
//! Standing-interest storage and rating-derived weights.
|
||||||
|
//!
|
||||||
|
//! Interest queries stay here so the central database layer remains focused on
|
||||||
|
//! the pipeline's shared records.
|
||||||
|
|
||||||
|
use std::collections::{BTreeSet, HashMap};
|
||||||
|
use std::fmt::Write as _;
|
||||||
|
|
||||||
|
use anyhow::{Context as _, Result, bail};
|
||||||
|
use jiff::Timestamp;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use sqlx::Row as _;
|
||||||
|
|
||||||
|
use crate::config::Config;
|
||||||
|
use crate::curate::llm::{LlmClient, Llms, provider_meters};
|
||||||
|
use crate::curate::profile;
|
||||||
|
use crate::curate::signals::TopInterest;
|
||||||
|
use crate::db::{Db, fmt_ts};
|
||||||
|
use crate::types::ArticleId;
|
||||||
|
|
||||||
|
/// One standing interest and its optional prompt category.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct Interest {
|
||||||
|
pub id: i64,
|
||||||
|
pub name: String,
|
||||||
|
pub category: Option<String>,
|
||||||
|
pub created_at: String,
|
||||||
|
pub categorized_at: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Result of adding a name whose uniqueness is case-insensitive.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum AddOutcome {
|
||||||
|
Added(i64),
|
||||||
|
Duplicate,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One stored article-to-interest match.
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
pub struct MatchRow {
|
||||||
|
pub article_id: ArticleId,
|
||||||
|
pub interest_id: i64,
|
||||||
|
pub name: String,
|
||||||
|
pub cos: f64,
|
||||||
|
pub z: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rating credit accumulated for one interest.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq)]
|
||||||
|
pub struct Rate {
|
||||||
|
pub up: f64,
|
||||||
|
pub down: f64,
|
||||||
|
pub n: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Rate {
|
||||||
|
/// Beta smoothing keeps an unrated interest neutral.
|
||||||
|
pub fn weight(&self) -> f64 {
|
||||||
|
(self.up + 1.0) / (self.up + self.down + 2.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Interest rates plus the number of ratings that could affect them.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq)]
|
||||||
|
pub struct Rates {
|
||||||
|
pub by_interest: HashMap<i64, Rate>,
|
||||||
|
pub attributable: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
const INTEREST_COLUMNS: &str = "id, name, category, created_at, categorized_at";
|
||||||
|
|
||||||
|
/// Dashboard article search for one interest name.
|
||||||
|
pub fn articles_href(name: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"/dashboard/articles?interest={}",
|
||||||
|
crate::web::encode_component(name)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse an OPML export for the one-time interests importer.
|
||||||
|
pub fn parse_opml(raw: &str) -> Vec<String> {
|
||||||
|
let mut seen = BTreeSet::new();
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for chunk in raw.split("text=\"").skip(1) {
|
||||||
|
let Some((value, _)) = chunk.split_once('"') else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let name = xml_unescape(value).trim().to_string();
|
||||||
|
if !name.is_empty() && seen.insert(name.to_lowercase()) {
|
||||||
|
out.push(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn xml_unescape(value: &str) -> String {
|
||||||
|
if !value.contains('&') {
|
||||||
|
return value.to_string();
|
||||||
|
}
|
||||||
|
value
|
||||||
|
.replace("<", "<")
|
||||||
|
.replace(">", ">")
|
||||||
|
.replace(""", "\"")
|
||||||
|
.replace("'", "'")
|
||||||
|
.replace("'", "'")
|
||||||
|
.replace("&", "&")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn interest_from(row: &sqlx::sqlite::SqliteRow) -> Interest {
|
||||||
|
Interest {
|
||||||
|
id: row.get("id"),
|
||||||
|
name: row.get("name"),
|
||||||
|
category: row.get("category"),
|
||||||
|
created_at: row.get("created_at"),
|
||||||
|
categorized_at: row.get("categorized_at"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// All interests, ordered case-insensitively by name.
|
||||||
|
pub async fn list(db: &Db) -> Result<Vec<Interest>> {
|
||||||
|
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||||
|
"SELECT {INTEREST_COLUMNS} FROM interests ORDER BY name COLLATE NOCASE, name"
|
||||||
|
)))
|
||||||
|
.fetch_all(db.pool())
|
||||||
|
.await?;
|
||||||
|
Ok(rows.iter().map(interest_from).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Add one trimmed, non-empty name of at most 80 characters.
|
||||||
|
pub async fn add(
|
||||||
|
db: &Db,
|
||||||
|
name: &str,
|
||||||
|
category: Option<&str>,
|
||||||
|
now: Timestamp,
|
||||||
|
) -> Result<AddOutcome> {
|
||||||
|
let name = name.trim();
|
||||||
|
let len = name.chars().count();
|
||||||
|
if !(1..=80).contains(&len) {
|
||||||
|
bail!("interest name must be 1–80 characters");
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = sqlx::query(
|
||||||
|
"INSERT OR IGNORE INTO interests (name, category, created_at, categorized_at)
|
||||||
|
VALUES (?, ?, ?, ?)",
|
||||||
|
)
|
||||||
|
.bind(name)
|
||||||
|
.bind(category)
|
||||||
|
.bind(fmt_ts(now))
|
||||||
|
.bind(category.map(|_| fmt_ts(now)))
|
||||||
|
.execute(db.pool())
|
||||||
|
.await?;
|
||||||
|
if result.rows_affected() == 0 {
|
||||||
|
Ok(AddOutcome::Duplicate)
|
||||||
|
} else {
|
||||||
|
Ok(AddOutcome::Added(result.last_insert_rowid()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set or clear an interest category and its categorization timestamp together.
|
||||||
|
pub async fn set_category(db: &Db, id: i64, category: Option<&str>, now: Timestamp) -> Result<()> {
|
||||||
|
sqlx::query("UPDATE interests SET category = ?, categorized_at = ? WHERE id = ?")
|
||||||
|
.bind(category)
|
||||||
|
.bind(category.map(|_| fmt_ts(now)))
|
||||||
|
.bind(id)
|
||||||
|
.execute(db.pool())
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Delete an interest, its match rows, and its name-keyed cached embedding.
|
||||||
|
pub async fn delete(db: &Db, id: i64) -> Result<()> {
|
||||||
|
let mut tx = db.pool().begin().await?;
|
||||||
|
let name: Option<String> = sqlx::query_scalar("SELECT name FROM interests WHERE id = ?")
|
||||||
|
.bind(id)
|
||||||
|
.fetch_optional(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
sqlx::query("DELETE FROM interests WHERE id = ?")
|
||||||
|
.bind(id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
if let Some(name) = name {
|
||||||
|
sqlx::query("DELETE FROM interest_embeddings WHERE interest = ?")
|
||||||
|
.bind(name)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// All names in the stable order used for embedding requests.
|
||||||
|
pub async fn names(db: &Db) -> Result<Vec<String>> {
|
||||||
|
Ok(
|
||||||
|
sqlx::query_scalar("SELECT name FROM interests ORDER BY name COLLATE NOCASE, name")
|
||||||
|
.fetch_all(db.pool())
|
||||||
|
.await?,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Names grouped for the prompt, with uncategorized interests last.
|
||||||
|
pub async fn grouped(db: &Db) -> Result<Vec<(String, Vec<String>)>> {
|
||||||
|
let mut by_category: HashMap<String, Vec<String>> = HashMap::new();
|
||||||
|
let mut other = Vec::new();
|
||||||
|
for interest in list(db).await? {
|
||||||
|
if let Some(category) = interest.category {
|
||||||
|
by_category.entry(category).or_default().push(interest.name);
|
||||||
|
} else {
|
||||||
|
other.push(interest.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut groups: Vec<_> = by_category.into_iter().collect();
|
||||||
|
groups.sort_by(|left, right| {
|
||||||
|
left.0
|
||||||
|
.to_lowercase()
|
||||||
|
.cmp(&right.0.to_lowercase())
|
||||||
|
.then_with(|| left.0.cmp(&right.0))
|
||||||
|
});
|
||||||
|
for (_, members) in &mut groups {
|
||||||
|
members.sort_by(|left, right| {
|
||||||
|
left.to_lowercase()
|
||||||
|
.cmp(&right.to_lowercase())
|
||||||
|
.then_with(|| left.cmp(right))
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if !other.is_empty() {
|
||||||
|
groups.push(("Other standing interests".to_string(), other));
|
||||||
|
}
|
||||||
|
Ok(groups)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Interests awaiting the categorizer, ordered case-insensitively by name.
|
||||||
|
pub async fn uncategorized(db: &Db) -> Result<Vec<Interest>> {
|
||||||
|
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||||
|
"SELECT {INTEREST_COLUMNS} FROM interests WHERE category IS NULL
|
||||||
|
ORDER BY name COLLATE NOCASE, name"
|
||||||
|
)))
|
||||||
|
.fetch_all(db.pool())
|
||||||
|
.await?;
|
||||||
|
Ok(rows.iter().map(interest_from).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Upsert the current run's recorded top-interest matches in one transaction.
|
||||||
|
pub async fn replace_matches(
|
||||||
|
db: &Db,
|
||||||
|
run_id: Option<i64>,
|
||||||
|
matches: &[(ArticleId, Vec<TopInterest>)],
|
||||||
|
ids: &HashMap<String, i64>,
|
||||||
|
) -> Result<()> {
|
||||||
|
write_matches(db, run_id, matches, ids, true).await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert backfilled matches without disturbing rows a real run wrote.
|
||||||
|
pub async fn insert_matches_if_absent(
|
||||||
|
db: &Db,
|
||||||
|
matches: &[(ArticleId, Vec<TopInterest>)],
|
||||||
|
ids: &HashMap<String, i64>,
|
||||||
|
) -> Result<u64> {
|
||||||
|
write_matches(db, None, matches, ids, false).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One transaction over the top interests of every article, either upserting
|
||||||
|
/// (a run's own rows) or leaving whatever is already stored alone (a backfill).
|
||||||
|
async fn write_matches(
|
||||||
|
db: &Db,
|
||||||
|
run_id: Option<i64>,
|
||||||
|
matches: &[(ArticleId, Vec<TopInterest>)],
|
||||||
|
ids: &HashMap<String, i64>,
|
||||||
|
replace: bool,
|
||||||
|
) -> Result<u64> {
|
||||||
|
let sql = if replace {
|
||||||
|
"INSERT INTO article_interests (article_id, interest_id, cos, z, run_id)
|
||||||
|
VALUES (?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(article_id, interest_id) DO UPDATE SET
|
||||||
|
cos = excluded.cos, z = excluded.z, run_id = excluded.run_id"
|
||||||
|
} else {
|
||||||
|
"INSERT OR IGNORE INTO article_interests (article_id, interest_id, cos, z, run_id)
|
||||||
|
VALUES (?, ?, ?, ?, ?)"
|
||||||
|
};
|
||||||
|
let mut tx = db.pool().begin().await?;
|
||||||
|
let mut written = 0;
|
||||||
|
for (article_id, top_interests) in matches {
|
||||||
|
for top in top_interests {
|
||||||
|
let Some(interest_id) = ids.get(&top.name) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
written += sqlx::query(sql)
|
||||||
|
.bind(article_id)
|
||||||
|
.bind(interest_id)
|
||||||
|
.bind(top.cos)
|
||||||
|
.bind(top.z)
|
||||||
|
.bind(run_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(written)
|
||||||
|
}
|
||||||
|
|
||||||
|
const CATEGORIZE_PROMPT: &str = r#"TASK: file each new standing interest under one of the reader's interest categories.
|
||||||
|
Existing categories (reuse these names verbatim): {categories}
|
||||||
|
Create a new category only when none of the existing ones fits; a new category must be broad enough to hold several interests and named like the existing ones (two to five words, sentence case). Every interest gets exactly one category.
|
||||||
|
New interests: {interests}
|
||||||
|
Return JSON exactly: {"assignments":[{"interest":"…","category":"…"}]}"#;
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct CategorizeResponse {
|
||||||
|
#[serde(default)]
|
||||||
|
assignments: Vec<CategoryAssignment>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct CategoryAssignment {
|
||||||
|
interest: String,
|
||||||
|
category: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// File every currently uncategorized interest in one bulk-model call.
|
||||||
|
pub async fn categorize(config: &Config, db: &Db) -> Result<String> {
|
||||||
|
let pending = uncategorized(db).await?;
|
||||||
|
if pending.is_empty() {
|
||||||
|
return Ok("nothing to categorize".into());
|
||||||
|
}
|
||||||
|
let taste = profile::load_or_build(
|
||||||
|
db,
|
||||||
|
&config.profile_path,
|
||||||
|
config.curation.feedback.verdicts_in_prompt,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let llms = Llms::from_config(config, taste.text, &provider_meters(config));
|
||||||
|
let llm = llms
|
||||||
|
.bulk
|
||||||
|
.as_ref()
|
||||||
|
.or_else(|| llms.editor_or_bulk())
|
||||||
|
.context("no LLM provider is available for interest categorization")?;
|
||||||
|
categorize_with_llm(db, &pending, llm).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn categorize_with_llm(db: &Db, pending: &[Interest], llm: &LlmClient) -> Result<String> {
|
||||||
|
let all = list(db).await?;
|
||||||
|
let categories = all
|
||||||
|
.iter()
|
||||||
|
.filter_map(|interest| interest.category.as_deref())
|
||||||
|
.collect::<BTreeSet<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let names = pending
|
||||||
|
.iter()
|
||||||
|
.map(|interest| interest.name.as_str())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let prompt = CATEGORIZE_PROMPT
|
||||||
|
.replace("{categories}", &categories.join(", "))
|
||||||
|
.replace("{interests}", &format!("\n{}", names.join("\n")));
|
||||||
|
let response: CategorizeResponse = llm.complete_json(&prompt, 0.2).await?;
|
||||||
|
|
||||||
|
let pending_by_name = pending
|
||||||
|
.iter()
|
||||||
|
.map(|interest| (interest.name.to_lowercase(), interest))
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
let existing_categories = categories
|
||||||
|
.iter()
|
||||||
|
.map(|category| (category.to_lowercase(), *category))
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
let mut assigned = BTreeSet::new();
|
||||||
|
let mut new_categories = BTreeSet::new();
|
||||||
|
let now = Timestamp::now();
|
||||||
|
for assignment in response.assignments {
|
||||||
|
let Some(interest) = pending_by_name.get(&assignment.interest.trim().to_lowercase()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let category = assignment.category.trim();
|
||||||
|
if !(1..=60).contains(&category.chars().count()) || !assigned.insert(interest.id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// A model that answers "software" for an existing "Software" must not
|
||||||
|
// split the category.
|
||||||
|
let category = match existing_categories.get(&category.to_lowercase()) {
|
||||||
|
Some(existing) => existing,
|
||||||
|
None => {
|
||||||
|
new_categories.insert(category.to_string());
|
||||||
|
category
|
||||||
|
}
|
||||||
|
};
|
||||||
|
set_category(db, interest.id, Some(category), now).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut message = format!(
|
||||||
|
"categorized {} ({} new categories:",
|
||||||
|
assigned.len(),
|
||||||
|
new_categories.len()
|
||||||
|
);
|
||||||
|
if !new_categories.is_empty() {
|
||||||
|
let _ = write!(
|
||||||
|
message,
|
||||||
|
" {}",
|
||||||
|
new_categories.into_iter().collect::<Vec<_>>().join(", ")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
message.push(')');
|
||||||
|
tracing::info!(%message);
|
||||||
|
Ok(message)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stored matches for the requested articles.
|
||||||
|
pub async fn matches_for_articles(db: &Db, article_ids: &[ArticleId]) -> Result<Vec<MatchRow>> {
|
||||||
|
let mut matches = Vec::new();
|
||||||
|
for chunk in article_ids.chunks(500) {
|
||||||
|
let placeholders = vec!["?"; chunk.len()].join(", ");
|
||||||
|
let mut query = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||||
|
"SELECT ai.article_id, ai.interest_id, i.name, ai.cos, ai.z
|
||||||
|
FROM article_interests ai
|
||||||
|
JOIN interests i ON i.id = ai.interest_id
|
||||||
|
WHERE ai.article_id IN ({placeholders})"
|
||||||
|
)));
|
||||||
|
for article_id in chunk {
|
||||||
|
query = query.bind(article_id);
|
||||||
|
}
|
||||||
|
for row in query.fetch_all(db.pool()).await? {
|
||||||
|
matches.push(MatchRow {
|
||||||
|
article_id: row.get("article_id"),
|
||||||
|
interest_id: row.get("interest_id"),
|
||||||
|
name: row.get("name"),
|
||||||
|
cos: row.get("cos"),
|
||||||
|
z: row.get("z"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
matches.sort_by(|left, right| {
|
||||||
|
left.article_id
|
||||||
|
.cmp(&right.article_id)
|
||||||
|
.then_with(|| right.z.total_cmp(&left.z))
|
||||||
|
.then_with(|| left.interest_id.cmp(&right.interest_id))
|
||||||
|
});
|
||||||
|
Ok(matches)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Number of stored article matches for each interest.
|
||||||
|
pub async fn match_counts(db: &Db) -> Result<HashMap<i64, i64>> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT interest_id, COUNT(*) AS matches FROM article_interests GROUP BY interest_id",
|
||||||
|
)
|
||||||
|
.fetch_all(db.pool())
|
||||||
|
.await?;
|
||||||
|
Ok(rows
|
||||||
|
.iter()
|
||||||
|
.map(|row| (row.get("interest_id"), row.get("matches")))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Derive smoothed interest rates from current ratings and their match rows.
|
||||||
|
pub fn rates(ratings: &[(ArticleId, f64, f64)], rows: &[(ArticleId, i64, f64)]) -> Rates {
|
||||||
|
let mut rows_by_article: HashMap<ArticleId, Vec<(i64, f64)>> = HashMap::new();
|
||||||
|
for &(article_id, interest_id, z) in rows {
|
||||||
|
rows_by_article
|
||||||
|
.entry(article_id)
|
||||||
|
.or_default()
|
||||||
|
.push((interest_id, z));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut result = Rates::default();
|
||||||
|
for &(article_id, value, decay) in ratings {
|
||||||
|
let mut attributed = false;
|
||||||
|
if let Some(matches) = rows_by_article.get(&article_id) {
|
||||||
|
for &(interest_id, z) in matches {
|
||||||
|
let strength = (z / 3.0).clamp(0.0, 1.0);
|
||||||
|
if strength <= 0.0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
attributed = true;
|
||||||
|
let credit = value * decay * strength;
|
||||||
|
let rate = result.by_interest.entry(interest_id).or_default();
|
||||||
|
rate.up += credit.max(0.0);
|
||||||
|
rate.down += (-credit).max(0.0);
|
||||||
|
rate.n += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if attributed {
|
||||||
|
result.attributable += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn ts(value: &str) -> Timestamp {
|
||||||
|
value.parse().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn test_db() -> (tempfile::TempDir, Db) {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
(dir, db)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn opml_parser_unescapes_trims_and_deduplicates_names() {
|
||||||
|
let interests = parse_opml(
|
||||||
|
r#"<opml><body>
|
||||||
|
<outline text=" Rust "/>
|
||||||
|
<outline text="E-Ink & RSS"/>
|
||||||
|
<outline text="rust"/>
|
||||||
|
<outline text="Quotes "and" apostrophes 'x' 'y'"/>
|
||||||
|
<outline text="Markup <tag>"/>
|
||||||
|
<outline text=""/>
|
||||||
|
</body></opml>"#,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
interests,
|
||||||
|
[
|
||||||
|
"Rust",
|
||||||
|
"E-Ink & RSS",
|
||||||
|
"Quotes \"and\" apostrophes 'x' 'y'",
|
||||||
|
"Markup <tag>",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seed_article(db: &Db, id: ArticleId) {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO articles (id, canonical_url, title, first_seen) VALUES (?, ?, ?, ?)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(format!("https://example.com/{id}"))
|
||||||
|
.bind(format!("Article {id}"))
|
||||||
|
.bind("2026-09-12T00:00:00Z")
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn add_trims_names_and_uniqueness_is_case_insensitive() {
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
let now = ts("2026-09-12T12:00:00Z");
|
||||||
|
let AddOutcome::Added(id) = add(&db, " Rust ", Some("Software"), now).await.unwrap()
|
||||||
|
else {
|
||||||
|
panic!("first insert should succeed");
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
add(&db, "rust", None, now).await.unwrap(),
|
||||||
|
AddOutcome::Duplicate
|
||||||
|
);
|
||||||
|
assert!(add(&db, " ", None, now).await.is_err());
|
||||||
|
assert!(add(&db, &"x".repeat(81), None, now).await.is_err());
|
||||||
|
|
||||||
|
let interests = list(&db).await.unwrap();
|
||||||
|
assert_eq!(interests.len(), 1);
|
||||||
|
assert_eq!(interests[0].id, id);
|
||||||
|
assert_eq!(interests[0].name, "Rust");
|
||||||
|
assert_eq!(interests[0].category.as_deref(), Some("Software"));
|
||||||
|
assert_eq!(
|
||||||
|
interests[0].categorized_at.as_deref(),
|
||||||
|
Some("2026-09-12T12:00:00Z")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rates_apply_value_decay_strength_and_negative_credit() {
|
||||||
|
let ratings = [
|
||||||
|
(1, 1.0, 1.0),
|
||||||
|
(2, 0.35, 1.0),
|
||||||
|
(3, 1.0, 0.5),
|
||||||
|
(4, -1.0, 0.5),
|
||||||
|
(5, -1.0, 1.0),
|
||||||
|
];
|
||||||
|
let rows = [
|
||||||
|
(1, 10, 3.0),
|
||||||
|
(2, 10, 1.5),
|
||||||
|
(3, 11, 3.0),
|
||||||
|
(4, 10, 0.9),
|
||||||
|
(5, 12, 0.0),
|
||||||
|
];
|
||||||
|
let rates = rates(&ratings, &rows);
|
||||||
|
|
||||||
|
assert_eq!(rates.attributable, 4);
|
||||||
|
let ten = rates.by_interest[&10];
|
||||||
|
assert!((ten.up - 1.175).abs() < 1e-12);
|
||||||
|
assert!((ten.down - 0.15).abs() < 1e-12);
|
||||||
|
assert_eq!(ten.n, 3);
|
||||||
|
assert!((ten.weight() - 2.175 / 3.325).abs() < 1e-12);
|
||||||
|
assert_eq!(
|
||||||
|
rates.by_interest[&11],
|
||||||
|
Rate {
|
||||||
|
up: 0.5,
|
||||||
|
down: 0.0,
|
||||||
|
n: 1
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert!(!rates.by_interest.contains_key(&12));
|
||||||
|
assert_eq!(Rate::default().weight(), 0.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn delete_cascades_matches_and_removes_the_embedding() {
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
seed_article(&db, 1).await;
|
||||||
|
let now = ts("2026-09-12T12:00:00Z");
|
||||||
|
let AddOutcome::Added(id) = add(&db, "Rust", None, now).await.unwrap() else {
|
||||||
|
unreachable!();
|
||||||
|
};
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO interest_embeddings
|
||||||
|
(interest, model, dimension, embedding, created_at) VALUES (?, ?, ?, ?, ?)",
|
||||||
|
)
|
||||||
|
.bind("Rust")
|
||||||
|
.bind("test")
|
||||||
|
.bind(1_i64)
|
||||||
|
.bind(vec![0_u8; 4])
|
||||||
|
.bind("2026-09-12T12:00:00Z")
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let ids = HashMap::from([("Rust".to_string(), id)]);
|
||||||
|
replace_matches(
|
||||||
|
&db,
|
||||||
|
Some(7),
|
||||||
|
&[(
|
||||||
|
1,
|
||||||
|
vec![
|
||||||
|
TopInterest {
|
||||||
|
name: "Rust".into(),
|
||||||
|
cos: 0.7,
|
||||||
|
z: 1.2,
|
||||||
|
},
|
||||||
|
TopInterest {
|
||||||
|
name: "Unknown".into(),
|
||||||
|
cos: 0.9,
|
||||||
|
z: 2.0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)],
|
||||||
|
&ids,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
replace_matches(
|
||||||
|
&db,
|
||||||
|
None,
|
||||||
|
&[(
|
||||||
|
1,
|
||||||
|
vec![TopInterest {
|
||||||
|
name: "Rust".into(),
|
||||||
|
cos: 0.8,
|
||||||
|
z: 1.5,
|
||||||
|
}],
|
||||||
|
)],
|
||||||
|
&ids,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(match_counts(&db).await.unwrap(), HashMap::from([(id, 1)]));
|
||||||
|
assert_eq!(
|
||||||
|
matches_for_articles(&db, &[1]).await.unwrap(),
|
||||||
|
[MatchRow {
|
||||||
|
article_id: 1,
|
||||||
|
interest_id: id,
|
||||||
|
name: "Rust".into(),
|
||||||
|
cos: 0.8,
|
||||||
|
z: 1.5
|
||||||
|
}]
|
||||||
|
);
|
||||||
|
let run_id: Option<i64> = sqlx::query_scalar(
|
||||||
|
"SELECT run_id FROM article_interests WHERE article_id = 1 AND interest_id = ?",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.fetch_one(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(run_id, None);
|
||||||
|
|
||||||
|
delete(&db, id).await.unwrap();
|
||||||
|
let match_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM article_interests")
|
||||||
|
.fetch_one(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let embedding_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM interest_embeddings")
|
||||||
|
.fetch_one(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(match_rows, 0);
|
||||||
|
assert_eq!(embedding_rows, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn grouped_sorts_categories_and_puts_uncategorized_last() {
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
let now = ts("2026-09-12T12:00:00Z");
|
||||||
|
add(&db, "zebra", Some("Animals"), now).await.unwrap();
|
||||||
|
add(&db, "Alpaca", Some("Animals"), now).await.unwrap();
|
||||||
|
let AddOutcome::Added(id) = add(&db, "rust", None, now).await.unwrap() else {
|
||||||
|
unreachable!();
|
||||||
|
};
|
||||||
|
add(&db, "Baking", Some("cooking"), now).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
grouped(&db).await.unwrap(),
|
||||||
|
[
|
||||||
|
("Animals".into(), vec!["Alpaca".into(), "zebra".into()]),
|
||||||
|
("cooking".into(), vec!["Baking".into()]),
|
||||||
|
("Other standing interests".into(), vec!["rust".into()]),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(uncategorized(&db).await.unwrap()[0].id, id);
|
||||||
|
|
||||||
|
set_category(&db, id, Some("Software"), now).await.unwrap();
|
||||||
|
assert!(uncategorized(&db).await.unwrap().is_empty());
|
||||||
|
set_category(&db, id, None, now).await.unwrap();
|
||||||
|
let rust = uncategorized(&db).await.unwrap().pop().unwrap();
|
||||||
|
assert_eq!(rust.categorized_at, None);
|
||||||
|
assert_eq!(
|
||||||
|
names(&db).await.unwrap(),
|
||||||
|
["Alpaca", "Baking", "rust", "zebra"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn empty_article_lookup_is_a_no_op() {
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
assert!(matches_for_articles(&db, &[]).await.unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn categorizer_short_circuits_without_uncategorized_interests() {
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
add(
|
||||||
|
&db,
|
||||||
|
"Databases",
|
||||||
|
Some("Software"),
|
||||||
|
ts("2026-09-12T12:00:00Z"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
categorize(&Config::default(), &db).await.unwrap(),
|
||||||
|
"nothing to categorize"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn categorizer_assigns_known_names_and_tracks_new_categories() {
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use crate::config::ProviderConfig;
|
||||||
|
use crate::curate::llm::{MockBackend, UsageMeter};
|
||||||
|
use crate::types::TokenUsage;
|
||||||
|
|
||||||
|
let (_dir, db) = test_db().await;
|
||||||
|
let now = ts("2026-09-12T12:00:00Z");
|
||||||
|
add(&db, "Databases", Some("Software"), now).await.unwrap();
|
||||||
|
add(&db, "Rust macros", None, now).await.unwrap();
|
||||||
|
add(&db, "Wheel-thrown pottery", None, now).await.unwrap();
|
||||||
|
let pending = uncategorized(&db).await.unwrap();
|
||||||
|
|
||||||
|
let backend = Arc::new(MockBackend::new());
|
||||||
|
backend.push(
|
||||||
|
r#"{"assignments":[
|
||||||
|
{"interest":"RUST MACROS","category":" software "},
|
||||||
|
{"interest":"Wheel-thrown pottery","category":"Creative crafts"},
|
||||||
|
{"interest":"Not in the batch","category":"Made up"}
|
||||||
|
]}"#,
|
||||||
|
TokenUsage::default(),
|
||||||
|
);
|
||||||
|
let llm = LlmClient::with_backend(
|
||||||
|
"mock",
|
||||||
|
"taste prompt".into(),
|
||||||
|
UsageMeter::for_provider(&ProviderConfig::deepseek()),
|
||||||
|
backend.clone(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let message = categorize_with_llm(&db, &pending, &llm).await.unwrap();
|
||||||
|
assert_eq!(message, "categorized 2 (1 new categories: Creative crafts)");
|
||||||
|
let stored = list(&db).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
stored
|
||||||
|
.iter()
|
||||||
|
.find(|interest| interest.name == "Rust macros")
|
||||||
|
.and_then(|interest| interest.category.as_deref()),
|
||||||
|
Some("Software")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
stored
|
||||||
|
.iter()
|
||||||
|
.find(|interest| interest.name == "Wheel-thrown pottery")
|
||||||
|
.and_then(|interest| interest.category.as_deref()),
|
||||||
|
Some("Creative crafts")
|
||||||
|
);
|
||||||
|
assert_eq!(backend.calls(), 1);
|
||||||
|
let request = &backend.prompts()[0];
|
||||||
|
assert_eq!(request.temperature, 0.2);
|
||||||
|
assert!(request.json);
|
||||||
|
assert!(
|
||||||
|
request
|
||||||
|
.user
|
||||||
|
.contains("Existing categories (reuse these names verbatim): Software")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
request
|
||||||
|
.user
|
||||||
|
.contains("New interests: \nRust macros\nWheel-thrown pottery")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
+20
-2
@@ -44,11 +44,13 @@ pub enum Job {
|
|||||||
FeaturesPrune,
|
FeaturesPrune,
|
||||||
/// `import-ratings` → process ratings-dashboard URL imports.
|
/// `import-ratings` → process ratings-dashboard URL imports.
|
||||||
ImportRatings,
|
ImportRatings,
|
||||||
|
/// `interests-categorize` → file uncategorized standing interests.
|
||||||
|
InterestsCategorize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Job {
|
impl Job {
|
||||||
/// The catalogue in the order the Jobs page lists it.
|
/// The catalogue in the order the Jobs page lists it.
|
||||||
pub const CATALOGUE: [Job; 7] = [
|
pub const CATALOGUE: [Job; 8] = [
|
||||||
Job::Generate { date: None },
|
Job::Generate { date: None },
|
||||||
Job::DryRun,
|
Job::DryRun,
|
||||||
Job::ProfileRebuild,
|
Job::ProfileRebuild,
|
||||||
@@ -56,6 +58,7 @@ impl Job {
|
|||||||
Job::BackfillSocial,
|
Job::BackfillSocial,
|
||||||
Job::FeaturesPrune,
|
Job::FeaturesPrune,
|
||||||
Job::ImportRatings,
|
Job::ImportRatings,
|
||||||
|
Job::InterestsCategorize,
|
||||||
];
|
];
|
||||||
|
|
||||||
/// `^[a-z0-9-]+$`: the only characters a job (and so a unit instance) name
|
/// `^[a-z0-9-]+$`: the only characters a job (and so a unit instance) name
|
||||||
@@ -81,6 +84,7 @@ impl Job {
|
|||||||
"backfill-social" => Some(Job::BackfillSocial),
|
"backfill-social" => Some(Job::BackfillSocial),
|
||||||
"features-prune" => Some(Job::FeaturesPrune),
|
"features-prune" => Some(Job::FeaturesPrune),
|
||||||
"import-ratings" => Some(Job::ImportRatings),
|
"import-ratings" => Some(Job::ImportRatings),
|
||||||
|
"interests-categorize" => Some(Job::InterestsCategorize),
|
||||||
_ => {
|
_ => {
|
||||||
let date = name.strip_prefix("generate-")?;
|
let date = name.strip_prefix("generate-")?;
|
||||||
// Exactly `YYYY-MM-DD`; the round trip rejects `2026-9-3`.
|
// Exactly `YYYY-MM-DD`; the round trip rejects `2026-9-3`.
|
||||||
@@ -100,6 +104,7 @@ impl Job {
|
|||||||
Job::BackfillSocial => "backfill-social".into(),
|
Job::BackfillSocial => "backfill-social".into(),
|
||||||
Job::FeaturesPrune => "features-prune".into(),
|
Job::FeaturesPrune => "features-prune".into(),
|
||||||
Job::ImportRatings => "import-ratings".into(),
|
Job::ImportRatings => "import-ratings".into(),
|
||||||
|
Job::InterestsCategorize => "interests-categorize".into(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,6 +133,9 @@ impl Job {
|
|||||||
"Drop stale embeddings, old candidate telemetry and old assessments per the retention config."
|
"Drop stale embeddings, old candidate telemetry and old assessments per the retention config."
|
||||||
}
|
}
|
||||||
Job::ImportRatings => "Fetch, embed and rate the URLs queued from the Ratings page.",
|
Job::ImportRatings => "Fetch, embed and rate the URLs queued from the Ratings page.",
|
||||||
|
Job::InterestsCategorize => {
|
||||||
|
"File uncategorized interests under categories with the bulk model, creating new ones only when needed."
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -139,7 +147,7 @@ impl Job {
|
|||||||
Job::ProfileRebuild => Some("profile rebuild"),
|
Job::ProfileRebuild => Some("profile rebuild"),
|
||||||
Job::FeaturesBackfill => Some("features backfill"),
|
Job::FeaturesBackfill => Some("features backfill"),
|
||||||
Job::BackfillSocial => Some("backfill-social"),
|
Job::BackfillSocial => Some("backfill-social"),
|
||||||
Job::FeaturesPrune | Job::ImportRatings => None,
|
Job::FeaturesPrune | Job::ImportRatings | Job::InterestsCategorize => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -459,6 +467,16 @@ mod tests {
|
|||||||
Job::ImportRatings.description(),
|
Job::ImportRatings.description(),
|
||||||
"Fetch, embed and rate the URLs queued from the Ratings page."
|
"Fetch, embed and rate the URLs queued from the Ratings page."
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
Job::parse("interests-categorize"),
|
||||||
|
Some(Job::InterestsCategorize)
|
||||||
|
);
|
||||||
|
assert_eq!(Job::InterestsCategorize.takes_lock(), None);
|
||||||
|
assert_eq!(
|
||||||
|
Job::InterestsCategorize.description(),
|
||||||
|
"File uncategorized interests under categories with the bulk model, creating new ones only when needed."
|
||||||
|
);
|
||||||
|
assert!(!Job::InterestsCategorize.dangerous());
|
||||||
assert!(Job::parse("generate-2026-09-03").unwrap().dangerous());
|
assert!(Job::parse("generate-2026-09-03").unwrap().dangerous());
|
||||||
assert!(!Job::parse("dry-run").unwrap().dangerous());
|
assert!(!Job::parse("dry-run").unwrap().dangerous());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ pub mod html;
|
|||||||
pub mod http;
|
pub mod http;
|
||||||
pub mod images;
|
pub mod images;
|
||||||
pub mod imports;
|
pub mod imports;
|
||||||
|
pub mod interests;
|
||||||
pub mod jobs;
|
pub mod jobs;
|
||||||
pub mod lock;
|
pub mod lock;
|
||||||
pub mod mail;
|
pub mod mail;
|
||||||
|
|||||||
+360
-7
@@ -3,6 +3,7 @@
|
|||||||
//! Everything of substance lives in the library (`src/lib.rs`); this binary only
|
//! Everything of substance lives in the library (`src/lib.rs`); this binary only
|
||||||
//! parses flags, loads config, opens the database and dispatches.
|
//! parses flags, loads config, opens the database and dispatches.
|
||||||
|
|
||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::io::Write as _;
|
use std::io::Write as _;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
@@ -17,7 +18,7 @@ use daily_epub::db::Db;
|
|||||||
use daily_epub::pipeline::{self, GenerateOptions, GenerateOutcome};
|
use daily_epub::pipeline::{self, GenerateOptions, GenerateOutcome};
|
||||||
use daily_epub::report::{RunReport, VOYAGE_PROVIDER};
|
use daily_epub::report::{RunReport, VOYAGE_PROVIDER};
|
||||||
use daily_epub::types::{ArticleId, Vote};
|
use daily_epub::types::{ArticleId, Vote};
|
||||||
use daily_epub::{curate, discovery, http, imports, jobs, lock, rate, server, social};
|
use daily_epub::{curate, discovery, http, imports, interests, jobs, lock, rate, server, social};
|
||||||
|
|
||||||
/// A personalized daily newspaper, delivered as an EPUB.
|
/// A personalized daily newspaper, delivered as an EPUB.
|
||||||
#[derive(Debug, Parser)]
|
#[derive(Debug, Parser)]
|
||||||
@@ -64,6 +65,9 @@ enum Command {
|
|||||||
/// Feed subscription candidates (feed discovery plan §4 step 6).
|
/// Feed subscription candidates (feed discovery plan §4 step 6).
|
||||||
#[command(subcommand)]
|
#[command(subcommand)]
|
||||||
Feeds(FeedsCommand),
|
Feeds(FeedsCommand),
|
||||||
|
/// Import and backfill standing interests.
|
||||||
|
#[command(subcommand)]
|
||||||
|
Interests(InterestsCommand),
|
||||||
/// Operator jobs (what `daily-epub-job@<name>.service` runs).
|
/// Operator jobs (what `daily-epub-job@<name>.service` runs).
|
||||||
#[command(subcommand)]
|
#[command(subcommand)]
|
||||||
Job(JobCommand),
|
Job(JobCommand),
|
||||||
@@ -74,7 +78,8 @@ enum JobCommand {
|
|||||||
/// Run one catalogue job in-process and record it in the `jobs` table.
|
/// Run one catalogue job in-process and record it in the `jobs` table.
|
||||||
Run {
|
Run {
|
||||||
/// `generate`, `generate-YYYY-MM-DD`, `dry-run`, `profile-rebuild`,
|
/// `generate`, `generate-YYYY-MM-DD`, `dry-run`, `profile-rebuild`,
|
||||||
/// `features-backfill`, `backfill-social`, `features-prune` or `import-ratings`.
|
/// `features-backfill`, `backfill-social`, `features-prune`, `import-ratings`
|
||||||
|
/// or `interests-categorize`.
|
||||||
name: String,
|
name: String,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -86,6 +91,24 @@ enum FeedsCommand {
|
|||||||
Discover(FeedsDiscoverArgs),
|
Discover(FeedsDiscoverArgs),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Subcommand)]
|
||||||
|
enum InterestsCommand {
|
||||||
|
/// Import standing interests from OPML and the profile's Interests section.
|
||||||
|
Import(InterestsImportArgs),
|
||||||
|
/// Match every compatible cached article embedding to standing interests.
|
||||||
|
Backfill,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, clap::Args)]
|
||||||
|
struct InterestsImportArgs {
|
||||||
|
/// OPML input (defaults to data/scour-interests.opml).
|
||||||
|
#[arg(long, value_name = "PATH")]
|
||||||
|
opml: Option<PathBuf>,
|
||||||
|
/// Profile input (defaults to profile_path from the configuration).
|
||||||
|
#[arg(long, value_name = "PATH")]
|
||||||
|
profile: Option<PathBuf>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, clap::Args)]
|
#[derive(Debug, clap::Args)]
|
||||||
struct FeedsDiscoverArgs {
|
struct FeedsDiscoverArgs {
|
||||||
/// How far back to look for articles.
|
/// How far back to look for articles.
|
||||||
@@ -410,6 +433,10 @@ async fn main() -> Result<()> {
|
|||||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||||
println!("{}", cmd_feeds_discover(&config, &db, args).await?);
|
println!("{}", cmd_feeds_discover(&config, &db, args).await?);
|
||||||
}
|
}
|
||||||
|
Command::Interests(command) => {
|
||||||
|
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||||
|
println!("{}", cmd_interests(&config, &db, command).await?);
|
||||||
|
}
|
||||||
Command::Job(JobCommand::Run { name }) => {
|
Command::Job(JobCommand::Run { name }) => {
|
||||||
let Some(job) = jobs::Job::parse(&name) else {
|
let Some(job) = jobs::Job::parse(&name) else {
|
||||||
eprintln!("unknown job {name:?}; the catalogue is:");
|
eprintln!("unknown job {name:?}; the catalogue is:");
|
||||||
@@ -435,8 +462,8 @@ async fn main() -> Result<()> {
|
|||||||
|
|
||||||
/// The commands that write the database and provider budgets and so hold the
|
/// The commands that write the database and provider budgets and so hold the
|
||||||
/// run lock (§5): `generate`, `profile rebuild`, `features backfill`,
|
/// run lock (§5): `generate`, `profile rebuild`, `features backfill`,
|
||||||
/// `backfill-social`, and a `job run` of any of them. Everything else is
|
/// `backfill-social`, `interests backfill`, and a `job run` of any of them.
|
||||||
/// read-only or its own writer.
|
/// Everything else is read-only or its own writer.
|
||||||
fn lock_holder(command: &Command) -> Option<&'static str> {
|
fn lock_holder(command: &Command) -> Option<&'static str> {
|
||||||
match command {
|
match command {
|
||||||
Command::Generate(_) => Some("generate"),
|
Command::Generate(_) => Some("generate"),
|
||||||
@@ -445,6 +472,7 @@ fn lock_holder(command: &Command) -> Option<&'static str> {
|
|||||||
Command::Profile(ProfileCommand::Rebuild) => Some("profile rebuild"),
|
Command::Profile(ProfileCommand::Rebuild) => Some("profile rebuild"),
|
||||||
Command::Features(FeaturesCommand::Backfill(_)) => Some("features backfill"),
|
Command::Features(FeaturesCommand::Backfill(_)) => Some("features backfill"),
|
||||||
Command::BackfillSocial(_) => Some("backfill-social"),
|
Command::BackfillSocial(_) => Some("backfill-social"),
|
||||||
|
Command::Interests(InterestsCommand::Backfill) => Some("interests backfill"),
|
||||||
// An unknown name takes no lock; the dispatch exits 2 before opening
|
// An unknown name takes no lock; the dispatch exits 2 before opening
|
||||||
// the database.
|
// the database.
|
||||||
Command::Job(JobCommand::Run { name }) => {
|
Command::Job(JobCommand::Run { name }) => {
|
||||||
@@ -457,7 +485,8 @@ fn lock_holder(command: &Command) -> Option<&'static str> {
|
|||||||
| Command::Features(FeaturesCommand::Prune)
|
| Command::Features(FeaturesCommand::Prune)
|
||||||
| Command::Db(_)
|
| Command::Db(_)
|
||||||
| Command::Config(_)
|
| Command::Config(_)
|
||||||
| Command::Users(_) => None,
|
| Command::Users(_)
|
||||||
|
| Command::Interests(InterestsCommand::Import(_)) => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -494,6 +523,152 @@ async fn cmd_feeds_discover(config: &Config, db: &Db, args: FeedsDiscoverArgs) -
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn cmd_interests(config: &Config, db: &Db, command: InterestsCommand) -> Result<String> {
|
||||||
|
match command {
|
||||||
|
InterestsCommand::Import(args) => cmd_interests_import(config, db, args).await,
|
||||||
|
InterestsCommand::Backfill => cmd_interests_backfill(config, db).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cmd_interests_import(
|
||||||
|
config: &Config,
|
||||||
|
db: &Db,
|
||||||
|
args: InterestsImportArgs,
|
||||||
|
) -> Result<String> {
|
||||||
|
let opml_path = args
|
||||||
|
.opml
|
||||||
|
.unwrap_or_else(|| PathBuf::from("data/scour-interests.opml"));
|
||||||
|
let profile_path = args.profile.unwrap_or_else(|| config.profile_path.clone());
|
||||||
|
let opml = std::fs::read_to_string(&opml_path)
|
||||||
|
.with_context(|| format!("reading interests OPML at {}", opml_path.display()))?;
|
||||||
|
let profile = curate::profile::load_profile(&profile_path)?;
|
||||||
|
let mut seen = HashSet::new();
|
||||||
|
let names = interests::parse_opml(&opml)
|
||||||
|
.into_iter()
|
||||||
|
.chain(profile.interests)
|
||||||
|
.filter(|name| seen.insert(name.to_lowercase()))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
let now = jiff::Timestamp::now();
|
||||||
|
let mut added = Vec::new();
|
||||||
|
let mut skipped = 0;
|
||||||
|
for name in names {
|
||||||
|
match interests::add(db, &name, None, now).await? {
|
||||||
|
interests::AddOutcome::Added(id) => added.push((id, name)),
|
||||||
|
interests::AddOutcome::Duplicate => skipped += 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let new_names = added
|
||||||
|
.iter()
|
||||||
|
.map(|(_, name)| name.clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let categories = curate::profile::themes::group_into_themes(&new_names)
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|(category, members)| {
|
||||||
|
members.into_iter().map(move |name| {
|
||||||
|
let category = (category != "Other standing interests").then(|| category.clone());
|
||||||
|
(name.to_lowercase(), category)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
let mut uncategorized = 0;
|
||||||
|
for (id, name) in &added {
|
||||||
|
match categories
|
||||||
|
.get(&name.to_lowercase())
|
||||||
|
.and_then(Option::as_deref)
|
||||||
|
{
|
||||||
|
Some(category) => interests::set_category(db, *id, Some(category), now).await?,
|
||||||
|
None => uncategorized += 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(format!(
|
||||||
|
"imported {}, skipped {} existing, {} left for the categorizer",
|
||||||
|
added.len(),
|
||||||
|
skipped,
|
||||||
|
uncategorized
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Backfill uses z-scores over the whole compatible cache as a stand-in for a
|
||||||
|
/// run's per-day article cohort.
|
||||||
|
async fn cmd_interests_backfill(config: &Config, db: &Db) -> Result<String> {
|
||||||
|
use sqlx::Row as _;
|
||||||
|
|
||||||
|
let mut article_embeddings = HashMap::new();
|
||||||
|
let mut after = i64::MIN;
|
||||||
|
loop {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT article_id, embedding FROM article_embeddings
|
||||||
|
WHERE model = ? AND dimension = ? AND article_id > ?
|
||||||
|
ORDER BY article_id LIMIT 500",
|
||||||
|
)
|
||||||
|
.bind(&config.voyage.model)
|
||||||
|
.bind(config.voyage.output_dimension as i64)
|
||||||
|
.bind(after)
|
||||||
|
.fetch_all(db.pool())
|
||||||
|
.await?;
|
||||||
|
if rows.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
for row in &rows {
|
||||||
|
let article_id: ArticleId = row.get("article_id");
|
||||||
|
after = article_id;
|
||||||
|
match embedding::decode_blob(
|
||||||
|
&row.get::<Vec<u8>, _>("embedding"),
|
||||||
|
config.voyage.output_dimension,
|
||||||
|
) {
|
||||||
|
Ok(vector) => {
|
||||||
|
article_embeddings.insert(article_id, vector);
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
tracing::warn!(article_id, %error, "ignoring a malformed cached embedding")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows = interests::list(db).await?;
|
||||||
|
let names = rows
|
||||||
|
.iter()
|
||||||
|
.map(|interest| interest.name.clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let ids = rows
|
||||||
|
.into_iter()
|
||||||
|
.map(|interest| (interest.name, interest.id))
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
// Like the pipeline: a missing key means cached vectors only, not a failure.
|
||||||
|
let service = match config.voyage.enabled {
|
||||||
|
true => match embedding::EmbeddingService::real(db.clone(), config.voyage.clone()) {
|
||||||
|
Ok(service) => Some(service),
|
||||||
|
Err(embedding::EmbeddingError::MissingApiKey) => None,
|
||||||
|
Err(error) => return Err(error).context("building the Voyage client"),
|
||||||
|
},
|
||||||
|
false => None,
|
||||||
|
};
|
||||||
|
let cached_only = service.is_none();
|
||||||
|
let service = service.unwrap_or_else(|| {
|
||||||
|
embedding::EmbeddingService::cached_only(db.clone(), config.voyage.clone())
|
||||||
|
});
|
||||||
|
let interest_embeddings = service.interests(&names).await?;
|
||||||
|
let mut matches = curate::signals::interest_matches(&article_embeddings, &interest_embeddings)
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(article_id, matched)| {
|
||||||
|
(!matched.top_interests.is_empty()).then_some((article_id, matched.top_interests))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
matches.sort_by_key(|(article_id, _)| *article_id);
|
||||||
|
let inserted = interests::insert_matches_if_absent(db, &matches, &ids).await?;
|
||||||
|
if cached_only {
|
||||||
|
Ok(format!(
|
||||||
|
"no Voyage client; used cached interest vectors and wrote {inserted} interest match rows"
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
Ok(format!("wrote {inserted} interest match rows"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn cmd_users(db: &Db, command: UsersCommand) -> Result<()> {
|
async fn cmd_users(db: &Db, command: UsersCommand) -> Result<()> {
|
||||||
use daily_epub::web::users;
|
use daily_epub::web::users;
|
||||||
match command {
|
match command {
|
||||||
@@ -724,7 +899,6 @@ async fn cmd_profile_rebuild(config: &Config, db: &Db) -> Result<String> {
|
|||||||
use curate::llm::{Llms, provider_meters};
|
use curate::llm::{Llms, provider_meters};
|
||||||
let profile = curate::profile::load_or_build(
|
let profile = curate::profile::load_or_build(
|
||||||
db,
|
db,
|
||||||
&config.interests_opml,
|
|
||||||
&config.profile_path,
|
&config.profile_path,
|
||||||
config.curation.feedback.verdicts_in_prompt,
|
config.curation.feedback.verdicts_in_prompt,
|
||||||
)
|
)
|
||||||
@@ -749,7 +923,6 @@ async fn cmd_profile_rebuild(config: &Config, db: &Db) -> Result<String> {
|
|||||||
let rebuilt = curate::profile::rebuild(
|
let rebuilt = curate::profile::rebuild(
|
||||||
db,
|
db,
|
||||||
llm,
|
llm,
|
||||||
&config.interests_opml,
|
|
||||||
&config.profile_path,
|
&config.profile_path,
|
||||||
config.curation.feedback.verdicts_in_prompt,
|
config.curation.feedback.verdicts_in_prompt,
|
||||||
)
|
)
|
||||||
@@ -1008,6 +1181,7 @@ async fn run_job(config: &Config, db: &Db, job: &jobs::Job) -> Result<(String, O
|
|||||||
None,
|
None,
|
||||||
)),
|
)),
|
||||||
jobs::Job::ImportRatings => Ok((imports::run(config, db).await?, None)),
|
jobs::Job::ImportRatings => Ok((imports::run(config, db).await?, None)),
|
||||||
|
jobs::Job::InterestsCategorize => Ok((interests::categorize(config, db).await?, None)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1151,6 +1325,10 @@ mod tests {
|
|||||||
lock_holder(&parse(&["backfill-social"])),
|
lock_holder(&parse(&["backfill-social"])),
|
||||||
Some("backfill-social")
|
Some("backfill-social")
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
lock_holder(&parse(&["interests", "backfill"])),
|
||||||
|
Some("interests backfill")
|
||||||
|
);
|
||||||
for args in [
|
for args in [
|
||||||
vec!["serve"],
|
vec!["serve"],
|
||||||
vec!["explain", "--date", "2026-09-02", "--near-misses"],
|
vec!["explain", "--date", "2026-09-02", "--near-misses"],
|
||||||
@@ -1159,7 +1337,9 @@ mod tests {
|
|||||||
vec!["db", "migrate"],
|
vec!["db", "migrate"],
|
||||||
vec!["features", "prune"],
|
vec!["features", "prune"],
|
||||||
vec!["config", "check"],
|
vec!["config", "check"],
|
||||||
|
vec!["interests", "import"],
|
||||||
vec!["job", "run", "features-prune"],
|
vec!["job", "run", "features-prune"],
|
||||||
|
vec!["job", "run", "interests-categorize"],
|
||||||
vec!["job", "run", "not-a-job"],
|
vec!["job", "run", "not-a-job"],
|
||||||
] {
|
] {
|
||||||
assert_eq!(lock_holder(&parse(&args)), None, "{args:?}");
|
assert_eq!(lock_holder(&parse(&args)), None, "{args:?}");
|
||||||
@@ -1211,6 +1391,34 @@ mod tests {
|
|||||||
assert!(Cli::try_parse_from(["daily-epub", "job"]).is_err());
|
assert!(Cli::try_parse_from(["daily-epub", "job"]).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_interests_commands() {
|
||||||
|
match Cli::try_parse_from([
|
||||||
|
"daily-epub",
|
||||||
|
"interests",
|
||||||
|
"import",
|
||||||
|
"--opml",
|
||||||
|
"/tmp/interests.opml",
|
||||||
|
"--profile",
|
||||||
|
"/tmp/profile.md",
|
||||||
|
])
|
||||||
|
.unwrap()
|
||||||
|
.command
|
||||||
|
{
|
||||||
|
Command::Interests(InterestsCommand::Import(args)) => {
|
||||||
|
assert_eq!(args.opml, Some(PathBuf::from("/tmp/interests.opml")));
|
||||||
|
assert_eq!(args.profile, Some(PathBuf::from("/tmp/profile.md")));
|
||||||
|
}
|
||||||
|
other => panic!("expected interests import, got {other:?}"),
|
||||||
|
}
|
||||||
|
assert!(matches!(
|
||||||
|
Cli::try_parse_from(["daily-epub", "interests", "backfill"])
|
||||||
|
.unwrap()
|
||||||
|
.command,
|
||||||
|
Command::Interests(InterestsCommand::Backfill)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
/// Dashboard plan §17 "Jobs": `job run` flips the dashboard's `requested`
|
/// Dashboard plan §17 "Jobs": `job run` flips the dashboard's `requested`
|
||||||
/// row to `running` and then `ok` with the command's message, in-process
|
/// row to `running` and then `ok` with the command's message, in-process
|
||||||
/// and without systemd.
|
/// and without systemd.
|
||||||
@@ -1497,4 +1705,149 @@ mod tests {
|
|||||||
assert_eq!(rows[1].get::<f64, _>("value"), 0.0);
|
assert_eq!(rows[1].get::<f64, _>("value"), 0.0);
|
||||||
assert!(db.current_ratings(36500).await.unwrap().is_empty());
|
assert!(db.current_ratings(36500).await.unwrap().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn interests_import_is_idempotent_and_preserves_theme_groups() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let db = Db::open_and_migrate(&dir.path().join("interests.db"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let opml_path = dir.path().join("interests.opml");
|
||||||
|
let profile_path = dir.path().join("profile.md");
|
||||||
|
std::fs::write(
|
||||||
|
&opml_path,
|
||||||
|
r#"<opml><body><outline text="Rust"/><outline text="Flibbertigibbet"/></body></opml>"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
&profile_path,
|
||||||
|
"# Reader\n\n## Interests\n- rust\n- Postgres query plans\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let config = Config {
|
||||||
|
profile_path: profile_path.clone(),
|
||||||
|
..Config::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let args = || InterestsImportArgs {
|
||||||
|
opml: Some(opml_path.clone()),
|
||||||
|
profile: None,
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
cmd_interests_import(&config, &db, args()).await.unwrap(),
|
||||||
|
"imported 3, skipped 0 existing, 1 left for the categorizer"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cmd_interests_import(&config, &db, args()).await.unwrap(),
|
||||||
|
"imported 0, skipped 3 existing, 0 left for the categorizer"
|
||||||
|
);
|
||||||
|
|
||||||
|
let rows = interests::list(&db).await.unwrap();
|
||||||
|
assert_eq!(rows.len(), 3);
|
||||||
|
assert_eq!(
|
||||||
|
rows.iter()
|
||||||
|
.find(|interest| interest.name == "Rust")
|
||||||
|
.and_then(|interest| interest.category.as_deref()),
|
||||||
|
Some("Systems & languages")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
rows.iter()
|
||||||
|
.find(|interest| interest.name == "Flibbertigibbet")
|
||||||
|
.and_then(|interest| interest.category.as_deref()),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn interests_backfill_inserts_matches_without_replacing_run_rows() {
|
||||||
|
use daily_epub::curate::embedding::encode_blob;
|
||||||
|
use daily_epub::curate::signals::TopInterest;
|
||||||
|
use sqlx::Row as _;
|
||||||
|
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let db = Db::open_and_migrate(&dir.path().join("interests.db"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO articles (id, canonical_url, title, first_seen) VALUES
|
||||||
|
(1, 'https://example.com/1', 'One', '2026-09-12T00:00:00Z'),
|
||||||
|
(2, 'https://example.com/2', 'Two', '2026-09-12T00:00:00Z')",
|
||||||
|
)
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let now = jiff::Timestamp::now();
|
||||||
|
let interests::AddOutcome::Added(one_id) =
|
||||||
|
interests::add(&db, "First axis", None, now).await.unwrap()
|
||||||
|
else {
|
||||||
|
unreachable!();
|
||||||
|
};
|
||||||
|
let interests::AddOutcome::Added(two_id) =
|
||||||
|
interests::add(&db, "Second axis", None, now).await.unwrap()
|
||||||
|
else {
|
||||||
|
unreachable!();
|
||||||
|
};
|
||||||
|
for (article_id, vector) in [(1, [1.0_f32, 0.0]), (2, [0.0, 1.0])] {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO article_embeddings
|
||||||
|
(article_id, model, dimension, input_hash, embedding, created_at)
|
||||||
|
VALUES (?, 'test-model', 2, 'hash', ?, '2026-09-12T00:00:00Z')",
|
||||||
|
)
|
||||||
|
.bind(article_id)
|
||||||
|
.bind(encode_blob(&vector).unwrap())
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
for (name, vector) in [("First axis", [1.0_f32, 0.0]), ("Second axis", [0.0, 1.0])] {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO interest_embeddings
|
||||||
|
(interest, model, dimension, embedding, created_at)
|
||||||
|
VALUES (?, 'test-model', 2, ?, '2026-09-12T00:00:00Z')",
|
||||||
|
)
|
||||||
|
.bind(name)
|
||||||
|
.bind(encode_blob(&vector).unwrap())
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
interests::replace_matches(
|
||||||
|
&db,
|
||||||
|
Some(77),
|
||||||
|
&[(
|
||||||
|
1,
|
||||||
|
vec![TopInterest {
|
||||||
|
name: "First axis".into(),
|
||||||
|
cos: 0.75,
|
||||||
|
z: 1.5,
|
||||||
|
}],
|
||||||
|
)],
|
||||||
|
&HashMap::from([("First axis".to_string(), one_id)]),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut config = Config::default();
|
||||||
|
config.voyage.enabled = false;
|
||||||
|
config.voyage.model = "test-model".into();
|
||||||
|
config.voyage.output_dimension = 2;
|
||||||
|
|
||||||
|
let message = cmd_interests_backfill(&config, &db).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
message,
|
||||||
|
"no Voyage client; used cached interest vectors and wrote 1 interest match rows"
|
||||||
|
);
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT article_id, interest_id, run_id, cos, z
|
||||||
|
FROM article_interests ORDER BY article_id",
|
||||||
|
)
|
||||||
|
.fetch_all(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(rows.len(), 2);
|
||||||
|
assert_eq!(rows[0].get::<i64, _>("interest_id"), one_id);
|
||||||
|
assert_eq!(rows[0].get::<Option<i64>, _>("run_id"), Some(77));
|
||||||
|
assert_eq!(rows[0].get::<f64, _>("cos"), 0.75);
|
||||||
|
assert_eq!(rows[1].get::<i64, _>("interest_id"), two_id);
|
||||||
|
assert_eq!(rows[1].get::<Option<i64>, _>("run_id"), None);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+82
-18
@@ -46,7 +46,7 @@ use crate::types::{
|
|||||||
Article, ArticleId, Artifact, BehindThePaper, Candidate, Colophon, Edition, Issue, IssueMeta,
|
Article, ArticleId, Artifact, BehindThePaper, Candidate, Colophon, Edition, Issue, IssueMeta,
|
||||||
Lineup, Models, TokenUsage, reading_minutes,
|
Lineup, Models, TokenUsage, reading_minutes,
|
||||||
};
|
};
|
||||||
use crate::{comments, dedupe, discovery, epub, http, miniflux, publish, social, world};
|
use crate::{comments, dedupe, discovery, epub, http, interests, miniflux, publish, social, world};
|
||||||
|
|
||||||
/// One `generate` invocation's inputs — the CLI flags, already parsed (§2).
|
/// One `generate` invocation's inputs — the CLI flags, already parsed (§2).
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
@@ -943,15 +943,22 @@ async fn prepare_features(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
report.counts.embedded = article_embeddings.len() as i64;
|
report.counts.embedded = article_embeddings.len() as i64;
|
||||||
let interests =
|
let interest_rows = match interests::list(db).await {
|
||||||
match profile::load_standing_interests(&config.interests_opml, &config.profile_path) {
|
Ok(interests) => interests,
|
||||||
Ok(interests) => interests,
|
Err(error) => {
|
||||||
Err(error) => {
|
tracing::warn!(%error, "could not load standing interests for embeddings");
|
||||||
tracing::warn!(%error, "could not load standing interests for embeddings");
|
Vec::new()
|
||||||
Vec::new()
|
}
|
||||||
}
|
};
|
||||||
};
|
let interest_names = interest_rows
|
||||||
let interest_embeddings = match service.interests(&interests).await {
|
.iter()
|
||||||
|
.map(|interest| interest.name.clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let interest_ids = interest_rows
|
||||||
|
.into_iter()
|
||||||
|
.map(|interest| (interest.name, interest.id))
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
let interest_embeddings = match service.interests(&interest_names).await {
|
||||||
Ok(embeddings) => embeddings,
|
Ok(embeddings) => embeddings,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
report.warn(format!("interest embedding stage degraded: {error}"));
|
report.warn(format!("interest embedding stage degraded: {error}"));
|
||||||
@@ -1009,6 +1016,21 @@ async fn prepare_features(
|
|||||||
.remove(&candidate.article.id)
|
.remove(&candidate.article.id)
|
||||||
.unwrap_or_else(|| signals::Signals::baseline(&candidate.article));
|
.unwrap_or_else(|| signals::Signals::baseline(&candidate.article));
|
||||||
}
|
}
|
||||||
|
let matches = candidates
|
||||||
|
.iter()
|
||||||
|
.filter(|candidate| !candidate.signals.top_interests.is_empty())
|
||||||
|
.map(|candidate| {
|
||||||
|
(
|
||||||
|
candidate.article.id,
|
||||||
|
candidate.signals.top_interests.clone(),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if let Err(error) =
|
||||||
|
interests::replace_matches(db, Some(ctx.run_id), &matches, &interest_ids).await
|
||||||
|
{
|
||||||
|
report.warn(format!("could not record interest matches: {error}"));
|
||||||
|
}
|
||||||
if let Err(error) = record_candidates(ctx, candidates).await {
|
if let Err(error) = record_candidates(ctx, candidates).await {
|
||||||
report.warn(format!("could not record eligible candidates: {error}"));
|
report.warn(format!("could not record eligible candidates: {error}"));
|
||||||
}
|
}
|
||||||
@@ -1138,7 +1160,6 @@ async fn build_llms(
|
|||||||
) -> Llms {
|
) -> Llms {
|
||||||
let profile = match profile::load_or_build(
|
let profile = match profile::load_or_build(
|
||||||
ctx.db,
|
ctx.db,
|
||||||
&ctx.config.interests_opml,
|
|
||||||
&ctx.config.profile_path,
|
&ctx.config.profile_path,
|
||||||
ctx.config.curation.feedback.verdicts_in_prompt,
|
ctx.config.curation.feedback.verdicts_in_prompt,
|
||||||
)
|
)
|
||||||
@@ -1161,6 +1182,39 @@ async fn build_llms(
|
|||||||
let make_clients = |prompt: String| Llms::from_config(ctx.config, prompt, meters);
|
let make_clients = |prompt: String| Llms::from_config(ctx.config, prompt, meters);
|
||||||
|
|
||||||
let mut llms = make_clients(profile.text);
|
let mut llms = make_clients(profile.text);
|
||||||
|
// Categorizing regroups the prompt's standing interests, so the prompt is
|
||||||
|
// reloaded and the clients remade before anything uses them.
|
||||||
|
if !llms.is_empty() {
|
||||||
|
match interests::uncategorized(ctx.db).await {
|
||||||
|
Ok(pending) if !pending.is_empty() => {
|
||||||
|
match interests::categorize(ctx.config, ctx.db).await {
|
||||||
|
Ok(message) => {
|
||||||
|
tracing::info!(%message);
|
||||||
|
match profile::load_or_build(
|
||||||
|
ctx.db,
|
||||||
|
&ctx.config.profile_path,
|
||||||
|
ctx.config.curation.feedback.verdicts_in_prompt,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(regrouped) => {
|
||||||
|
report.counts.verdicts_in_prompt = regrouped.verdicts as i64;
|
||||||
|
llms = make_clients(regrouped.text);
|
||||||
|
}
|
||||||
|
Err(error) => report.warn(format!(
|
||||||
|
"could not rebuild the taste prompt after categorizing: {error:#}"
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => report.warn(format!("interest categorization failed: {error:#}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(error) => report.warn(format!(
|
||||||
|
"could not check for uncategorized interests: {error:#}"
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
let Some(rebuild_client) = llms.editor_or_bulk() else {
|
let Some(rebuild_client) = llms.editor_or_bulk() else {
|
||||||
report.warn("no LLM provider is available; curating heuristically");
|
report.warn("no LLM provider is available; curating heuristically");
|
||||||
return llms;
|
return llms;
|
||||||
@@ -1168,7 +1222,6 @@ async fn build_llms(
|
|||||||
match profile::weekly_rebuild_if_due(
|
match profile::weekly_rebuild_if_due(
|
||||||
ctx.db,
|
ctx.db,
|
||||||
rebuild_client,
|
rebuild_client,
|
||||||
&ctx.config.interests_opml,
|
|
||||||
&ctx.config.profile_path,
|
&ctx.config.profile_path,
|
||||||
ctx.config.curation.feedback.verdicts_in_prompt,
|
ctx.config.curation.feedback.verdicts_in_prompt,
|
||||||
)
|
)
|
||||||
@@ -1577,12 +1630,9 @@ mod tests {
|
|||||||
config.curation.blocked_domains = vec!["blocked.example".into()];
|
config.curation.blocked_domains = vec!["blocked.example".into()];
|
||||||
config.voyage.output_dimension = 4;
|
config.voyage.output_dimension = 4;
|
||||||
config.target_article_count = 1;
|
config.target_article_count = 1;
|
||||||
config.interests_opml = dir.path().join("interests.opml");
|
interests::add(&db, "Writerdeck", Some("Publishing"), Timestamp::now())
|
||||||
std::fs::write(
|
.await
|
||||||
&config.interests_opml,
|
.unwrap();
|
||||||
"<opml><body><outline text=\"Writerdeck\"/></body></opml>",
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
config.profile_path = dir.path().join("profile.md");
|
config.profile_path = dir.path().join("profile.md");
|
||||||
std::fs::write(&config.profile_path, "# Reader profile\n").unwrap();
|
std::fs::write(&config.profile_path, "# Reader profile\n").unwrap();
|
||||||
|
|
||||||
@@ -1708,6 +1758,7 @@ mod tests {
|
|||||||
assert!(report.voyage_tokens > 0);
|
assert!(report.voyage_tokens > 0);
|
||||||
// One batch for the two articles, one for the interest.
|
// One batch for the two articles, one for the interest.
|
||||||
assert_eq!(backend.calls(), 2);
|
assert_eq!(backend.calls(), 2);
|
||||||
|
assert_eq!(backend.requests()[1].input, ["Writerdeck"]);
|
||||||
let signals = &features
|
let signals = &features
|
||||||
.iter()
|
.iter()
|
||||||
.find(|candidate| candidate.article.id == a)
|
.find(|candidate| candidate.article.id == a)
|
||||||
@@ -1724,6 +1775,19 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert!(signals.preliminary.is_some());
|
assert!(signals.preliminary.is_some());
|
||||||
|
|
||||||
|
let expected_matches = features
|
||||||
|
.iter()
|
||||||
|
.map(|candidate| candidate.signals.top_interests.len() as i64)
|
||||||
|
.sum::<i64>();
|
||||||
|
let stored_matches: i64 =
|
||||||
|
sqlx::query_scalar("SELECT COUNT(*) FROM article_interests WHERE run_id = ?")
|
||||||
|
.bind(h.run_id)
|
||||||
|
.fetch_one(h.db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(expected_matches > 0);
|
||||||
|
assert_eq!(stored_matches, expected_matches);
|
||||||
|
|
||||||
let rows = stage_rows(&h.db, h.run_id).await;
|
let rows = stage_rows(&h.db, h.run_id).await;
|
||||||
assert_eq!(rows.len(), 4, "one row per considered article");
|
assert_eq!(rows.len(), 4, "one row per considered article");
|
||||||
assert_eq!(rows[&blocked].0, "excluded");
|
assert_eq!(rows[&blocked].0, "excluded");
|
||||||
|
|||||||
+179
-16
@@ -53,6 +53,7 @@ pub fn routes() -> Router<AppState> {
|
|||||||
#[derive(Debug, Default, Deserialize)]
|
#[derive(Debug, Default, Deserialize)]
|
||||||
pub struct ArticlesQuery {
|
pub struct ArticlesQuery {
|
||||||
pub q: Option<String>,
|
pub q: Option<String>,
|
||||||
|
pub interest: Option<String>,
|
||||||
pub feed: Option<String>,
|
pub feed: Option<String>,
|
||||||
pub stage: Option<String>,
|
pub stage: Option<String>,
|
||||||
pub reason: Option<String>,
|
pub reason: Option<String>,
|
||||||
@@ -68,8 +69,9 @@ pub struct ArticlesQuery {
|
|||||||
const RATED: [&str; 7] = ["any", "loved", "good", "down", "slop", "cleared", "none"];
|
const RATED: [&str; 7] = ["any", "loved", "good", "down", "slop", "cleared", "none"];
|
||||||
const PUBLISHED: [&str; 2] = ["yes", "no"];
|
const PUBLISHED: [&str; 2] = ["yes", "no"];
|
||||||
|
|
||||||
const ARTICLE_SORTS: [(&str, &str); 7] = [
|
const ARTICLE_SORTS: [(&str, &str); 8] = [
|
||||||
("first_seen", "x.first_seen DESC, x.id DESC"),
|
("first_seen", "x.first_seen DESC, x.id DESC"),
|
||||||
|
("match", "x.match_cos DESC, x.id DESC"),
|
||||||
("utility", "x.utility DESC, x.id DESC"),
|
("utility", "x.utility DESC, x.id DESC"),
|
||||||
("quality", "x.quality DESC, x.id DESC"),
|
("quality", "x.quality DESC, x.id DESC"),
|
||||||
("fit", "x.fit DESC, x.id DESC"),
|
("fit", "x.fit DESC, x.id DESC"),
|
||||||
@@ -82,6 +84,7 @@ const ARTICLE_SORTS: [(&str, &str); 7] = [
|
|||||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
pub struct ArticleFilters {
|
pub struct ArticleFilters {
|
||||||
pub q: Option<String>,
|
pub q: Option<String>,
|
||||||
|
pub interest: Option<InterestFilter>,
|
||||||
pub feed: Option<i64>,
|
pub feed: Option<i64>,
|
||||||
pub stage: Option<String>,
|
pub stage: Option<String>,
|
||||||
pub reason: Option<String>,
|
pub reason: Option<String>,
|
||||||
@@ -93,8 +96,14 @@ pub struct ArticleFilters {
|
|||||||
pub sort: &'static str,
|
pub sort: &'static str,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct InterestFilter {
|
||||||
|
pub id: Option<i64>,
|
||||||
|
pub name: String,
|
||||||
|
}
|
||||||
|
|
||||||
impl ArticleFilters {
|
impl ArticleFilters {
|
||||||
pub fn from_query(query: &ArticlesQuery) -> Self {
|
pub fn from_query(query: &ArticlesQuery, interest: Option<InterestFilter>) -> Self {
|
||||||
let owned = |value: Option<&str>| value.map(str::to_string);
|
let owned = |value: Option<&str>| value.map(str::to_string);
|
||||||
let date = |value: Option<&str>| {
|
let date = |value: Option<&str>| {
|
||||||
non_empty(value)
|
non_empty(value)
|
||||||
@@ -103,8 +112,19 @@ impl ArticleFilters {
|
|||||||
};
|
};
|
||||||
let mut kinds: Vec<&str> = TRIAGE_KINDS.to_vec();
|
let mut kinds: Vec<&str> = TRIAGE_KINDS.to_vec();
|
||||||
kinds.push(PROVIDER_REJECTED);
|
kinds.push(PROVIDER_REJECTED);
|
||||||
|
let sort = ARTICLE_SORTS
|
||||||
|
.iter()
|
||||||
|
.find(|(name, _)| Some(*name) == query.sort.as_deref())
|
||||||
|
.map(|(name, _)| *name)
|
||||||
|
.filter(|sort| *sort != "match" || interest.is_some())
|
||||||
|
.unwrap_or(if interest.is_some() {
|
||||||
|
"match"
|
||||||
|
} else {
|
||||||
|
ARTICLE_SORTS[0].0
|
||||||
|
});
|
||||||
Self {
|
Self {
|
||||||
q: owned(non_empty(query.q.as_deref())),
|
q: owned(non_empty(query.q.as_deref())),
|
||||||
|
interest,
|
||||||
feed: non_empty(query.feed.as_deref()).and_then(|feed| feed.parse::<i64>().ok()),
|
feed: non_empty(query.feed.as_deref()).and_then(|feed| feed.parse::<i64>().ok()),
|
||||||
stage: owned(allow_listed(query.stage.as_deref(), &STAGES)),
|
stage: owned(allow_listed(query.stage.as_deref(), &STAGES)),
|
||||||
reason: owned(allow_listed(query.reason.as_deref(), &REASONS)),
|
reason: owned(allow_listed(query.reason.as_deref(), &REASONS)),
|
||||||
@@ -113,11 +133,7 @@ impl ArticleFilters {
|
|||||||
from: date(query.from.as_deref()),
|
from: date(query.from.as_deref()),
|
||||||
to: date(query.to.as_deref()),
|
to: date(query.to.as_deref()),
|
||||||
kind: owned(allow_listed(query.kind.as_deref(), &kinds)),
|
kind: owned(allow_listed(query.kind.as_deref(), &kinds)),
|
||||||
sort: ARTICLE_SORTS
|
sort,
|
||||||
.iter()
|
|
||||||
.find(|(name, _)| Some(*name) == query.sort.as_deref())
|
|
||||||
.map(|(name, _)| *name)
|
|
||||||
.unwrap_or(ARTICLE_SORTS[0].0),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -182,6 +198,10 @@ impl ArticleFilters {
|
|||||||
fn params(&self) -> Vec<(&'static str, Option<String>)> {
|
fn params(&self) -> Vec<(&'static str, Option<String>)> {
|
||||||
vec![
|
vec![
|
||||||
("q", self.q.clone()),
|
("q", self.q.clone()),
|
||||||
|
(
|
||||||
|
"interest",
|
||||||
|
self.interest.as_ref().map(|interest| interest.name.clone()),
|
||||||
|
),
|
||||||
("feed", self.feed.map(|feed| feed.to_string())),
|
("feed", self.feed.map(|feed| feed.to_string())),
|
||||||
("stage", self.stage.clone()),
|
("stage", self.stage.clone()),
|
||||||
("reason", self.reason.clone()),
|
("reason", self.reason.clone()),
|
||||||
@@ -219,6 +239,13 @@ pub struct ArticleListRow {
|
|||||||
pub rating: Option<String>,
|
pub rating: Option<String>,
|
||||||
pub rating_class: &'static str,
|
pub rating_class: &'static str,
|
||||||
pub published: Option<String>,
|
pub published: Option<String>,
|
||||||
|
pub interests: Vec<ArticleInterest>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ArticleInterest {
|
||||||
|
pub name: String,
|
||||||
|
pub href: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
const ARTICLE_INNER: &str =
|
const ARTICLE_INNER: &str =
|
||||||
@@ -226,12 +253,14 @@ const ARTICLE_INNER: &str =
|
|||||||
a.word_count, e.feed_id, COALESCE(e.feed_title, '') AS feed_title,
|
a.word_count, e.feed_id, COALESCE(e.feed_title, '') AS feed_title,
|
||||||
l.run_id, l.stage, l.excluded_reason, l.utility, r.date AS run_date,
|
l.run_id, l.stage, l.excluded_reason, l.utility, r.date AS run_date,
|
||||||
t.score AS triage, t.kind AS triage_kind, d.score AS quality, d.fit AS fit,
|
t.score AS triage, t.kind AS triage_kind, d.score AS quality, d.fit AS fit,
|
||||||
|
{match_cos} AS match_cos,
|
||||||
(SELECT re.label FROM rating_events re
|
(SELECT re.label FROM rating_events re
|
||||||
WHERE re.article_id = a.id AND re.kind = 'explicit'
|
WHERE re.article_id = a.id AND re.kind = 'explicit'
|
||||||
ORDER BY re.event_at DESC, re.id DESC LIMIT 1) AS rating,
|
ORDER BY re.event_at DESC, re.id DESC LIMIT 1) AS rating,
|
||||||
(SELECT ia.issue_date FROM issue_articles ia
|
(SELECT ia.issue_date FROM issue_articles ia
|
||||||
WHERE ia.article_id = a.id ORDER BY ia.issue_date DESC LIMIT 1) AS published
|
WHERE ia.article_id = a.id ORDER BY ia.issue_date DESC LIMIT 1) AS published
|
||||||
FROM articles a
|
FROM articles a
|
||||||
|
{interest_join}
|
||||||
LEFT JOIN entries e ON e.id = a.best_entry_id
|
LEFT JOIN entries e ON e.id = a.best_entry_id
|
||||||
LEFT JOIN candidate_runs l ON l.article_id = a.id
|
LEFT JOIN candidate_runs l ON l.article_id = a.id
|
||||||
AND l.run_id = (SELECT MAX(c2.run_id) FROM candidate_runs c2
|
AND l.run_id = (SELECT MAX(c2.run_id) FROM candidate_runs c2
|
||||||
@@ -245,10 +274,16 @@ pub async fn list_articles(
|
|||||||
config: &Config,
|
config: &Config,
|
||||||
filters: &ArticleFilters,
|
filters: &ArticleFilters,
|
||||||
page: u32,
|
page: u32,
|
||||||
) -> Result<(Vec<ArticleListRow>, Pagination), sqlx::Error> {
|
) -> anyhow::Result<(Vec<ArticleListRow>, Pagination)> {
|
||||||
let (clauses, binds) = filters.where_clauses();
|
let (clauses, binds) = filters.where_clauses();
|
||||||
let count_sql = format!("SELECT COUNT(*) FROM ({ARTICLE_INNER}) x WHERE 1 = 1{clauses}");
|
let (inner, interest_binds) = article_inner(filters);
|
||||||
let total: i64 = bind_all(dynamic_query(count_sql), &binds)
|
let all_binds = interest_binds
|
||||||
|
.iter()
|
||||||
|
.cloned()
|
||||||
|
.chain(binds.iter().cloned())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let count_sql = format!("SELECT COUNT(*) FROM ({inner}) x WHERE 1 = 1{clauses}");
|
||||||
|
let total: i64 = bind_all(dynamic_query(count_sql), &all_binds)
|
||||||
.fetch_one(db.pool())
|
.fetch_one(db.pool())
|
||||||
.await?
|
.await?
|
||||||
.get(0);
|
.get(0);
|
||||||
@@ -258,15 +293,15 @@ pub async fn list_articles(
|
|||||||
total,
|
total,
|
||||||
};
|
};
|
||||||
let select_sql = format!(
|
let select_sql = format!(
|
||||||
"SELECT * FROM ({ARTICLE_INNER}) x WHERE 1 = 1{clauses} ORDER BY {} LIMIT ? OFFSET ?",
|
"SELECT * FROM ({inner}) x WHERE 1 = 1{clauses} ORDER BY {} LIMIT ? OFFSET ?",
|
||||||
filters.order_by()
|
filters.order_by()
|
||||||
);
|
);
|
||||||
let rows = bind_all(dynamic_query(select_sql), &binds)
|
let rows = bind_all(dynamic_query(select_sql), &all_binds)
|
||||||
.bind(i64::from(ARTICLES_PER_PAGE))
|
.bind(i64::from(ARTICLES_PER_PAGE))
|
||||||
.bind(pagination.offset())
|
.bind(pagination.offset())
|
||||||
.fetch_all(db.pool())
|
.fetch_all(db.pool())
|
||||||
.await?;
|
.await?;
|
||||||
let rows = rows
|
let mut rows: Vec<ArticleListRow> = rows
|
||||||
.iter()
|
.iter()
|
||||||
.map(|row| {
|
.map(|row| {
|
||||||
let id: ArticleId = row.get("id");
|
let id: ArticleId = row.get("id");
|
||||||
@@ -290,12 +325,45 @@ pub async fn list_articles(
|
|||||||
rating_class: widget_label(rating.as_deref()),
|
rating_class: widget_label(rating.as_deref()),
|
||||||
rating,
|
rating,
|
||||||
published: row.get("published"),
|
published: row.get("published"),
|
||||||
|
interests: Vec::new(),
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
let ids = rows.iter().map(|row| row.id).collect::<Vec<_>>();
|
||||||
|
let mut by_article: HashMap<ArticleId, Vec<ArticleInterest>> = HashMap::new();
|
||||||
|
for matched in crate::interests::matches_for_articles(db, &ids).await? {
|
||||||
|
by_article
|
||||||
|
.entry(matched.article_id)
|
||||||
|
.or_default()
|
||||||
|
.push(ArticleInterest {
|
||||||
|
href: crate::interests::articles_href(&matched.name),
|
||||||
|
name: matched.name,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for row in &mut rows {
|
||||||
|
row.interests = by_article.remove(&row.id).unwrap_or_default();
|
||||||
|
}
|
||||||
Ok((rows, pagination))
|
Ok((rows, pagination))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn article_inner(filters: &ArticleFilters) -> (String, Vec<Bind>) {
|
||||||
|
match &filters.interest {
|
||||||
|
Some(interest) => (
|
||||||
|
ARTICLE_INNER.replace("{match_cos}", "ai.cos").replace(
|
||||||
|
"{interest_join}",
|
||||||
|
"JOIN article_interests ai ON ai.article_id = a.id AND ai.interest_id = ?",
|
||||||
|
),
|
||||||
|
vec![Bind::Int(interest.id.unwrap_or(-1))],
|
||||||
|
),
|
||||||
|
None => (
|
||||||
|
ARTICLE_INNER
|
||||||
|
.replace("{match_cos}", "NULL")
|
||||||
|
.replace("{interest_join}", ""),
|
||||||
|
Vec::new(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Template)]
|
#[derive(Template)]
|
||||||
#[template(path = "dashboard/articles.html")]
|
#[template(path = "dashboard/articles.html")]
|
||||||
struct ArticlesTemplate {
|
struct ArticlesTemplate {
|
||||||
@@ -308,6 +376,7 @@ struct ArticlesTemplate {
|
|||||||
rated: Vec<&'static str>,
|
rated: Vec<&'static str>,
|
||||||
kinds: Vec<&'static str>,
|
kinds: Vec<&'static str>,
|
||||||
sorts: Vec<&'static str>,
|
sorts: Vec<&'static str>,
|
||||||
|
interest_options: Vec<crate::interests::Interest>,
|
||||||
pager: Pager,
|
pager: Pager,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -319,11 +388,24 @@ async fn list(
|
|||||||
) -> Result<Response, WebError> {
|
) -> Result<Response, WebError> {
|
||||||
let viewer = auth.user().await.map(Viewer::from);
|
let viewer = auth.user().await.map(Viewer::from);
|
||||||
let config = state.config();
|
let config = state.config();
|
||||||
let filters = ArticleFilters::from_query(&query);
|
let interest_options = crate::interests::list(&state.db)
|
||||||
|
.await
|
||||||
|
.map_err(WebError::Internal)?;
|
||||||
|
let requested_interest = non_empty(query.interest.as_deref()).map(str::to_string);
|
||||||
|
let interest = requested_interest.map(|name| {
|
||||||
|
let found = interest_options
|
||||||
|
.iter()
|
||||||
|
.find(|interest| interest.name.eq_ignore_ascii_case(&name));
|
||||||
|
InterestFilter {
|
||||||
|
id: found.map(|interest| interest.id),
|
||||||
|
name: found.map(|interest| interest.name.clone()).unwrap_or(name),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let filters = ArticleFilters::from_query(&query, interest);
|
||||||
let page_no = page_number(query.page);
|
let page_no = page_number(query.page);
|
||||||
let (articles, pagination) = list_articles(&state.db, &config, &filters, page_no)
|
let (articles, pagination) = list_articles(&state.db, &config, &filters, page_no)
|
||||||
.await
|
.await
|
||||||
.map_err(db_err)?;
|
.map_err(WebError::Internal)?;
|
||||||
let pager = Pager::new(pagination, "/dashboard/articles", &filters.params());
|
let pager = Pager::new(pagination, "/dashboard/articles", &filters.params());
|
||||||
let mut kinds: Vec<&'static str> = TRIAGE_KINDS.to_vec();
|
let mut kinds: Vec<&'static str> = TRIAGE_KINDS.to_vec();
|
||||||
kinds.push(PROVIDER_REJECTED);
|
kinds.push(PROVIDER_REJECTED);
|
||||||
@@ -342,6 +424,7 @@ async fn list(
|
|||||||
rated: RATED.to_vec(),
|
rated: RATED.to_vec(),
|
||||||
kinds,
|
kinds,
|
||||||
sorts: ARTICLE_SORTS.iter().map(|(name, _)| *name).collect(),
|
sorts: ARTICLE_SORTS.iter().map(|(name, _)| *name).collect(),
|
||||||
|
interest_options,
|
||||||
pager,
|
pager,
|
||||||
})
|
})
|
||||||
.into_response())
|
.into_response())
|
||||||
@@ -845,7 +928,7 @@ mod tests {
|
|||||||
fn query(f: impl FnOnce(&mut ArticlesQuery)) -> ArticleFilters {
|
fn query(f: impl FnOnce(&mut ArticlesQuery)) -> ArticleFilters {
|
||||||
let mut query = ArticlesQuery::default();
|
let mut query = ArticlesQuery::default();
|
||||||
f(&mut query);
|
f(&mut query);
|
||||||
ArticleFilters::from_query(&query)
|
ArticleFilters::from_query(&query, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -939,6 +1022,86 @@ mod tests {
|
|||||||
assert_eq!(rows[0].id, 1);
|
assert_eq!(rows[0].id, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_match_sort_needs_an_interest_and_round_trips_in_the_pager_links() {
|
||||||
|
let without = query(|q| q.sort = Some("match".into()));
|
||||||
|
assert_eq!(without.sort, "first_seen");
|
||||||
|
assert!(without.params().contains(&("sort", None)));
|
||||||
|
|
||||||
|
let with = ArticlesQuery {
|
||||||
|
interest: Some("Rust".into()),
|
||||||
|
..ArticlesQuery::default()
|
||||||
|
};
|
||||||
|
let with = ArticleFilters::from_query(
|
||||||
|
&with,
|
||||||
|
Some(InterestFilter {
|
||||||
|
id: Some(7),
|
||||||
|
name: "Rust".into(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert_eq!(with.sort, "match", "the interest filter sorts by cosine");
|
||||||
|
let params = with.params();
|
||||||
|
assert!(params.contains(&("interest", Some("Rust".into()))));
|
||||||
|
assert!(params.contains(&("sort", Some("match".into()))));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn interest_filter_finds_matches_unknown_is_empty_and_badges_link() {
|
||||||
|
let seed = seed().await;
|
||||||
|
let crate::interests::AddOutcome::Added(interest_id) = crate::interests::add(
|
||||||
|
&seed.db,
|
||||||
|
"Rust & Systems",
|
||||||
|
Some("Software"),
|
||||||
|
"2026-09-12T12:00:00Z".parse().unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap() else {
|
||||||
|
unreachable!();
|
||||||
|
};
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO article_interests (article_id, interest_id, cos, z, run_id)
|
||||||
|
VALUES (1, ?, 0.82, 2.1, ?)",
|
||||||
|
)
|
||||||
|
.bind(interest_id)
|
||||||
|
.bind(seed.run_id)
|
||||||
|
.execute(seed.db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let app = app_with_users(&seed.db).await;
|
||||||
|
let admin = login_cookie(&app, "admin", "correct horse battery").await;
|
||||||
|
let filtered = get(
|
||||||
|
&app,
|
||||||
|
"/dashboard/articles?interest=rust+%26+systems",
|
||||||
|
Some(&admin),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(filtered.status(), axum::http::StatusCode::OK);
|
||||||
|
let filtered = crate::web::dashboard::tests::response_text(filtered).await;
|
||||||
|
assert!(filtered.contains("Article 1 about prose"), "{filtered}");
|
||||||
|
assert!(!filtered.contains("Article 2 about graphs"), "{filtered}");
|
||||||
|
assert!(filtered.contains("value=\"match\" selected"), "{filtered}");
|
||||||
|
assert!(
|
||||||
|
filtered.contains(
|
||||||
|
"href=\"/dashboard/articles?interest=Rust+%26+Systems\">Rust & Systems</a>"
|
||||||
|
),
|
||||||
|
"{filtered}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let unknown = get(
|
||||||
|
&app,
|
||||||
|
"/dashboard/articles?interest=not-a-real-interest",
|
||||||
|
Some(&admin),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(unknown.status(), axum::http::StatusCode::OK);
|
||||||
|
let unknown = crate::web::dashboard::tests::response_text(unknown).await;
|
||||||
|
assert!(
|
||||||
|
unknown.contains("No articles match this filter."),
|
||||||
|
"{unknown}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn article_detail_shows_assessments_run_history_and_rating_events() {
|
async fn article_detail_shows_assessments_run_history_and_rating_events() {
|
||||||
let seed = seed().await;
|
let seed = seed().await;
|
||||||
|
|||||||
@@ -69,6 +69,12 @@ struct WhyArticle {
|
|||||||
title: String,
|
title: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct WhyInterest {
|
||||||
|
name: String,
|
||||||
|
href: String,
|
||||||
|
}
|
||||||
|
|
||||||
/// One table row.
|
/// One table row.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct FeedRow {
|
struct FeedRow {
|
||||||
@@ -79,7 +85,7 @@ struct FeedRow {
|
|||||||
/// The candidate's title, or its feed URL when it has none.
|
/// The candidate's title, or its feed URL when it has none.
|
||||||
label: String,
|
label: String,
|
||||||
host: String,
|
host: String,
|
||||||
interests: Vec<String>,
|
interests: Vec<WhyInterest>,
|
||||||
articles: Vec<WhyArticle>,
|
articles: Vec<WhyArticle>,
|
||||||
article_count: usize,
|
article_count: usize,
|
||||||
first_seen: String,
|
first_seen: String,
|
||||||
@@ -383,7 +389,13 @@ fn row(
|
|||||||
) -> FeedRow {
|
) -> FeedRow {
|
||||||
let mut scored = evidence_of(candidate, evidence);
|
let mut scored = evidence_of(candidate, evidence);
|
||||||
let score = discovery::score(&scored);
|
let score = discovery::score(&scored);
|
||||||
let interests = discovery::why(&scored);
|
let interests = discovery::why(&scored)
|
||||||
|
.into_iter()
|
||||||
|
.map(|name| WhyInterest {
|
||||||
|
href: crate::interests::articles_href(&name),
|
||||||
|
name,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
// Best evidence first; the articles nothing is known about come last, in
|
// Best evidence first; the articles nothing is known about come last, in
|
||||||
// title order.
|
// title order.
|
||||||
|
|||||||
@@ -0,0 +1,485 @@
|
|||||||
|
//! Dashboard: standing interests and their rating-derived weights.
|
||||||
|
|
||||||
|
use std::cmp::Ordering;
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
|
||||||
|
use askama::Template;
|
||||||
|
use axum::Router;
|
||||||
|
use axum::extract::{Extension, Form, Path, Query, State};
|
||||||
|
use axum::response::{IntoResponse, Redirect, Response};
|
||||||
|
use axum::routing::{get, post};
|
||||||
|
use axum_login::tower_sessions::Session;
|
||||||
|
use jiff::Timestamp;
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
use super::jobs::set_flash;
|
||||||
|
use crate::curate::signals;
|
||||||
|
use crate::interests as interest_store;
|
||||||
|
use crate::server::AppState;
|
||||||
|
use crate::types::ArticleId;
|
||||||
|
use crate::web::session::{AuthSession, Viewer};
|
||||||
|
use crate::web::{Html, Page, WebError, encode_component, take_flash};
|
||||||
|
|
||||||
|
const PATH: &str = "/dashboard/interests";
|
||||||
|
const SORTS: [&str; 4] = ["weight", "name", "matches", "added"];
|
||||||
|
|
||||||
|
/// Routes contributed by the interests page.
|
||||||
|
pub fn routes() -> Router<AppState> {
|
||||||
|
Router::new()
|
||||||
|
.route(PATH, get(index).post(add))
|
||||||
|
.route("/dashboard/interests/{id}/category", post(set_category))
|
||||||
|
.route("/dashboard/interests/{id}/delete", post(delete))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Deserialize)]
|
||||||
|
struct InterestsQuery {
|
||||||
|
category: Option<String>,
|
||||||
|
sort: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct AddForm {
|
||||||
|
name: String,
|
||||||
|
#[serde(default)]
|
||||||
|
category: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct CategoryForm {
|
||||||
|
#[serde(default)]
|
||||||
|
category: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct CategoryOption {
|
||||||
|
name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct InterestRow {
|
||||||
|
id: i64,
|
||||||
|
name: String,
|
||||||
|
href: String,
|
||||||
|
category: Option<String>,
|
||||||
|
category_href: String,
|
||||||
|
weight: String,
|
||||||
|
weight_value: f64,
|
||||||
|
up: String,
|
||||||
|
down: String,
|
||||||
|
rated_matches: usize,
|
||||||
|
matched_articles: i64,
|
||||||
|
added: String,
|
||||||
|
created_at: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Template)]
|
||||||
|
#[template(path = "dashboard/interests.html")]
|
||||||
|
struct InterestsTemplate {
|
||||||
|
page: Page,
|
||||||
|
rows: Vec<InterestRow>,
|
||||||
|
categories: Vec<CategoryOption>,
|
||||||
|
selected_category: String,
|
||||||
|
selected_sort: String,
|
||||||
|
sorts: Vec<&'static str>,
|
||||||
|
total: usize,
|
||||||
|
uncategorized: usize,
|
||||||
|
lookback_days: i64,
|
||||||
|
half_life_days: String,
|
||||||
|
affinity_gate: String,
|
||||||
|
attributable: usize,
|
||||||
|
affinity_full: usize,
|
||||||
|
jobs_enabled: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn index(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
auth: AuthSession,
|
||||||
|
Extension(session): Extension<Session>,
|
||||||
|
Query(query): Query<InterestsQuery>,
|
||||||
|
) -> Result<Response, WebError> {
|
||||||
|
let viewer = auth
|
||||||
|
.user()
|
||||||
|
.await
|
||||||
|
.map(Viewer::from)
|
||||||
|
.ok_or_else(|| WebError::Unauthenticated { next: PATH.into() })?;
|
||||||
|
let config = state.config();
|
||||||
|
let ranking = &config.curation.ranking;
|
||||||
|
let db = &state.db;
|
||||||
|
let now = Timestamp::now();
|
||||||
|
let interests = interest_store::list(db).await.map_err(WebError::Internal)?;
|
||||||
|
let total = interests.len();
|
||||||
|
let uncategorized = interests
|
||||||
|
.iter()
|
||||||
|
.filter(|interest| interest.category.is_none())
|
||||||
|
.count();
|
||||||
|
|
||||||
|
let mut category_names = interests
|
||||||
|
.iter()
|
||||||
|
.filter_map(|interest| interest.category.clone())
|
||||||
|
.collect::<BTreeSet<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
category_names.sort_by(|left, right| {
|
||||||
|
left.to_lowercase()
|
||||||
|
.cmp(&right.to_lowercase())
|
||||||
|
.then_with(|| left.cmp(right))
|
||||||
|
});
|
||||||
|
let categories = category_names
|
||||||
|
.iter()
|
||||||
|
.map(|name| CategoryOption { name: name.clone() })
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
let selected_category = query
|
||||||
|
.category
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|category| !category.is_empty())
|
||||||
|
.map(str::to_string)
|
||||||
|
.unwrap_or_default();
|
||||||
|
let selected_sort = query
|
||||||
|
.sort
|
||||||
|
.as_deref()
|
||||||
|
.filter(|sort| SORTS.contains(sort))
|
||||||
|
.unwrap_or("weight")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let ratings = db.current_ratings(ranking.rating_lookback_days).await?;
|
||||||
|
let rated_ids = ratings
|
||||||
|
.iter()
|
||||||
|
.map(|rating| rating.article_id)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let matches = interest_store::matches_for_articles(db, &rated_ids)
|
||||||
|
.await
|
||||||
|
.map_err(WebError::Internal)?;
|
||||||
|
let rated = ratings
|
||||||
|
.iter()
|
||||||
|
.map(|rating| {
|
||||||
|
let age_days = (now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0;
|
||||||
|
(
|
||||||
|
rating.article_id,
|
||||||
|
rating.value,
|
||||||
|
signals::decay(age_days, ranking.rating_half_life_days),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<(ArticleId, f64, f64)>>();
|
||||||
|
let matched = matches
|
||||||
|
.iter()
|
||||||
|
.map(|row| (row.article_id, row.interest_id, row.z))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let rates = interest_store::rates(&rated, &matched);
|
||||||
|
let counts = interest_store::match_counts(db)
|
||||||
|
.await
|
||||||
|
.map_err(WebError::Internal)?;
|
||||||
|
|
||||||
|
let mut rows = interests
|
||||||
|
.into_iter()
|
||||||
|
.filter(|interest| match selected_category.as_str() {
|
||||||
|
"" => true,
|
||||||
|
"uncategorized" => interest.category.is_none(),
|
||||||
|
category => interest
|
||||||
|
.category
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|current| current.eq_ignore_ascii_case(category)),
|
||||||
|
})
|
||||||
|
.map(|interest| {
|
||||||
|
let rate = rates
|
||||||
|
.by_interest
|
||||||
|
.get(&interest.id)
|
||||||
|
.copied()
|
||||||
|
.unwrap_or_default();
|
||||||
|
let category_href = interest
|
||||||
|
.category
|
||||||
|
.as_deref()
|
||||||
|
.map(|category| format!("{PATH}?category={}", encode_component(category)))
|
||||||
|
.unwrap_or_else(|| format!("{PATH}?category=uncategorized"));
|
||||||
|
InterestRow {
|
||||||
|
id: interest.id,
|
||||||
|
href: interest_store::articles_href(&interest.name),
|
||||||
|
name: interest.name,
|
||||||
|
category: interest.category,
|
||||||
|
category_href,
|
||||||
|
weight: if rate.n > 0 {
|
||||||
|
format!("{:.2}", rate.weight())
|
||||||
|
} else {
|
||||||
|
"—".into()
|
||||||
|
},
|
||||||
|
weight_value: rate.weight(),
|
||||||
|
up: format!("{:.2}", rate.up),
|
||||||
|
down: format!("{:.2}", rate.down),
|
||||||
|
rated_matches: rate.n,
|
||||||
|
matched_articles: counts.get(&interest.id).copied().unwrap_or(0),
|
||||||
|
added: super::fmt_stored_time(Some(&interest.created_at), &config),
|
||||||
|
created_at: interest.created_at,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
sort_rows(&mut rows, &selected_sort);
|
||||||
|
|
||||||
|
let mut page = Page::new("Interests", Some(viewer), "interests");
|
||||||
|
page.flash = take_flash(&session).await?;
|
||||||
|
Ok(Html(InterestsTemplate {
|
||||||
|
page,
|
||||||
|
rows,
|
||||||
|
categories,
|
||||||
|
selected_category,
|
||||||
|
selected_sort,
|
||||||
|
sorts: SORTS.to_vec(),
|
||||||
|
total,
|
||||||
|
uncategorized,
|
||||||
|
lookback_days: ranking.rating_lookback_days,
|
||||||
|
half_life_days: format!("{}", ranking.rating_half_life_days),
|
||||||
|
affinity_gate: format!(
|
||||||
|
"{:.2}",
|
||||||
|
signals::gate(
|
||||||
|
rates.attributable,
|
||||||
|
ranking.affinity_floor,
|
||||||
|
ranking.affinity_full
|
||||||
|
)
|
||||||
|
),
|
||||||
|
attributable: rates.attributable,
|
||||||
|
affinity_full: ranking.affinity_full,
|
||||||
|
jobs_enabled: config.server.jobs_enabled,
|
||||||
|
})
|
||||||
|
.into_response())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sort_rows(rows: &mut [InterestRow], sort: &str) {
|
||||||
|
rows.sort_by(|left, right| {
|
||||||
|
let selected = match sort {
|
||||||
|
"name" => Ordering::Equal,
|
||||||
|
"matches" => right.matched_articles.cmp(&left.matched_articles),
|
||||||
|
"added" => right.created_at.cmp(&left.created_at),
|
||||||
|
_ => right
|
||||||
|
.weight_value
|
||||||
|
.partial_cmp(&left.weight_value)
|
||||||
|
.unwrap_or(Ordering::Equal),
|
||||||
|
};
|
||||||
|
selected.then_with(|| {
|
||||||
|
left.name
|
||||||
|
.to_lowercase()
|
||||||
|
.cmp(&right.name.to_lowercase())
|
||||||
|
.then_with(|| left.name.cmp(&right.name))
|
||||||
|
})
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn form_category(value: &str) -> Option<&str> {
|
||||||
|
let value = value.trim();
|
||||||
|
(!value.is_empty()).then_some(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn add(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(session): Extension<Session>,
|
||||||
|
Form(form): Form<AddForm>,
|
||||||
|
) -> Result<Response, WebError> {
|
||||||
|
match interest_store::add(
|
||||||
|
&state.db,
|
||||||
|
&form.name,
|
||||||
|
form_category(&form.category),
|
||||||
|
Timestamp::now(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(interest_store::AddOutcome::Added(_)) => {
|
||||||
|
set_flash(&session, "success", format!("Added {}.", form.name.trim())).await?;
|
||||||
|
}
|
||||||
|
Ok(interest_store::AddOutcome::Duplicate) => {
|
||||||
|
set_flash(
|
||||||
|
&session,
|
||||||
|
"error",
|
||||||
|
format!("{} already exists.", form.name.trim()),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
Err(error) => set_flash(&session, "error", error.to_string()).await?,
|
||||||
|
}
|
||||||
|
Ok(Redirect::to(PATH).into_response())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn set_category(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(session): Extension<Session>,
|
||||||
|
Path(id): Path<i64>,
|
||||||
|
Form(form): Form<CategoryForm>,
|
||||||
|
) -> Result<Response, WebError> {
|
||||||
|
interest_store::set_category(
|
||||||
|
&state.db,
|
||||||
|
id,
|
||||||
|
form_category(&form.category),
|
||||||
|
Timestamp::now(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(WebError::Internal)?;
|
||||||
|
set_flash(&session, "success", "Category saved.".into()).await?;
|
||||||
|
Ok(Redirect::to(PATH).into_response())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(session): Extension<Session>,
|
||||||
|
Path(id): Path<i64>,
|
||||||
|
) -> Result<Response, WebError> {
|
||||||
|
let name: Option<String> = sqlx::query_scalar("SELECT name FROM interests WHERE id = ?")
|
||||||
|
.bind(id)
|
||||||
|
.fetch_optional(state.db.pool())
|
||||||
|
.await
|
||||||
|
.map_err(super::db_err)?;
|
||||||
|
interest_store::delete(&state.db, id)
|
||||||
|
.await
|
||||||
|
.map_err(WebError::Internal)?;
|
||||||
|
set_flash(
|
||||||
|
&session,
|
||||||
|
"success",
|
||||||
|
name.map(|name| format!("Deleted {name}."))
|
||||||
|
.unwrap_or_else(|| "Interest already deleted.".into()),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(Redirect::to(PATH).into_response())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use axum::body::Body;
|
||||||
|
use axum::http::{Method, Request, StatusCode, header};
|
||||||
|
use tower::ServiceExt;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::interests::AddOutcome;
|
||||||
|
use crate::web::dashboard::tests::{
|
||||||
|
app_with_users, assert_admin_only, login_cookie, response_text, seed,
|
||||||
|
};
|
||||||
|
|
||||||
|
async fn post_form(
|
||||||
|
app: &axum::Router,
|
||||||
|
uri: &str,
|
||||||
|
cookie: Option<&str>,
|
||||||
|
body: &str,
|
||||||
|
) -> Response {
|
||||||
|
let mut request = Request::builder()
|
||||||
|
.method(Method::POST)
|
||||||
|
.uri(uri)
|
||||||
|
.header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
||||||
|
.header("sec-fetch-site", "same-origin");
|
||||||
|
if let Some(cookie) = cookie {
|
||||||
|
request = request.header(header::COOKIE, cookie);
|
||||||
|
}
|
||||||
|
app.clone()
|
||||||
|
.oneshot(request.body(Body::from(body.to_string())).unwrap())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn page_sorts_rating_weights_descending_and_is_admin_only() {
|
||||||
|
let seed = seed().await;
|
||||||
|
let now = Timestamp::now();
|
||||||
|
let AddOutcome::Added(rust) = interest_store::add(&seed.db, "Rust", Some("Software"), now)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
else {
|
||||||
|
unreachable!();
|
||||||
|
};
|
||||||
|
interest_store::add(&seed.db, "Cooking", None, now)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO article_interests (article_id, interest_id, cos, z, run_id)
|
||||||
|
VALUES (1, ?, 0.8, 3.0, ?)",
|
||||||
|
)
|
||||||
|
.bind(rust)
|
||||||
|
.bind(seed.run_id)
|
||||||
|
.execute(seed.db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let app = app_with_users(&seed.db).await;
|
||||||
|
let body = assert_admin_only(&app, PATH).await;
|
||||||
|
let table = body.split_once("<tbody>").unwrap().1;
|
||||||
|
assert!(table.find(">Rust</a>").unwrap() < table.find(">Cooking</a>").unwrap());
|
||||||
|
assert!(
|
||||||
|
table.contains("/dashboard/articles?interest=Rust"),
|
||||||
|
"{table}"
|
||||||
|
);
|
||||||
|
assert!(body.contains("affinity gate"), "{body}");
|
||||||
|
|
||||||
|
let reader = login_cookie(&app, "reader", "correct horse battery").await;
|
||||||
|
for (uri, body) in [
|
||||||
|
(PATH.to_string(), "name=New&category="),
|
||||||
|
(
|
||||||
|
format!("/dashboard/interests/{rust}/category"),
|
||||||
|
"category=Other",
|
||||||
|
),
|
||||||
|
(format!("/dashboard/interests/{rust}/delete"), ""),
|
||||||
|
] {
|
||||||
|
let forbidden = post_form(&app, &uri, Some(&reader), body).await;
|
||||||
|
assert_eq!(forbidden.status(), StatusCode::FORBIDDEN, "{uri}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn add_reports_a_case_insensitive_duplicate() {
|
||||||
|
let seed = seed().await;
|
||||||
|
interest_store::add(&seed.db, "Rust", None, Timestamp::now())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let app = app_with_users(&seed.db).await;
|
||||||
|
let admin = login_cookie(&app, "admin", "correct horse battery").await;
|
||||||
|
let response = post_form(&app, PATH, Some(&admin), "name=rust&category=").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::SEE_OTHER);
|
||||||
|
let body =
|
||||||
|
response_text(crate::web::dashboard::tests::get(&app, PATH, Some(&admin)).await).await;
|
||||||
|
assert!(body.contains("rust already exists."), "{body}");
|
||||||
|
assert_eq!(interest_store::list(&seed.db).await.unwrap().len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn delete_action_removes_matches_and_the_embedding() {
|
||||||
|
let seed = seed().await;
|
||||||
|
let AddOutcome::Added(id) = interest_store::add(&seed.db, "Rust", None, Timestamp::now())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
else {
|
||||||
|
unreachable!();
|
||||||
|
};
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO article_interests (article_id, interest_id, cos, z)
|
||||||
|
VALUES (1, ?, 0.8, 2.0)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.execute(seed.db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO interest_embeddings
|
||||||
|
(interest, model, dimension, embedding, created_at)
|
||||||
|
VALUES ('Rust', 'test', 1, X'00000000', '2026-09-12T00:00:00Z')",
|
||||||
|
)
|
||||||
|
.execute(seed.db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let app = app_with_users(&seed.db).await;
|
||||||
|
let admin = login_cookie(&app, "admin", "correct horse battery").await;
|
||||||
|
let response = post_form(
|
||||||
|
&app,
|
||||||
|
&format!("/dashboard/interests/{id}/delete"),
|
||||||
|
Some(&admin),
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::SEE_OTHER);
|
||||||
|
let interests: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM interests")
|
||||||
|
.fetch_one(seed.db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let matches: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM article_interests")
|
||||||
|
.fetch_one(seed.db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let embeddings: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM interest_embeddings")
|
||||||
|
.fetch_one(seed.db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!((interests, matches, embeddings), (0, 0, 0));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,6 +11,7 @@
|
|||||||
|
|
||||||
pub mod articles;
|
pub mod articles;
|
||||||
pub mod feeds;
|
pub mod feeds;
|
||||||
|
pub mod interests;
|
||||||
pub mod jobs;
|
pub mod jobs;
|
||||||
pub mod profile;
|
pub mod profile;
|
||||||
pub mod ratings;
|
pub mod ratings;
|
||||||
@@ -45,6 +46,7 @@ pub fn router() -> Router<AppState> {
|
|||||||
.merge(runs::routes())
|
.merge(runs::routes())
|
||||||
.merge(articles::routes())
|
.merge(articles::routes())
|
||||||
.merge(ratings::routes())
|
.merge(ratings::routes())
|
||||||
|
.merge(interests::routes())
|
||||||
.merge(feeds::routes())
|
.merge(feeds::routes())
|
||||||
.merge(profile::routes())
|
.merge(profile::routes())
|
||||||
.merge(settings::routes())
|
.merge(settings::routes())
|
||||||
@@ -58,10 +60,11 @@ pub fn router() -> Router<AppState> {
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/// Signal names in the order of curation plan §7.5.
|
/// Signal names in the order of curation plan §7.5.
|
||||||
pub const SIGNAL_NAMES: [&str; 8] = [
|
pub const SIGNAL_NAMES: [&str; 9] = [
|
||||||
"interest",
|
"interest",
|
||||||
"knn",
|
"knn",
|
||||||
"feed",
|
"feed",
|
||||||
|
"affinity",
|
||||||
"social",
|
"social",
|
||||||
"heuristic",
|
"heuristic",
|
||||||
"triage",
|
"triage",
|
||||||
@@ -276,6 +279,7 @@ pub struct SignalLine {
|
|||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct InterestLine {
|
pub struct InterestLine {
|
||||||
pub name: String,
|
pub name: String,
|
||||||
|
pub href: String,
|
||||||
pub z: String,
|
pub z: String,
|
||||||
pub cos: String,
|
pub cos: String,
|
||||||
}
|
}
|
||||||
@@ -343,6 +347,7 @@ impl SignalsView {
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|interest| InterestLine {
|
.map(|interest| InterestLine {
|
||||||
name: interest.name.clone(),
|
name: interest.name.clone(),
|
||||||
|
href: crate::interests::articles_href(&interest.name),
|
||||||
z: format!("{:.2}", interest.z),
|
z: format!("{:.2}", interest.z),
|
||||||
cos: format!("{:.3}", interest.cos),
|
cos: format!("{:.3}", interest.cos),
|
||||||
})
|
})
|
||||||
@@ -428,6 +433,8 @@ struct OverviewTemplate {
|
|||||||
ratings_total: i64,
|
ratings_total: i64,
|
||||||
access_requests: i64,
|
access_requests: i64,
|
||||||
feed_candidates: i64,
|
feed_candidates: i64,
|
||||||
|
interests_total: usize,
|
||||||
|
uncategorized_interests: usize,
|
||||||
unrated: Vec<UnratedPick>,
|
unrated: Vec<UnratedPick>,
|
||||||
active_jobs: Vec<JobLine>,
|
active_jobs: Vec<JobLine>,
|
||||||
finished_jobs: Vec<JobLine>,
|
finished_jobs: Vec<JobLine>,
|
||||||
@@ -455,6 +462,14 @@ async fn overview(
|
|||||||
.await
|
.await
|
||||||
.map_err(db_err)?;
|
.map_err(db_err)?;
|
||||||
let feed_candidates = crate::discovery::count(db, "candidate").await?;
|
let feed_candidates = crate::discovery::count(db, "candidate").await?;
|
||||||
|
let interests = crate::interests::list(db)
|
||||||
|
.await
|
||||||
|
.map_err(WebError::Internal)?;
|
||||||
|
let interests_total = interests.len();
|
||||||
|
let uncategorized_interests = interests
|
||||||
|
.iter()
|
||||||
|
.filter(|interest| interest.category.is_none())
|
||||||
|
.count();
|
||||||
let unrated = unrated_picks(db).await?;
|
let unrated = unrated_picks(db).await?;
|
||||||
let (active_jobs, finished_jobs) = jobs_summary(db, &config).await?;
|
let (active_jobs, finished_jobs) = jobs_summary(db, &config).await?;
|
||||||
let sparklines = overview_sparklines(db).await?;
|
let sparklines = overview_sparklines(db).await?;
|
||||||
@@ -474,6 +489,8 @@ async fn overview(
|
|||||||
ratings_total,
|
ratings_total,
|
||||||
access_requests,
|
access_requests,
|
||||||
feed_candidates,
|
feed_candidates,
|
||||||
|
interests_total,
|
||||||
|
uncategorized_interests,
|
||||||
unrated,
|
unrated,
|
||||||
active_jobs,
|
active_jobs,
|
||||||
finished_jobs,
|
finished_jobs,
|
||||||
@@ -1225,6 +1242,8 @@ pub(crate) mod tests {
|
|||||||
"/dashboard/articles/1".to_string(),
|
"/dashboard/articles/1".to_string(),
|
||||||
"/dashboard/ratings".to_string(),
|
"/dashboard/ratings".to_string(),
|
||||||
"/dashboard/ratings?tab=events".to_string(),
|
"/dashboard/ratings?tab=events".to_string(),
|
||||||
|
"/dashboard/interests".to_string(),
|
||||||
|
"/dashboard/interests?category=uncategorized&sort=name".to_string(),
|
||||||
"/dashboard/profile".to_string(),
|
"/dashboard/profile".to_string(),
|
||||||
"/dashboard/stats?days=14".to_string(),
|
"/dashboard/stats?days=14".to_string(),
|
||||||
"/dashboard/settings".to_string(),
|
"/dashboard/settings".to_string(),
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
//! Dashboard: the profile page (`/dashboard/profile`, web plan §11).
|
//! Dashboard: the profile page (`/dashboard/profile`, web plan §11).
|
||||||
//!
|
//!
|
||||||
//! Edits `profile.md` with version history, shows what the loader parses out
|
//! Edits `profile.md` with version history and shows the stored interests,
|
||||||
//! of it, the standing OPML interests by theme, the stored system prompt and
|
//! system prompt, and weekly learned adjustments.
|
||||||
//! the weekly learned adjustments, and offers the `profile-rebuild` job.
|
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
@@ -18,6 +17,7 @@ use sqlx::Row;
|
|||||||
|
|
||||||
use crate::curate::profile::{self, KV_LEARNED_ADJUSTMENTS, ProfileFile, REBUILD_INTERVAL_DAYS};
|
use crate::curate::profile::{self, KV_LEARNED_ADJUSTMENTS, ProfileFile, REBUILD_INTERVAL_DAYS};
|
||||||
use crate::db::{Db, DbError, KV_TASTE_PROFILE};
|
use crate::db::{Db, DbError, KV_TASTE_PROFILE};
|
||||||
|
use crate::interests;
|
||||||
use crate::server::AppState;
|
use crate::server::AppState;
|
||||||
use crate::web::session::{AuthSession, Viewer};
|
use crate::web::session::{AuthSession, Viewer};
|
||||||
use crate::web::{Flash, Html, Page, WebError, format_time, take_flash};
|
use crate::web::{Flash, Html, Page, WebError, format_time, take_flash};
|
||||||
@@ -104,8 +104,7 @@ fn read_profile(path: &Path) -> anyhow::Result<Option<String>> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The live preview of what the loader extracts (§11): the passthrough body
|
/// The live preview of the prose that reaches the prompt.
|
||||||
/// and the `## Interests` lines.
|
|
||||||
pub fn preview(content: &str) -> ProfileFile {
|
pub fn preview(content: &str) -> ProfileFile {
|
||||||
profile::parse_profile_str(content)
|
profile::parse_profile_str(content)
|
||||||
}
|
}
|
||||||
@@ -191,7 +190,7 @@ async fn versions(db: &Db, config: &crate::config::Config) -> Result<Vec<Version
|
|||||||
// Page
|
// Page
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
struct ThemeView {
|
struct CategoryView {
|
||||||
name: String,
|
name: String,
|
||||||
members: String,
|
members: String,
|
||||||
count: usize,
|
count: usize,
|
||||||
@@ -207,12 +206,10 @@ struct ProfileTemplate {
|
|||||||
bytes: usize,
|
bytes: usize,
|
||||||
max_bytes: usize,
|
max_bytes: usize,
|
||||||
preview_body: String,
|
preview_body: String,
|
||||||
preview_interests: Vec<String>,
|
|
||||||
versions: Vec<VersionView>,
|
versions: Vec<VersionView>,
|
||||||
opml_path: String,
|
interest_count: usize,
|
||||||
opml_count: usize,
|
category_count: usize,
|
||||||
opml_error: String,
|
categories: Vec<CategoryView>,
|
||||||
themes: Vec<ThemeView>,
|
|
||||||
prompt: String,
|
prompt: String,
|
||||||
prompt_chars: usize,
|
prompt_chars: usize,
|
||||||
prompt_version: String,
|
prompt_version: String,
|
||||||
@@ -254,20 +251,17 @@ async fn show(
|
|||||||
let content = stored.unwrap_or_default();
|
let content = stored.unwrap_or_default();
|
||||||
let parsed = preview(&content);
|
let parsed = preview(&content);
|
||||||
|
|
||||||
let (opml_count, opml_error, themes) = match profile::parse_interests(&config.interests_opml) {
|
let grouped = interests::grouped(db).await.map_err(WebError::Internal)?;
|
||||||
Ok(interests) => {
|
let interest_count = grouped.iter().map(|(_, members)| members.len()).sum();
|
||||||
let themes = profile::group_into_themes(&interests)
|
let category_count = grouped.len();
|
||||||
.into_iter()
|
let categories = grouped
|
||||||
.map(|(name, members)| ThemeView {
|
.into_iter()
|
||||||
name,
|
.map(|(name, members)| CategoryView {
|
||||||
count: members.len(),
|
name,
|
||||||
members: members.join(", "),
|
count: members.len(),
|
||||||
})
|
members: members.join(", "),
|
||||||
.collect();
|
})
|
||||||
(interests.len(), String::new(), themes)
|
.collect();
|
||||||
}
|
|
||||||
Err(error) => (0, format!("{error:#}"), Vec::new()),
|
|
||||||
};
|
|
||||||
|
|
||||||
let prompt = db.kv_get(KV_TASTE_PROFILE).await?.unwrap_or_default();
|
let prompt = db.kv_get(KV_TASTE_PROFILE).await?.unwrap_or_default();
|
||||||
let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default();
|
let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default();
|
||||||
@@ -297,12 +291,10 @@ async fn show(
|
|||||||
max_bytes: MAX_PROFILE_BYTES,
|
max_bytes: MAX_PROFILE_BYTES,
|
||||||
content,
|
content,
|
||||||
preview_body: parsed.body,
|
preview_body: parsed.body,
|
||||||
preview_interests: parsed.interests,
|
|
||||||
versions: versions(db, &config).await?,
|
versions: versions(db, &config).await?,
|
||||||
opml_path: config.interests_opml.display().to_string(),
|
interest_count,
|
||||||
opml_count,
|
category_count,
|
||||||
opml_error,
|
categories,
|
||||||
themes,
|
|
||||||
prompt_chars: prompt.len(),
|
prompt_chars: prompt.len(),
|
||||||
prompt_verdicts: count_verdict_lines(&prompt),
|
prompt_verdicts: count_verdict_lines(&prompt),
|
||||||
prompt,
|
prompt,
|
||||||
@@ -516,15 +508,15 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
let config = Config {
|
let config = Config {
|
||||||
profile_path: dir.path().join("profile.md"),
|
profile_path: dir.path().join("profile.md"),
|
||||||
interests_opml: dir.path().join("interests.opml"),
|
|
||||||
..Config::default()
|
..Config::default()
|
||||||
};
|
};
|
||||||
std::fs::write(&config.profile_path, "# Original\n\nProse.\n").unwrap();
|
std::fs::write(&config.profile_path, "# Original\n\nProse.\n").unwrap();
|
||||||
std::fs::write(
|
interests::add(&db, "Rust", Some("Software"), Timestamp::now())
|
||||||
&config.interests_opml,
|
.await
|
||||||
r#"<outline text="Rust"/><outline text="Boston"/>"#,
|
.unwrap();
|
||||||
)
|
interests::add(&db, "Boston", Some("Places"), Timestamp::now())
|
||||||
.unwrap();
|
.await
|
||||||
|
.unwrap();
|
||||||
db.kv_set(
|
db.kv_set(
|
||||||
KV_TASTE_PROFILE,
|
KV_TASTE_PROFILE,
|
||||||
"system prompt text\n\n## Recent verdicts\n\nLOVED | x\n",
|
"system prompt text\n\n## Recent verdicts\n\nLOVED | x\n",
|
||||||
@@ -617,7 +609,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn profile_page_shows_editor_preview_interests_prompt_and_rebuild_form() {
|
async fn profile_page_shows_editor_standing_interests_prompt_and_rebuild_form() {
|
||||||
let (_dir, _state, app, cookie) = setup().await;
|
let (_dir, _state, app, cookie) = setup().await;
|
||||||
let response = get(&app, Some(&cookie)).await;
|
let response = get(&app, Some(&cookie)).await;
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
@@ -625,7 +617,8 @@ mod tests {
|
|||||||
assert!(body.contains("# Original"));
|
assert!(body.contains("# Original"));
|
||||||
assert!(body.contains("Prose."));
|
assert!(body.contains("Prose."));
|
||||||
assert!(body.contains("Rust, Boston") || body.contains("Rust") && body.contains("Boston"));
|
assert!(body.contains("Rust, Boston") || body.contains("Rust") && body.contains("Boston"));
|
||||||
assert!(body.contains("2 interests"));
|
assert!(body.contains("2 standing interests in 2 categories"));
|
||||||
|
assert!(body.contains("Interests page"));
|
||||||
assert!(body.contains("system prompt text"));
|
assert!(body.contains("system prompt text"));
|
||||||
assert!(body.contains("Rank depth higher."));
|
assert!(body.contains("Rank depth higher."));
|
||||||
assert!(body.contains("never built"));
|
assert!(body.contains("never built"));
|
||||||
@@ -666,7 +659,7 @@ mod tests {
|
|||||||
|
|
||||||
let page = text(get(&app, Some(&cookie)).await).await;
|
let page = text(get(&app, Some(&cookie)).await).await;
|
||||||
assert!(page.contains("Saved; the next run rebuilds the system prompt."));
|
assert!(page.contains("Saved; the next run rebuilds the system prompt."));
|
||||||
assert!(page.contains("Writerdeck"));
|
assert!(page.contains("section is ignored"));
|
||||||
assert!(page.contains("# Original"));
|
assert!(page.contains("# Original"));
|
||||||
assert!(page.contains(">tyler<"));
|
assert!(page.contains(">tyler<"));
|
||||||
|
|
||||||
|
|||||||
@@ -246,7 +246,6 @@ const PATH_KEYS: &[&str] = &[
|
|||||||
"database_path",
|
"database_path",
|
||||||
"out_dir",
|
"out_dir",
|
||||||
"profile_path",
|
"profile_path",
|
||||||
"interests_opml",
|
|
||||||
"publish.epub_dir",
|
"publish.epub_dir",
|
||||||
"publish.xtc_dir",
|
"publish.xtc_dir",
|
||||||
"xtc.settings",
|
"xtc.settings",
|
||||||
@@ -295,7 +294,6 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[
|
|||||||
("database_path", "SQLite file; parent directories are created on demand."),
|
("database_path", "SQLite file; parent directories are created on demand."),
|
||||||
("out_dir", "Where generate writes artifacts before publishing (overridden by --out)."),
|
("out_dir", "Where generate writes artifacts before publishing (overridden by --out)."),
|
||||||
("profile_path", "Hand-maintained reader profile, loaded every run."),
|
("profile_path", "Hand-maintained reader profile, loaded every run."),
|
||||||
("interests_opml", "Scour interests OPML merged with the profile interests."),
|
|
||||||
("miniflux.base_url", "Miniflux root (no /v1)."),
|
("miniflux.base_url", "Miniflux root (no /v1)."),
|
||||||
("miniflux.public_url", "Browser-facing Miniflux web UI URL for dashboard links. Defaults to miniflux.base_url."),
|
("miniflux.public_url", "Browser-facing Miniflux web UI URL for dashboard links. Defaults to miniflux.base_url."),
|
||||||
("miniflux.api_key", "X-Auth-Token for Miniflux. Required; environment only."),
|
("miniflux.api_key", "X-Auth-Token for Miniflux. Required; environment only."),
|
||||||
@@ -351,6 +349,8 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[
|
|||||||
("curation.ranking.knn_full", "Rated articles at which the knn signal reaches full weight. Must be > knn_floor."),
|
("curation.ranking.knn_full", "Rated articles at which the knn signal reaches full weight. Must be > knn_floor."),
|
||||||
("curation.ranking.feed_floor", "Attributable ratings before the feed-affinity signal starts to count."),
|
("curation.ranking.feed_floor", "Attributable ratings before the feed-affinity signal starts to count."),
|
||||||
("curation.ranking.feed_full", "Attributable ratings at which feed affinity reaches full weight. Must be > feed_floor."),
|
("curation.ranking.feed_full", "Attributable ratings at which feed affinity reaches full weight. Must be > feed_floor."),
|
||||||
|
("curation.ranking.affinity_floor", "Interest-attributable ratings before the affinity signal starts to count."),
|
||||||
|
("curation.ranking.affinity_full", "Interest-attributable ratings at which affinity reaches full weight. Must be > affinity_floor."),
|
||||||
("curation.ranking.semantic_min_words", "Bodies shorter than this are not embedded."),
|
("curation.ranking.semantic_min_words", "Bodies shorter than this are not embedded."),
|
||||||
("curation.ranking.exploration_slots", "Shortlist slots reserved for exploration picks."),
|
("curation.ranking.exploration_slots", "Shortlist slots reserved for exploration picks."),
|
||||||
("curation.ranking.embedding_retention_days", "features prune: unrated, unpublished vectors older than this are deleted."),
|
("curation.ranking.embedding_retention_days", "features prune: unrated, unpublished vectors older than this are deleted."),
|
||||||
@@ -360,12 +360,14 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[
|
|||||||
("curation.ranking.quotas.knn", "Deep-set slots filled by rated-neighbour preference."),
|
("curation.ranking.quotas.knn", "Deep-set slots filled by rated-neighbour preference."),
|
||||||
("curation.ranking.weights.preliminary.interest", "Interest similarity in the preliminary blend."),
|
("curation.ranking.weights.preliminary.interest", "Interest similarity in the preliminary blend."),
|
||||||
("curation.ranking.weights.preliminary.knn", "Rated-neighbour preference in the preliminary blend."),
|
("curation.ranking.weights.preliminary.knn", "Rated-neighbour preference in the preliminary blend."),
|
||||||
|
("curation.ranking.weights.preliminary.affinity", "Rating-derived interest affinity in the preliminary blend."),
|
||||||
("curation.ranking.weights.preliminary.heuristic", "Heuristic score in the preliminary blend."),
|
("curation.ranking.weights.preliminary.heuristic", "Heuristic score in the preliminary blend."),
|
||||||
("curation.ranking.weights.preliminary.feed", "Feed affinity in the preliminary blend."),
|
("curation.ranking.weights.preliminary.feed", "Feed affinity in the preliminary blend."),
|
||||||
("curation.ranking.weights.preliminary.social", "Social signal in the preliminary blend."),
|
("curation.ranking.weights.preliminary.social", "Social signal in the preliminary blend."),
|
||||||
("curation.ranking.weights.utility.quality", "Deep-assessment quality in the utility score."),
|
("curation.ranking.weights.utility.quality", "Deep-assessment quality in the utility score."),
|
||||||
("curation.ranking.weights.utility.fit", "Deep-assessment fit in the utility score."),
|
("curation.ranking.weights.utility.fit", "Deep-assessment fit in the utility score."),
|
||||||
("curation.ranking.weights.utility.knn", "Rated-neighbour preference in the utility score."),
|
("curation.ranking.weights.utility.knn", "Rated-neighbour preference in the utility score."),
|
||||||
|
("curation.ranking.weights.utility.affinity", "Rating-derived interest affinity in the utility score."),
|
||||||
("curation.ranking.weights.utility.interest", "Interest similarity in the utility score."),
|
("curation.ranking.weights.utility.interest", "Interest similarity in the utility score."),
|
||||||
("curation.ranking.weights.utility.feed", "Feed affinity in the utility score."),
|
("curation.ranking.weights.utility.feed", "Feed affinity in the utility score."),
|
||||||
("curation.ranking.weights.utility.triage", "Triage score in the utility score."),
|
("curation.ranking.weights.utility.triage", "Triage score in the utility score."),
|
||||||
|
|||||||
@@ -2012,6 +2012,8 @@ mod tests {
|
|||||||
|
|
||||||
let article_id = source.lineup.picks[0].article.id;
|
let article_id = source.lineup.picks[0].article.id;
|
||||||
let dashboard_href = format!("/dashboard/articles/{article_id}");
|
let dashboard_href = format!("/dashboard/articles/{article_id}");
|
||||||
|
let interest_href = crate::interests::articles_href("Filesystems");
|
||||||
|
assert!(!issue.contains(&interest_href));
|
||||||
assert!(!issue.contains(&dashboard_href));
|
assert!(!issue.contains(&dashboard_href));
|
||||||
let article = app
|
let article = app
|
||||||
.clone()
|
.clone()
|
||||||
@@ -2741,6 +2743,7 @@ mod tests {
|
|||||||
assert!(admin_issue.contains("Was this a good pick?"));
|
assert!(admin_issue.contains("Was this a good pick?"));
|
||||||
assert!(admin_issue.contains("value=\"loved\" data-label=\"loved\" class=\"active\""));
|
assert!(admin_issue.contains("value=\"loved\" data-label=\"loved\" class=\"active\""));
|
||||||
assert!(admin_issue.contains(&dashboard_href));
|
assert!(admin_issue.contains(&dashboard_href));
|
||||||
|
assert!(admin_issue.contains(&crate::interests::articles_href("Filesystems")));
|
||||||
|
|
||||||
let admin_article = app
|
let admin_article = app
|
||||||
.clone()
|
.clone()
|
||||||
|
|||||||
@@ -322,6 +322,7 @@ impl Page {
|
|||||||
| "runs"
|
| "runs"
|
||||||
| "articles"
|
| "articles"
|
||||||
| "ratings"
|
| "ratings"
|
||||||
|
| "interests"
|
||||||
| "profile"
|
| "profile"
|
||||||
| "stats"
|
| "stats"
|
||||||
| "jobs"
|
| "jobs"
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
<div class="scroll-x"><table class="signals"><thead><tr><th>signal</th><th class="num">raw</th><th class="num">norm</th><th class="num">weight</th><th>present</th></tr></thead>
|
<div class="scroll-x"><table class="signals"><thead><tr><th>signal</th><th class="num">raw</th><th class="num">norm</th><th class="num">weight</th><th>present</th></tr></thead>
|
||||||
<tbody>{% for line in signals.lines %}<tr{% if !line.present %} class="muted"{% endif %}><td>{{ line.name }}</td><td class="num">{{ line.raw }}</td><td class="num">{{ line.norm }}</td><td class="num">{{ line.weight }}</td><td>{% if line.present %}yes{% else %}<span class="muted">absent</span>{% endif %}</td></tr>{% endfor %}</tbody></table></div>
|
<tbody>{% for line in signals.lines %}<tr{% if !line.present %} class="muted"{% endif %}><td>{{ line.name }}</td><td class="num">{{ line.raw }}</td><td class="num">{{ line.norm }}</td><td class="num">{{ line.weight }}</td><td>{% if line.present %}yes{% else %}<span class="muted">absent</span>{% endif %}</td></tr>{% endfor %}</tbody></table></div>
|
||||||
<p class="muted">Preliminary blend <span class="tabular-nums text-ink">{{ signals.blend }}</span>{% if let Some(cos) = signals.top1_cos %} · interest top-1 cosine <span class="tabular-nums">{{ cos }}</span>{% endif %}{% if signals.exploration %} · <span class="badge">exploration</span>{% endif %}{% if signals.auto_include %} · <span class="badge">auto-include</span>{% endif %}{% if signals.slop_author %} · <span class="badge down">slop author</span>{% endif %}</p>
|
<p class="muted">Preliminary blend <span class="tabular-nums text-ink">{{ signals.blend }}</span>{% if let Some(cos) = signals.top1_cos %} · interest top-1 cosine <span class="tabular-nums">{{ cos }}</span>{% endif %}{% if signals.exploration %} · <span class="badge">exploration</span>{% endif %}{% if signals.auto_include %} · <span class="badge">auto-include</span>{% endif %}{% if signals.slop_author %} · <span class="badge down">slop author</span>{% endif %}</p>
|
||||||
{% if !signals.top_interests.is_empty() %}<p class="page-eyebrow">Top interests</p><ul>{% for interest in signals.top_interests %}<li>{{ interest.name }} <span class="muted">· z {{ interest.z }} · cos {{ interest.cos }}</span></li>{% endfor %}</ul>{% endif %}
|
{% if !signals.top_interests.is_empty() %}<p class="page-eyebrow">Top interests</p><ul>{% for interest in signals.top_interests %}<li><a href="{{ interest.href }}">{{ interest.name }}</a> <span class="muted">· z {{ interest.z }} · cos {{ interest.cos }}</span></li>{% endfor %}</ul>{% endif %}
|
||||||
{% if !signals.neighbours.is_empty() %}<p class="page-eyebrow">Nearest rated neighbours</p><ul>{% for neighbour in signals.neighbours %}<li><span class="badge {{ neighbour.label }}">{{ neighbour.label }}</span> <a href="/dashboard/articles/{{ neighbour.article_id }}">{{ neighbour.title }}</a> <span class="muted">· cos {{ neighbour.cos }}</span></li>{% endfor %}</ul>{% endif %}
|
{% if !signals.neighbours.is_empty() %}<p class="page-eyebrow">Nearest rated neighbours</p><ul>{% for neighbour in signals.neighbours %}<li><span class="badge {{ neighbour.label }}">{{ neighbour.label }}</span> <a href="/dashboard/articles/{{ neighbour.article_id }}">{{ neighbour.title }}</a> <span class="muted">· cos {{ neighbour.cos }}</span></li>{% endfor %}</ul>{% endif %}
|
||||||
{% if !signals.notes.is_empty() %}<ul class="muted">{% for note in signals.notes %}<li>{{ note }}</li>{% endfor %}</ul>{% endif %}
|
{% if !signals.notes.is_empty() %}<ul class="muted">{% for note in signals.notes %}<li>{{ note }}</li>{% endfor %}</ul>{% endif %}
|
||||||
</div>{% endif %}
|
</div>{% endif %}
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{% if understanding.kicker.is_some() || understanding.topics.is_some() %}<p class="mt-3 font-sans text-sm text-muted">{% if let Some(kicker) = understanding.kicker %}<span class="text-[0.72rem] uppercase tracking-[0.12em]">{{ kicker }}</span>{% if let Some(topics) = understanding.topics %}<span class="ml-3">{{ topics }}</span>{% endif %}{% else %}{% if let Some(topics) = understanding.topics %}<span>{{ topics }}</span>{% endif %}{% endif %}</p>{% endif %}{% if why.is_some() || understanding.interests.is_some() %}<div class="mt-4 border-l-2 border-accent pl-3">{% if let Some(why) = why %}<p class="italic text-ink-2">Why it's here: {{ why }}</p>{% endif %}{% if let Some(interests) = understanding.interests %}<p class="mt-1 font-sans text-sm text-muted">Matches: {{ interests }}</p>{% endif %}</div>{% endif %}
|
{% if understanding.kicker.is_some() || understanding.topics.is_some() %}<p class="mt-3 font-sans text-sm text-muted">{% if let Some(kicker) = understanding.kicker %}<span class="text-[0.72rem] uppercase tracking-[0.12em]">{{ kicker }}</span>{% if let Some(topics) = understanding.topics %}<span class="ml-3">{{ topics }}</span>{% endif %}{% else %}{% if let Some(topics) = understanding.topics %}<span>{{ topics }}</span>{% endif %}{% endif %}</p>{% endif %}{% if why.is_some() || !understanding.interests.is_empty() %}<div class="mt-4 border-l-2 border-accent pl-3">{% if let Some(why) = why %}<p class="italic text-ink-2">Why it's here: {{ why }}</p>{% endif %}{% if !understanding.interests.is_empty() %}<p class="mt-1 font-sans text-sm text-muted">Matches: {% for interest in understanding.interests %}{% if page.is_admin() %}<a href="{{ interest.href }}">{{ interest.name }}</a>{% else %}{{ interest.name }}{% endif %}{% if !loop.last %} · {% endif %}{% endfor %}</p>{% endif %}</div>{% endif %}
|
||||||
|
|||||||
@@ -58,7 +58,7 @@
|
|||||||
|
|
||||||
<h2>Neighbours and interests</h2>
|
<h2>Neighbours and interests</h2>
|
||||||
{% if let Some(signals) = latest_signals %}{% if signals.top_interests.is_empty() && signals.neighbours.is_empty() %}<p class="muted text-sm">The latest run recorded no interests or neighbours for this article.</p>{% else %}<div class="cards">
|
{% if let Some(signals) = latest_signals %}{% if signals.top_interests.is_empty() && signals.neighbours.is_empty() %}<p class="muted text-sm">The latest run recorded no interests or neighbours for this article.</p>{% else %}<div class="cards">
|
||||||
<section class="card"><h3>Top interests</h3>{% if signals.top_interests.is_empty() %}<p class="muted">None.</p>{% else %}<div class="scroll-x"><table><thead><tr><th>interest</th><th class="num">z</th><th class="num">cos</th></tr></thead><tbody>{% for interest in signals.top_interests %}<tr><td>{{ interest.name }}</td><td class="num">{{ interest.z }}</td><td class="num">{{ interest.cos }}</td></tr>{% endfor %}</tbody></table></div>{% endif %}</section>
|
<section class="card"><h3>Top interests</h3>{% if signals.top_interests.is_empty() %}<p class="muted">None.</p>{% else %}<div class="scroll-x"><table><thead><tr><th>interest</th><th class="num">z</th><th class="num">cos</th></tr></thead><tbody>{% for interest in signals.top_interests %}<tr><td><a href="{{ interest.href }}">{{ interest.name }}</a></td><td class="num">{{ interest.z }}</td><td class="num">{{ interest.cos }}</td></tr>{% endfor %}</tbody></table></div>{% endif %}</section>
|
||||||
<section class="card"><h3>Nearest rated neighbours</h3>{% if signals.neighbours.is_empty() %}<p class="muted">None.</p>{% else %}<div class="scroll-x"><table><thead><tr><th>label</th><th class="num">cos</th><th>article</th></tr></thead><tbody>{% for neighbour in signals.neighbours %}<tr><td><span class="badge {{ neighbour.label }}">{{ neighbour.label }}</span></td><td class="num">{{ neighbour.cos }}</td><td class="cell-wrap"><a href="/dashboard/articles/{{ neighbour.article_id }}">{{ neighbour.title }}</a></td></tr>{% endfor %}</tbody></table></div>{% endif %}</section>
|
<section class="card"><h3>Nearest rated neighbours</h3>{% if signals.neighbours.is_empty() %}<p class="muted">None.</p>{% else %}<div class="scroll-x"><table><thead><tr><th>label</th><th class="num">cos</th><th>article</th></tr></thead><tbody>{% for neighbour in signals.neighbours %}<tr><td><span class="badge {{ neighbour.label }}">{{ neighbour.label }}</span></td><td class="num">{{ neighbour.cos }}</td><td class="cell-wrap"><a href="/dashboard/articles/{{ neighbour.article_id }}">{{ neighbour.title }}</a></td></tr>{% endfor %}</tbody></table></div>{% endif %}</section>
|
||||||
</div>{% endif %}{% else %}<p class="muted text-sm">No signals recorded.</p>{% endif %}
|
</div>{% endif %}{% else %}<p class="muted text-sm">No signals recorded.</p>{% endif %}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
<form class="filters" method="get" action="/dashboard/articles">
|
<form class="filters" method="get" action="/dashboard/articles">
|
||||||
<label>Title or URL <input type="search" name="q" value="{% if let Some(q) = filters.q %}{{ q }}{% endif %}" placeholder="contains…"></label>
|
<label>Title or URL <input type="search" name="q" value="{% if let Some(q) = filters.q %}{{ q }}{% endif %}" placeholder="contains…"></label>
|
||||||
<label>Feed id <input type="number" name="feed" value="{{ feed_value }}" min="1"></label>
|
<label>Feed id <input type="number" name="feed" value="{{ feed_value }}" min="1"></label>
|
||||||
|
<label>Interest <select name="interest"><option value="">any</option>{% for interest in interest_options %}<option value="{{ interest.name }}"{% if let Some(current) = filters.interest %}{% if current.name.as_str() == interest.name.as_str() %} selected{% endif %}{% endif %}>{{ interest.name }}</option>{% endfor %}</select></label>
|
||||||
<label>Stage <select name="stage"><option value="">any</option>{% for name in stages %}<option value="{{ name }}"{% if let Some(current) = filters.stage %}{% if current.as_str() == *name %} selected{% endif %}{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
<label>Stage <select name="stage"><option value="">any</option>{% for name in stages %}<option value="{{ name }}"{% if let Some(current) = filters.stage %}{% if current.as_str() == *name %} selected{% endif %}{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
||||||
<label>Reason <select name="reason"><option value="">any</option>{% for name in reasons %}<option value="{{ name }}"{% if let Some(current) = filters.reason %}{% if current.as_str() == *name %} selected{% endif %}{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
<label>Reason <select name="reason"><option value="">any</option>{% for name in reasons %}<option value="{{ name }}"{% if let Some(current) = filters.reason %}{% if current.as_str() == *name %} selected{% endif %}{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
||||||
<label>Rated <select name="rated"><option value="">all</option>{% for name in rated %}<option value="{{ name }}"{% if let Some(current) = filters.rated %}{% if current.as_str() == *name %} selected{% endif %}{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
<label>Rated <select name="rated"><option value="">all</option>{% for name in rated %}<option value="{{ name }}"{% if let Some(current) = filters.rated %}{% if current.as_str() == *name %} selected{% endif %}{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
||||||
@@ -18,11 +19,12 @@
|
|||||||
</form>
|
</form>
|
||||||
{% include "dashboard/_pager.html" %}
|
{% include "dashboard/_pager.html" %}
|
||||||
{% if articles.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table class="articles" data-filter>
|
{% if articles.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table class="articles" data-filter>
|
||||||
<thead><tr><th>first seen</th><th>title</th><th>feed</th><th class="num">words</th><th>last stage</th><th>reason</th><th class="num">utility</th><th class="num">triage</th><th class="num">quality</th><th class="num">fit</th><th>rating</th><th>published</th></tr></thead>
|
<thead><tr><th>first seen</th><th>title</th><th>feed</th><th>interests</th><th class="num">words</th><th>last stage</th><th>reason</th><th class="num">utility</th><th class="num">triage</th><th class="num">quality</th><th class="num">fit</th><th>rating</th><th>published</th></tr></thead>
|
||||||
<tbody>{% for article in articles %}<tr>
|
<tbody>{% for article in articles %}<tr>
|
||||||
<td class="cell-tight text-muted">{{ article.first_seen }}</td>
|
<td class="cell-tight text-muted">{{ article.first_seen }}</td>
|
||||||
<td class="cell-wrap"><a href="{{ article.href }}">{{ article.title }}</a></td>
|
<td class="cell-wrap"><a href="{{ article.href }}">{{ article.title }}</a></td>
|
||||||
<td class="cell-tight">{% if let Some(feed_id) = article.feed_id %}<a href="/dashboard/articles?feed={{ feed_id }}">{{ article.feed }}</a>{% else %}{{ article.feed }}{% endif %}</td>
|
<td class="cell-tight">{% if let Some(feed_id) = article.feed_id %}<a href="/dashboard/articles?feed={{ feed_id }}">{{ article.feed }}</a>{% else %}{{ article.feed }}{% endif %}</td>
|
||||||
|
<td class="cell-wrap">{% for interest in article.interests %}<a class="badge" href="{{ interest.href }}">{{ interest.name }}</a>{% if !loop.last %} {% endif %}{% endfor %}</td>
|
||||||
<td class="num">{{ article.words }}</td>
|
<td class="num">{{ article.words }}</td>
|
||||||
<td class="cell-tight">{% if let Some(stage) = article.stage %}<span class="badge {{ stage }}">{{ stage }}</span>{% if let Some(run_id) = article.run_id %} <a class="muted text-xs" href="/dashboard/runs/{{ run_id }}">{% if let Some(date) = article.run_date %}{{ date }}{% else %}run {{ run_id }}{% endif %}</a>{% endif %}{% else %}<span class="muted text-xs">never considered</span>{% endif %}</td>
|
<td class="cell-tight">{% if let Some(stage) = article.stage %}<span class="badge {{ stage }}">{{ stage }}</span>{% if let Some(run_id) = article.run_id %} <a class="muted text-xs" href="/dashboard/runs/{{ run_id }}">{% if let Some(date) = article.run_date %}{{ date }}{% else %}run {{ run_id }}{% endif %}</a>{% endif %}{% else %}<span class="muted text-xs">never considered</span>{% endif %}</td>
|
||||||
<td class="cell-tight text-muted">{% if let Some(reason) = article.reason %}{{ reason }}{% endif %}</td>
|
<td class="cell-tight text-muted">{% if let Some(reason) = article.reason %}{{ reason }}{% endif %}</td>
|
||||||
@@ -32,6 +34,6 @@
|
|||||||
<td class="num">{{ article.fit }}</td>
|
<td class="num">{{ article.fit }}</td>
|
||||||
<td class="cell-tight">{% if let Some(rating) = article.rating %}<span class="badge {{ article.rating_class }}">{{ rating }}</span>{% endif %}</td>
|
<td class="cell-tight">{% if let Some(rating) = article.rating %}<span class="badge {{ article.rating_class }}">{{ rating }}</span>{% endif %}</td>
|
||||||
<td class="cell-tight">{% if let Some(date) = article.published %}<a href="/issues/{{ date }}">{{ date }}</a>{% endif %}</td>
|
<td class="cell-tight">{% if let Some(date) = article.published %}<a href="/issues/{{ date }}">{{ date }}</a>{% endif %}</td>
|
||||||
</tr>{% endfor %}{% if articles.is_empty() %}<tr><td colspan="12" class="text-muted">No articles match this filter.</td></tr>{% endif %}</tbody></table></div>
|
</tr>{% endfor %}{% if articles.is_empty() %}<tr><td colspan="13" class="text-muted">No articles match this filter.</td></tr>{% endif %}</tbody></table></div>
|
||||||
{% include "dashboard/_pager.html" %}
|
{% include "dashboard/_pager.html" %}
|
||||||
</section>{% endblock %}
|
</section>{% endblock %}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
<tbody>{% for row in rows %}<tr>
|
<tbody>{% for row in rows %}<tr>
|
||||||
<td class="num tabular-nums">{{ row.score }}</td>
|
<td class="num tabular-nums">{{ row.score }}</td>
|
||||||
<td class="cell-wrap"><a class="line-clamp-2" href="{{ row.feed_url }}" rel="noopener" target="_blank" title="{{ row.feed_url }}">{{ row.label }}</a><span class="muted block truncate text-xs" title="{{ row.host }}">{{ row.host }}</span></td>
|
<td class="cell-wrap"><a class="line-clamp-2" href="{{ row.feed_url }}" rel="noopener" target="_blank" title="{{ row.feed_url }}">{{ row.label }}</a><span class="muted block truncate text-xs" title="{{ row.host }}">{{ row.host }}</span></td>
|
||||||
<td class="cell-wrap">{% for interest in row.interests %}<span class="badge">{{ interest }}</span> {% endfor %}{% for article in row.articles %}<div class="line-clamp-1 text-xs"><a href="/dashboard/articles/{{ article.id }}" title="{{ article.title }}">{{ article.title }}</a></div>{% endfor %}</td>
|
<td class="cell-wrap">{% for interest in row.interests %}<a class="badge" href="{{ interest.href }}">{{ interest.name }}</a> {% endfor %}{% for article in row.articles %}<div class="line-clamp-1 text-xs"><a href="/dashboard/articles/{{ article.id }}" title="{{ article.title }}">{{ article.title }}</a></div>{% endfor %}</td>
|
||||||
<td class="num">{{ row.article_count }}</td>
|
<td class="num">{{ row.article_count }}</td>
|
||||||
<td class="cell-tight text-xs text-muted"><span title="First seen {{ row.first_seen }} · last seen {{ row.last_seen }}">{{ row.seen }}</span></td>
|
<td class="cell-tight text-xs text-muted"><span title="First seen {{ row.first_seen }} · last seen {{ row.last_seen }}">{{ row.seen }}</span></td>
|
||||||
<td>{% if status == "candidate" %}<div class="flex flex-wrap items-start gap-2"><form class="form-inline flex-nowrap" method="post" action="/dashboard/feeds/{{ row.id }}/add"><select class="min-w-0 max-w-36 truncate" name="category_id" aria-label="Category for {{ row.label }}"{% if categories.is_empty() %} disabled{% endif %}>{% for category in categories %}<option value="{{ category.id }}"{% if category.id == selected_category %} selected{% endif %}>{{ category.title }}</option>{% endfor %}</select><button class="btn-primary" type="submit"{% if categories.is_empty() %} disabled title="Miniflux categories are unavailable"{% endif %}>Add</button></form><form method="post" action="/dashboard/feeds/{{ row.id }}/dismiss"><button class="btn" type="submit">Dismiss</button></form></div>{% else %}<span class="muted text-xs">{{ row.decided_at }}{% if let Some(href) = row.miniflux_href %} · <a href="{{ href }}" rel="noopener" target="_blank">in Miniflux</a>{% endif %}</span>{% endif %}</td>
|
<td>{% if status == "candidate" %}<div class="flex flex-wrap items-start gap-2"><form class="form-inline flex-nowrap" method="post" action="/dashboard/feeds/{{ row.id }}/add"><select class="min-w-0 max-w-36 truncate" name="category_id" aria-label="Category for {{ row.label }}"{% if categories.is_empty() %} disabled{% endif %}>{% for category in categories %}<option value="{{ category.id }}"{% if category.id == selected_category %} selected{% endif %}>{{ category.title }}</option>{% endfor %}</select><button class="btn-primary" type="submit"{% if categories.is_empty() %} disabled title="Miniflux categories are unavailable"{% endif %}>Add</button></form><form method="post" action="/dashboard/feeds/{{ row.id }}/dismiss"><button class="btn" type="submit">Dismiss</button></form></div>{% else %}<span class="muted text-xs">{{ row.decided_at }}{% if let Some(href) = row.miniflux_href %} · <a href="{{ href }}" rel="noopener" target="_blank">in Miniflux</a>{% endif %}</span>{% endif %}</td>
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
{% extends "layout.html" %}{% block content %}<section class="dashboard interests">
|
||||||
|
<header class="page-head"><div>
|
||||||
|
<h1>Interests</h1>
|
||||||
|
<p class="page-desc">{{ total }} standing interests. Weights use the last {{ lookback_days }} days of ratings with a {{ half_life_days }}-day half-life · affinity gate {{ affinity_gate }} · {{ attributable }} of {{ affinity_full }} attributable ratings.</p>
|
||||||
|
</div><div class="page-actions"><span class="badge">{{ uncategorized }} uncategorized</span><form method="post" action="/dashboard/jobs/interests-categorize" class="form-inline"><button class="btn" type="submit"{% if !jobs_enabled %} disabled{% endif %}>Categorize now</button>{% if !jobs_enabled %} <span class="meta text-xs">Jobs are disabled on this server (<code>server.jobs_enabled = false</code>); run <code>daily-epub job run interests-categorize</code> instead.</span>{% endif %}</form></div></header>
|
||||||
|
|
||||||
|
<section class="card"><h2>Add an interest</h2><form class="filters" method="post" action="/dashboard/interests">
|
||||||
|
<label>Interest <input name="name" type="text" minlength="1" maxlength="80" required placeholder="e.g. Rust macros"></label>
|
||||||
|
<label>Category <select name="category"><option value="">let the categorizer decide</option>{% for category in categories %}<option value="{{ category.name }}">{{ category.name }}</option>{% endfor %}</select></label>
|
||||||
|
<div class="filter-actions"><button class="btn btn-primary" type="submit">Add</button></div>
|
||||||
|
</form></section>
|
||||||
|
|
||||||
|
<form class="filters" method="get" action="/dashboard/interests">
|
||||||
|
<label>Category <select name="category"><option value="">all</option><option value="uncategorized"{% if selected_category == "uncategorized" %} selected{% endif %}>uncategorized</option>{% for category in categories %}<option value="{{ category.name }}"{% if selected_category == category.name.as_str() %} selected{% endif %}>{{ category.name }}</option>{% endfor %}</select></label>
|
||||||
|
<label>Sort <select name="sort">{% for sort in sorts %}<option value="{{ sort }}"{% if selected_sort == *sort %} selected{% endif %}>{{ sort }}</option>{% endfor %}</select></label>
|
||||||
|
<div class="filter-actions"><button class="btn" type="submit">Apply</button> <a class="btn-quiet" href="/dashboard/interests">Reset</a></div>
|
||||||
|
</form>
|
||||||
|
{% if rows.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter interests" aria-label="Filter interests" data-table-filter>{% endif %}
|
||||||
|
<div class="scroll-x tall"><table data-filter>
|
||||||
|
<thead><tr><th>interest</th><th>category</th><th class="num">weight</th><th class="num">up</th><th class="num">down</th><th class="num">rated matches</th><th class="num">matched articles</th><th>added</th><th>actions</th></tr></thead>
|
||||||
|
<tbody>{% for row in rows %}<tr>
|
||||||
|
<td class="cell-wrap"><a href="{{ row.href }}">{{ row.name }}</a></td>
|
||||||
|
<td class="cell-tight">{% if let Some(category) = row.category %}<a href="{{ row.category_href }}">{{ category }}</a>{% else %}<a class="muted" href="{{ row.category_href }}">uncategorized</a>{% endif %}</td>
|
||||||
|
<td class="num tabular-nums">{{ row.weight }}</td>
|
||||||
|
<td class="num tabular-nums">{{ row.up }}</td>
|
||||||
|
<td class="num tabular-nums">{{ row.down }}</td>
|
||||||
|
<td class="num tabular-nums">{{ row.rated_matches }}</td>
|
||||||
|
<td class="num tabular-nums">{{ row.matched_articles }}</td>
|
||||||
|
<td class="cell-tight text-muted">{{ row.added }}</td>
|
||||||
|
<td><div class="flex flex-wrap items-start gap-2"><form class="form-inline flex-nowrap" method="post" action="/dashboard/interests/{{ row.id }}/category"><select class="min-w-0 max-w-44 truncate" name="category" aria-label="Category for {{ row.name }}"><option value="">uncategorized</option>{% for category in categories %}<option value="{{ category.name }}"{% if let Some(current) = row.category %}{% if current.as_str() == category.name.as_str() %} selected{% endif %}{% endif %}>{{ category.name }}</option>{% endfor %}</select><button class="btn" type="submit">Save</button></form><form method="post" action="/dashboard/interests/{{ row.id }}/delete" onsubmit="return confirm('Delete this interest? Its article matches and cached embedding will also be removed.')"><button class="btn" type="submit">Delete</button></form></div></td>
|
||||||
|
</tr>{% endfor %}{% if rows.is_empty() %}<tr><td colspan="9" class="text-muted">No interests match this filter.</td></tr>{% endif %}</tbody>
|
||||||
|
</table></div>
|
||||||
|
</section>{% endblock %}
|
||||||
@@ -11,6 +11,7 @@
|
|||||||
<div class="tile"><span class="tile-label">Unrated picks</span><span class="tile-num">{{ unrated.len() }}</span><span class="tile-delta">from the last three issues</span></div>
|
<div class="tile"><span class="tile-label">Unrated picks</span><span class="tile-num">{{ unrated.len() }}</span><span class="tile-delta">from the last three issues</span></div>
|
||||||
<div class="tile"><span class="tile-label">Active jobs</span><span class="tile-num">{{ active_jobs.len() }}</span><span class="tile-delta"><a href="/dashboard/jobs">all jobs</a></span></div>
|
<div class="tile"><span class="tile-label">Active jobs</span><span class="tile-num">{{ active_jobs.len() }}</span><span class="tile-delta"><a href="/dashboard/jobs">all jobs</a></span></div>
|
||||||
<div class="tile"><span class="tile-label">Feed candidates</span><span class="tile-num">{{ feed_candidates }}</span><span class="tile-delta"><a href="/dashboard/feeds">review feeds</a></span></div>
|
<div class="tile"><span class="tile-label">Feed candidates</span><span class="tile-num">{{ feed_candidates }}</span><span class="tile-delta"><a href="/dashboard/feeds">review feeds</a></span></div>
|
||||||
|
<div class="tile"><span class="tile-label">Interests</span><span class="tile-num">{{ interests_total }}</span><span class="tile-delta"><a href="/dashboard/interests?category=uncategorized">{{ uncategorized_interests }} uncategorized</a></span></div>
|
||||||
<div class="tile"><span class="tile-label">Access requests</span><span class="tile-num">{{ access_requests }}</span><span class="tile-delta"><a href="/dashboard/users">review requests</a></span></div>
|
<div class="tile"><span class="tile-label">Access requests</span><span class="tile-num">{{ access_requests }}</span><span class="tile-delta"><a href="/dashboard/users">review requests</a></span></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
{% extends "layout.html" %}{% block content %}<section class="dashboard profile">
|
{% extends "layout.html" %}{% block content %}<section class="dashboard profile">
|
||||||
<header class="page-head"><div>
|
<header class="page-head"><div>
|
||||||
<h1>Profile</h1>
|
<h1>Profile</h1>
|
||||||
<p class="page-desc">The standing taste file the curator reads before every run: what you like, what the OPML declares, and what the editor model has learned from your verdicts.</p>
|
<p class="page-desc">The standing taste file the curator reads before every run, alongside stored interests and what the editor model has learned from your verdicts.</p>
|
||||||
</div><div class="page-actions"><a class="btn" href="/dashboard/ratings">Ratings</a><a class="btn" href="/dashboard/settings#curation.feedback">Feedback settings</a></div></header>
|
</div><div class="page-actions"><a class="btn" href="/dashboard/ratings">Ratings</a><a class="btn" href="/dashboard/settings#curation.feedback">Feedback settings</a></div></header>
|
||||||
|
|
||||||
<div class="profile-grid grid gap-6 lg:grid-cols-2">
|
<div class="profile-grid grid gap-6 lg:grid-cols-2">
|
||||||
<div class="profile-editor min-w-0">
|
<div class="profile-editor min-w-0">
|
||||||
<h3 class="font-mono text-base">profile.md</h3>
|
<h3 class="font-mono text-base">profile.md</h3>
|
||||||
<p class="meta mt-1 text-xs"><code>{{ path }}</code>{% if !exists %} — <strong class="text-down">missing</strong>; saving creates it{% else %} · {{ bytes }} bytes{% endif %} · limit {{ max_bytes }} bytes. Any <code>## Interests</code> section is parsed one interest per line; everything else goes into the system prompt verbatim.</p>
|
<p class="meta mt-1 text-xs"><code>{{ path }}</code>{% if !exists %} — <strong class="text-down">missing</strong>; saving creates it{% else %} · {{ bytes }} bytes{% endif %} · limit {{ max_bytes }} bytes. Any <code>## Interests</code> section is ignored; everything else goes into the system prompt verbatim.</p>
|
||||||
<form method="post" action="/dashboard/profile" class="profile-form mt-3">
|
<form method="post" action="/dashboard/profile" class="profile-form mt-3">
|
||||||
<textarea name="content" rows="24" spellcheck="true" required class="w-full font-mono text-sm">{{ content }}</textarea>
|
<textarea name="content" rows="24" spellcheck="true" required class="w-full font-mono text-sm">{{ content }}</textarea>
|
||||||
<div class="mt-3 flex flex-wrap items-center gap-3"><button class="btn-primary" type="submit">Save</button> <span class="meta text-xs">The next run rebuilds the system prompt from the saved file.</span></div>
|
<div class="mt-3 flex flex-wrap items-center gap-3"><button class="btn-primary" type="submit">Save</button> <span class="meta text-xs">The next run rebuilds the system prompt from the saved file.</span></div>
|
||||||
@@ -19,8 +19,6 @@
|
|||||||
<p class="meta mt-1 text-xs">A live read of the text on the left, exactly as <code>curate::profile</code> splits it.</p>
|
<p class="meta mt-1 text-xs">A live read of the text on the left, exactly as <code>curate::profile</code> splits it.</p>
|
||||||
<h4 class="mt-4 page-eyebrow">Passthrough sections</h4>
|
<h4 class="mt-4 page-eyebrow">Passthrough sections</h4>
|
||||||
{% if preview_body.trim().is_empty() %}<p class="meta mt-1 text-sm">Nothing passes through — the file is empty or only has an Interests section.</p>{% else %}<pre class="preview mt-2">{{ preview_body }}</pre>{% endif %}
|
{% if preview_body.trim().is_empty() %}<p class="meta mt-1 text-sm">Nothing passes through — the file is empty or only has an Interests section.</p>{% else %}<pre class="preview mt-2">{{ preview_body }}</pre>{% endif %}
|
||||||
<h4 class="mt-5 page-eyebrow">Extracted <code>## Interests</code> lines</h4>
|
|
||||||
{% if preview_interests.is_empty() %}<p class="meta mt-1 text-sm">None. The prompt uses the OPML interests alone.</p>{% else %}<ul class="mt-2 list-disc space-y-1 pl-5 text-sm marker:text-muted">{% for interest in preview_interests %}<li>{{ interest }}</li>{% endfor %}</ul>{% endif %}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -40,9 +38,8 @@
|
|||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
<h2>Standing interests</h2>
|
<h2>Standing interests</h2>
|
||||||
<p class="meta text-sm"><code>{{ opml_path }}</code> · {{ opml_count }} interests, grouped the way the system prompt lists them. The union of these and the <code>## Interests</code> lines above is what the prompt uses; edit the OPML file to change them.</p>
|
<p class="meta text-sm">{{ interest_count }} standing interests in {{ category_count }} categories — manage them on the <a href="/dashboard/interests">Interests page</a>.</p>
|
||||||
{% if !opml_error.is_empty() %}<p class="error">{{ opml_error }}</p>{% endif %}
|
{% if !categories.is_empty() %}<dl class="kv themes">{% for category in categories %}<dt class="text-ink">{{ category.name }} <span class="meta">({{ category.count }})</span></dt><dd class="text-muted">{{ category.members }}</dd>{% endfor %}</dl>{% endif %}
|
||||||
{% if !themes.is_empty() %}<dl class="kv themes">{% for theme in themes %}<dt class="text-ink">{{ theme.name }} <span class="meta">({{ theme.count }})</span></dt><dd class="text-muted">{{ theme.members }}</dd>{% endfor %}</dl>{% endif %}
|
|
||||||
|
|
||||||
<h2>Learned adjustments</h2>
|
<h2>Learned adjustments</h2>
|
||||||
<p class="meta text-sm">Rebuilt weekly from ratings by the editor model (prompt version {{ prompt_version }}, built {{ prompt_built_at }}, {{ learned_age }}). {% if rebuild_due %}<strong class="text-warn">A rebuild is due</strong> — the next run performs it, or start it now.{% else %}The next scheduled rebuild is at least {{ rebuild_interval_days }} days after the last one; the next run performs it when due.{% endif %}</p>
|
<p class="meta text-sm">Rebuilt weekly from ratings by the editor model (prompt version {{ prompt_version }}, built {{ prompt_built_at }}, {{ learned_age }}). {% if rebuild_due %}<strong class="text-warn">A rebuild is due</strong> — the next run performs it, or start it now.{% else %}The next scheduled rebuild is at least {{ rebuild_interval_days }} days after the last one; the next run performs it when due.{% endif %}</p>
|
||||||
|
|||||||
@@ -47,6 +47,7 @@
|
|||||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "runs" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/runs"{% if page.active_nav == "runs" %} aria-current="page"{% endif %}>Runs</a>
|
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "runs" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/runs"{% if page.active_nav == "runs" %} aria-current="page"{% endif %}>Runs</a>
|
||||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "articles" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/articles"{% if page.active_nav == "articles" %} aria-current="page"{% endif %}>Articles</a>
|
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "articles" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/articles"{% if page.active_nav == "articles" %} aria-current="page"{% endif %}>Articles</a>
|
||||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "ratings" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/ratings"{% if page.active_nav == "ratings" %} aria-current="page"{% endif %}>Ratings</a>
|
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "ratings" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/ratings"{% if page.active_nav == "ratings" %} aria-current="page"{% endif %}>Ratings</a>
|
||||||
|
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "interests" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/interests"{% if page.active_nav == "interests" %} aria-current="page"{% endif %}>Interests</a>
|
||||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "feeds" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/feeds"{% if page.active_nav == "feeds" %} aria-current="page"{% endif %}>Feeds</a>
|
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "feeds" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/feeds"{% if page.active_nav == "feeds" %} aria-current="page"{% endif %}>Feeds</a>
|
||||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "profile" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/profile"{% if page.active_nav == "profile" %} aria-current="page"{% endif %}>Profile</a>
|
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "profile" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/profile"{% if page.active_nav == "profile" %} aria-current="page"{% endif %}>Profile</a>
|
||||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "stats" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/stats"{% if page.active_nav == "stats" %} aria-current="page"{% endif %}>Stats</a>
|
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "stats" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/stats"{% if page.active_nav == "stats" %} aria-current="page"{% endif %}>Stats</a>
|
||||||
|
|||||||
@@ -35,7 +35,6 @@ fn config_check_prints_the_facts_and_exits_zero_without_keys() {
|
|||||||
"config: ",
|
"config: ",
|
||||||
"database_path: /var/lib/daily-epub/daily-epub.db",
|
"database_path: /var/lib/daily-epub/daily-epub.db",
|
||||||
"profile_path: data/profile.md",
|
"profile_path: data/profile.md",
|
||||||
"interests_opml: data/scour-interests.opml",
|
|
||||||
"llm.bulk: deepseek · openai · deepseek-v4-flash",
|
"llm.bulk: deepseek · openai · deepseek-v4-flash",
|
||||||
"key MISSING (set DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY)",
|
"key MISSING (set DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY)",
|
||||||
"llm.editor: anthropic · anthropic · claude-opus-5 · effort high · max_daily_usd $3.00",
|
"llm.editor: anthropic · anthropic · claude-opus-5 · effort high · max_daily_usd $3.00",
|
||||||
@@ -66,4 +65,8 @@ fn config_check_exits_non_zero_on_an_invalid_config() {
|
|||||||
let (code, _, stderr) = run("[deepseek]\nmodel = \"x\"\n");
|
let (code, _, stderr) = run("[deepseek]\nmodel = \"x\"\n");
|
||||||
assert_ne!(code, 0);
|
assert_ne!(code, 0);
|
||||||
assert!(stderr.contains("[providers.deepseek]"), "{stderr}");
|
assert!(stderr.contains("[providers.deepseek]"), "{stderr}");
|
||||||
|
|
||||||
|
let (code, _, stderr) = run("interests_opml = \"data/scour-interests.opml\"\n");
|
||||||
|
assert_ne!(code, 0);
|
||||||
|
assert!(stderr.contains("interests_opml"), "{stderr}");
|
||||||
}
|
}
|
||||||
|
|||||||
+10
-7
@@ -21,6 +21,7 @@ use daily_epub::curate::assess::parse_deep_response;
|
|||||||
use daily_epub::curate::editor::parse_selection_response;
|
use daily_epub::curate::editor::parse_selection_response;
|
||||||
use daily_epub::curate::editorial::BriefResponse;
|
use daily_epub::curate::editorial::BriefResponse;
|
||||||
use daily_epub::curate::profile;
|
use daily_epub::curate::profile;
|
||||||
|
use daily_epub::interests;
|
||||||
|
|
||||||
fn repo(rel: &str) -> PathBuf {
|
fn repo(rel: &str) -> PathBuf {
|
||||||
Path::new(env!("CARGO_MANIFEST_DIR")).join(rel)
|
Path::new(env!("CARGO_MANIFEST_DIR")).join(rel)
|
||||||
@@ -183,13 +184,14 @@ fn stage_c_fixture_parses_into_the_brief() {
|
|||||||
assert!(value.get("section_intros").is_none());
|
assert!(value.get("section_intros").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The taste profile is seeded from this file; a broken export would silently
|
/// The importer relies on this file; a broken export would silently lose
|
||||||
/// gut the system prompt (§3.6a).
|
/// standing interests (§3.6a).
|
||||||
#[test]
|
#[test]
|
||||||
fn scour_opml_still_yields_the_interest_list() {
|
fn scour_opml_still_yields_the_interest_list() {
|
||||||
let interests = profile::parse_interests(&repo("data/scour-interests.opml"))
|
let raw = std::fs::read_to_string(repo("data/scour-interests.opml"))
|
||||||
.expect("the shipped OPML must parse");
|
.expect("the shipped OPML must be readable");
|
||||||
let unique: BTreeSet<String> = interests.iter().map(|n| n.to_lowercase()).collect();
|
let names = interests::parse_opml(&raw);
|
||||||
|
let unique: BTreeSet<String> = names.iter().map(|n| n.to_lowercase()).collect();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
unique.len() > 180,
|
unique.len() > 180,
|
||||||
@@ -206,7 +208,7 @@ fn scour_opml_still_yields_the_interest_list() {
|
|||||||
assert!(unique.contains(expected), "{expected} disappeared");
|
assert!(unique.contains(expected), "{expected} disappeared");
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
!interests.iter().any(|n| n.contains("token=")),
|
!names.iter().any(|n| n.contains("token=")),
|
||||||
"interest names must not leak the Scour token"
|
"interest names must not leak the Scour token"
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -217,9 +219,10 @@ fn scour_opml_still_yields_the_interest_list() {
|
|||||||
"/data/profile.md"
|
"/data/profile.md"
|
||||||
)))
|
)))
|
||||||
.expect("profile file");
|
.expect("profile file");
|
||||||
|
let grouped = vec![("Imported interests".to_string(), names)];
|
||||||
let document = profile::build(
|
let document = profile::build(
|
||||||
&profile_file.body,
|
&profile_file.body,
|
||||||
&interests,
|
&grouped,
|
||||||
profile::NO_LEARNED_ADJUSTMENTS,
|
profile::NO_LEARNED_ADJUSTMENTS,
|
||||||
&[],
|
&[],
|
||||||
60,
|
60,
|
||||||
|
|||||||
Reference in New Issue
Block a user