Highlight code blocks in article bodies

arborium (tree-sitter) renders each token as a flat <span class="hl-…">,
which the X4's no-descendant-selector stylesheet can still target.

The sanitizer strips class attributes, so language hints never reached
the database. annotate_hints now copies them onto <pre data-lang> before
readability and sanitizing: pre/code classes, Rouge and GitHub wrapper
divs, SyntaxHighlighter's brush:, and data-lang style attributes.
ammonia keeps only canonical values.

Highlighting runs at render time for the web article page and both EPUB
editions, so stored articles benefit too. Unhinted blocks are guessed
from content (JSON that parses, shebangs, leading <script>/<style>, then
betlang at >= 0.5). A borderline block also takes the article's prevailing
language when that language is among its top candidates. Anything else
stays plain, since wrong colours are worse than none.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Cargo.toml

@@ -6,11 +6,13 @@ edition = "2024"
 [dependencies]
 ammonia = "4.1.4"
 anyhow = "1.0.104"
+arborium = { version = "2.18.2", default-features = false, features = ["lang-asm", "lang-awk", "lang-bash", "lang-c", "lang-c-sharp", "lang-clojure", "lang-cmake", "lang-commonlisp", "lang-cpp", "lang-css", "lang-d", "lang-dart", "lang-diff", "lang-dockerfile", "lang-elisp", "lang-elixir", "lang-elm", "lang-erlang", "lang-fish", "lang-fsharp", "lang-gleam", "lang-glsl", "lang-go", "lang-graphql", "lang-groovy", "lang-haskell", "lang-hcl", "lang-html", "lang-ini", "lang-java", "lang-javascript", "lang-jq", "lang-json", "lang-julia", "lang-kotlin", "lang-lean", "lang-lua", "lang-make", "lang-matlab", "lang-nix", "lang-objc", "lang-ocaml", "lang-odin", "lang-perl", "lang-php", "lang-powershell", "lang-prolog", "lang-proto", "lang-python", "lang-r", "lang-ruby", "lang-rust", "lang-scala", "lang-scheme", "lang-scss", "lang-solidity", "lang-sql", "lang-starlark", "lang-swift", "lang-toml", "lang-tsx", "lang-typescript", "lang-verilog", "lang-vhdl", "lang-vim", "lang-x86asm", "lang-xml", "lang-yaml", "lang-zig", "lang-zsh"] }
 askama = "0.16.0"
 async-trait = "0.1.92"
 axum = "0.8.9"
 axum-login = { git = "https://github.com/maxcountryman/axum-login.git", rev = "151c72d7a1b4646830f86b4332e6bd6e34d719a7", version = "0.18.0" }
 base64 = "0.23.1"
+betlang = "0.1.1"
 clap = { version = "4.6.6", features = ["derive"] }
 dom_smoothie = "0.18.0"
 epub-builder = "0.8.3"

src/epub/chapters.rs

@@ -150,9 +150,10 @@ struct ProviderCostLine {
 
 /// Prepare article markup for XHTML: rewrite images, sanitize, self-close voids.
 pub fn prepare_body(html: &str, images_: &[ImageAsset]) -> String {
-    // Sanitize first: image rewriting emits our own trusted markup (including the
+    // Sanitize first: highlighting and image rewriting emit our own trusted
-    // `image-placeholder` class, which ammonia would otherwise strip).
+    // markup (the `hl-*` token classes and `image-placeholder`, which ammonia
-    let cleaned = ammonia::clean(html);
+    // would otherwise strip).
+    let cleaned = crate::highlight::clean_and_highlight(html);
     let rewritten = images::rewrite_img_srcs(&cleaned, images_);
     to_xhtml(&rewritten)
 }

src/extract.rs

@@ -15,6 +15,7 @@ use std::time::Duration;
 use futures::StreamExt;
 use url::Url;
 
+use crate::highlight::annotate_hints;
 use crate::html::word_count;
 use crate::images::normalize::{normalize_img_tags, prepare_for_readability};
 use crate::images::refs::collect_image_urls;
@@ -338,7 +339,8 @@ fn readable_page(
     html: &str,
     url: &str,
 ) -> Result<(String, String, Option<String>, Option<String>), ExtractError> {
-    let html = prepare_for_readability(html);
+    // Readability strips classes, and with them code blocks' language hints.
+    let html = prepare_for_readability(&annotate_hints(html));
     let config = dom_smoothie::Config {
         max_elements_to_parse: 60_000,
         ..Default::default()
@@ -447,6 +449,7 @@ fn builder() -> ammonia::Builder<'static> {
         .generic_attributes(HashSet::new())
         .link_rel(None)
         .strip_comments(true);
+    crate::highlight::keep_lang_attr(&mut b);
     b
 }
 
@@ -454,8 +457,10 @@ fn builder() -> ammonia::Builder<'static> {
 ///
 /// Allowed: `p`, `h1`–`h4`, `ul`/`ol`/`li`, `blockquote`, `pre`, `code`, `em`,
 /// `strong`, `a`, `img`, `figure`, `figcaption`, table basics, `hr`, `br`.
+/// The one attribute kept beyond links and images is a code block's language,
+/// as `<pre data-lang>` (see [`crate::highlight`]).
 pub fn sanitize(html: &str) -> String {
-    builder().clean(html).to_string()
+    builder().clean(&annotate_hints(html)).to_string()
 }
 
 /// [`sanitize`], additionally rewriting relative `href`/`src` against `base_url`
@@ -464,7 +469,7 @@ pub fn sanitize_with_base(html: &str, base_url: &str) -> String {
     match Url::parse(base_url) {
         Ok(base) => builder()
             .url_relative(ammonia::UrlRelative::RewriteWithBase(base))
-            .clean(html)
+            .clean(&annotate_hints(html))
             .to_string(),
         Err(_) => sanitize(html),
     }
@@ -546,6 +551,26 @@ mod tests {
         format!("<p>{}</p>", "lorem ".repeat(words))
     }
 
+    #[test]
+    fn sanitize_keeps_code_block_languages_as_data_lang_only() {
+        let dirty = r#"<div class="language-ruby highlighter-rouge"><pre class="highlight"><code>puts 1</code></pre></div>
+            <pre><code class="language-rust" data-x="y">fn main() {}</code></pre>
+            <pre data-lang="&quot; onload=&quot;x">forged</pre>
+            <p data-lang="rust">not a block</p>"#;
+        let clean = sanitize(dirty);
+        assert!(
+            clean.contains(r#"<pre data-lang="ruby"><code>puts 1</code></pre>"#),
+            "{clean}"
+        );
+        assert!(
+            clean.contains(r#"<pre data-lang="rust"><code>fn main() {}</code></pre>"#),
+            "{clean}"
+        );
+        assert!(clean.contains("<pre>forged</pre>"), "{clean}");
+        assert!(clean.contains("<p>not a block</p>"), "{clean}");
+        assert!(!clean.contains("class="));
+    }
+
     #[test]
     fn sanitize_keeps_the_allowlist_and_drops_everything_else() {
         let dirty = r#"

src/highlight.rs

@@ -0,0 +1,743 @@
+//! Syntax highlighting for code blocks in article bodies.
+//!
+//! Two halves, run at different times:
+//!
+//! * [`annotate_hints`] runs on source markup at extraction, before
+//!   readability and the sanitizer strip every `class`. It reads whatever
+//!   language hint the page carries (`language-rust`, Rouge's wrapper `<div>`,
+//!   SyntaxHighlighter's `brush: py`, `data-lang`, …) and records it as one
+//!   normalized `data-lang` on the `<pre>`, which both of them keep.
+//! * [`highlight_blocks`] runs at render time, on sanitized markup, for the web
+//!   article page and both EPUB editions. Blocks with a hint use it; blocks
+//!   without one are guessed from their content ([`guess_language`]), and
+//!   anything not confidently code is left exactly as it was.
+//!
+//! Tokens come out as flat `<span class="hl-…">` elements, one class each, so
+//! the X4 stylesheet (no descendant selectors) can style them too. Colours are
+//! the stylesheets' business: the web theme is in `src/web/tailwind.css`, the
+//! monochrome e-ink themes in `src/epub/templates/style*.css`.
+
+use std::borrow::Cow;
+use std::sync::{Arc, LazyLock};
+
+use arborium::{Config, GrammarStore, Highlighter, HtmlFormat};
+
+use crate::html::{attr_escape, decode_entities, parse_attrs, tag_end, tag_name};
+
+/// The attribute that carries a block's language from extraction to render.
+pub const LANG_ATTR: &str = "data-lang";
+
+/// The `data-lang` value for a block its source marked as not code (plain
+/// text, program output): never highlighted, never guessed.
+pub const PLAIN: &str = "text";
+
+/// Prefix of every token class, e.g. `hl-keyword`.
+const CLASS_PREFIX: &str = "hl";
+
+/// Hints that mean "this is not source code".
+const PLAIN_HINTS: &[&str] = &[
+    "text",
+    "txt",
+    "plain",
+    "plaintext",
+    "none",
+    "nohighlight",
+    "no-highlight",
+    "output",
+    "console",
+    "terminal",
+    "log",
+];
+
+/// Attributes that name a block's language outright.
+const LANG_ATTRIBUTES: &[&str] = &[
+    "data-lang",
+    "data-language",
+    "data-code-block-lang",
+    "data-code-language",
+];
+
+/// Class prefixes highlighters put in front of a language name.
+const CLASS_PREFIXES: &[&str] = &[
+    "language-",
+    "lang-",
+    "highlight-source-",
+    "highlight-",
+    "source-",
+];
+
+/// Minimum model confidence for a guessed language. Measured on short real
+/// snippets, guesses at or above this were right; below it they were coin flips.
+const MIN_GUESS_SCORE: f32 = 0.5;
+
+/// Unhinted blocks shorter than this are left alone: one-liners are usually a
+/// command or a path, and too short to classify.
+const MIN_GUESS_CHARS: usize = 40;
+
+/// A block that is only plausibly code is weighed against the article's
+/// prevailing language using at most this many of its best candidates…
+const MAX_CANDIDATES: usize = 3;
+
+/// …each scoring at least this much.
+const MIN_CANDIDATE_SCORE: f32 = 0.1;
+
+/// The detector reads at most this much of a block.
+const MAX_GUESS_BYTES: usize = 4096;
+
+/// Compiled grammars, shared across threads; each call gets its own parser.
+static GRAMMARS: LazyLock<Arc<GrammarStore>> = LazyLock::new(|| Arc::new(GrammarStore::new()));
+
+fn new_highlighter() -> Highlighter {
+    Highlighter::with_store_and_config(
+        GRAMMARS.clone(),
+        Config {
+            html_format: HtmlFormat::ClassNamesWithPrefix(CLASS_PREFIX.into()),
+            ..Default::default()
+        },
+    )
+}
+
+/// Whether `name` (a language or one of arborium's aliases) has a grammar.
+fn supported(name: &str) -> bool {
+    GRAMMARS.get(name).is_some()
+}
+
+// ---------------------------------------------------------------------------
+// Hints (extraction time)
+// ---------------------------------------------------------------------------
+
+/// Normalize one hint token (`language-rust`, `py`, `nohighlight`) to the
+/// value stored in `data-lang`: a supported language name or [`PLAIN`].
+///
+/// A bare token (no `language-` style prefix) must be at least two characters,
+/// so stray one-letter classes don't turn into C or R.
+pub fn normalize_hint(token: &str) -> Option<String> {
+    let token = token.trim().trim_matches(';').to_ascii_lowercase();
+    let (name, prefixed) = CLASS_PREFIXES
+        .iter()
+        .find_map(|p| token.strip_prefix(p).map(|rest| (rest, true)))
+        .unwrap_or((token.as_str(), false));
+    if name.is_empty() || (!prefixed && name.len() < 2) {
+        return None;
+    }
+    if PLAIN_HINTS.contains(&name) {
+        return Some(PLAIN.to_string());
+    }
+    let name = canonical_name(name);
+    supported(name).then(|| name.to_string())
+}
+
+/// The name detection uses for the common aliases a hint may carry, so a
+/// `py` block and a guessed `python` block count as the same language.
+fn canonical_name(name: &str) -> &str {
+    match name {
+        "py" | "python3" => "python",
+        "js" | "node" | "jsx" => "javascript",
+        "ts" => "typescript",
+        "sh" | "shell" => "bash",
+        "rs" => "rust",
+        "golang" => "go",
+        "rb" => "ruby",
+        "yml" => "yaml",
+        "c++" | "cxx" | "cc" | "hpp" => "cpp",
+        "cs" | "csharp" | "c#" => "c-sharp",
+        "objective-c" | "objectivec" => "objc",
+        "kt" => "kotlin",
+        "ex" | "exs" => "elixir",
+        "hs" => "haskell",
+        "ps1" | "pwsh" => "powershell",
+        other => other,
+    }
+}
+
+/// The language hint carried by one element's attributes, if any.
+fn hint_from_attrs(attrs: &[(String, String)]) -> Option<String> {
+    for (name, value) in attrs {
+        if LANG_ATTRIBUTES.contains(&name.as_str())
+            && let Some(hint) = normalize_hint(value)
+        {
+            return Some(hint);
+        }
+    }
+    let class = attrs.iter().find(|(name, _)| name == "class")?.1.as_str();
+    // SyntaxHighlighter writes `class="brush: python; gutter: false"`.
+    let tokens: Vec<&str> = class
+        .split(|c: char| c.is_whitespace() || c == ';')
+        .filter(|t| !t.is_empty())
+        .collect();
+    for (i, token) in tokens.iter().enumerate() {
+        let candidate = match token.strip_prefix("brush:") {
+            Some("") => tokens.get(i + 1).copied(),
+            Some(rest) => Some(rest),
+            None if i > 0 && tokens[i - 1] == "brush:" => None,
+            None => Some(*token),
+        };
+        if let Some(hint) = candidate.and_then(normalize_hint) {
+            return Some(hint);
+        }
+    }
+    None
+}
+
+/// Record each `<pre>`'s language hint as `data-lang`, before readability and
+/// the sanitizer strip the classes that carry it.
+///
+/// The hint may sit on the `<pre>`, on the `<code>` right inside it, or on an
+/// enclosing `<div>` (Jekyll's Rouge, GitHub's `highlight-source-*`). Markup is
+/// otherwise returned byte-identical.
+pub fn annotate_hints(html: &str) -> String {
+    let mut out = String::with_capacity(html.len() + 64);
+    let mut cursor = 0usize;
+    // Hint of every open `<div>`, innermost last.
+    let mut divs: Vec<Option<String>> = Vec::new();
+    while let Some(rel) = html[cursor..].find('<') {
+        let start = cursor + rel;
+        out.push_str(&html[cursor..start]);
+        let Some(end) = tag_end(html, start) else {
+            out.push_str(&html[start..]);
+            return out;
+        };
+        let raw = &html[start..end];
+        cursor = end;
+        let inner = raw.trim_start_matches('<').trim_end_matches('>');
+        let name = tag_name(inner);
+        if inner.starts_with('/') {
+            if name == "div" {
+                divs.pop();
+            }
+            out.push_str(raw);
+            continue;
+        }
+        match name.as_str() {
+            "div" => {
+                if !inner.trim_end().ends_with('/') {
+                    divs.push(hint_from_attrs(&parse_attrs(inner)));
+                }
+                out.push_str(raw);
+            }
+            "pre" => {
+                let attrs = parse_attrs(inner.trim_end_matches('/'));
+                let hint = hint_from_attrs(&attrs)
+                    .or_else(|| first_code_hint(html, end))
+                    .or_else(|| divs.iter().rev().find_map(Clone::clone));
+                match hint {
+                    Some(hint) => out.push_str(&pre_tag(&attrs, &hint)),
+                    None => out.push_str(raw),
+                }
+            }
+            _ => out.push_str(raw),
+        }
+    }
+    out.push_str(&html[cursor..]);
+    out
+}
+
+/// The hint on a `<code>` that opens right after a `<pre>` ending at `after`.
+fn first_code_hint(html: &str, after: usize) -> Option<String> {
+    let rest = &html[after..];
+    let start = after + (rest.len() - rest.trim_start().len());
+    if !html[start..].starts_with('<') {
+        return None;
+    }
+    let end = tag_end(html, start)?;
+    let inner = html[start..end]
+        .trim_start_matches('<')
+        .trim_end_matches('>');
+    (tag_name(inner) == "code").then(|| hint_from_attrs(&parse_attrs(inner)))?
+}
+
+/// Rebuild a `<pre>` tag with `data-lang` set to `hint`.
+fn pre_tag(attrs: &[(String, String)], hint: &str) -> String {
+    let mut tag = String::from("<pre");
+    for (name, value) in attrs {
+        if LANG_ATTRIBUTES.contains(&name.as_str()) {
+            continue;
+        }
+        tag.push_str(&format!(" {name}=\"{}\"", attr_escape(value)));
+    }
+    tag.push_str(&format!(" {LANG_ATTR}=\"{}\">", attr_escape(hint)));
+    tag
+}
+
+/// Sanitizer filter for `data-lang`: keep it only if it is a value
+/// [`annotate_hints`] could have written.
+pub fn filter_lang_attr(value: &str) -> Option<String> {
+    normalize_hint(value).filter(|hint| hint == &value.to_ascii_lowercase())
+}
+
+/// Let an ammonia builder keep a valid `data-lang` on `<pre>` (and nothing else
+/// it did not already allow).
+pub fn keep_lang_attr(builder: &mut ammonia::Builder<'_>) {
+    builder
+        .add_tag_attributes("pre", &[LANG_ATTR])
+        .attribute_filter(|element, attribute, value| {
+            if element == "pre" && attribute == LANG_ATTR {
+                filter_lang_attr(value).map(Cow::Owned)
+            } else {
+                Some(Cow::Borrowed(value))
+            }
+        });
+}
+
+/// `ammonia::clean` that keeps block languages, then [`highlight_blocks`]:
+/// the render-time entry point for article bodies.
+pub fn clean_and_highlight(html: &str) -> String {
+    let mut builder = ammonia::Builder::default();
+    keep_lang_attr(&mut builder);
+    highlight_blocks(&builder.clean(html).to_string())
+}
+
+// ---------------------------------------------------------------------------
+// Detection
+// ---------------------------------------------------------------------------
+
+/// What an unhinted block's content says about its language.
+#[derive(Debug, Clone, PartialEq)]
+enum Detected {
+    /// Confidently this language.
+    Sure(&'static str),
+    /// Plausibly one of these, best first: taken only when the rest of the
+    /// article is in one of them (see [`highlight_blocks`]).
+    Maybe(Vec<&'static str>),
+    /// Not code, or too little of it to tell.
+    Unknown,
+}
+
+/// Best guess at the language of an unhinted block, or `None` when it is not
+/// confidently code. A wrong colour scheme is worse than none.
+pub fn guess_language(code: &str) -> Option<&'static str> {
+    match detect(code) {
+        Detected::Sure(language) => Some(language),
+        _ => None,
+    }
+}
+
+fn detect(code: &str) -> Detected {
+    let text = code.trim();
+    // Parsing is proof, at any length.
+    if (text.starts_with('{') || text.starts_with('['))
+        && serde_json::from_str::<serde_json::Value>(text).is_ok()
+    {
+        return Detected::Sure("json");
+    }
+    if text.chars().count() < MIN_GUESS_CHARS && !text.starts_with("#!") {
+        return Detected::Unknown;
+    }
+    if let Some(language) = first_line_language(text) {
+        return Detected::Sure(language);
+    }
+    // A shell session is prompts interleaved with output; neither grammar fits.
+    if text.starts_with("$ ") || text.starts_with("> ") || text.starts_with("% ") {
+        return Detected::Unknown;
+    }
+    let sample = crate::html::truncate_utf8(text, MAX_GUESS_BYTES);
+    let detection = betlang::detect(sample);
+    let mut ranked = detection.top_languages().take(MAX_CANDIDATES);
+    let Some((score, top)) = ranked.next() else {
+        return Detected::Unknown;
+    };
+    // Prose, logs and program output all come out as Markdown.
+    let Some(top) = betlang_language(top) else {
+        return Detected::Unknown;
+    };
+    if score >= MIN_GUESS_SCORE {
+        return Detected::Sure(top);
+    }
+    let mut candidates = vec![top];
+    candidates.extend(
+        ranked
+            .filter(|(score, _)| *score >= MIN_CANDIDATE_SCORE)
+            .filter_map(|(_, language)| betlang_language(language)),
+    );
+    Detected::Maybe(candidates)
+}
+
+/// The grammar for a betlang label, if we have one.
+fn betlang_language(language: betlang::Language) -> Option<&'static str> {
+    let name = match language.slug() {
+        "markdown" => return None,
+        "cs" => "c-sharp",
+        "objectivec" => "objc",
+        "lisp" => "commonlisp",
+        "shell" => "bash",
+        "gemfile" | "gemspec" => "ruby",
+        "gradle" => "groovy",
+        other => other,
+    };
+    supported(name).then_some(name)
+}
+
+/// Languages a block announces on its first line: shebangs and XML-ish preludes.
+fn first_line_language(text: &str) -> Option<&'static str> {
+    let first = text.lines().next()?.trim().to_ascii_lowercase();
+    if let Some(shebang) = first.strip_prefix("#!") {
+        let interpreter = shebang
+            .split_whitespace()
+            .flat_map(|word| word.rsplit('/').next())
+            .find(|word| *word != "env" && !word.starts_with('-'))?;
+        let name = match interpreter.trim_end_matches(|c: char| c.is_ascii_digit() || c == '.') {
+            "sh" | "bash" | "dash" | "ksh" => "bash",
+            "zsh" => "zsh",
+            "fish" => "fish",
+            "python" => "python",
+            "node" | "deno" | "bun" => "javascript",
+            "ruby" => "ruby",
+            "perl" => "perl",
+            "php" => "php",
+            "lua" => "lua",
+            _ => return None,
+        };
+        return supported(name).then_some(name);
+    }
+    if first.starts_with("<?php") {
+        return Some("php");
+    }
+    if first.starts_with("<?xml") {
+        return Some("xml");
+    }
+    // Pages and the tags posts most often quote on their own.
+    if ["<!doctype html", "<html", "<script", "<style", "<template"]
+        .iter()
+        .any(|prefix| first.starts_with(prefix))
+    {
+        return Some("html");
+    }
+    None
+}
+
+// ---------------------------------------------------------------------------
+// Rendering
+// ---------------------------------------------------------------------------
+
+/// One `<pre>` block found by [`highlight_blocks`].
+struct Block {
+    /// Byte range of the opening tag, and where `</pre>` starts.
+    start: usize,
+    open_end: usize,
+    close: usize,
+    attrs: Vec<(String, String)>,
+    hint: Option<String>,
+    code: Option<String>,
+    detected: Detected,
+}
+
+/// Highlight every `<pre>` block in sanitized markup.
+///
+/// A block's language is its `data-lang`, else what its content confidently
+/// says ([`guess_language`]). A block whose content is only plausibly code
+/// takes the article's prevailing language when that is one of its
+/// candidates, so a post in Go does not alternate between highlighted and
+/// plain listings. A highlighted block becomes `<pre><code>…spans…</code></pre>`;
+/// any other block is copied through untouched apart from losing `data-lang`,
+/// which is only for this step.
+pub fn highlight_blocks(html: &str) -> String {
+    if !html.contains("<pre") {
+        return html.to_string();
+    }
+    let blocks = find_blocks(html);
+    let prevailing = prevailing_language(&blocks);
+    let mut highlighter: Option<Highlighter> = None;
+    let mut out = String::with_capacity(html.len() * 2);
+    let mut cursor = 0usize;
+    for block in &blocks {
+        out.push_str(&html[cursor..block.start]);
+        let language = match (block.hint.as_deref(), &block.detected) {
+            (Some(PLAIN), _) => None,
+            (Some(hint), _) => Some(hint),
+            (None, Detected::Sure(language)) => Some(*language),
+            (None, Detected::Maybe(candidates)) => {
+                prevailing.filter(|language| candidates.contains(language))
+            }
+            (None, Detected::Unknown) => None,
+        };
+        let highlighted = match (language, &block.code) {
+            (Some(language), Some(code)) => highlighter
+                .get_or_insert_with(new_highlighter)
+                .highlight(language, code)
+                .ok()
+                .filter(|spans| spans.contains("<span")),
+            _ => None,
+        };
+        match highlighted {
+            Some(spans) => {
+                out.push_str("<pre><code>");
+                out.push_str(&spans);
+                out.push_str("</code></pre>");
+            }
+            None => {
+                out.push_str(&pre_without_lang(
+                    &block.attrs,
+                    block.hint.is_some(),
+                    &html[block.start..block.open_end],
+                ));
+                out.push_str(&html[block.open_end..block.close]);
+                out.push_str("</pre>");
+            }
+        }
+        cursor = block.close + "</pre>".len();
+    }
+    out.push_str(&html[cursor..]);
+    out
+}
+
+/// Every complete `<pre>` block, in order, with its hint and detection.
+fn find_blocks(html: &str) -> Vec<Block> {
+    let mut blocks = Vec::new();
+    let mut cursor = 0usize;
+    while let Some(rel) = html[cursor..].find("<pre") {
+        let start = cursor + rel;
+        let Some(open_end) = tag_end(html, start) else {
+            break;
+        };
+        let inner = html[start..open_end]
+            .trim_start_matches('<')
+            .trim_end_matches('>');
+        if tag_name(inner) != "pre" {
+            cursor = open_end;
+            continue;
+        }
+        let Some(close_rel) = html[open_end..].find("</pre>") else {
+            break;
+        };
+        let close = open_end + close_rel;
+        let attrs = parse_attrs(inner);
+        let hint = attrs
+            .iter()
+            .find(|(name, _)| name == LANG_ATTR)
+            .map(|(_, value)| value.clone());
+        let code = block_text(&html[open_end..close]);
+        let detected = match (&hint, &code) {
+            (None, Some(code)) => detect(code),
+            _ => Detected::Unknown,
+        };
+        blocks.push(Block {
+            start,
+            open_end,
+            close,
+            attrs,
+            hint,
+            code,
+            detected,
+        });
+        cursor = close + "</pre>".len();
+    }
+    blocks
+}
+
+/// The language most of an article's hinted and confidently detected blocks
+/// share, earliest first on a tie.
+fn prevailing_language(blocks: &[Block]) -> Option<&str> {
+    let mut counts: Vec<(&str, usize)> = Vec::new();
+    for block in blocks {
+        let language = match (block.hint.as_deref(), &block.detected) {
+            (Some(PLAIN), _) => continue,
+            (Some(hint), _) => hint,
+            (None, Detected::Sure(language)) => language,
+            _ => continue,
+        };
+        match counts.iter_mut().find(|(name, _)| *name == language) {
+            Some((_, count)) => *count += 1,
+            None => counts.push((language, 1)),
+        }
+    }
+    counts
+        .iter()
+        .rev()
+        .max_by_key(|(_, count)| *count)
+        .map(|(language, _)| *language)
+}
+
+/// The `<pre>` opening tag minus `data-lang`, or `raw` when it had none.
+fn pre_without_lang(attrs: &[(String, String)], had_lang: bool, raw: &str) -> String {
+    if !had_lang {
+        return raw.to_string();
+    }
+    let mut tag = String::from("<pre");
+    for (name, value) in attrs.iter().filter(|(name, _)| name != LANG_ATTR) {
+        tag.push_str(&format!(" {name}=\"{}\"", attr_escape(value)));
+    }
+    tag.push('>');
+    tag
+}
+
+/// The source text of a `<pre>` body: an optional wrapping `<code>` removed,
+/// `<br>` as newlines, inline markup dropped, entities decoded, and the one
+/// leading newline HTML ignores stripped.
+///
+/// `None` when the body holds block markup (a table, a list) — that is a
+/// layout, not a listing, and is left alone.
+fn block_text(body: &str) -> Option<String> {
+    let mut text = String::with_capacity(body.len());
+    let mut cursor = 0usize;
+    while let Some(rel) = body[cursor..].find('<') {
+        let start = cursor + rel;
+        text.push_str(&body[cursor..start]);
+        let end = tag_end(body, start)?;
+        let inner = body[start..end]
+            .trim_start_matches('<')
+            .trim_end_matches('>');
+        match tag_name(inner).as_str() {
+            "br" => text.push('\n'),
+            "code" | "span" | "em" | "strong" | "a" | "b" | "i" | "u" | "s" | "mark" | "kbd"
+            | "samp" | "var" | "sub" | "sup" | "small" => {}
+            _ => return None,
+        }
+        cursor = end;
+    }
+    text.push_str(&body[cursor..]);
+    let text = decode_entities(&text);
+    let text = text
+        .strip_prefix("\r\n")
+        .or_else(|| text.strip_prefix('\n'))
+        .unwrap_or(&text);
+    let text = text.trim_end_matches(['\n', '\r']);
+    (!text.trim().is_empty()).then(|| text.to_string())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn normalizes_prefixed_bare_and_plain_hints() {
+        assert_eq!(normalize_hint("language-rust").as_deref(), Some("rust"));
+        assert_eq!(normalize_hint("lang-py").as_deref(), Some("python"));
+        assert_eq!(normalize_hint("highlight-source-go").as_deref(), Some("go"));
+        assert_eq!(normalize_hint("Python").as_deref(), Some("python"));
+        assert_eq!(normalize_hint("language-text").as_deref(), Some(PLAIN));
+        assert_eq!(normalize_hint("nohighlight").as_deref(), Some(PLAIN));
+        assert_eq!(normalize_hint("language-c").as_deref(), Some("c"));
+        // Bare one-letter classes and framework classes are not hints.
+        assert_eq!(normalize_hint("c"), None);
+        assert_eq!(normalize_hint("highlight"), None);
+        assert_eq!(normalize_hint("chroma"), None);
+        assert_eq!(normalize_hint("language-klingon"), None);
+    }
+
+    #[test]
+    fn annotates_hints_from_pre_code_and_wrapper_divs() {
+        assert_eq!(
+            annotate_hints(r#"<pre class="chroma"><code class="language-go">x</code></pre>"#),
+            r#"<pre class="chroma" data-lang="go"><code class="language-go">x</code></pre>"#
+        );
+        assert_eq!(
+            annotate_hints(r#"<pre class="brush: py; gutter: false">x</pre>"#),
+            r#"<pre class="brush: py; gutter: false" data-lang="python">x</pre>"#
+        );
+        // Jekyll/Rouge: the hint is on a wrapper two levels up.
+        assert_eq!(
+            annotate_hints(
+                r#"<div class="language-ruby highlighter-rouge"><div class="highlight"><pre class="highlight"><code>x</code></pre></div></div><pre>y</pre>"#
+            ),
+            r#"<div class="language-ruby highlighter-rouge"><div class="highlight"><pre class="highlight" data-lang="ruby"><code>x</code></pre></div></div><pre>y</pre>"#
+        );
+        assert_eq!(
+            annotate_hints(r#"<pre data-code-block-lang="JavaScript">x</pre>"#),
+            r#"<pre data-lang="javascript">x</pre>"#
+        );
+        let plain = "<p>a</p><pre><code>no hint</code></pre>";
+        assert_eq!(annotate_hints(plain), plain);
+    }
+
+    #[test]
+    fn the_sanitizer_filter_only_keeps_normalized_values() {
+        assert_eq!(filter_lang_attr("rust").as_deref(), Some("rust"));
+        assert_eq!(filter_lang_attr("text").as_deref(), Some(PLAIN));
+        assert_eq!(filter_lang_attr("language-rust"), None);
+        // Hints are stored canonical, so an alias is not a stored value.
+        assert_eq!(filter_lang_attr("py"), None);
+        assert_eq!(filter_lang_attr("\" onload=\"x"), None);
+    }
+
+    #[test]
+    fn highlights_hinted_blocks_with_flat_prefixed_spans() {
+        let out = highlight_blocks(
+            "<p>a</p><pre data-lang=\"rust\"><code>fn main() {\n    let s = \"&lt;&amp;&gt;\";\n}</code></pre><p>b</p>",
+        );
+        assert!(out.starts_with("<p>a</p><pre><code><span class=\"hl-keyword\">fn</span>"));
+        assert!(out.contains("<span class=\"hl-string\">&quot;&lt;&amp;&gt;&quot;</span>"));
+        assert!(out.ends_with("</code></pre><p>b</p>"));
+        assert!(!out.contains("data-lang"));
+    }
+
+    #[test]
+    fn br_line_breaks_and_inline_markup_become_plain_source() {
+        let out = highlight_blocks(
+            "<pre data-lang=\"python\"><code>def f():<br>    <em>return</em> 1</code></pre>",
+        );
+        assert!(out.contains("<span class=\"hl-keyword\">def</span>"));
+        assert!(out.contains("\n    <span class=\"hl-keyword\">return</span>"));
+        assert!(!out.contains("<em>"));
+    }
+
+    #[test]
+    fn plain_and_unrecognized_blocks_are_left_alone() {
+        assert_eq!(
+            highlight_blocks("<pre data-lang=\"text\"><code>def f(): return 1</code></pre>"),
+            "<pre><code>def f(): return 1</code></pre>"
+        );
+        let short = "<pre><code>cargo build</code></pre>";
+        assert_eq!(highlight_blocks(short), short);
+        let prose = "<pre>Dear reader, this is a poem set in a preformatted block\nbecause the author liked the look of it.</pre>";
+        assert_eq!(highlight_blocks(prose), prose);
+        let session = "<pre><code>$ cargo build --release\n   Compiling foo v0.1.0\n    Finished release</code></pre>";
+        assert_eq!(highlight_blocks(session), session);
+    }
+
+    #[test]
+    fn guesses_unhinted_code_by_content() {
+        assert_eq!(
+            guess_language(
+                "import os\n\ndef main():\n    print(os.getcwd())\n\nif __name__ == '__main__':\n    main()\n"
+            ),
+            Some("python")
+        );
+        assert_eq!(
+            guess_language(
+                "SELECT id, name FROM users WHERE created_at > NOW() - INTERVAL '1 day' ORDER BY id;"
+            ),
+            Some("sql")
+        );
+        assert_eq!(
+            guess_language("#!/usr/bin/env bash\nset -euo pipefail"),
+            Some("bash")
+        );
+        assert_eq!(
+            guess_language("{\"a\": [1, 2, 3], \"b\": {\"c\": null}}"),
+            Some("json")
+        );
+        let out = highlight_blocks(
+            "<pre><code>#include &lt;stdio.h&gt;\n\nint main(void) {\n    printf(\"hello\\n\");\n    return 0;\n}</code></pre>",
+        );
+        assert!(out.contains("hl-"), "{out}");
+    }
+
+    #[test]
+    fn an_uncertain_block_takes_the_articles_prevailing_language() {
+        // Both from go.dev/blog/range-functions, which marks no languages.
+        let sure = "<pre><code>// Add adds an element to a set.\nfunc (s *Set[E]) Add(v E) {\n    s.m[v] = struct{}{}\n}\n\n// Contains reports whether an element is in a set.\nfunc (s *Set[E]) Contains(v E) bool {\n    _, ok := s.m[v]\n    return ok\n}\n</code></pre>";
+        let unsure = "<pre><code>func (s *Set[E]) Push(f func(E) bool) {\n    for v := range s.m {\n        if !f(v) {\n            return\n        }\n    }\n}\n</code></pre>";
+        let text = |pre: &str| block_text(&pre[5..pre.len() - 6]).unwrap();
+        assert_eq!(guess_language(&text(sure)), Some("go"));
+        assert_eq!(guess_language(&text(unsure)), None);
+        assert!(matches!(detect(&text(unsure)), Detected::Maybe(c) if c.contains(&"go")));
+
+        let alone = highlight_blocks(unsure);
+        assert_eq!(alone, unsure);
+        let together = highlight_blocks(&format!("{sure}<p>and</p>{unsure}"));
+        assert_eq!(
+            together
+                .matches("<span class=\"hl-keyword\">func</span>")
+                .count(),
+            4
+        );
+    }
+
+    #[test]
+    fn quoted_style_and_script_tags_are_html() {
+        assert_eq!(
+            guess_language("<style>\n* {\n  box-sizing: border-box;\n}\n</style>"),
+            Some("html")
+        );
+    }
+}

src/lib.rs

@@ -22,6 +22,7 @@ pub mod dedupe;
 pub mod discovery;
 pub mod epub;
 pub mod extract;
+pub mod highlight;
 pub mod html;
 pub mod http;
 pub mod images;

src/web/issue.rs

@@ -1415,7 +1415,7 @@ fn thousands(value: i64) -> String {
 /// Sanitize article HTML exactly as the EPUB does, then add browser-only image
 /// loading/referrer attributes without converting the fragment to XHTML.
 fn prepare_body(html: &str) -> String {
-    let clean = ammonia::clean(html);
+    let clean = crate::highlight::clean_and_highlight(html);
     let mut output = String::with_capacity(clean.len() + 64);
     let mut cursor = 0usize;
     while let Some(relative) = clean[cursor..].find('<') {