//! Syntax highlighting for code blocks in article bodies. //! //! Two halves, run at different times: //! //! * [`annotate_hints`] runs on source markup at extraction, before //! readability and the sanitizer strip every `class`. It reads whatever //! language hint the page carries (`language-rust`, Rouge's wrapper `
`, //! SyntaxHighlighter's `brush: py`, `data-lang`, …) and records it as one //! normalized `data-lang` on the `
`, which both of them keep.
//! * [`highlight_blocks`] runs at render time, on sanitized markup, for the web
//!   article page and both EPUB editions. Blocks with a hint use it; blocks
//!   without one are guessed from their content ([`guess_language`]), and
//!   anything not confidently code is left exactly as it was.
//!
//! Tokens come out as flat `` elements, one class each, so
//! the X4 stylesheet (no descendant selectors) can style them too. Colours are
//! the stylesheets' business: the web theme is in `src/web/tailwind.css`, the
//! monochrome e-ink themes in `src/epub/templates/style*.css`.

use std::borrow::Cow;
use std::sync::{Arc, LazyLock};

use arborium::{Config, GrammarStore, Highlighter, HtmlFormat};

use crate::html::{attr_escape, decode_entities, parse_attrs, tag_end, tag_name};

/// The attribute that carries a block's language from extraction to render.
pub const LANG_ATTR: &str = "data-lang";

/// The `data-lang` value for a block its source marked as not code (plain
/// text, program output): never highlighted, never guessed.
pub const PLAIN: &str = "text";

/// Prefix of every token class, e.g. `hl-keyword`.
const CLASS_PREFIX: &str = "hl";

/// Hints that mean "this is not source code".
const PLAIN_HINTS: &[&str] = &[
    "text",
    "txt",
    "plain",
    "plaintext",
    "none",
    "nohighlight",
    "no-highlight",
    "output",
    "console",
    "terminal",
    "log",
];

/// Attributes that name a block's language outright.
const LANG_ATTRIBUTES: &[&str] = &[
    "data-lang",
    "data-language",
    "data-code-block-lang",
    "data-code-language",
];

/// Class prefixes highlighters put in front of a language name.
const CLASS_PREFIXES: &[&str] = &[
    "language-",
    "lang-",
    "highlight-source-",
    "highlight-",
    "source-",
];

/// Minimum model confidence for a guessed language. Measured on short real
/// snippets, guesses at or above this were right; below it they were coin flips.
const MIN_GUESS_SCORE: f32 = 0.5;

/// Unhinted blocks shorter than this are left alone: one-liners are usually a
/// command or a path, and too short to classify.
const MIN_GUESS_CHARS: usize = 40;

/// A block that is only plausibly code is weighed against the article's
/// prevailing language using at most this many of its best candidates…
const MAX_CANDIDATES: usize = 3;

/// …each scoring at least this much.
const MIN_CANDIDATE_SCORE: f32 = 0.1;

/// The detector reads at most this much of a block.
const MAX_GUESS_BYTES: usize = 4096;

/// Compiled grammars, shared across threads; each call gets its own parser.
static GRAMMARS: LazyLock> = LazyLock::new(|| Arc::new(GrammarStore::new()));

fn new_highlighter() -> Highlighter {
    Highlighter::with_store_and_config(
        GRAMMARS.clone(),
        Config {
            html_format: HtmlFormat::ClassNamesWithPrefix(CLASS_PREFIX.into()),
            ..Default::default()
        },
    )
}

/// Whether `name` (a language or one of arborium's aliases) has a grammar.
fn supported(name: &str) -> bool {
    GRAMMARS.get(name).is_some()
}

// ---------------------------------------------------------------------------
// Hints (extraction time)
// ---------------------------------------------------------------------------

/// Normalize one hint token (`language-rust`, `py`, `nohighlight`) to the
/// value stored in `data-lang`: a supported language name or [`PLAIN`].
///
/// A bare token (no `language-` style prefix) must be at least two characters,
/// so stray one-letter classes don't turn into C or R.
pub fn normalize_hint(token: &str) -> Option {
    let token = token.trim().trim_matches(';').to_ascii_lowercase();
    let (name, prefixed) = CLASS_PREFIXES
        .iter()
        .find_map(|p| token.strip_prefix(p).map(|rest| (rest, true)))
        .unwrap_or((token.as_str(), false));
    if name.is_empty() || (!prefixed && name.len() < 2) {
        return None;
    }
    if PLAIN_HINTS.contains(&name) {
        return Some(PLAIN.to_string());
    }
    let name = canonical_name(name);
    supported(name).then(|| name.to_string())
}

/// The name detection uses for the common aliases a hint may carry, so a
/// `py` block and a guessed `python` block count as the same language.
fn canonical_name(name: &str) -> &str {
    match name {
        "py" | "python3" => "python",
        "js" | "node" | "jsx" => "javascript",
        "ts" => "typescript",
        "sh" | "shell" => "bash",
        "rs" => "rust",
        "golang" => "go",
        "rb" => "ruby",
        "yml" => "yaml",
        "c++" | "cxx" | "cc" | "hpp" => "cpp",
        "cs" | "csharp" | "c#" => "c-sharp",
        "objective-c" | "objectivec" => "objc",
        "kt" => "kotlin",
        "ex" | "exs" => "elixir",
        "hs" => "haskell",
        "ps1" | "pwsh" => "powershell",
        other => other,
    }
}

/// The language hint carried by one element's attributes, if any.
fn hint_from_attrs(attrs: &[(String, String)]) -> Option {
    for (name, value) in attrs {
        if LANG_ATTRIBUTES.contains(&name.as_str())
            && let Some(hint) = normalize_hint(value)
        {
            return Some(hint);
        }
    }
    let class = attrs.iter().find(|(name, _)| name == "class")?.1.as_str();
    // SyntaxHighlighter writes `class="brush: python; gutter: false"`.
    let tokens: Vec<&str> = class
        .split(|c: char| c.is_whitespace() || c == ';')
        .filter(|t| !t.is_empty())
        .collect();
    for (i, token) in tokens.iter().enumerate() {
        let candidate = match token.strip_prefix("brush:") {
            Some("") => tokens.get(i + 1).copied(),
            Some(rest) => Some(rest),
            None if i > 0 && tokens[i - 1] == "brush:" => None,
            None => Some(*token),
        };
        if let Some(hint) = candidate.and_then(normalize_hint) {
            return Some(hint);
        }
    }
    None
}

/// Record each `
`'s language hint as `data-lang`, before readability and
/// the sanitizer strip the classes that carry it.
///
/// The hint may sit on the `
`, on the `` right inside it, or on an
/// enclosing `
` (Jekyll's Rouge, GitHub's `highlight-source-*`). Markup is /// otherwise returned byte-identical. pub fn annotate_hints(html: &str) -> String { let mut out = String::with_capacity(html.len() + 64); let mut cursor = 0usize; // Hint of every open `
`, innermost last. let mut divs: Vec> = Vec::new(); while let Some(rel) = html[cursor..].find('<') { let start = cursor + rel; out.push_str(&html[cursor..start]); let Some(end) = tag_end(html, start) else { out.push_str(&html[start..]); return out; }; let raw = &html[start..end]; cursor = end; let inner = raw.trim_start_matches('<').trim_end_matches('>'); let name = tag_name(inner); if inner.starts_with('/') { if name == "div" { divs.pop(); } out.push_str(raw); continue; } match name.as_str() { "div" => { if !inner.trim_end().ends_with('/') { divs.push(hint_from_attrs(&parse_attrs(inner))); } out.push_str(raw); } "pre" => { let attrs = parse_attrs(inner.trim_end_matches('/')); let hint = hint_from_attrs(&attrs) .or_else(|| first_code_hint(html, end)) .or_else(|| divs.iter().rev().find_map(Clone::clone)); match hint { Some(hint) => out.push_str(&pre_tag(&attrs, &hint)), None => out.push_str(raw), } } _ => out.push_str(raw), } } out.push_str(&html[cursor..]); out } /// The hint on a `` that opens right after a `
` ending at `after`.
fn first_code_hint(html: &str, after: usize) -> Option {
    let rest = &html[after..];
    let start = after + (rest.len() - rest.trim_start().len());
    if !html[start..].starts_with('<') {
        return None;
    }
    let end = tag_end(html, start)?;
    let inner = html[start..end]
        .trim_start_matches('<')
        .trim_end_matches('>');
    (tag_name(inner) == "code").then(|| hint_from_attrs(&parse_attrs(inner)))?
}

/// Rebuild a `
` tag with `data-lang` set to `hint`.
fn pre_tag(attrs: &[(String, String)], hint: &str) -> String {
    let mut tag = String::from("", attr_escape(hint)));
    tag
}

/// Sanitizer filter for `data-lang`: keep it only if it is a value
/// [`annotate_hints`] could have written.
pub fn filter_lang_attr(value: &str) -> Option {
    normalize_hint(value).filter(|hint| hint == &value.to_ascii_lowercase())
}

/// Let an ammonia builder keep a valid `data-lang` on `
` (and nothing else
/// it did not already allow).
pub fn keep_lang_attr(builder: &mut ammonia::Builder<'_>) {
    builder
        .add_tag_attributes("pre", &[LANG_ATTR])
        .attribute_filter(|element, attribute, value| {
            if element == "pre" && attribute == LANG_ATTR {
                filter_lang_attr(value).map(Cow::Owned)
            } else {
                Some(Cow::Borrowed(value))
            }
        });
}

/// `ammonia::clean` that keeps block languages, then [`highlight_blocks`]:
/// the render-time entry point for article bodies.
pub fn clean_and_highlight(html: &str) -> String {
    let mut builder = ammonia::Builder::default();
    keep_lang_attr(&mut builder);
    highlight_blocks(&builder.clean(html).to_string())
}

// ---------------------------------------------------------------------------
// Detection
// ---------------------------------------------------------------------------

/// What an unhinted block's content says about its language.
#[derive(Debug, Clone, PartialEq)]
enum Detected {
    /// Confidently this language.
    Sure(&'static str),
    /// Plausibly one of these, best first: taken only when the rest of the
    /// article is in one of them (see [`highlight_blocks`]).
    Maybe(Vec<&'static str>),
    /// Not code, or too little of it to tell.
    Unknown,
}

/// Best guess at the language of an unhinted block, or `None` when it is not
/// confidently code. A wrong colour scheme is worse than none.
pub fn guess_language(code: &str) -> Option<&'static str> {
    match detect(code) {
        Detected::Sure(language) => Some(language),
        _ => None,
    }
}

fn detect(code: &str) -> Detected {
    let text = code.trim();
    // Parsing is proof, at any length.
    if (text.starts_with('{') || text.starts_with('['))
        && serde_json::from_str::(text).is_ok()
    {
        return Detected::Sure("json");
    }
    if text.chars().count() < MIN_GUESS_CHARS && !text.starts_with("#!") {
        return Detected::Unknown;
    }
    if let Some(language) = first_line_language(text) {
        return Detected::Sure(language);
    }
    // A shell session is prompts interleaved with output; neither grammar fits.
    if text.starts_with("$ ") || text.starts_with("> ") || text.starts_with("% ") {
        return Detected::Unknown;
    }
    let sample = crate::html::truncate_utf8(text, MAX_GUESS_BYTES);
    let detection = betlang::detect(sample);
    let mut ranked = detection.top_languages().take(MAX_CANDIDATES);
    let Some((score, top)) = ranked.next() else {
        return Detected::Unknown;
    };
    // Prose, logs and program output all come out as Markdown.
    let Some(top) = betlang_language(top) else {
        return Detected::Unknown;
    };
    if score >= MIN_GUESS_SCORE {
        return Detected::Sure(top);
    }
    let mut candidates = vec![top];
    candidates.extend(
        ranked
            .filter(|(score, _)| *score >= MIN_CANDIDATE_SCORE)
            .filter_map(|(_, language)| betlang_language(language)),
    );
    Detected::Maybe(candidates)
}

/// The grammar for a betlang label, if we have one.
fn betlang_language(language: betlang::Language) -> Option<&'static str> {
    let name = match language.slug() {
        "markdown" => return None,
        "cs" => "c-sharp",
        "objectivec" => "objc",
        "lisp" => "commonlisp",
        "shell" => "bash",
        "gemfile" | "gemspec" => "ruby",
        "gradle" => "groovy",
        other => other,
    };
    supported(name).then_some(name)
}

/// Languages a block announces on its first line: shebangs and XML-ish preludes.
fn first_line_language(text: &str) -> Option<&'static str> {
    let first = text.lines().next()?.trim().to_ascii_lowercase();
    if let Some(shebang) = first.strip_prefix("#!") {
        let interpreter = shebang
            .split_whitespace()
            .flat_map(|word| word.rsplit('/').next())
            .find(|word| *word != "env" && !word.starts_with('-'))?;
        let name = match interpreter.trim_end_matches(|c: char| c.is_ascii_digit() || c == '.') {
            "sh" | "bash" | "dash" | "ksh" => "bash",
            "zsh" => "zsh",
            "fish" => "fish",
            "python" => "python",
            "node" | "deno" | "bun" => "javascript",
            "ruby" => "ruby",
            "perl" => "perl",
            "php" => "php",
            "lua" => "lua",
            _ => return None,
        };
        return supported(name).then_some(name);
    }
    if first.starts_with("` block found by [`highlight_blocks`].
struct Block {
    /// Byte range of the opening tag, and where `
` starts. start: usize, open_end: usize, close: usize, attrs: Vec<(String, String)>, hint: Option, code: Option, detected: Detected, } /// Highlight every `
` block in sanitized markup.
///
/// A block's language is its `data-lang`, else what its content confidently
/// says ([`guess_language`]). A block whose content is only plausibly code
/// takes the article's prevailing language when that is one of its
/// candidates, so a post in Go does not alternate between highlighted and
/// plain listings. A highlighted block becomes `
…spans…
`; /// any other block is copied through untouched apart from losing `data-lang`, /// which is only for this step. pub fn highlight_blocks(html: &str) -> String { if !html.contains(" = None; let mut out = String::with_capacity(html.len() * 2); let mut cursor = 0usize; for block in &blocks { out.push_str(&html[cursor..block.start]); let language = match (block.hint.as_deref(), &block.detected) { (Some(PLAIN), _) => None, (Some(hint), _) => Some(hint), (None, Detected::Sure(language)) => Some(*language), (None, Detected::Maybe(candidates)) => { prevailing.filter(|language| candidates.contains(language)) } (None, Detected::Unknown) => None, }; let highlighted = match (language, &block.code) { (Some(language), Some(code)) => highlighter .get_or_insert_with(new_highlighter) .highlight(language, code) .ok() .filter(|spans| spans.contains(" None, }; match highlighted { Some(spans) => { out.push_str("
");
                out.push_str(&spans);
                out.push_str("
"); } None => { out.push_str(&pre_without_lang( &block.attrs, block.hint.is_some(), &html[block.start..block.open_end], )); out.push_str(&html[block.open_end..block.close]); out.push_str("
"); } } cursor = block.close + "
".len(); } out.push_str(&html[cursor..]); out } /// Every complete `
` block, in order, with its hint and detection.
fn find_blocks(html: &str) -> Vec {
    let mut blocks = Vec::new();
    let mut cursor = 0usize;
    while let Some(rel) = html[cursor..].find("');
        if tag_name(inner) != "pre" {
            cursor = open_end;
            continue;
        }
        let Some(close_rel) = html[open_end..].find("
") else { break; }; let close = open_end + close_rel; let attrs = parse_attrs(inner); let hint = attrs .iter() .find(|(name, _)| name == LANG_ATTR) .map(|(_, value)| value.clone()); let code = block_text(&html[open_end..close]); let detected = match (&hint, &code) { (None, Some(code)) => detect(code), _ => Detected::Unknown, }; blocks.push(Block { start, open_end, close, attrs, hint, code, detected, }); cursor = close + "
".len(); } blocks } /// The language most of an article's hinted and confidently detected blocks /// share, earliest first on a tie. fn prevailing_language(blocks: &[Block]) -> Option<&str> { let mut counts: Vec<(&str, usize)> = Vec::new(); for block in blocks { let language = match (block.hint.as_deref(), &block.detected) { (Some(PLAIN), _) => continue, (Some(hint), _) => hint, (None, Detected::Sure(language)) => language, _ => continue, }; match counts.iter_mut().find(|(name, _)| *name == language) { Some((_, count)) => *count += 1, None => counts.push((language, 1)), } } counts .iter() .rev() .max_by_key(|(_, count)| *count) .map(|(language, _)| *language) } /// The `
` opening tag minus `data-lang`, or `raw` when it had none.
fn pre_without_lang(attrs: &[(String, String)], had_lang: bool, raw: &str) -> String {
    if !had_lang {
        return raw.to_string();
    }
    let mut tag = String::from("');
    tag
}

/// The source text of a `
` body: an optional wrapping `` removed,
/// `
` as newlines, inline markup dropped, entities decoded, and the one /// leading newline HTML ignores stripped. /// /// `None` when the body holds block markup (a table, a list) — that is a /// layout, not a listing, and is left alone. fn block_text(body: &str) -> Option { let mut text = String::with_capacity(body.len()); let mut cursor = 0usize; while let Some(rel) = body[cursor..].find('<') { let start = cursor + rel; text.push_str(&body[cursor..start]); let end = tag_end(body, start)?; let inner = body[start..end] .trim_start_matches('<') .trim_end_matches('>'); match tag_name(inner).as_str() { "br" => text.push('\n'), "code" | "span" | "em" | "strong" | "a" | "b" | "i" | "u" | "s" | "mark" | "kbd" | "samp" | "var" | "sub" | "sup" | "small" => {} _ => return None, } cursor = end; } text.push_str(&body[cursor..]); let text = decode_entities(&text); let text = text .strip_prefix("\r\n") .or_else(|| text.strip_prefix('\n')) .unwrap_or(&text); let text = text.trim_end_matches(['\n', '\r']); (!text.trim().is_empty()).then(|| text.to_string()) } #[cfg(test)] mod tests { use super::*; #[test] fn normalizes_prefixed_bare_and_plain_hints() { assert_eq!(normalize_hint("language-rust").as_deref(), Some("rust")); assert_eq!(normalize_hint("lang-py").as_deref(), Some("python")); assert_eq!(normalize_hint("highlight-source-go").as_deref(), Some("go")); assert_eq!(normalize_hint("Python").as_deref(), Some("python")); assert_eq!(normalize_hint("language-text").as_deref(), Some(PLAIN)); assert_eq!(normalize_hint("nohighlight").as_deref(), Some(PLAIN)); assert_eq!(normalize_hint("language-c").as_deref(), Some("c")); // Bare one-letter classes and framework classes are not hints. assert_eq!(normalize_hint("c"), None); assert_eq!(normalize_hint("highlight"), None); assert_eq!(normalize_hint("chroma"), None); assert_eq!(normalize_hint("language-klingon"), None); } #[test] fn annotates_hints_from_pre_code_and_wrapper_divs() { assert_eq!( annotate_hints(r#"
x
"#), r#"
x
"# ); assert_eq!( annotate_hints(r#"
x
"#), r#"
x
"# ); // Jekyll/Rouge: the hint is on a wrapper two levels up. assert_eq!( annotate_hints( r#"
x
y
"# ), r#"
x
y
"# ); assert_eq!( annotate_hints(r#"
x
"#), r#"
x
"# ); let plain = "

a

no hint
"; assert_eq!(annotate_hints(plain), plain); } #[test] fn the_sanitizer_filter_only_keeps_normalized_values() { assert_eq!(filter_lang_attr("rust").as_deref(), Some("rust")); assert_eq!(filter_lang_attr("text").as_deref(), Some(PLAIN)); assert_eq!(filter_lang_attr("language-rust"), None); // Hints are stored canonical, so an alias is not a stored value. assert_eq!(filter_lang_attr("py"), None); assert_eq!(filter_lang_attr("\" onload=\"x"), None); } #[test] fn highlights_hinted_blocks_with_flat_prefixed_spans() { let out = highlight_blocks( "

a

fn main() {\n    let s = \"<&>\";\n}

b

", ); assert!(out.starts_with("

a

fn"));
        assert!(out.contains(""<&>""));
        assert!(out.ends_with("

b

")); assert!(!out.contains("data-lang")); } #[test] fn br_line_breaks_and_inline_markup_become_plain_source() { let out = highlight_blocks( "
def f():
return 1
", ); assert!(out.contains("def")); assert!(out.contains("\n return")); assert!(!out.contains("")); } #[test] fn plain_and_unrecognized_blocks_are_left_alone() { assert_eq!( highlight_blocks("
def f(): return 1
"), "
def f(): return 1
" ); let short = "
cargo build
"; assert_eq!(highlight_blocks(short), short); let prose = "
Dear reader, this is a poem set in a preformatted block\nbecause the author liked the look of it.
"; assert_eq!(highlight_blocks(prose), prose); let session = "
$ cargo build --release\n   Compiling foo v0.1.0\n    Finished release
"; assert_eq!(highlight_blocks(session), session); } #[test] fn guesses_unhinted_code_by_content() { assert_eq!( guess_language( "import os\n\ndef main():\n print(os.getcwd())\n\nif __name__ == '__main__':\n main()\n" ), Some("python") ); assert_eq!( guess_language( "SELECT id, name FROM users WHERE created_at > NOW() - INTERVAL '1 day' ORDER BY id;" ), Some("sql") ); assert_eq!( guess_language("#!/usr/bin/env bash\nset -euo pipefail"), Some("bash") ); assert_eq!( guess_language("{\"a\": [1, 2, 3], \"b\": {\"c\": null}}"), Some("json") ); let out = highlight_blocks( "
#include <stdio.h>\n\nint main(void) {\n    printf(\"hello\\n\");\n    return 0;\n}
", ); assert!(out.contains("hl-"), "{out}"); } #[test] fn an_uncertain_block_takes_the_articles_prevailing_language() { // Both from go.dev/blog/range-functions, which marks no languages. let sure = "
// Add adds an element to a set.\nfunc (s *Set[E]) Add(v E) {\n    s.m[v] = struct{}{}\n}\n\n// Contains reports whether an element is in a set.\nfunc (s *Set[E]) Contains(v E) bool {\n    _, ok := s.m[v]\n    return ok\n}\n
"; let unsure = "
func (s *Set[E]) Push(f func(E) bool) {\n    for v := range s.m {\n        if !f(v) {\n            return\n        }\n    }\n}\n
"; let text = |pre: &str| block_text(&pre[5..pre.len() - 6]).unwrap(); assert_eq!(guess_language(&text(sure)), Some("go")); assert_eq!(guess_language(&text(unsure)), None); assert!(matches!(detect(&text(unsure)), Detected::Maybe(c) if c.contains(&"go"))); let alone = highlight_blocks(unsure); assert_eq!(alone, unsure); let together = highlight_blocks(&format!("{sure}

and

{unsure}")); assert_eq!( together .matches("func") .count(), 4 ); } #[test] fn quoted_style_and_script_tags_are_html() { assert_eq!( guess_language(""), Some("html") ); } }