//! Syntax highlighting for code blocks in article bodies. //! //! Two halves, run at different times: //! //! * [`annotate_hints`] runs on source markup at extraction, before //! readability and the sanitizer strip every `class`. It reads whatever //! language hint the page carries (`language-rust`, Rouge's wrapper `
`, which both of them keep.
//! * [`highlight_blocks`] runs at render time, on sanitized markup, for the web
//! article page and both EPUB editions. Blocks with a hint use it; blocks
//! without one are guessed from their content ([`guess_language`]), and
//! anything not confidently code is left exactly as it was.
//!
//! Tokens come out as flat `` elements, one class each, so
//! the X4 stylesheet (no descendant selectors) can style them too. Colours are
//! the stylesheets' business: the web theme is in `src/web/tailwind.css`, the
//! monochrome e-ink themes in `src/epub/templates/style*.css`.
use std::borrow::Cow;
use std::sync::{Arc, LazyLock};
use arborium::{Config, GrammarStore, Highlighter, HtmlFormat};
use crate::html::{attr_escape, decode_entities, parse_attrs, tag_end, tag_name};
/// The attribute that carries a block's language from extraction to render.
pub const LANG_ATTR: &str = "data-lang";
/// The `data-lang` value for a block its source marked as not code (plain
/// text, program output): never highlighted, never guessed.
pub const PLAIN: &str = "text";
/// Prefix of every token class, e.g. `hl-keyword`.
const CLASS_PREFIX: &str = "hl";
/// Hints that mean "this is not source code".
const PLAIN_HINTS: &[&str] = &[
"text",
"txt",
"plain",
"plaintext",
"none",
"nohighlight",
"no-highlight",
"output",
"console",
"terminal",
"log",
];
/// Attributes that name a block's language outright.
const LANG_ATTRIBUTES: &[&str] = &[
"data-lang",
"data-language",
"data-code-block-lang",
"data-code-language",
];
/// Class prefixes highlighters put in front of a language name.
const CLASS_PREFIXES: &[&str] = &[
"language-",
"lang-",
"highlight-source-",
"highlight-",
"source-",
];
/// Minimum model confidence for a guessed language. Measured on short real
/// snippets, guesses at or above this were right; below it they were coin flips.
const MIN_GUESS_SCORE: f32 = 0.5;
/// Unhinted blocks shorter than this are left alone: one-liners are usually a
/// command or a path, and too short to classify.
const MIN_GUESS_CHARS: usize = 40;
/// A block that is only plausibly code is weighed against the article's
/// prevailing language using at most this many of its best candidates…
const MAX_CANDIDATES: usize = 3;
/// …each scoring at least this much.
const MIN_CANDIDATE_SCORE: f32 = 0.1;
/// The detector reads at most this much of a block.
const MAX_GUESS_BYTES: usize = 4096;
/// Compiled grammars, shared across threads; each call gets its own parser.
static GRAMMARS: LazyLock> = LazyLock::new(|| Arc::new(GrammarStore::new()));
fn new_highlighter() -> Highlighter {
Highlighter::with_store_and_config(
GRAMMARS.clone(),
Config {
html_format: HtmlFormat::ClassNamesWithPrefix(CLASS_PREFIX.into()),
..Default::default()
},
)
}
/// Whether `name` (a language or one of arborium's aliases) has a grammar.
fn supported(name: &str) -> bool {
GRAMMARS.get(name).is_some()
}
// ---------------------------------------------------------------------------
// Hints (extraction time)
// ---------------------------------------------------------------------------
/// Normalize one hint token (`language-rust`, `py`, `nohighlight`) to the
/// value stored in `data-lang`: a supported language name or [`PLAIN`].
///
/// A bare token (no `language-` style prefix) must be at least two characters,
/// so stray one-letter classes don't turn into C or R.
pub fn normalize_hint(token: &str) -> Option {
let token = token.trim().trim_matches(';').to_ascii_lowercase();
let (name, prefixed) = CLASS_PREFIXES
.iter()
.find_map(|p| token.strip_prefix(p).map(|rest| (rest, true)))
.unwrap_or((token.as_str(), false));
if name.is_empty() || (!prefixed && name.len() < 2) {
return None;
}
if PLAIN_HINTS.contains(&name) {
return Some(PLAIN.to_string());
}
let name = canonical_name(name);
supported(name).then(|| name.to_string())
}
/// The name detection uses for the common aliases a hint may carry, so a
/// `py` block and a guessed `python` block count as the same language.
fn canonical_name(name: &str) -> &str {
match name {
"py" | "python3" => "python",
"js" | "node" | "jsx" => "javascript",
"ts" => "typescript",
"sh" | "shell" => "bash",
"rs" => "rust",
"golang" => "go",
"rb" => "ruby",
"yml" => "yaml",
"c++" | "cxx" | "cc" | "hpp" => "cpp",
"cs" | "csharp" | "c#" => "c-sharp",
"objective-c" | "objectivec" => "objc",
"kt" => "kotlin",
"ex" | "exs" => "elixir",
"hs" => "haskell",
"ps1" | "pwsh" => "powershell",
other => other,
}
}
/// The language hint carried by one element's attributes, if any.
fn hint_from_attrs(attrs: &[(String, String)]) -> Option {
for (name, value) in attrs {
if LANG_ATTRIBUTES.contains(&name.as_str())
&& let Some(hint) = normalize_hint(value)
{
return Some(hint);
}
}
let class = attrs.iter().find(|(name, _)| name == "class")?.1.as_str();
// SyntaxHighlighter writes `class="brush: python; gutter: false"`.
let tokens: Vec<&str> = class
.split(|c: char| c.is_whitespace() || c == ';')
.filter(|t| !t.is_empty())
.collect();
for (i, token) in tokens.iter().enumerate() {
let candidate = match token.strip_prefix("brush:") {
Some("") => tokens.get(i + 1).copied(),
Some(rest) => Some(rest),
None if i > 0 && tokens[i - 1] == "brush:" => None,
None => Some(*token),
};
if let Some(hint) = candidate.and_then(normalize_hint) {
return Some(hint);
}
}
None
}
/// Record each ``'s language hint as `data-lang`, before readability and
/// the sanitizer strip the classes that carry it.
///
/// The hint may sit on the ``, on the `` right inside it, or on an
/// enclosing `` (Jekyll's Rouge, GitHub's `highlight-source-*`). Markup is
/// otherwise returned byte-identical.
pub fn annotate_hints(html: &str) -> String {
let mut out = String::with_capacity(html.len() + 64);
let mut cursor = 0usize;
// Hint of every open ``, innermost last.
let mut divs: Vec