// This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. //! Syntax highlighting producing independently-valid HTML per line. //! //! Two hard requirements drive the design. First, **line-addressable output**: the //! blob view anchors on line numbers, so highlighting yields a `Vec<`[`Line`]`>` //! where each line is a `Vec<`[`Span`]`>` that a template renders into balanced //! markup on its own — a span that crosses a newline is split and its class //! reopened on the next line. Second, **per-side diff highlighting**: a diff //! highlights the pre- and post-image of a file as whole documents (never //! per-hunk, which would feed a context-dependent grammar garbage) and then zips //! the resulting lines with the hunks. //! //! Highlighting **must not** be able to fail a page render (§8): oversized, binary, //! and minified inputs skip straight to plain text, and a grammar panic is caught //! and degraded the same way. The output carries no colours — only the stable //! `hl-*` classes from [`classes`], which themes style. mod classes; mod lang; use std::panic::{AssertUnwindSafe, catch_unwind}; use inkjet::Highlighter; use inkjet::constants::HIGHLIGHT_NAMES; use inkjet::tree_sitter_highlight::HighlightEvent; pub use crate::classes::{HL_CLASSES, class_for}; pub use crate::lang::{Lang, resolve}; /// Files larger than this are rendered as plain text (matches the /// `ui.max_highlight_bytes` default; the web layer may pre-filter with the /// configured value). pub const DEFAULT_MAX_BYTES: usize = 1_048_576; /// A single line longer than this marks the file minified; it is rendered plain. pub const MINIFIED_LINE_BYTES: usize = 5_000; /// Bytes sniffed from the head of the content when deciding whether it is binary. const BINARY_SNIFF_BYTES: usize = 8_192; /// One styled run of text within a line. `class` is a stable `hl-*` name or `None` /// for unstyled text; `text` is raw (the renderer escapes it) and never contains a /// newline. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Span { /// The highlight class, or `None` for text with no styling. pub class: Option<&'static str>, /// The raw text of this run, newline-free. pub text: String, } /// One rendered source line: its spans in order, without the trailing newline. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct Line { /// The line's styled runs, left to right. pub spans: Vec, } /// The result of highlighting a whole blob. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Highlighted { /// One entry per source line. pub lines: Vec, /// The resolved language's display name, or `None` when rendered plain. pub language: Option, } /// Highlight a blob for the file at `path`, applying every guard. /// /// `attr_override` is the `.gitattributes` language override the git layer /// resolved. Oversized, binary, minified, or unrecognized-language inputs — and any /// grammar failure — degrade to plain text; the call never fails. #[must_use] pub fn highlight( path: &std::path::Path, content: &[u8], attr_override: Option<&str>, ) -> Highlighted { let lang = resolve(path, content, attr_override); if lang.is_plain() || is_unhighlightable(content) { return Highlighted { lines: plain_lines(&String::from_utf8_lossy(content)), language: None, }; } // `is_unhighlightable` already rejected non-UTF-8 (a NUL sniff), but guard the // decode anyway; a lossy fallback would misalign byte offsets. let Ok(source) = std::str::from_utf8(content) else { return Highlighted { lines: plain_lines(&String::from_utf8_lossy(content)), language: None, }; }; match lang .language() .and_then(|inner| highlight_events(inner, source)) { Some(lines) => Highlighted { lines, language: lang.name().map(str::to_string), }, None => Highlighted { lines: plain_lines(source), language: None, }, } } /// Highlight a whole document (one side of a diff) with an already-resolved /// language, for zipping with hunks. Applies the size and minified guards and falls /// back to plain text on anything unhighlightable. #[must_use] pub fn highlight_document(lang: &Lang, source: &str) -> Vec { if lang.is_plain() || is_unhighlightable(source.as_bytes()) { return plain_lines(source); } lang.language() .and_then(|inner| highlight_events(inner, source)) .unwrap_or_else(|| plain_lines(source)) } /// Whether `content` should skip highlighting outright: too large, binary (a NUL in /// the sniff window), or minified (a very long line). fn is_unhighlightable(content: &[u8]) -> bool { if content.len() > DEFAULT_MAX_BYTES { return true; } let window = &content[..content.len().min(BINARY_SNIFF_BYTES)]; if window.contains(&0) { return true; } content .split(|b| *b == b'\n') .any(|line| line.len() > MINIFIED_LINE_BYTES) } /// Run the grammar and fold its events into lines, or `None` on any grammar error /// or panic (the caller falls back to plain text). fn highlight_events(language: inkjet::Language, source: &str) -> Option> { let outcome = catch_unwind(AssertUnwindSafe(|| { let mut highlighter = Highlighter::new(); let events = highlighter.highlight_raw(language, &source).ok()?; let mut lines: Vec = Vec::new(); let mut current: Vec = Vec::new(); // A stack of classes, innermost last; `None` for captures we do not style. let mut stack: Vec> = Vec::new(); for event in events { match event.ok()? { HighlightEvent::HighlightStart(highlight) => { let class = HIGHLIGHT_NAMES .get(highlight.0) .copied() .and_then(class_for); stack.push(class); } HighlightEvent::HighlightEnd => { stack.pop(); } HighlightEvent::Source { start, end } => { let text = source.get(start..end).unwrap_or_default(); let class = stack.iter().rev().flatten().next().copied(); push_text(&mut lines, &mut current, text, class); } } } if !current.is_empty() { lines.push(Line { spans: current }); } Some(lines) })); outcome.ok().flatten() } /// Render unhighlighted `source` into one plain span per line. fn plain_lines(source: &str) -> Vec { let mut lines: Vec = Vec::new(); let mut current: Vec = Vec::new(); push_text(&mut lines, &mut current, source, None); if !current.is_empty() { lines.push(Line { spans: current }); } lines } /// Append `text` (with class `class`) to the line being built, splitting on /// newlines. Each embedded `\n` finishes the current line — preserving blank lines /// — and the class carries onto the next line, which is the crux of producing /// balanced per-line markup for a span that crosses a break. fn push_text( lines: &mut Vec, current: &mut Vec, text: &str, class: Option<&'static str>, ) { for (i, segment) in text.split('\n').enumerate() { if i > 0 { lines.push(Line { spans: std::mem::take(current), }); } if !segment.is_empty() { current.push(Span { class, text: segment.to_string(), }); } } } #[cfg(test)] mod tests { #![allow(clippy::unwrap_used)] use std::path::Path; use super::*; /// Flatten a line back to its text, for assertions independent of span layout. fn line_text(line: &Line) -> String { line.spans.iter().map(|s| s.text.as_str()).collect() } #[test] fn highlights_rust_and_marks_keywords() { let out = highlight(Path::new("main.rs"), b"fn main() {}\n", None); assert_eq!(out.language.as_deref(), Some("rust")); assert_eq!(out.lines.len(), 1); // `fn` should carry the keyword class somewhere on the line. let classes: Vec<_> = out.lines[0].spans.iter().filter_map(|s| s.class).collect(); assert!(classes.contains(&"hl-keyword"), "classes: {classes:?}"); assert_eq!(line_text(&out.lines[0]), "fn main() {}"); } #[test] fn every_line_is_independently_balanced_across_a_multiline_string() { // A multi-line string literal: the string class must reopen on each line, // so every line stands alone with balanced spans. let src = "fn f() -> &'static str {\n \"line one\nline two\nline three\"\n}\n"; let out = highlight(Path::new("s.rs"), src.as_bytes(), None); // The three literal lines each carry the string class. let string_lines: Vec = out .lines .iter() .enumerate() .filter(|(_, l)| l.spans.iter().any(|s| s.class == Some("hl-string"))) .map(|(i, _)| i) .collect(); assert!( string_lines.len() >= 3, "string spans should reopen on each continuation line: {string_lines:?}" ); // No span text ever contains a newline. for line in &out.lines { for span in &line.spans { assert!(!span.text.contains('\n'), "span leaked a newline: {span:?}"); } } } #[test] fn preserves_blank_lines_and_line_count() { let out = highlight(Path::new("a.rs"), b"fn a() {}\n\nfn b() {}\n", None); assert_eq!(out.lines.len(), 3); assert_eq!(line_text(&out.lines[1]), ""); } #[test] fn binary_content_degrades_to_plain() { let out = highlight(Path::new("x.rs"), b"fn main() {\0}\n", None); assert_eq!(out.language, None, "NUL byte marks it binary → plain"); } #[test] fn minified_content_degrades_to_plain() { let mut src = String::from("let x = "); src.push_str(&"a".repeat(MINIFIED_LINE_BYTES + 1)); src.push('\n'); let out = highlight(Path::new("min.js"), src.as_bytes(), None); assert_eq!(out.language, None); } #[test] fn unknown_language_is_plain_but_still_line_split() { let out = highlight(Path::new("notes.zzz"), b"one\ntwo\n", None); assert_eq!(out.language, None); assert_eq!(out.lines.len(), 2); assert_eq!(line_text(&out.lines[0]), "one"); assert!(out.lines[0].spans.iter().all(|s| s.class.is_none())); } #[test] fn highlight_document_matches_source_lines() { let lang = resolve(Path::new("main.rs"), b"fn main() {}\n", None); let lines = highlight_document(&lang, "fn main() {}\nlet x = 1;\n"); assert_eq!(lines.len(), 2); assert_eq!(line_text(&lines[0]), "fn main() {}"); assert_eq!(line_text(&lines[1]), "let x = 1;"); } /// Render lines to the same span markup the web layer emits, for snapshotting. fn render(lines: &[Line]) -> String { use std::fmt::Write as _; let mut out = String::new(); for line in lines { for span in &line.spans { match span.class { Some(class) => { let _ = write!(out, "{}", span.text); } None => out.push_str(&span.text), } } out.push('\n'); } out } #[test] fn snapshot_rust_fixture() { let src = "use std::io;\n\nfn add(a: i32, b: i32) -> i32 {\n a + b // sum\n}\n"; let out = highlight(Path::new("fixture.rs"), src.as_bytes(), None); insta::assert_snapshot!(render(&out.lines)); } }