diff --git a/Cargo.lock b/Cargo.lock index 3c9da50..3617926 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,6 +127,919 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arborium" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230e826fe3204178030e420751396c99431e4611fc67af63baf7767b4317bd75" +dependencies = [ + "arborium-asm", + "arborium-awk", + "arborium-bash", + "arborium-c", + "arborium-c-sharp", + "arborium-clojure", + "arborium-cmake", + "arborium-commonlisp", + "arborium-cpp", + "arborium-css", + "arborium-d", + "arborium-dart", + "arborium-diff", + "arborium-dockerfile", + "arborium-elisp", + "arborium-elixir", + "arborium-elm", + "arborium-erlang", + "arborium-fish", + "arborium-fsharp", + "arborium-gleam", + "arborium-glsl", + "arborium-go", + "arborium-graphql", + "arborium-groovy", + "arborium-haskell", + "arborium-hcl", + "arborium-highlight", + "arborium-html", + "arborium-ini", + "arborium-java", + "arborium-javascript", + "arborium-jq", + "arborium-json", + "arborium-julia", + "arborium-kotlin", + "arborium-lean", + "arborium-lua", + "arborium-make", + "arborium-matlab", + "arborium-nix", + "arborium-objc", + "arborium-ocaml", + "arborium-odin", + "arborium-perl", + "arborium-php", + "arborium-powershell", + "arborium-prolog", + "arborium-proto", + "arborium-python", + "arborium-r", + "arborium-ruby", + "arborium-rust", + "arborium-scala", + "arborium-scheme", + "arborium-scss", + "arborium-solidity", + "arborium-sql", + "arborium-starlark", + "arborium-swift", + "arborium-theme", + "arborium-toml", + "arborium-tree-sitter", + "arborium-tsx", + "arborium-typescript", + "arborium-verilog", + "arborium-vhdl", + "arborium-vim", + "arborium-x86asm", + "arborium-xml", + "arborium-yaml", + "arborium-zig", + "arborium-zsh", + "dlmalloc", +] + +[[package]] +name = "arborium-asm" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4bf1bbe5b0c6bb55a429f48b17ed336b0ce27f17010b22d9d47e3ab848c4226" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-awk" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "968632acf457b0367049f4cacb0a53016eac979ccce2218b2056b911dfd84ecc" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-bash" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2a2c191c27ee915e5ae65b619f9f7c1345565d511ce7761d30b4b243b2e333" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-c" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37615879a3956f8918d43be52c9c4d11d60af5b7bedabca87ab8b47f769fa8c7" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-c-sharp" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1da8f9108bf98f334190243c27080476b126c54981fe2cb9b04d4f25d092960e" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-clojure" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9d188f152dae834e9ae69694089028373cd9099a339666cd36385e6beda19f" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-cmake" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "428a67eb72ef4bae3d59a588832ef34a6c22d0dc315ded3cbc5a74d01f855021" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-commonlisp" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68fd8337ca9dc1393b058200198cb8f47d0edd7af2580a7f81444860eb27270f" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-cpp" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736c482444d8d58e7f3cd8df7bc88bf6c4b450e9039a55c43b3aec39d6a805ce" +dependencies = [ + "arborium-c", + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-css" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3bf02fc773bafaac2cdc27862d6eb52e86927e740c69e0d87b4123f2589b9af" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-d" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5f76f6c8f0fc569543f53253837a2f275a30c74aed8e566540372fc7686f85" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-dart" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8479324e58c3fa5953006e79a79bf74547bb12b618f3856c270e9808a7a34b" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-diff" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f8e67cdf000ca1e099f71a28b32e17e9749fa27f2366c36eeb47c31ebee237" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-dockerfile" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54f567b2f106cea694126516b8ad20d44cddc40df8c16c77acf61ada303bfd9e" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-elisp" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8eaf4b0673fdcc8905d7d361bfcc07e687c64fab01ae2f4d37afb2b505476d" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-elixir" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad30809acaa5d56cc1a660cf9684884d37f86f994d35d6238ba2e1eaf9b409c1" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-elm" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a5be48b61b088274b924aab89d4869610c394f231ec9442bfdbcb741467a899" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-erlang" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa77f055957475a25051d32f291b9522e2466f22f976f7d94604788787e3fa6" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-fish" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b657a121d90b62c3d278f2ba7c67790e62131931194236e11096e674196d203" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-fsharp" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a99c43c646333704fe7f0af48269046fb8593c48244749aa2ad11c7a1a2c0e1e" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-gleam" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56bdfb4669245eaf62355baf64d88d3f974f5116a445598cb1e5a43297544c9" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-glsl" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71ddb4b9b05ec5f72984b3004fe008d78895405e103f2d8655e78dccef0a6794" +dependencies = [ + "arborium-c", + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-go" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f297634bca52a796de2725149dc684c18286c78f0a096fa27d532f9627bb1e02" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-graphql" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a1df39113685dfc32cf16089c3cf7f0f0e78596e46815503a11681f940987" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-groovy" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e06dd1c205f32687e45733c516a52e9864183724b8fe3b5ea79b5605ff2448f8" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-haskell" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e7f4f0976568a87bedddeebd30724577674751fd275dab7c98c63616d9961c" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-hcl" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30dc0e04b767c8345c41cc96b350e0e991826ff340d973bd52b1f27fc7963a12" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-highlight" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee400d694bd935c513c50f9b1fda469b6d9cd192d80fede7bfa9e5851a54cb8" +dependencies = [ + "arborium-theme", + "arborium-tree-sitter", + "streaming-iterator", +] + +[[package]] +name = "arborium-html" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed785a76d53a63ff144b26a68b789fe4d10169a5f683d573ee72f3234d8a9a4" +dependencies = [ + "arborium-css", + "arborium-javascript", + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-ini" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7023ba09a01b681ebac2d78c5d1747d58b548ec3cb010e16f852fe8e62ed869" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-java" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad9a5d6a278548b7a18bbaf79600402c167cec39d3b63be86daa1f3756b52a82" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-javascript" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b63db8c449ac227061f2bb4ed62673d85c3f567be8f01483b34f266298d061" +dependencies = [ + "arborium-jsdoc", + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-jq" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "befae6cef2501d589f31ca08ab96fa640f57299eb05bfddf2ebcc772c1632a20" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-jsdoc" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73945c45bdbbed8c137556d3a833999463e707078ca877597756dc394689e8b5" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-json" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1167c5abde6c8d953cc20d88713006c94a009926d39e2217c68cbffbdbf7f0" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-julia" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f96e64b392495c5958528a62d5ccb32829727cab0f678c5a0efbb9dd6afb5a84" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-kotlin" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2be9de4405957a7e6f9684d4774a8e743bb501ce5956d3e91da77e8e18a6fe6" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-lean" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e3e3af99597613fc97e63696fdec5020506ffd9849bc4172b8e01056afa936" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-lua" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5ffae9459733aa8457f88e43672ccae485a77f09b15da83f8655d954c75365f" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-make" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bed9575f27d799d26b522d101f50abdb7f4f06d6a7065dbed41a6834b5dfe21" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-matlab" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "810e898c7240a3aee12a18d3a49607a900a18024ca4b7d913aa5f9d51015b5c5" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-nix" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c89b161cf29773dff8abc46e0bed263ecd3195f7a4f17c9b1bc0903ec72602f" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-objc" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1988635123d7a6947c4596610c52dab1a8319e791517a8c61e0c0e5e0463f45" +dependencies = [ + "arborium-c", + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-ocaml" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761307ca8598a125b78523bd3d802426f69ddfb549b616bfad55b31100545e1a" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-odin" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6274fafd5cc98688bb9ca9839c24bf5c18da17e4b1dd7cf76953495693e022af" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-perl" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e858599e4fb53d710b4123fd2bba5e3625d5eaf9495fcddabb1154cffd39faf" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-php" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1772da0cd2f525332a958f5ebda27eda35be22dfa82b052774b6162b2c20b359" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-powershell" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e087e691371a5b63c514c7a5c85c213932be5c52ada6107c5fb2314106be07" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-prolog" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "945174b0c4b0d859d040bdfc8416acc95a777674ce0017a97a874980667a2701" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-proto" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19631b348e2b41507bbd00e272c9e4962cac2f78ab371d55ec72d5a5bbe48c1" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-python" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4da0da209e87d12f5922c5a7c005883d48d06baf0bf9b5d05bbe607b532e599f" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-r" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69ce5bd2f1c773d131393a7caa9cc9a3cf6d8e4432ee8452f53047245496fdb8" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-ruby" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f791dc0377b8df659b98aed534dddfd4dcb591735cafd05b12e1e1ce3a859a9" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-rust" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8471397c67c4164b93d6a23ae80fb5b7af5dcc00fa9b143015efb1ea927ca8da" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-scala" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c02e14b416b76db7ede4935d5306e9ba5d42cd931396fb7ffc791de44ab46005" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-scheme" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30fa0d8e10248c131e2040200114adadba049375b8e75f1090cc3f32642afc48" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-scss" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5baa8a1fe94fdf3a32859d34fb5b0283bcc521cc08a62ca006c42d3dfad5780" +dependencies = [ + "arborium-css", + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-solidity" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f08b59a3250b10f247284d5735d473fb31e0c3d933003f5ffc469da9c24456e1" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-sql" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ca021dab5e61d6eb07bad9cf40f5f5e0fb4c6f89f32aa14a1db07431d657f9" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-starlark" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0101c83d354421235ff3de1a5c593d56314cc41d47e93755404a2ac721ecb0fd" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-swift" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db83273e075d168a853cb480805b0614803d43c20c7d5fc3f655dac836dd88e1" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-sysroot" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd9d120dd5df8b918cd96676a88f1233a66f10687d415b69f70b3396acddddf" +dependencies = [ + "cc", + "dlmalloc", +] + +[[package]] +name = "arborium-theme" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "042f9d025c80b2e625b4f0f10eda56a652444ef916025e855a292937bb8a949a" + +[[package]] +name = "arborium-toml" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9f6144d675fbfe2e510c5886955581b6838eb613a3d1af4250ddf10de5bfa5d" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-tree-sitter" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd8de97695c76508ab780e03bd1db009f5db8567c985f5da5ae91a2ad57ef2bc" +dependencies = [ + "arborium-sysroot", + "cc", + "regex", + "regex-syntax", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "arborium-tsx" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3533e40fe9f843290d4ff481c67ed1494595395fb60318dbc1da7e58bba42bb" +dependencies = [ + "arborium-javascript", + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-typescript" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0024689641490077765b291be46197f48d23e75cbade86768b1abed64d406ed1" +dependencies = [ + "arborium-javascript", + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-verilog" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee5743742336f9b22e24c5bba1f43841462447f6743c6db97cf00fe49e14d643" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-vhdl" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1086bc0a068cfb18f466c66c4282095ed67318a006d949fa5caaec9df5da9151" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-vim" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6db7ec81e7365eed3467a90f2a803f055f687f954a40478b3f73c7be44cbf2" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-x86asm" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709db6d9704b006f77f46ce318f88e77790a6ec0fce1c5ddc38b315334f41a4e" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-xml" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01fb3f68edd5d901632cfbc9aa02ba537e16aafc3eda4603573acb2aeef7ae3" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-yaml" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528f9c28a2703384a66079b18f29e990f1ed749014a6556242e7c14ed5b29ea9" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-zig" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2303afe531f320075135ee152c04dfd6703b2e20d071662539dd4df587aff995" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + +[[package]] +name = "arborium-zsh" +version = "2.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e7f2e02c18813b556a0c3d0b64a10f40bf835b856ddb8058cf4840666c279d3" +dependencies = [ + "arborium-sysroot", + "cc", + "tree-sitter-language", +] + [[package]] name = "arg_enum_proc_macro" version = "0.3.4" @@ -442,6 +1355,15 @@ dependencies = [ "serde", ] +[[package]] +name = "betlang" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f89b0929539eaee70109704ae4e345df438be6ab02e4dc8ac060e05098ad1b7" +dependencies = [ + "fearless_simd", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -888,11 +1810,13 @@ version = "0.3.0" dependencies = [ "ammonia", "anyhow", + "arborium", "askama", "async-trait", "axum", "axum-login", "base64 0.23.1", + "betlang", "clap", "dom_smoothie", "epub-builder", @@ -1059,6 +1983,17 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "dlmalloc" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad5208a115eaba24916f7456929832e310a81518c641f93fee4f89aa93aa3675" +dependencies = [ + "cfg-if", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "dom_query" version = "0.28.0" @@ -1280,6 +2215,12 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + [[package]] name = "figment" version = "0.10.19" @@ -3211,6 +4152,18 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.18" @@ -3988,6 +4941,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "strict-num" version = "0.1.1" @@ -4681,6 +5640,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tree-sitter-language" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081" + [[package]] name = "try-lock" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index bb617fd..8162e02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,11 +6,13 @@ edition = "2024" [dependencies] ammonia = "4.1.4" anyhow = "1.0.104" +arborium = { version = "2.18.2", default-features = false, features = ["lang-asm", "lang-awk", "lang-bash", "lang-c", "lang-c-sharp", "lang-clojure", "lang-cmake", "lang-commonlisp", "lang-cpp", "lang-css", "lang-d", "lang-dart", "lang-diff", "lang-dockerfile", "lang-elisp", "lang-elixir", "lang-elm", "lang-erlang", "lang-fish", "lang-fsharp", "lang-gleam", "lang-glsl", "lang-go", "lang-graphql", "lang-groovy", "lang-haskell", "lang-hcl", "lang-html", "lang-ini", "lang-java", "lang-javascript", "lang-jq", "lang-json", "lang-julia", "lang-kotlin", "lang-lean", "lang-lua", "lang-make", "lang-matlab", "lang-nix", "lang-objc", "lang-ocaml", "lang-odin", "lang-perl", "lang-php", "lang-powershell", "lang-prolog", "lang-proto", "lang-python", "lang-r", "lang-ruby", "lang-rust", "lang-scala", "lang-scheme", "lang-scss", "lang-solidity", "lang-sql", "lang-starlark", "lang-swift", "lang-toml", "lang-tsx", "lang-typescript", "lang-verilog", "lang-vhdl", "lang-vim", "lang-x86asm", "lang-xml", "lang-yaml", "lang-zig", "lang-zsh"] } askama = "0.16.0" async-trait = "0.1.92" axum = "0.8.9" axum-login = { git = "https://github.com/maxcountryman/axum-login.git", rev = "151c72d7a1b4646830f86b4332e6bd6e34d719a7", version = "0.18.0" } base64 = "0.23.1" +betlang = "0.1.1" clap = { version = "4.6.6", features = ["derive"] } dom_smoothie = "0.18.0" epub-builder = "0.8.3" diff --git a/src/epub/chapters.rs b/src/epub/chapters.rs index dc87ff7..25ee148 100644 --- a/src/epub/chapters.rs +++ b/src/epub/chapters.rs @@ -150,9 +150,10 @@ struct ProviderCostLine { /// Prepare article markup for XHTML: rewrite images, sanitize, self-close voids. pub fn prepare_body(html: &str, images_: &[ImageAsset]) -> String { - // Sanitize first: image rewriting emits our own trusted markup (including the - // `image-placeholder` class, which ammonia would otherwise strip). - let cleaned = ammonia::clean(html); + // Sanitize first: highlighting and image rewriting emit our own trusted + // markup (the `hl-*` token classes and `image-placeholder`, which ammonia + // would otherwise strip). + let cleaned = crate::highlight::clean_and_highlight(html); let rewritten = images::rewrite_img_srcs(&cleaned, images_); to_xhtml(&rewritten) } diff --git a/src/extract.rs b/src/extract.rs index 30edc42..3c8a8ce 100644 --- a/src/extract.rs +++ b/src/extract.rs @@ -15,6 +15,7 @@ use std::time::Duration; use futures::StreamExt; use url::Url; +use crate::highlight::annotate_hints; use crate::html::word_count; use crate::images::normalize::{normalize_img_tags, prepare_for_readability}; use crate::images::refs::collect_image_urls; @@ -338,7 +339,8 @@ fn readable_page( html: &str, url: &str, ) -> Result<(String, String, Option, Option), ExtractError> { - let html = prepare_for_readability(html); + // Readability strips classes, and with them code blocks' language hints. + let html = prepare_for_readability(&annotate_hints(html)); let config = dom_smoothie::Config { max_elements_to_parse: 60_000, ..Default::default() @@ -447,6 +449,7 @@ fn builder() -> ammonia::Builder<'static> { .generic_attributes(HashSet::new()) .link_rel(None) .strip_comments(true); + crate::highlight::keep_lang_attr(&mut b); b } @@ -454,8 +457,10 @@ fn builder() -> ammonia::Builder<'static> { /// /// Allowed: `p`, `h1`–`h4`, `ul`/`ol`/`li`, `blockquote`, `pre`, `code`, `em`, /// `strong`, `a`, `img`, `figure`, `figcaption`, table basics, `hr`, `br`. +/// The one attribute kept beyond links and images is a code block's language, +/// as `
` (see [`crate::highlight`]).
 pub fn sanitize(html: &str) -> String {
-    builder().clean(html).to_string()
+    builder().clean(&annotate_hints(html)).to_string()
 }
 
 /// [`sanitize`], additionally rewriting relative `href`/`src` against `base_url`
@@ -464,7 +469,7 @@ pub fn sanitize_with_base(html: &str, base_url: &str) -> String {
     match Url::parse(base_url) {
         Ok(base) => builder()
             .url_relative(ammonia::UrlRelative::RewriteWithBase(base))
-            .clean(html)
+            .clean(&annotate_hints(html))
             .to_string(),
         Err(_) => sanitize(html),
     }
@@ -546,6 +551,26 @@ mod tests {
         format!("

{}

", "lorem ".repeat(words)) } + #[test] + fn sanitize_keeps_code_block_languages_as_data_lang_only() { + let dirty = r#"
puts 1
+
fn main() {}
+
forged
+

not a block

"#; + let clean = sanitize(dirty); + assert!( + clean.contains(r#"
puts 1
"#), + "{clean}" + ); + assert!( + clean.contains(r#"
fn main() {}
"#), + "{clean}" + ); + assert!(clean.contains("
forged
"), "{clean}"); + assert!(clean.contains("

not a block

"), "{clean}"); + assert!(!clean.contains("class=")); + } + #[test] fn sanitize_keeps_the_allowlist_and_drops_everything_else() { let dirty = r#" diff --git a/src/highlight.rs b/src/highlight.rs new file mode 100644 index 0000000..4c70780 --- /dev/null +++ b/src/highlight.rs @@ -0,0 +1,743 @@ +//! Syntax highlighting for code blocks in article bodies. +//! +//! Two halves, run at different times: +//! +//! * [`annotate_hints`] runs on source markup at extraction, before +//! readability and the sanitizer strip every `class`. It reads whatever +//! language hint the page carries (`language-rust`, Rouge's wrapper `
`, +//! SyntaxHighlighter's `brush: py`, `data-lang`, …) and records it as one +//! normalized `data-lang` on the `
`, which both of them keep.
+//! * [`highlight_blocks`] runs at render time, on sanitized markup, for the web
+//!   article page and both EPUB editions. Blocks with a hint use it; blocks
+//!   without one are guessed from their content ([`guess_language`]), and
+//!   anything not confidently code is left exactly as it was.
+//!
+//! Tokens come out as flat `` elements, one class each, so
+//! the X4 stylesheet (no descendant selectors) can style them too. Colours are
+//! the stylesheets' business: the web theme is in `src/web/tailwind.css`, the
+//! monochrome e-ink themes in `src/epub/templates/style*.css`.
+
+use std::borrow::Cow;
+use std::sync::{Arc, LazyLock};
+
+use arborium::{Config, GrammarStore, Highlighter, HtmlFormat};
+
+use crate::html::{attr_escape, decode_entities, parse_attrs, tag_end, tag_name};
+
+/// The attribute that carries a block's language from extraction to render.
+pub const LANG_ATTR: &str = "data-lang";
+
+/// The `data-lang` value for a block its source marked as not code (plain
+/// text, program output): never highlighted, never guessed.
+pub const PLAIN: &str = "text";
+
+/// Prefix of every token class, e.g. `hl-keyword`.
+const CLASS_PREFIX: &str = "hl";
+
+/// Hints that mean "this is not source code".
+const PLAIN_HINTS: &[&str] = &[
+    "text",
+    "txt",
+    "plain",
+    "plaintext",
+    "none",
+    "nohighlight",
+    "no-highlight",
+    "output",
+    "console",
+    "terminal",
+    "log",
+];
+
+/// Attributes that name a block's language outright.
+const LANG_ATTRIBUTES: &[&str] = &[
+    "data-lang",
+    "data-language",
+    "data-code-block-lang",
+    "data-code-language",
+];
+
+/// Class prefixes highlighters put in front of a language name.
+const CLASS_PREFIXES: &[&str] = &[
+    "language-",
+    "lang-",
+    "highlight-source-",
+    "highlight-",
+    "source-",
+];
+
+/// Minimum model confidence for a guessed language. Measured on short real
+/// snippets, guesses at or above this were right; below it they were coin flips.
+const MIN_GUESS_SCORE: f32 = 0.5;
+
+/// Unhinted blocks shorter than this are left alone: one-liners are usually a
+/// command or a path, and too short to classify.
+const MIN_GUESS_CHARS: usize = 40;
+
+/// A block that is only plausibly code is weighed against the article's
+/// prevailing language using at most this many of its best candidates…
+const MAX_CANDIDATES: usize = 3;
+
+/// …each scoring at least this much.
+const MIN_CANDIDATE_SCORE: f32 = 0.1;
+
+/// The detector reads at most this much of a block.
+const MAX_GUESS_BYTES: usize = 4096;
+
+/// Compiled grammars, shared across threads; each call gets its own parser.
+static GRAMMARS: LazyLock> = LazyLock::new(|| Arc::new(GrammarStore::new()));
+
+fn new_highlighter() -> Highlighter {
+    Highlighter::with_store_and_config(
+        GRAMMARS.clone(),
+        Config {
+            html_format: HtmlFormat::ClassNamesWithPrefix(CLASS_PREFIX.into()),
+            ..Default::default()
+        },
+    )
+}
+
+/// Whether `name` (a language or one of arborium's aliases) has a grammar.
+fn supported(name: &str) -> bool {
+    GRAMMARS.get(name).is_some()
+}
+
+// ---------------------------------------------------------------------------
+// Hints (extraction time)
+// ---------------------------------------------------------------------------
+
+/// Normalize one hint token (`language-rust`, `py`, `nohighlight`) to the
+/// value stored in `data-lang`: a supported language name or [`PLAIN`].
+///
+/// A bare token (no `language-` style prefix) must be at least two characters,
+/// so stray one-letter classes don't turn into C or R.
+pub fn normalize_hint(token: &str) -> Option {
+    let token = token.trim().trim_matches(';').to_ascii_lowercase();
+    let (name, prefixed) = CLASS_PREFIXES
+        .iter()
+        .find_map(|p| token.strip_prefix(p).map(|rest| (rest, true)))
+        .unwrap_or((token.as_str(), false));
+    if name.is_empty() || (!prefixed && name.len() < 2) {
+        return None;
+    }
+    if PLAIN_HINTS.contains(&name) {
+        return Some(PLAIN.to_string());
+    }
+    let name = canonical_name(name);
+    supported(name).then(|| name.to_string())
+}
+
+/// The name detection uses for the common aliases a hint may carry, so a
+/// `py` block and a guessed `python` block count as the same language.
+fn canonical_name(name: &str) -> &str {
+    match name {
+        "py" | "python3" => "python",
+        "js" | "node" | "jsx" => "javascript",
+        "ts" => "typescript",
+        "sh" | "shell" => "bash",
+        "rs" => "rust",
+        "golang" => "go",
+        "rb" => "ruby",
+        "yml" => "yaml",
+        "c++" | "cxx" | "cc" | "hpp" => "cpp",
+        "cs" | "csharp" | "c#" => "c-sharp",
+        "objective-c" | "objectivec" => "objc",
+        "kt" => "kotlin",
+        "ex" | "exs" => "elixir",
+        "hs" => "haskell",
+        "ps1" | "pwsh" => "powershell",
+        other => other,
+    }
+}
+
+/// The language hint carried by one element's attributes, if any.
+fn hint_from_attrs(attrs: &[(String, String)]) -> Option {
+    for (name, value) in attrs {
+        if LANG_ATTRIBUTES.contains(&name.as_str())
+            && let Some(hint) = normalize_hint(value)
+        {
+            return Some(hint);
+        }
+    }
+    let class = attrs.iter().find(|(name, _)| name == "class")?.1.as_str();
+    // SyntaxHighlighter writes `class="brush: python; gutter: false"`.
+    let tokens: Vec<&str> = class
+        .split(|c: char| c.is_whitespace() || c == ';')
+        .filter(|t| !t.is_empty())
+        .collect();
+    for (i, token) in tokens.iter().enumerate() {
+        let candidate = match token.strip_prefix("brush:") {
+            Some("") => tokens.get(i + 1).copied(),
+            Some(rest) => Some(rest),
+            None if i > 0 && tokens[i - 1] == "brush:" => None,
+            None => Some(*token),
+        };
+        if let Some(hint) = candidate.and_then(normalize_hint) {
+            return Some(hint);
+        }
+    }
+    None
+}
+
+/// Record each `
`'s language hint as `data-lang`, before readability and
+/// the sanitizer strip the classes that carry it.
+///
+/// The hint may sit on the `
`, on the `` right inside it, or on an
+/// enclosing `
` (Jekyll's Rouge, GitHub's `highlight-source-*`). Markup is +/// otherwise returned byte-identical. +pub fn annotate_hints(html: &str) -> String { + let mut out = String::with_capacity(html.len() + 64); + let mut cursor = 0usize; + // Hint of every open `
`, innermost last. + let mut divs: Vec> = Vec::new(); + while let Some(rel) = html[cursor..].find('<') { + let start = cursor + rel; + out.push_str(&html[cursor..start]); + let Some(end) = tag_end(html, start) else { + out.push_str(&html[start..]); + return out; + }; + let raw = &html[start..end]; + cursor = end; + let inner = raw.trim_start_matches('<').trim_end_matches('>'); + let name = tag_name(inner); + if inner.starts_with('/') { + if name == "div" { + divs.pop(); + } + out.push_str(raw); + continue; + } + match name.as_str() { + "div" => { + if !inner.trim_end().ends_with('/') { + divs.push(hint_from_attrs(&parse_attrs(inner))); + } + out.push_str(raw); + } + "pre" => { + let attrs = parse_attrs(inner.trim_end_matches('/')); + let hint = hint_from_attrs(&attrs) + .or_else(|| first_code_hint(html, end)) + .or_else(|| divs.iter().rev().find_map(Clone::clone)); + match hint { + Some(hint) => out.push_str(&pre_tag(&attrs, &hint)), + None => out.push_str(raw), + } + } + _ => out.push_str(raw), + } + } + out.push_str(&html[cursor..]); + out +} + +/// The hint on a `` that opens right after a `
` ending at `after`.
+fn first_code_hint(html: &str, after: usize) -> Option {
+    let rest = &html[after..];
+    let start = after + (rest.len() - rest.trim_start().len());
+    if !html[start..].starts_with('<') {
+        return None;
+    }
+    let end = tag_end(html, start)?;
+    let inner = html[start..end]
+        .trim_start_matches('<')
+        .trim_end_matches('>');
+    (tag_name(inner) == "code").then(|| hint_from_attrs(&parse_attrs(inner)))?
+}
+
+/// Rebuild a `
` tag with `data-lang` set to `hint`.
+fn pre_tag(attrs: &[(String, String)], hint: &str) -> String {
+    let mut tag = String::from("", attr_escape(hint)));
+    tag
+}
+
+/// Sanitizer filter for `data-lang`: keep it only if it is a value
+/// [`annotate_hints`] could have written.
+pub fn filter_lang_attr(value: &str) -> Option {
+    normalize_hint(value).filter(|hint| hint == &value.to_ascii_lowercase())
+}
+
+/// Let an ammonia builder keep a valid `data-lang` on `
` (and nothing else
+/// it did not already allow).
+pub fn keep_lang_attr(builder: &mut ammonia::Builder<'_>) {
+    builder
+        .add_tag_attributes("pre", &[LANG_ATTR])
+        .attribute_filter(|element, attribute, value| {
+            if element == "pre" && attribute == LANG_ATTR {
+                filter_lang_attr(value).map(Cow::Owned)
+            } else {
+                Some(Cow::Borrowed(value))
+            }
+        });
+}
+
+/// `ammonia::clean` that keeps block languages, then [`highlight_blocks`]:
+/// the render-time entry point for article bodies.
+pub fn clean_and_highlight(html: &str) -> String {
+    let mut builder = ammonia::Builder::default();
+    keep_lang_attr(&mut builder);
+    highlight_blocks(&builder.clean(html).to_string())
+}
+
+// ---------------------------------------------------------------------------
+// Detection
+// ---------------------------------------------------------------------------
+
+/// What an unhinted block's content says about its language.
+#[derive(Debug, Clone, PartialEq)]
+enum Detected {
+    /// Confidently this language.
+    Sure(&'static str),
+    /// Plausibly one of these, best first: taken only when the rest of the
+    /// article is in one of them (see [`highlight_blocks`]).
+    Maybe(Vec<&'static str>),
+    /// Not code, or too little of it to tell.
+    Unknown,
+}
+
+/// Best guess at the language of an unhinted block, or `None` when it is not
+/// confidently code. A wrong colour scheme is worse than none.
+pub fn guess_language(code: &str) -> Option<&'static str> {
+    match detect(code) {
+        Detected::Sure(language) => Some(language),
+        _ => None,
+    }
+}
+
+fn detect(code: &str) -> Detected {
+    let text = code.trim();
+    // Parsing is proof, at any length.
+    if (text.starts_with('{') || text.starts_with('['))
+        && serde_json::from_str::(text).is_ok()
+    {
+        return Detected::Sure("json");
+    }
+    if text.chars().count() < MIN_GUESS_CHARS && !text.starts_with("#!") {
+        return Detected::Unknown;
+    }
+    if let Some(language) = first_line_language(text) {
+        return Detected::Sure(language);
+    }
+    // A shell session is prompts interleaved with output; neither grammar fits.
+    if text.starts_with("$ ") || text.starts_with("> ") || text.starts_with("% ") {
+        return Detected::Unknown;
+    }
+    let sample = crate::html::truncate_utf8(text, MAX_GUESS_BYTES);
+    let detection = betlang::detect(sample);
+    let mut ranked = detection.top_languages().take(MAX_CANDIDATES);
+    let Some((score, top)) = ranked.next() else {
+        return Detected::Unknown;
+    };
+    // Prose, logs and program output all come out as Markdown.
+    let Some(top) = betlang_language(top) else {
+        return Detected::Unknown;
+    };
+    if score >= MIN_GUESS_SCORE {
+        return Detected::Sure(top);
+    }
+    let mut candidates = vec![top];
+    candidates.extend(
+        ranked
+            .filter(|(score, _)| *score >= MIN_CANDIDATE_SCORE)
+            .filter_map(|(_, language)| betlang_language(language)),
+    );
+    Detected::Maybe(candidates)
+}
+
+/// The grammar for a betlang label, if we have one.
+fn betlang_language(language: betlang::Language) -> Option<&'static str> {
+    let name = match language.slug() {
+        "markdown" => return None,
+        "cs" => "c-sharp",
+        "objectivec" => "objc",
+        "lisp" => "commonlisp",
+        "shell" => "bash",
+        "gemfile" | "gemspec" => "ruby",
+        "gradle" => "groovy",
+        other => other,
+    };
+    supported(name).then_some(name)
+}
+
+/// Languages a block announces on its first line: shebangs and XML-ish preludes.
+fn first_line_language(text: &str) -> Option<&'static str> {
+    let first = text.lines().next()?.trim().to_ascii_lowercase();
+    if let Some(shebang) = first.strip_prefix("#!") {
+        let interpreter = shebang
+            .split_whitespace()
+            .flat_map(|word| word.rsplit('/').next())
+            .find(|word| *word != "env" && !word.starts_with('-'))?;
+        let name = match interpreter.trim_end_matches(|c: char| c.is_ascii_digit() || c == '.') {
+            "sh" | "bash" | "dash" | "ksh" => "bash",
+            "zsh" => "zsh",
+            "fish" => "fish",
+            "python" => "python",
+            "node" | "deno" | "bun" => "javascript",
+            "ruby" => "ruby",
+            "perl" => "perl",
+            "php" => "php",
+            "lua" => "lua",
+            _ => return None,
+        };
+        return supported(name).then_some(name);
+    }
+    if first.starts_with("` block found by [`highlight_blocks`].
+struct Block {
+    /// Byte range of the opening tag, and where `
` starts. + start: usize, + open_end: usize, + close: usize, + attrs: Vec<(String, String)>, + hint: Option, + code: Option, + detected: Detected, +} + +/// Highlight every `
` block in sanitized markup.
+///
+/// A block's language is its `data-lang`, else what its content confidently
+/// says ([`guess_language`]). A block whose content is only plausibly code
+/// takes the article's prevailing language when that is one of its
+/// candidates, so a post in Go does not alternate between highlighted and
+/// plain listings. A highlighted block becomes `
…spans…
`; +/// any other block is copied through untouched apart from losing `data-lang`, +/// which is only for this step. +pub fn highlight_blocks(html: &str) -> String { + if !html.contains(" = None; + let mut out = String::with_capacity(html.len() * 2); + let mut cursor = 0usize; + for block in &blocks { + out.push_str(&html[cursor..block.start]); + let language = match (block.hint.as_deref(), &block.detected) { + (Some(PLAIN), _) => None, + (Some(hint), _) => Some(hint), + (None, Detected::Sure(language)) => Some(*language), + (None, Detected::Maybe(candidates)) => { + prevailing.filter(|language| candidates.contains(language)) + } + (None, Detected::Unknown) => None, + }; + let highlighted = match (language, &block.code) { + (Some(language), Some(code)) => highlighter + .get_or_insert_with(new_highlighter) + .highlight(language, code) + .ok() + .filter(|spans| spans.contains(" None, + }; + match highlighted { + Some(spans) => { + out.push_str("
");
+                out.push_str(&spans);
+                out.push_str("
"); + } + None => { + out.push_str(&pre_without_lang( + &block.attrs, + block.hint.is_some(), + &html[block.start..block.open_end], + )); + out.push_str(&html[block.open_end..block.close]); + out.push_str("
"); + } + } + cursor = block.close + "
".len(); + } + out.push_str(&html[cursor..]); + out +} + +/// Every complete `
` block, in order, with its hint and detection.
+fn find_blocks(html: &str) -> Vec {
+    let mut blocks = Vec::new();
+    let mut cursor = 0usize;
+    while let Some(rel) = html[cursor..].find("');
+        if tag_name(inner) != "pre" {
+            cursor = open_end;
+            continue;
+        }
+        let Some(close_rel) = html[open_end..].find("
") else { + break; + }; + let close = open_end + close_rel; + let attrs = parse_attrs(inner); + let hint = attrs + .iter() + .find(|(name, _)| name == LANG_ATTR) + .map(|(_, value)| value.clone()); + let code = block_text(&html[open_end..close]); + let detected = match (&hint, &code) { + (None, Some(code)) => detect(code), + _ => Detected::Unknown, + }; + blocks.push(Block { + start, + open_end, + close, + attrs, + hint, + code, + detected, + }); + cursor = close + "
".len(); + } + blocks +} + +/// The language most of an article's hinted and confidently detected blocks +/// share, earliest first on a tie. +fn prevailing_language(blocks: &[Block]) -> Option<&str> { + let mut counts: Vec<(&str, usize)> = Vec::new(); + for block in blocks { + let language = match (block.hint.as_deref(), &block.detected) { + (Some(PLAIN), _) => continue, + (Some(hint), _) => hint, + (None, Detected::Sure(language)) => language, + _ => continue, + }; + match counts.iter_mut().find(|(name, _)| *name == language) { + Some((_, count)) => *count += 1, + None => counts.push((language, 1)), + } + } + counts + .iter() + .rev() + .max_by_key(|(_, count)| *count) + .map(|(language, _)| *language) +} + +/// The `
` opening tag minus `data-lang`, or `raw` when it had none.
+fn pre_without_lang(attrs: &[(String, String)], had_lang: bool, raw: &str) -> String {
+    if !had_lang {
+        return raw.to_string();
+    }
+    let mut tag = String::from("');
+    tag
+}
+
+/// The source text of a `
` body: an optional wrapping `` removed,
+/// `
` as newlines, inline markup dropped, entities decoded, and the one +/// leading newline HTML ignores stripped. +/// +/// `None` when the body holds block markup (a table, a list) — that is a +/// layout, not a listing, and is left alone. +fn block_text(body: &str) -> Option { + let mut text = String::with_capacity(body.len()); + let mut cursor = 0usize; + while let Some(rel) = body[cursor..].find('<') { + let start = cursor + rel; + text.push_str(&body[cursor..start]); + let end = tag_end(body, start)?; + let inner = body[start..end] + .trim_start_matches('<') + .trim_end_matches('>'); + match tag_name(inner).as_str() { + "br" => text.push('\n'), + "code" | "span" | "em" | "strong" | "a" | "b" | "i" | "u" | "s" | "mark" | "kbd" + | "samp" | "var" | "sub" | "sup" | "small" => {} + _ => return None, + } + cursor = end; + } + text.push_str(&body[cursor..]); + let text = decode_entities(&text); + let text = text + .strip_prefix("\r\n") + .or_else(|| text.strip_prefix('\n')) + .unwrap_or(&text); + let text = text.trim_end_matches(['\n', '\r']); + (!text.trim().is_empty()).then(|| text.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalizes_prefixed_bare_and_plain_hints() { + assert_eq!(normalize_hint("language-rust").as_deref(), Some("rust")); + assert_eq!(normalize_hint("lang-py").as_deref(), Some("python")); + assert_eq!(normalize_hint("highlight-source-go").as_deref(), Some("go")); + assert_eq!(normalize_hint("Python").as_deref(), Some("python")); + assert_eq!(normalize_hint("language-text").as_deref(), Some(PLAIN)); + assert_eq!(normalize_hint("nohighlight").as_deref(), Some(PLAIN)); + assert_eq!(normalize_hint("language-c").as_deref(), Some("c")); + // Bare one-letter classes and framework classes are not hints. + assert_eq!(normalize_hint("c"), None); + assert_eq!(normalize_hint("highlight"), None); + assert_eq!(normalize_hint("chroma"), None); + assert_eq!(normalize_hint("language-klingon"), None); + } + + #[test] + fn annotates_hints_from_pre_code_and_wrapper_divs() { + assert_eq!( + annotate_hints(r#"
x
"#), + r#"
x
"# + ); + assert_eq!( + annotate_hints(r#"
x
"#), + r#"
x
"# + ); + // Jekyll/Rouge: the hint is on a wrapper two levels up. + assert_eq!( + annotate_hints( + r#"
x
y
"# + ), + r#"
x
y
"# + ); + assert_eq!( + annotate_hints(r#"
x
"#), + r#"
x
"# + ); + let plain = "

a

no hint
"; + assert_eq!(annotate_hints(plain), plain); + } + + #[test] + fn the_sanitizer_filter_only_keeps_normalized_values() { + assert_eq!(filter_lang_attr("rust").as_deref(), Some("rust")); + assert_eq!(filter_lang_attr("text").as_deref(), Some(PLAIN)); + assert_eq!(filter_lang_attr("language-rust"), None); + // Hints are stored canonical, so an alias is not a stored value. + assert_eq!(filter_lang_attr("py"), None); + assert_eq!(filter_lang_attr("\" onload=\"x"), None); + } + + #[test] + fn highlights_hinted_blocks_with_flat_prefixed_spans() { + let out = highlight_blocks( + "

a

fn main() {\n    let s = \"<&>\";\n}

b

", + ); + assert!(out.starts_with("

a

fn"));
+        assert!(out.contains(""<&>""));
+        assert!(out.ends_with("

b

")); + assert!(!out.contains("data-lang")); + } + + #[test] + fn br_line_breaks_and_inline_markup_become_plain_source() { + let out = highlight_blocks( + "
def f():
return 1
", + ); + assert!(out.contains("def")); + assert!(out.contains("\n return")); + assert!(!out.contains("")); + } + + #[test] + fn plain_and_unrecognized_blocks_are_left_alone() { + assert_eq!( + highlight_blocks("
def f(): return 1
"), + "
def f(): return 1
" + ); + let short = "
cargo build
"; + assert_eq!(highlight_blocks(short), short); + let prose = "
Dear reader, this is a poem set in a preformatted block\nbecause the author liked the look of it.
"; + assert_eq!(highlight_blocks(prose), prose); + let session = "
$ cargo build --release\n   Compiling foo v0.1.0\n    Finished release
"; + assert_eq!(highlight_blocks(session), session); + } + + #[test] + fn guesses_unhinted_code_by_content() { + assert_eq!( + guess_language( + "import os\n\ndef main():\n print(os.getcwd())\n\nif __name__ == '__main__':\n main()\n" + ), + Some("python") + ); + assert_eq!( + guess_language( + "SELECT id, name FROM users WHERE created_at > NOW() - INTERVAL '1 day' ORDER BY id;" + ), + Some("sql") + ); + assert_eq!( + guess_language("#!/usr/bin/env bash\nset -euo pipefail"), + Some("bash") + ); + assert_eq!( + guess_language("{\"a\": [1, 2, 3], \"b\": {\"c\": null}}"), + Some("json") + ); + let out = highlight_blocks( + "
#include <stdio.h>\n\nint main(void) {\n    printf(\"hello\\n\");\n    return 0;\n}
", + ); + assert!(out.contains("hl-"), "{out}"); + } + + #[test] + fn an_uncertain_block_takes_the_articles_prevailing_language() { + // Both from go.dev/blog/range-functions, which marks no languages. + let sure = "
// Add adds an element to a set.\nfunc (s *Set[E]) Add(v E) {\n    s.m[v] = struct{}{}\n}\n\n// Contains reports whether an element is in a set.\nfunc (s *Set[E]) Contains(v E) bool {\n    _, ok := s.m[v]\n    return ok\n}\n
"; + let unsure = "
func (s *Set[E]) Push(f func(E) bool) {\n    for v := range s.m {\n        if !f(v) {\n            return\n        }\n    }\n}\n
"; + let text = |pre: &str| block_text(&pre[5..pre.len() - 6]).unwrap(); + assert_eq!(guess_language(&text(sure)), Some("go")); + assert_eq!(guess_language(&text(unsure)), None); + assert!(matches!(detect(&text(unsure)), Detected::Maybe(c) if c.contains(&"go"))); + + let alone = highlight_blocks(unsure); + assert_eq!(alone, unsure); + let together = highlight_blocks(&format!("{sure}

and

{unsure}")); + assert_eq!( + together + .matches("func") + .count(), + 4 + ); + } + + #[test] + fn quoted_style_and_script_tags_are_html() { + assert_eq!( + guess_language(""), + Some("html") + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index dc01ca6..1bd75c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,7 @@ pub mod dedupe; pub mod discovery; pub mod epub; pub mod extract; +pub mod highlight; pub mod html; pub mod http; pub mod images; diff --git a/src/web/issue.rs b/src/web/issue.rs index 15123b7..2f46bbf 100644 --- a/src/web/issue.rs +++ b/src/web/issue.rs @@ -1415,7 +1415,7 @@ fn thousands(value: i64) -> String { /// Sanitize article HTML exactly as the EPUB does, then add browser-only image /// loading/referrer attributes without converting the fragment to XHTML. fn prepare_body(html: &str) -> String { - let clean = ammonia::clean(html); + let clean = crate::highlight::clean_and_highlight(html); let mut output = String::with_capacity(clean.len() + 64); let mut cursor = 0usize; while let Some(relative) = clean[cursor..].find('<') {