Compare commits
5
Commits
d6447d47b7
...
ce4a1a127d
| Author | SHA1 | Date | |
|---|---|---|---|
 thallada
|
ce4a1a127d | ||
|
thallada
|
e2eb50eb85 | ||
|
thallada
|
a728c758a7 | ||
 thalladaandGitHub
|
d539c8abe7 | ||
|
thalladaandTyler Hallada
|
e018a742a3 |
@@ -0,0 +1,55 @@
|
||||
# Build configuration for keydr.
|
||||
#
|
||||
# HISTORY: this file once contained aggressive workarounds (LTO disabled,
|
||||
# opt-level dropped to 1) because release builds consumed 5+ GB and never
|
||||
# finished on a 4-core / 7.6 GB VM. That was NOT a hardware limitation —
|
||||
# it was a codegen bug in rust-i18n v3, fixed by upgrading to v4.
|
||||
#
|
||||
# With rust-i18n v4 the same machine does a full opt-level=3 + thin-LTO
|
||||
# release build with a 2.08 GB peak in 3m25s. See docs/BUILDING.md.
|
||||
#
|
||||
# What remains here is ordinary small-machine hygiene, not a handicap:
|
||||
# nothing below reduces the optimisation level of shipped code.
|
||||
|
||||
[build]
|
||||
# Cap parallelism at 3 of 4 cores. This is about keeping the machine
|
||||
# interactive (SSH stays responsive during a build), not about memory.
|
||||
# Remove it if you don't care about using the box while it compiles.
|
||||
jobs = 3
|
||||
|
||||
rustflags = [
|
||||
# Use the lld that ships with the Rust toolchain. GNU ld is
|
||||
# single-threaded and holds the whole link graph in memory; lld is
|
||||
# faster and leaner. The -B flag is the stable-Rust way to select it
|
||||
# (`-Clink-self-contained=+linker` requires nightly).
|
||||
#
|
||||
# If you move this repo to another machine, update this path or
|
||||
# install system lld and use plain `-fuse-ld=lld`.
|
||||
"-Clink-arg=-fuse-ld=lld",
|
||||
"-Clink-arg=-B/home/thallada/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/bin/gcc-ld",
|
||||
]
|
||||
|
||||
[profile.dev]
|
||||
# Default is debug = 2 (full debug info), the dominant contributor to both
|
||||
# rustc peak memory and target/ size. Level 1 keeps line numbers, so
|
||||
# backtraces and panics still point at real source lines; you only lose
|
||||
# full variable inspection in a debugger.
|
||||
debug = 1
|
||||
split-debuginfo = "unpacked"
|
||||
|
||||
[profile.dev.package."*"]
|
||||
# Dependencies are compiled once and rarely stepped through in a debugger.
|
||||
# Dropping their debug info shrinks target/ and speeds up linking.
|
||||
debug = 0
|
||||
|
||||
[profile.test]
|
||||
debug = 1
|
||||
split-debuginfo = "unpacked"
|
||||
|
||||
[profile.test.package."*"]
|
||||
debug = 0
|
||||
|
||||
# NOTE: [profile.release] is deliberately absent — cargo's defaults
|
||||
# (opt-level = 3, codegen-units = 16, no LTO) are correct for this project
|
||||
# now. Add `lto = "thin"` if you want it; measured at 2.08 GB peak / 3m25s
|
||||
# on this box, which is comfortable.
|
||||
Generated
+83
-14
@@ -2,6 +2,19 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.4"
|
||||
@@ -32,6 +45,17 @@ version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
|
||||
|
||||
[[package]]
|
||||
name = "annotate-snippets"
|
||||
version = "0.12.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"memchr",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.6.21"
|
||||
@@ -97,6 +121,12 @@ dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arraydeque"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236"
|
||||
|
||||
[[package]]
|
||||
name = "atomic"
|
||||
version = "0.6.1"
|
||||
@@ -633,6 +663,15 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs_io"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
@@ -817,9 +856,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"r-efi",
|
||||
"wasip2",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -852,6 +893,16 @@ dependencies = [
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "granit-parser"
|
||||
version = "0.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d03f81ad4732830d85cfd417a9f62cde6dadda4354d37d078a6084a19560aa2d"
|
||||
dependencies = [
|
||||
"arraydeque",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.13"
|
||||
@@ -1491,6 +1542,12 @@ dependencies = [
|
||||
"memoffset",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nohash-hasher"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
@@ -2123,12 +2180,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rust-i18n"
|
||||
version = "3.1.5"
|
||||
version = "4.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fda2551fdfaf6cc5ee283adc15e157047b92ae6535cf80f6d4962d05717dc332"
|
||||
checksum = "f10cee36dd3b1f7929ea12b759de9eea9eff83bfccbc71f387ef4d41a57c64a4"
|
||||
dependencies = [
|
||||
"globwalk",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"rust-i18n-macro",
|
||||
"rust-i18n-support",
|
||||
@@ -2137,39 +2193,33 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rust-i18n-macro"
|
||||
version = "3.1.5"
|
||||
version = "4.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22baf7d7f56656d23ebe24f6bb57a5d40d2bce2a5f1c503e692b5b2fa450f965"
|
||||
checksum = "f0bb1ed4e04fe26c2a2652cad1c6595efaf7196f4445c0d6e13c67154347fb7e"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rust-i18n-support",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-i18n-support"
|
||||
version = "3.1.5"
|
||||
version = "4.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940ed4f52bba4c0152056d771e563b7133ad9607d4384af016a134b58d758f19"
|
||||
checksum = "ba1c083408a2733180ae0acf2897612f8ceec7b0c0dcd065a0a87103bfb3b1d9"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"base62",
|
||||
"globwalk",
|
||||
"itertools 0.11.0",
|
||||
"lazy_static",
|
||||
"normpath",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde-saphyr",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"siphasher",
|
||||
"toml",
|
||||
"triomphe",
|
||||
@@ -2318,6 +2368,25 @@ dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde-saphyr"
|
||||
version = "0.0.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7bd22781911de0ca6debda95f073c8f18bec65d1a94f1fa9573f3102e514cea4"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"annotate-snippets",
|
||||
"base64",
|
||||
"encoding_rs_io",
|
||||
"getrandom 0.3.4",
|
||||
"granit-parser",
|
||||
"nohash-hasher",
|
||||
"num-traits",
|
||||
"serde_core",
|
||||
"smallvec",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ anyhow = "1.0"
|
||||
thiserror = "2.0"
|
||||
reqwest = { version = "0.12", features = ["blocking"], optional = true }
|
||||
icu_normalizer = { version = "2.1", default-features = false, features = ["compiled_data"] }
|
||||
rust-i18n = "3"
|
||||
rust-i18n = "4"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
# keydr
|
||||
|
||||
A terminal typing tutor with adaptive learning, a skill-tree progression
|
||||
system, and live statistics — built in Rust on [ratatui]. Heavily inspired by
|
||||
[keybr.com], extended for code practice and the terminal.
|
||||
|
||||
Warning: still very much a WIP and is heavily vibe-coded so there may be bugs.
|
||||
Backwards compatibility is not guaranteed in this phase of development.
|
||||
|
||||
<img width="1187" height="797" alt="Adaptive drill screen" src="https://github.com/user-attachments/assets/74c358d6-976f-42bc-ae24-efd1feab5579" />
|
||||
|
||||
<details>
|
||||
<summary><h2>Screenshots</h2></summary>
|
||||
|
||||
<img width="1187" height="797" alt="Main menu" src="https://github.com/user-attachments/assets/64512852-c379-4e4b-94c6-6e62ef7d0526" />
|
||||
|
||||
<img width="1437" height="988" alt="Skill tree" src="https://github.com/user-attachments/assets/d975ac93-ec63-4fff-a275-27a71754d3bb" />
|
||||
|
||||
<img width="1437" height="988" alt="Statistics" src="https://github.com/user-attachments/assets/d98e67e8-1f85-46db-a22b-fe8b13cbe3d0" />
|
||||
|
||||
<img width="1437" height="988" alt="Keyboard explorer" src="https://github.com/user-attachments/assets/68eb3127-0046-499c-9a11-a641a82debf3" />
|
||||
|
||||
<img width="1437" height="988" alt="Code drill (farout theme)" src="https://github.com/user-attachments/assets/b079037b-2ce8-4bba-93b1-38151ebd7b68" />
|
||||
|
||||
<img width="1437" height="988" alt="Passage drill (kanagawa lotus theme)" src="https://github.com/user-attachments/assets/78d0eb61-4be0-4c11-aadb-72c355458314" />
|
||||
|
||||
<img width="1437" height="988" alt="Settings" src="https://github.com/user-attachments/assets/912b743e-72e3-4a19-84cd-d0ee9267b75e" />
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
## What makes it different
|
||||
|
||||
Most TUI typing apps either drop you on a fixed word list or randomize from a
|
||||
dictionary. keydr keeps a model of _your_ typing: per-key timing, per-bigram
|
||||
error rates, and a confidence score for every key on the keyboard. The text it
|
||||
generates for you is shaped by that model — words bias toward the keys you're
|
||||
slowest at, new keys unlock only when the keys you already have are solid, and
|
||||
the difficulty grows with you.
|
||||
|
||||
## Features
|
||||
|
||||
### Adaptive engine
|
||||
|
||||
- **Per-key confidence model.** Every keystroke updates an exponential moving
|
||||
average of your per-key time. Your `confidence` for a key is your target
|
||||
speed divided by that filtered time — a key with `confidence >= 1.0` is at
|
||||
or above your target speed.
|
||||
- **Gradual letter unlocking.** You start with 6 letters in English-frequency
|
||||
order (`e t a o i n …`). A new letter unlocks only when every letter you
|
||||
already have is confident, so you're never thrown into the full alphabet at
|
||||
once.
|
||||
- **Focused-key biasing.** Whichever included key has the lowest confidence
|
||||
becomes your "focused" key. The generator searches its Markov chain for
|
||||
states containing the focused letter and starts pseudo-words from there, so
|
||||
drills naturally pile up practice on your weak spots.
|
||||
- **Phonetic pseudo-word generation.** Words are generated from a Markov
|
||||
transition table (the same approach as keybr) so they're pronounceable and
|
||||
language-shaped, not random consonant noise.
|
||||
- **N-gram error tracking.** Beyond single keys, keydr tracks bigrams and
|
||||
trigrams with Laplace-smoothed error rates and a redundancy formula that
|
||||
separates _genuine_ transition difficulties (e.g. awkward same-finger
|
||||
bigrams) from errors that are just proxies for a single weak key. Hard
|
||||
bigrams get folded into drill selection.
|
||||
|
||||
### Skill tree
|
||||
|
||||
Once you've mastered the lowercase alphabet, you don't just keep grinding
|
||||
prose — five sibling branches unlock at once and you pick what to drill next:
|
||||
|
||||
- **Capitals** (A–Z, 3 levels) — sentence-start and proper-noun rules
|
||||
- **Numbers** (0–9, 2 levels)
|
||||
- **Prose punctuation** (`. , ' " ? !` …, 3 levels)
|
||||
- **Whitespace** (Tab, Enter, …, 2 levels)
|
||||
- **Code symbols** (`{ } [ ] ( ) ; : = + - * /` …, 4 levels)
|
||||
|
||||
Each branch tracks its own current level and its own focused key. You can have
|
||||
multiple branches "in progress" at once, and drills can be scoped globally
|
||||
(everything you've unlocked) or to a single branch.
|
||||
|
||||
### Three drill modes
|
||||
|
||||
- **Adaptive** — the default. Generated text from the phonetic model, biased
|
||||
by your skill-tree scope and focused key.
|
||||
- **Code** — language-specific syntax practice. Bundled snippets for Rust,
|
||||
Python, JavaScript, and Go, with optional opt-in downloads of real code
|
||||
samples from GitHub for more variety. Configurable per-language, with an
|
||||
onboarding screen on first use and a download progress screen.
|
||||
- **Passage** — type real prose from public-domain books (Project
|
||||
Gutenberg). Opt-in downloads, configurable paragraphs-per-book limit, falls
|
||||
back to bundled content when offline.
|
||||
|
||||
### Statistics dashboard
|
||||
|
||||
A multi-tab dashboard with everything the engine has learned about you:
|
||||
|
||||
- **Dashboard** — current WPM, accuracy, confident-key count, level/streak
|
||||
- **History** — recent drills with WPM/accuracy
|
||||
- **Activity** — a GitHub-style heatmap of drilling activity over time
|
||||
- **Accuracy** — per-key accuracy heatmap laid out on the keyboard
|
||||
- **Timing** — per-key timing heatmap
|
||||
- **N-grams** — your worst bigrams/trigrams, redundancy scores, watchlist of
|
||||
emerging weak transitions, and what's currently driving drill focus
|
||||
|
||||
### Visual keyboard
|
||||
|
||||
A live keyboard diagram colors each key by the finger that should press it,
|
||||
highlights your focused key, and (in terminals that support the Kitty keyboard
|
||||
protocol) lights up modifier keys as you press them. A separate **Keyboard
|
||||
Explorer** screen lets you click around the layout to inspect any key's
|
||||
stats.
|
||||
|
||||
### Theming
|
||||
|
||||
14 built-in themes: Catppuccin (Mocha/Latte), Dracula, Gruvbox (Dark/Darkest),
|
||||
Kanagawa (Wave/Dragon/Lotus), Nord, One Dark, Solarized Dark, Tokyo Night,
|
||||
Farout, plus an ANSI-safe `terminal-default` that respects your terminal's
|
||||
own colors.
|
||||
|
||||
### Internationalization
|
||||
|
||||
- **UI translated into 21 languages** (`en`, `de`, `es`, `fr`, `it`, `pt`,
|
||||
`nl`, `sv`, `da`, `nb`, `fi`, `pl`, `cs`, `ro`, `hr`, `hu`, `lt`, `lv`,
|
||||
`sl`, `et`, `tr`).
|
||||
- **Dictionaries for the same 21 languages**, sourced from
|
||||
[keybr-content-words](https://github.com/aradzie/keybr.com).
|
||||
- **Keyboard layout profiles**: QWERTY, Dvorak, Colemak, German QWERTZ,
|
||||
French AZERTY — with per-layout finger maps so the diagram stays accurate.
|
||||
- Unicode-normalized (NFC) input matching, so composed and decomposed forms
|
||||
of accented characters compare equal.
|
||||
|
||||
### Import / export
|
||||
|
||||
Back up everything — config, profile, key stats, ranked stats, drill history
|
||||
— to a single timestamped JSON file from the Settings page, and restore it on
|
||||
another machine. Machine-local paths (download directories) are preserved
|
||||
from the target machine on import. Versioned format with a clear error if
|
||||
schemas don't match.
|
||||
|
||||
### Other niceties
|
||||
|
||||
- Atomic JSON writes (temp file → fsync → rename) so a crash mid-save can't
|
||||
corrupt your stats.
|
||||
- Auto-continue between drills with a brief input lock so a trailing
|
||||
keystroke doesn't start the next drill prematurely.
|
||||
- Mouse support for menus, tabs, and the keyboard explorer.
|
||||
|
||||
## Build and run
|
||||
|
||||
keydr is a regular Cargo project. You need a recent Rust toolchain (Rust
|
||||
edition 2024, i.e. a 2026-era stable compiler).
|
||||
|
||||
```sh
|
||||
git clone <repo-url> keydr
|
||||
cd keydr
|
||||
cargo run --release
|
||||
```
|
||||
|
||||
Or build once and run the binary:
|
||||
|
||||
```sh
|
||||
cargo build --release
|
||||
./target/release/keydr
|
||||
```
|
||||
|
||||
### CLI flags
|
||||
|
||||
```sh
|
||||
keydr [--theme <name>] [--layout <layout>] [--words <n>]
|
||||
```
|
||||
|
||||
- `--theme, -t` — pick a theme by name for this run (e.g. `dracula`,
|
||||
`nord`, `catppuccin-mocha`)
|
||||
- `--layout, -l` — pick a keyboard layout: `qwerty`, `dvorak`, `colemak`,
|
||||
`de_qwertz`, `fr_azerty`
|
||||
- `--words, -w` — number of words per drill
|
||||
|
||||
Everything else is set from inside the app, on the Settings screen.
|
||||
|
||||
### Features
|
||||
|
||||
The `network` Cargo feature (on by default) pulls in `reqwest` and enables the
|
||||
optional GitHub code download and Project Gutenberg passage download flows. To
|
||||
build a fully offline binary:
|
||||
|
||||
```sh
|
||||
cargo build --release --no-default-features
|
||||
```
|
||||
|
||||
### Files on disk
|
||||
|
||||
- Config: `~/.config/keydr/config.toml`
|
||||
- Profile, key stats, drill history: `~/.local/share/keydr/*.json`
|
||||
- Downloaded passages: `~/.local/share/keydr/passages/`
|
||||
- Downloaded code samples: `~/.local/share/keydr/code/`
|
||||
|
||||
(`dirs` is used to resolve these, so the exact paths follow XDG/macOS/Windows
|
||||
conventions on your platform.)
|
||||
|
||||
### Tests and benchmarks
|
||||
|
||||
```sh
|
||||
cargo test
|
||||
cargo bench # n-gram benchmarks in benches/
|
||||
```
|
||||
|
||||
## Terminal requirements
|
||||
|
||||
keydr works in any reasonably modern terminal. For the best experience —
|
||||
modifier-key highlighting on the keyboard diagram, unambiguous key
|
||||
disambiguation, and per-key release events — use a terminal that supports the
|
||||
[Kitty keyboard protocol] (Kitty, WezTerm, foot, recent Ghostty, recent
|
||||
Alacritty). keydr falls back gracefully on terminals that don't (tmux, mosh,
|
||||
SSH to older hosts) using timer-based heuristics.
|
||||
|
||||
A true-color terminal is recommended so the bundled themes render as
|
||||
intended; the `terminal-default` theme is provided for ANSI-only setups.
|
||||
|
||||
## License and attribution
|
||||
|
||||
keydr is licensed under the GNU Affero General Public License v3.0 only
|
||||
(AGPL-3.0-only) — see [LICENSE](LICENSE).
|
||||
|
||||
It incorporates content and ideas from [keybr.com] (dictionaries, the core
|
||||
adaptive algorithm). See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for
|
||||
attribution and [docs/license-compliance.md](docs/license-compliance.md) for
|
||||
the compliance process.
|
||||
|
||||
[ratatui]: https://ratatui.rs/
|
||||
[keybr.com]: https://www.keybr.com/
|
||||
[Kitty keyboard protocol]: https://sw.kovidgoyal.net/kitty/keyboard-protocol/
|
||||
@@ -0,0 +1,209 @@
|
||||
# Building keydr
|
||||
|
||||
## TL;DR
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
cargo test
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Plain cargo. No wrapper, no memory ceiling, no reduced optimisation.
|
||||
A clean release build peaks at **1.92 GB** and takes **2m17s** on a
|
||||
4-core / 7.6 GB VM.
|
||||
|
||||
If that's not what you're seeing, read on.
|
||||
|
||||
---
|
||||
|
||||
## The build used to be unusable — here's what was actually wrong
|
||||
|
||||
Release builds consumed 5+ GB and never finished; the VM thrashed so
|
||||
badly that SSH stopped responding and needed a console reboot.
|
||||
|
||||
The obvious explanations were all wrong:
|
||||
|
||||
- **Not too many parallel jobs.** The peak came from a *single* rustc
|
||||
process. A single process's memory is unaffected by `--jobs`.
|
||||
- **Not insufficient RAM.** 8 GB is fine for a project this size.
|
||||
- **Not LTO** (though LTO made it worse).
|
||||
|
||||
**The cause was a codegen bug in `rust-i18n` v3.** Its macro emitted one
|
||||
`HashMap::from([...])` construction *and* one `add_translations()` call
|
||||
**per translation string** — all inside a single function body. With 21
|
||||
locales and ~9,000 keys that's 8,652 separate HashMap constructions in
|
||||
one function. LLVM's optimiser scales superlinearly on function size, so
|
||||
it consumed ~4.5 GB trying to optimise that one initialiser.
|
||||
|
||||
Verified by expanding the macro:
|
||||
|
||||
| | rust-i18n v3.1.5 | rust-i18n v4.2.1 |
|
||||
|---|---|---|
|
||||
| `HashMap::from` constructions | 8,652 | **0** |
|
||||
| `add_translations` calls | 8,652 | **21** (one per locale) |
|
||||
| Expanded lines (lib target) | 93,026 | 73,444 |
|
||||
|
||||
Upstream fixed this in **v4.0.0**. Reproduce the check yourself:
|
||||
|
||||
```bash
|
||||
cargo install cargo-expand
|
||||
cargo expand --lib | grep -c 'HashMap::from'
|
||||
```
|
||||
|
||||
### The fix
|
||||
|
||||
```bash
|
||||
cargo add rust-i18n@4
|
||||
```
|
||||
|
||||
That's it. Measured on this box, same machine, clean builds:
|
||||
|
||||
| Configuration | Peak | Time |
|
||||
|---|---|---|
|
||||
| v3, `opt-level=3`, thin LTO | 5.2 GB | stalled >18 min, never finished |
|
||||
| v3, `opt-level=3`, no LTO | 4.5 GB | stalled >12 min, never finished |
|
||||
| v3, `opt-level=1`, no LTO (workaround) | 1.8 GB | 15m30s |
|
||||
| **v4, `opt-level=3`, thin LTO** | **2.08 GB** | **3m25s** |
|
||||
| **v4, stock cargo defaults** | **1.92 GB** | **2m17s** |
|
||||
|
||||
The earlier workaround — dropping `opt-level` to 1 and disabling LTO —
|
||||
has been **removed**. It was treating a symptom.
|
||||
|
||||
---
|
||||
|
||||
## What's still in `.cargo/config.toml`, and why
|
||||
|
||||
Nothing there reduces the optimisation of shipped code.
|
||||
|
||||
- **`jobs = 3`** — uses 3 of 4 cores so the box stays interactive while
|
||||
compiling. Purely about responsiveness; delete it if you don't care.
|
||||
- **`lld` linker** — ships with the Rust toolchain, no install needed.
|
||||
GNU ld is single-threaded and holds the whole link graph in memory.
|
||||
- **`debug = 1` for this crate, `debug = 0` for dependencies** (dev/test
|
||||
profiles only). Debug info is the largest contributor to `target/` size
|
||||
and link time. Level 1 keeps line numbers, so backtraces still work.
|
||||
|
||||
`[profile.release]` is deliberately absent — cargo's defaults are right.
|
||||
|
||||
---
|
||||
|
||||
## Known remaining inefficiency
|
||||
|
||||
`src/main.rs` re-declares the whole module tree (`mod app; mod config;
|
||||
...`) instead of depending on the `keydr` lib target, so **every module
|
||||
compiles twice** — once for the lib, once for the bin. The `i18n!` macro
|
||||
must therefore be invoked in both crate roots (`t!()` expands to
|
||||
`crate::_rust_i18n_t`, so it must exist in each crate).
|
||||
|
||||
Measured with `cargo build --release --timings` on a clean tree, the two
|
||||
duplicated units are the slowest in the whole graph:
|
||||
|
||||
```
|
||||
33.7s keydr (bin)
|
||||
26.0s keydr (lib)
|
||||
1.4s generate_test_profiles (bin)
|
||||
------
|
||||
61.0s of 292.2s total unit-seconds (21%)
|
||||
```
|
||||
|
||||
So restructuring `main.rs` to `use keydr::...` would save roughly 26s of
|
||||
compile work — real, but nowhere near half the build. Wall-clock saving is
|
||||
smaller still, since those units partly overlap with dependency
|
||||
compilation across 3 parallel jobs (292s of unit-work compressed into
|
||||
~2m15s wall). Worth doing for code hygiene; not a build-performance
|
||||
emergency.
|
||||
|
||||
Note the remaining ~79% is dependency compilation (`reqwest` 12.7s,
|
||||
`clap_builder` 11.4s, `toml_edit` 9.3s, `tokio` 8.0s...), which is
|
||||
one-time work that incremental builds skip entirely.
|
||||
|
||||
---
|
||||
|
||||
## If a build ever does go wild again
|
||||
|
||||
The machine-level protections below need no per-project changes.
|
||||
|
||||
### Keep SSH alive (recommended, needs root once)
|
||||
|
||||
```bash
|
||||
# Reserve memory for the SSH daemon so it can't be swapped out entirely.
|
||||
sudo systemctl edit ssh # [Service] / MemoryMin=128M
|
||||
```
|
||||
|
||||
**Do not bother with `systemd-oomd` on this box** — it is a separate
|
||||
package whose only reverse-dependencies are `ubuntu-desktop*`, so Ubuntu
|
||||
Server never installs it (`systemctl is-enabled systemd-oomd` → `not-found`
|
||||
here). Use **earlyoom** instead, which is in `universe` and kills only the
|
||||
single highest-scoring process rather than a whole cgroup:
|
||||
|
||||
```bash
|
||||
sudo apt install earlyoom
|
||||
# /etc/default/earlyoom
|
||||
EARLYOOM_ARGS="-m 8 -s 5 -r 60 \
|
||||
--avoid '(^|/)(systemd|sshd|mosh-server|tmux.*|bash|fish)$' \
|
||||
--prefer '(^|/)(rustc|cargo|ld|lld|collect2)$'"
|
||||
```
|
||||
|
||||
`--avoid` protects your shell; `--prefer` points it at the compiler. Fedora
|
||||
enabled earlyoom by default for exactly this "system becomes completely
|
||||
unresponsive, user has no choice but to force power off" scenario.
|
||||
|
||||
### Cap every build at once, forever (needs root once)
|
||||
|
||||
This is the global knob — no wrapper script, no per-project config:
|
||||
|
||||
```bash
|
||||
sudo systemctl set-property user-1000.slice MemoryHigh=5G MemoryMax=6500M
|
||||
```
|
||||
|
||||
Applies to every shell and every build you start, persists across reboots.
|
||||
Because `sshd` itself lives in `system.slice`, capping the user slice can
|
||||
never lock you out of a new login. (Ubuntu already ships `TasksMax=33%`
|
||||
this way via `/usr/lib/systemd/system/user-.slice.d/`, so it's a
|
||||
distro-blessed pattern.)
|
||||
|
||||
Note `MemoryHigh` at the *slice* level is reasonable — it throttles a
|
||||
sprawling session gradually. Do not put it on a single short-lived build
|
||||
scope, where it causes the reclaim-stall described below.
|
||||
|
||||
### Cap one build ad-hoc (no root)
|
||||
|
||||
`memory` is delegated to the user slice on this box, so you can confine a
|
||||
single command without any wrapper script:
|
||||
|
||||
```bash
|
||||
systemd-run --user --scope -p MemoryMax=4G -p MemorySwapMax=0 \
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
`MemorySwapMax=0` is the important half. The lockup was never really an
|
||||
OOM — it was *thrashing*. With swap available the kernel pages `sshd` out
|
||||
to feed the build and the box goes catatonic while `oom_kill` stays at 0.
|
||||
Denying the build swap turns a slow total failure into a fast contained one.
|
||||
|
||||
**Do not add `MemoryHigh` to a single build scope.** It sounds safer but
|
||||
traps the process in continuous reclaim so it grinds forever instead of
|
||||
dying. Measured against an identical 256 MB ceiling: `MemoryMax` alone →
|
||||
clean kill in seconds; `MemoryMax` + `MemoryHigh` → still spinning after 45
|
||||
seconds.
|
||||
|
||||
### Diagnosing which crate is expensive
|
||||
|
||||
```bash
|
||||
cargo build --release --timings # HTML report in target/cargo-timings/
|
||||
cargo install cargo-llvm-lines
|
||||
cargo llvm-lines --release | head -30 # which functions generate the most IR
|
||||
cargo expand --lib | wc -l # how much code a macro really emits
|
||||
```
|
||||
|
||||
`cargo llvm-lines` and `cargo expand` are what actually found this bug.
|
||||
Reach for them before touching `opt-level`.
|
||||
|
||||
## Upstream context
|
||||
|
||||
Cargo has no memory-aware job scheduling; it schedules on core count only.
|
||||
That's a known gap ([rust-lang/cargo#12912](https://github.com/rust-lang/cargo/issues/12912)),
|
||||
and maintainers have said they'd rather delegate resource limits to the OS
|
||||
(cgroups) than build it into cargo. So the systemd approach above isn't a
|
||||
hack — it's the sanctioned answer. But in this case none of it was needed:
|
||||
the real fix was a dependency upgrade.
|
||||
+10
@@ -1,3 +1,13 @@
|
||||
// The `i18n!` macro must be invoked in each crate root that uses `t!()`,
|
||||
// because `t!` expands to `crate::_rust_i18n_t`. main.rs re-declares the
|
||||
// module tree rather than depending on the lib target, so it is a separate
|
||||
// crate and needs its own invocation.
|
||||
//
|
||||
// This means the translation table is generated twice (once here, once in
|
||||
// lib.rs). With rust-i18n v4 that costs ~21 add_translations calls per
|
||||
// invocation instead of v3's ~8,650, so the duplication is now cheap.
|
||||
// Eliminating it entirely would require main.rs to `use keydr::...` instead
|
||||
// of re-declaring `mod app; mod config; ...` — a larger refactor.
|
||||
rust_i18n::i18n!("locales", fallback = "en");
|
||||
|
||||
mod app;
|
||||
|
||||
Reference in New Issue
Block a user