thallada/crawlnicle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding tokio-console support for debugging tokio tasks

Browse Source 
This isn't always that useful and I'm not sure what performance impact
it has so I'm going to keep this in a branch in case I need it in the
future.
This commit is contained in:
Tyler Hallada 2024-05-11 16:04:18 -04:00
parent 27ab839df4
commit cfc962c2cb
4 changed files with 274 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.cargo/config.toml Normal file
View File

@ -0,0 +1,3 @@
[build]
# needed so that tokio-console has the necessary data
rustflags = ["--cfg", "tokio_unstable"]

247
Cargo.lock generated
View File

@ -156,6 +156,28 @@ dependencies = [
"password-hash",
]
[[package]]
name = "async-stream"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
dependencies = [
"async-stream-impl",
"futures-core",
"pin-project-lite",
]
[[package]]
name = "async-stream-impl"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.61",
]
[[package]]
name = "async-trait"
version = "0.1.80"
@ -182,6 +204,34 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
[[package]]
name = "axum"
version = "0.6.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
dependencies = [
"async-trait",
"axum-core 0.3.4",
"bitflags 1.3.2",
"bytes",
"futures-util",
"http 0.2.12",
"http-body 0.4.6",
"hyper 0.14.28",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"sync_wrapper 0.1.2",
"tower",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum"
version = "0.7.5"
@ -189,7 +239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
dependencies = [
"async-trait",
"axum-core",
"axum-core 0.4.3",
"bytes",
"futures-util",
"http 1.1.0",
@ -223,11 +273,28 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72188bed20deb981f3a4a9fe674e5980fd9e9c2bd880baa94715ad5d60d64c67"
dependencies = [
"axum",
"axum 0.7.5",
"forwarded-header-value",
"serde",
]
[[package]]
name = "axum-core"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http 0.2.12",
"http-body 0.4.6",
"mime",
"rustversion",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.4.3"
@ -255,8 +322,8 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733"
dependencies = [
"axum",
"axum-core",
"axum 0.7.5",
"axum-core 0.4.3",
"bytes",
"futures-util",
"headers",
@ -279,7 +346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0fbc0d7bd2577dda9aa9cac096e53b30342725d8eea5798169ff2537a214f45"
dependencies = [
"async-trait",
"axum",
"axum 0.7.5",
"form_urlencoded",
"serde",
"subtle",
@ -469,6 +536,43 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
[[package]]
name = "console-api"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787"
dependencies = [
"futures-core",
"prost",
"prost-types",
"tonic",
"tracing-core",
]
[[package]]
name = "console-subscriber"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e"
dependencies = [
"console-api",
"crossbeam-channel",
"crossbeam-utils",
"futures-task",
"hdrhistogram",
"humantime",
"prost-types",
"serde",
"serde_json",
"thread_local",
"tokio",
"tokio-stream",
"tonic",
"tracing",
"tracing-core",
"tracing-subscriber",
]
[[package]]
name = "const-oid"
version = "0.9.6"
@ -533,7 +637,7 @@ dependencies = [
"ansi-to-html",
"anyhow",
"async-trait",
"axum",
"axum 0.7.5",
"axum-client-ip",
"axum-extra",
"axum-login",
@ -541,6 +645,7 @@ dependencies = [
"bytes",
"chrono",
"clap",
"console-subscriber",
"dotenvy",
"feed-rs",
"futures",
@ -596,6 +701,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff"
[[package]]
name = "crc32fast"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.12"
@ -812,6 +926,16 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6"
[[package]]
name = "flate2"
version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]]
name = "float-cmp"
version = "0.9.0"
@ -1134,6 +1258,19 @@ dependencies = [
"hashbrown 0.14.5",
]
[[package]]
name = "hdrhistogram"
version = "7.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
dependencies = [
"base64 0.21.7",
"byteorder",
"flate2",
"nom",
"num-traits",
]
[[package]]
name = "headers"
version = "0.4.0"
@ -1325,6 +1462,12 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.28"
@ -1370,6 +1513,18 @@ dependencies = [
"want",
]
[[package]]
name = "hyper-timeout"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
dependencies = [
"hyper 0.14.28",
"pin-project-lite",
"tokio",
"tokio-io-timeout",
]
[[package]]
name = "hyper-tls"
version = "0.5.0"
@ -1723,7 +1878,7 @@ name = "maud"
version = "0.25.0"
source = "git+https://github.com/vidhanio/maud?branch=patch-1#fb77d399d001a345f3a369eb2c7a12da243376a6"
dependencies = [
"axum-core",
"axum-core 0.4.3",
"http 1.1.0",
"itoa",
"maud_macros",
@ -2277,6 +2432,38 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prost"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-derive"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9554e3ab233f0a932403704f1a1d08c30d5ccd931adfdfa1e8b5a19b52c1d55a"
dependencies = [
"anyhow",
"itertools",
"proc-macro2",
"quote",
"syn 2.0.61",
]
[[package]]
name = "prost-types"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe"
dependencies = [
"prost",
]
[[package]]
name = "psm"
version = "0.1.21"
@ -3312,9 +3499,20 @@ dependencies = [
"signal-hook-registry",
"socket2",
"tokio-macros",
"tracing",
"windows-sys 0.48.0",
]
[[package]]
name = "tokio-io-timeout"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
dependencies = [
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-macros"
version = "2.2.0"
@ -3361,6 +3559,33 @@ dependencies = [
"tokio",
]
[[package]]
name = "tonic"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e"
dependencies = [
"async-stream",
"async-trait",
"axum 0.6.20",
"base64 0.21.7",
"bytes",
"h2 0.3.26",
"http 0.2.12",
"http-body 0.4.6",
"hyper 0.14.28",
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost",
"tokio",
"tokio-stream",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower"
version = "0.4.13"
@ -3369,9 +3594,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
"futures-core",
"futures-util",
"indexmap 1.9.3",
"pin-project",
"pin-project-lite",
"rand",
"slab",
"tokio",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",
@ -3384,7 +3613,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fd0118512cf0b3768f7fcccf0bef1ae41d68f2b45edc1e77432b36c97c56c6d"
dependencies = [
"async-trait",
"axum-core",
"axum-core 0.4.3",
"cookie",
"futures-util",
"http 1.1.0",
@ -3470,7 +3699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38767064990c327ec1d92bba2576dce0944750e9c9ae021f12ebc72de77ac406"
dependencies = [
"async-trait",
"axum-core",
"axum-core 0.4.3",
"base64 0.22.1",
"futures",
"http 1.1.0",

1
Cargo.toml
View File

@ -25,6 +25,7 @@ bytes = "1.4"
# TODO: replace chrono with time
chrono = { version = "0.4", features = ["serde"] }
clap = { version = "4.4", features = ["derive", "env"] }
console-subscriber = "0.2"
dotenvy = "0.15"
feed-rs = "1.3"
futures = "0.3"

38
src/log.rs
View File

@ -5,7 +5,10 @@ use anyhow::Result;
use bytes::Bytes;
use once_cell::sync::Lazy;
use tokio::sync::watch::Sender;
use tracing::level_filters::LevelFilter;
use tracing::Level;
use tracing_appender::non_blocking::WorkerGuard;
use tracing_subscriber::filter::Targets;
use tracing_subscriber::prelude::*;
use tracing_subscriber::EnvFilter;
@ -75,8 +78,8 @@ pub fn init_tracing(
config: &Config,
log_sender: Sender<Bytes>,
) -> Result<(WorkerGuard, WorkerGuard)> {
let fmt_layer = tracing_subscriber::fmt::layer();
let filter_layer = EnvFilter::from_default_env();
let console_layer = console_subscriber::spawn();
let stdout_layer = tracing_subscriber::fmt::layer().pretty();
let file_appender = tracing_appender::rolling::hourly("./logs", "log");
let (file_writer, file_writer_guard) = tracing_appender::non_blocking(file_appender);
let mem_writer = LimitedInMemoryBuffer::new(&MEM_LOG, log_sender, config.max_mem_log_size);
@ -84,10 +87,33 @@ pub fn init_tracing(
let file_writer_layer = tracing_subscriber::fmt::layer().with_writer(file_writer);
let mem_writer_layer = tracing_subscriber::fmt::layer().with_writer(mem_writer);
tracing_subscriber::registry()
.with(filter_layer)
.with(fmt_layer)
.with(file_writer_layer)
.with(mem_writer_layer)
.with(
EnvFilter::from_default_env()
.add_directive("tokio=trace".parse()?)
.add_directive("runtime=trace".parse()?),
)
.with(console_layer)
.with(
stdout_layer.with_filter(
EnvFilter::from_default_env()
.add_directive("tokio=off".parse()?)
.add_directive("runtime=off".parse()?),
),
)
.with(
file_writer_layer.with_filter(
EnvFilter::from_default_env()
.add_directive("tokio=off".parse()?)
.add_directive("runtime=off".parse()?),
),
)
.with(
mem_writer_layer.with_filter(
EnvFilter::from_default_env()
.add_directive("tokio=off".parse()?)
.add_directive("runtime=off".parse()?),
),
)
.init();
Ok((file_writer_guard, mem_writer_guard))
}