Streaming log page with colors

Using the ansi-to-html crate.

Cargo.lock

@@ -31,6 +31,16 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "ansi-to-html"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7bd918cc0ff933f0e6cf48a8f74584818ea43e07d1fba1f9251bb3df2a37ca2"
+dependencies = [
+ "regex",
+ "thiserror",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.71"
@@ -252,6 +262,7 @@ dependencies = [
 name = "crawlnicle"
 version = "0.1.0"
 dependencies = [
+ "ansi-to-html",
  "anyhow",
  "argh",
  "axum",

Cargo.toml

@@ -11,6 +11,7 @@ path = "src/lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+ansi-to-html = "0.1"
 anyhow = "1"
 argh = "0.1"
 axum = "0.6"

src/handlers/log.rs

@@ -1,13 +1,15 @@
 use std::convert::Infallible;
+use std::str::from_utf8;
 use std::time::Duration;
 
+use ansi_to_html::convert_escaped;
 use axum::extract::State;
 use axum::response::{
     sse::{Event, Sse},
     Response,
 };
 use bytes::Bytes;
-use maud::html;
+use maud::{html, PreEscaped};
 use tokio::sync::watch::Receiver;
 use tokio_stream::wrappers::WatchStream;
 use tokio_stream::Stream;
@@ -21,7 +23,7 @@ pub async fn get(layout: Layout) -> Result<Response> {
     let mem_buf = MEM_LOG.lock().unwrap();
     Ok(layout.render(html! {
         turbo-stream-source src="/log/stream" {}
-        pre id="log" { (std::str::from_utf8(mem_buf.as_slices().0).unwrap()) }
+        pre id="log" { (PreEscaped(convert_escaped(from_utf8(mem_buf.as_slices().0).unwrap()).unwrap())) }
     }))
 }
 
@@ -30,13 +32,16 @@ pub async fn stream(
 ) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
     let log_stream = WatchStream::new(log_receiver);
     let log_stream = log_stream.map(|line| {
-        Ok(Event::default().data(html! {
-            turbo-stream action="append" target="log" {
-                template {
-                    (std::str::from_utf8(&line).unwrap())
+        Ok(Event::default().data(
+            html! {
+                turbo-stream action="append" target="log" {
+                    template {
+                        (PreEscaped(convert_escaped(from_utf8(&line).unwrap()).unwrap()))
+                    }
                 }
             }
-        }.into_string()))
+            .into_string(),
+        ))
     });
     Sse::new(log_stream).keep_alive(
         axum::response::sse::KeepAlive::new()

src/log.rs

@@ -1,5 +1,5 @@
 use std::sync::Mutex;
-use std::{io::Write, collections::VecDeque};
+use std::{collections::VecDeque, io::Write};
 
 use anyhow::Result;
 use bytes::Bytes;
@@ -7,23 +7,23 @@ use once_cell::sync::Lazy;
 use tokio::sync::watch::Sender;
 use tracing_appender::non_blocking::WorkerGuard;
 use tracing_subscriber::prelude::*;
-use tracing_subscriber::{fmt::format, EnvFilter};
+use tracing_subscriber::EnvFilter;
 
 use crate::config::Config;
 
 /// A shared in-memory buffer to store log bytes
 pub static MEM_LOG: Lazy<Mutex<VecDeque<u8>>> = Lazy::new(|| Mutex::new(VecDeque::new()));
 
-/// A `Writer` to a shared static in-memory buffer that stores bytes up until `max` bytes, at which 
-/// point it will truncate the buffer from the front up to the first newline byte `\n` within the 
+/// A `Writer` to a shared static in-memory buffer that stores bytes up until `max` bytes, at which
+/// point it will truncate the buffer from the front up to the first newline byte `\n` within the
 /// size limit.
 ///
-/// This is useful for storing the last emitted log lines of an application in-memory without 
+/// This is useful for storing the last emitted log lines of an application in-memory without
 /// needing to worry about the memory growing infinitely large.
 ///
-/// `LimitedInMemoryBuffer` does not guarantee that the memory usage is less than `max`. 
-/// VecDeque`'s capacity may exceed `max` and it will only check and truncate the size of the 
-/// internal buffer *before* writing to it. It will continue to write, even if the size of the line 
+/// `LimitedInMemoryBuffer` does not guarantee that the memory usage is less than `max`.
+/// VecDeque`'s capacity may exceed `max` and it will only check and truncate the size of the
+/// internal buffer *before* writing to it. It will continue to write, even if the size of the line
 /// to write will make the buffer exceed `max`.
 struct LimitedInMemoryBuffer {
     pub buf: &'static Mutex<VecDeque<u8>>,
@@ -33,14 +33,10 @@ struct LimitedInMemoryBuffer {
 
 impl LimitedInMemoryBuffer {
     fn new(buf: &'static Mutex<VecDeque<u8>>, sender: Sender<Bytes>, max: usize) -> Self {
-        Self {
-            buf,
-            sender,
-            max,
-        }
+        Self { buf, sender, max }
     }
 
-    /// Truncate the buffer to max bytes plus bytes before the closest newline starting from the 
+    /// Truncate the buffer to max bytes plus bytes before the closest newline starting from the
     /// front
     fn truncate(&mut self) {
         let mut buf = self.buf.lock().unwrap();
@@ -75,21 +71,18 @@ impl Write for LimitedInMemoryBuffer {
     }
 }
 
-pub fn init_tracing(config: &Config, log_sender: Sender<Bytes>) -> Result<(WorkerGuard, WorkerGuard)> {
+pub fn init_tracing(
+    config: &Config,
+    log_sender: Sender<Bytes>,
+) -> Result<(WorkerGuard, WorkerGuard)> {
     let fmt_layer = tracing_subscriber::fmt::layer();
     let filter_layer = EnvFilter::from_default_env();
     let file_appender = tracing_appender::rolling::hourly("./logs", "log");
     let (file_writer, file_writer_guard) = tracing_appender::non_blocking(file_appender);
     let mem_writer = LimitedInMemoryBuffer::new(&MEM_LOG, log_sender, config.max_mem_log_size);
     let (mem_writer, mem_writer_guard) = tracing_appender::non_blocking(mem_writer);
-    let file_writer_layer = tracing_subscriber::fmt::layer()
-        .with_writer(file_writer)
-        .with_ansi(false)
-        .fmt_fields(format::PrettyFields::new().with_ansi(false));
-    let mem_writer_layer = tracing_subscriber::fmt::layer()
-        .with_writer(mem_writer)
-        .with_ansi(false)
-        .fmt_fields(format::PrettyFields::new().with_ansi(false));
+    let file_writer_layer = tracing_subscriber::fmt::layer().with_writer(file_writer);
+    let mem_writer_layer = tracing_subscriber::fmt::layer().with_writer(mem_writer);
     tracing_subscriber::registry()
         .with(filter_layer)
         .with(fmt_layer)

src/main.rs

@@ -17,8 +17,8 @@ use tracing::debug;
 
 use lib::config::Config;
 use lib::handlers;
-use lib::state::AppState;
 use lib::log::init_tracing;
+use lib::state::AppState;
 
 #[tokio::main]
 async fn main() -> Result<()> {
@@ -49,7 +49,11 @@ async fn main() -> Result<()> {
         .route("/entry/:id", get(handlers::entry::get))
         .route("/log", get(handlers::log::get))
         .route("/log/stream", get(handlers::log::stream))
-        .with_state(AppState { pool, config, log_receiver })
+        .with_state(AppState {
+            pool,
+            config,
+            log_receiver,
+        })
         .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()));
 
     #[cfg(debug_assertions)]

src/partials/header.rs

@@ -1,10 +1,10 @@
 use maud::{html, Markup};
 
-pub fn header() -> Markup {
+pub fn header(title: &str) -> Markup {
     html! {
         header {
             nav {
-                h1 { a href="/" data-turbo-frame="main" { "crawlnicle" } }
+                h1 { a href="/" data-turbo-frame="main" { (title) } }
                 ul {
                     li { a href="/feeds" data-turbo-frame="main" { "feeds" } }
                     li { a href="/log" data-turbo-frame="main" { "log" } }

src/partials/layout.rs

@@ -38,13 +38,13 @@ impl Layout {
             html lang="en" {
                 head {
                     meta charset="utf-8";
-                    title { "crawlnicle" }
+                    title { (self.title) }
                     script type="module" {
                         r#"import * as Turbo from 'https://cdn.skypack.dev/@hotwired/turbo';"#
                     }
                 }
                 body {
-                    (header())
+                    (header(&self.title))
                     turbo-frame id="main" data-turbo-action="advance" {
                         (template)
                     }