thallada/crawlnicle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace hotwire with htmx

Browse Source 
In the process, also improve the feedback from the import/add feed
forms.

I also replaced the frontend code to replace utc timestamps with local
time strings with @hotwired/stimulus with vanilla js.
This commit is contained in:
Tyler Hallada 2023-09-01 00:25:05 -04:00
parent ff0b218da1
commit 1d6f98c6bb
20 changed files with 215 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
frontend/bun.lockb
View File

Binary file not shown.

4
frontend/css/styles.css
View File

@ -156,7 +156,7 @@ form.feed-form .form-grid button {
grid-column: 3 / 4;
}
ul#add-feed-messages {
ul.stream-messages {
list-style: none;
padding: 0;
margin: 0;
@ -164,7 +164,7 @@ ul#add-feed-messages {
white-space: nowrap;
}
ul#add-feed-messages li {
ul.stream-messages li {
overflow: hidden;
white-space: no-wrap;
text-overflow: ellipsis;

7
frontend/js/index.ts
View File

@ -1,9 +1,4 @@
import { Application } from "@hotwired/stimulus";
import LocalTimeController from "./local_time_controller";
// import CSS so it gets named with a content hash that busts caches
import "../css/styles.css";
window.Stimulus = Application.start();
window.Stimulus.register("local-time", LocalTimeController);
import "./localTimeController";

24
frontend/js/localTimeController.ts Normal file
View File

@ -0,0 +1,24 @@
function convertTimeElements() {
const timeElements = document.querySelectorAll('time.local-time');
timeElements.forEach((element) => {
const utcString = element.getAttribute("datetime");
if (utcString) {
const utcTime = new Date(utcString);
element.textContent = utcTime.toLocaleDateString(window.navigator.language, {
year: "numeric",
month: "long",
day: "numeric",
});
} else {
console.error("Missing datetime attribute on time.local-time element", element);
}
});
}
document.addEventListener("DOMContentLoaded", function() {
convertTimeElements();
});
document.body.addEventListener('htmx:afterSwap', function() {
convertTimeElements();
});

28
frontend/js/local_time_controller.ts
View File

@ -1,28 +0,0 @@
import { Controller } from "@hotwired/stimulus";
// Replaces all UTC timestamps with time formated for the local timezone
export default class extends Controller {
connect() {
this.renderLocalTime();
}
renderLocalTime() {
this.element.textContent = this.localTimeString;
}
get localTimeString(): string {
if (this.utcTime) {
return this.utcTime.toLocaleDateString(window.navigator.language, {
year: "numeric",
month: "long",
day: "numeric",
});
}
return "Unknown datetime"
}
get utcTime(): Date | null {
const utcString = this.element.getAttribute("datetime");
return utcString ? new Date(utcString) : null;
}
}

7
frontend/package.json
View File

@ -1,14 +1,11 @@
{
"dependencies": {
"@hotwired/stimulus": "^3.2.1"
},
"name": "crawlnicle-frontend",
"module": "js/index.ts",
"type": "module",
"devDependencies": {
"bun-types": "^0.6.0"
},
"peerDependencies": {
"typescript": "^5.0.0"
}
},
"type": "module"
}

36
src/actors/importer.rs
View File

@ -5,11 +5,13 @@ use bytes::Bytes;
use opml::OPML;
use sqlx::PgPool;
use tokio::sync::{broadcast, mpsc};
use tokio::task::JoinSet;
use tracing::{debug, error, instrument};
use uuid::Uuid;
use crate::actors::crawl_scheduler::{CrawlSchedulerHandle, CrawlSchedulerHandleMessage};
use crate::models::feed::{Feed, UpsertFeed};
use crate::error::Error;
use crate::models::feed::{Feed, CreateFeed};
use crate::state::Imports;
use crate::uuid::Base62Uuid;
@ -51,11 +53,12 @@ async fn listen_to_crawl(
feed_id: Uuid,
crawl_scheduler: CrawlSchedulerHandle,
respond_to: broadcast::Sender<ImporterHandleMessage>,
) {
) -> Uuid {
let mut receiver = crawl_scheduler.schedule(feed_id).await;
while let Ok(msg) = receiver.recv().await {
let _ = respond_to.send(ImporterHandleMessage::CrawlScheduler(msg));
}
feed_id
}
/// An error type that enumerates possible failures during a crawl and is cloneable and can be sent
@ -95,25 +98,38 @@ impl Importer {
let document = OPML::from_reader(&mut Cursor::new(bytes)).map_err(|_| {
ImporterError::InvalidOPML(file_name.unwrap_or(Base62Uuid::from(import_id).to_string()))
})?;
let mut crawls = JoinSet::new();
for url in Self::gather_feed_urls(document.body.outlines) {
let feed = Feed::upsert(
dbg!(&url);
let feed = Feed::create(
&self.pool,
UpsertFeed {
CreateFeed {
url: url.clone(),
..Default::default()
},
)
.await
.map_err(|_| ImporterError::CreateFeedError(url))?;
if feed.updated_at.is_none() {
tokio::spawn(listen_to_crawl(
.await;
if let Err(Error::Sqlx(sqlx::error::Error::Database(err))) = feed {
if err.is_unique_violation() {
dbg!("already imported", &url);
let _ = respond_to.send(ImporterHandleMessage::AlreadyImported(url));
}
} else if let Ok(feed) = feed {
crawls.spawn(listen_to_crawl(
feed.feed_id,
self.crawl_scheduler.clone(),
respond_to.clone(),
));
} else {
let _ = respond_to.send(ImporterHandleMessage::CreateFeedError(url));
}
}
while let Some(feed_id) = crawls.join_next().await {
dbg!("done crawling feed", feed_id);
}
dbg!("done import_opml");
Ok(())
}
@ -137,6 +153,7 @@ impl Importer {
bytes,
respond_to,
} => {
dbg!("handle_message", import_id);
let result = self
.import_opml(import_id, file_name, bytes, respond_to.clone())
.await;
@ -178,6 +195,8 @@ pub struct ImporterHandle {
#[derive(Debug, Clone)]
pub enum ImporterHandleMessage {
Import(ImporterResult<()>),
CreateFeedError(String),
AlreadyImported(String),
CrawlScheduler(CrawlSchedulerHandleMessage),
}
@ -200,6 +219,7 @@ impl ImporterHandle {
file_name: Option<String>,
bytes: Bytes,
) -> broadcast::Receiver<ImporterHandleMessage> {
dbg!(import_id, &file_name, bytes.len());
let (sender, receiver) = broadcast::channel(8);
let msg = ImporterMessage::Import {
import_id,

2
src/handlers/entry.rs
View File

@ -29,7 +29,7 @@ pub async fn get(
div {
span class="published" {
strong { "Published: " }
time datetime=(published_at) data-controller="local-time" {
time datetime=(published_at) class="local-time" {
(published_at)
}
}

67
src/handlers/feed.rs
View File

@ -18,9 +18,10 @@ use crate::actors::feed_crawler::FeedCrawlerHandleMessage;
use crate::error::{Error, Result};
use crate::models::entry::Entry;
use crate::models::feed::{CreateFeed, Feed};
use crate::partials::add_feed_form::add_feed_form;
use crate::partials::entry_link::entry_link;
use crate::partials::{entry_list::entry_list, feed_link::feed_link, layout::Layout};
use crate::state::Crawls;
use crate::turbo_stream::TurboStream;
use crate::uuid::Base62Uuid;
pub async fn get(
@ -88,16 +89,13 @@ impl IntoResponse for AddFeedError {
fn into_response(self) -> Response {
(
self.status_code(),
TurboStream(
html! {
turbo-stream action="append" target="feeds" {
template {
(add_feed_form())
ul class="stream-messages" {
li { span class="error" { (self) } }
}
}
}
.into_string(),
),
)
.into_response()
}
@ -137,22 +135,18 @@ pub async fn post(
crawls.insert(feed.feed_id, receiver);
}
let feed_id = format!("feed-{}", Base62Uuid::from(feed.feed_id));
let feed_stream = format!("/feed/{}/stream", Base62Uuid::from(feed.feed_id));
let feed_stream = format!("connect:/feed/{}/stream", Base62Uuid::from(feed.feed_id));
Ok((
StatusCode::CREATED,
TurboStream(
html! {
turbo-stream-source src=(feed_stream) id="feed-stream" {}
turbo-stream action="append" target="feeds" {
template {
li id=(feed_id) { (feed_link(&feed, true)) }
(add_feed_form())
div hx-sse=(feed_stream) {
ul class="stream-messages" hx-sse="swap:message" hx-swap="beforeend" {
li { "Fetching feed..." }
}
}
turbo-stream action="remove" target="no-feeds";
}
.into_string(),
),
)
.into_response())
}
@ -174,52 +168,39 @@ pub async fn stream(
Ok::<Event, String>(
Event::default().data(
html! {
turbo-stream action="remove" target="feed-stream" {}
turbo-stream action="replace" target=(feed_id) {
template {
li id=(feed_id) { (feed_link(&feed, false)) }
}
}
li { "Crawled feed: " (feed_link(&feed, false)) }
}
.into_string(),
),
)
}
Ok(CrawlSchedulerHandleMessage::FeedCrawler(FeedCrawlerHandleMessage::Entry(Ok(
entry,
)))) => Ok(Event::default().data(
html! {
li { "Crawled entry: " (entry_link(entry)) }
}
.into_string(),
)),
Ok(CrawlSchedulerHandleMessage::FeedCrawler(FeedCrawlerHandleMessage::Feed(Err(
error,
)))) => Ok(Event::default().data(
html! {
turbo-stream action="remove" target="feed-stream" {}
turbo-stream action="replace" target=(feed_id) {
template {
li id=(feed_id) { span class="error" { (error) } }
}
}
}
.into_string(),
)),
// TODO: these Entry messages are not yet sent, need to handle them better
Ok(CrawlSchedulerHandleMessage::FeedCrawler(FeedCrawlerHandleMessage::Entry(Ok(_)))) => {
Ok(Event::default().data(
html! {
turbo-stream action="replace" target=(feed_id) {
template {
li id=(feed_id) { "fetched entry" }
}
}
}
.into_string(),
))
}
Ok(CrawlSchedulerHandleMessage::FeedCrawler(FeedCrawlerHandleMessage::Entry(Err(
error,
)))) => Ok(Event::default().data(
html! {
turbo-stream action="replace" target=(feed_id) {
template {
li id=(feed_id) { span class="error" { (error) } }
}
li { span class="error" { (error) } }
}
.into_string(),
)),
Ok(CrawlSchedulerHandleMessage::Schedule(Err(error))) => Ok(Event::default().data(
html! {
li { span class="error" { (error) } }
}
.into_string(),
)),

41
src/handlers/feeds.rs
View File

@ -4,48 +4,23 @@ use maud::html;
use sqlx::PgPool;
use crate::error::Result;
use crate::models::feed::{Feed, GetFeedsOptions, DEFAULT_FEEDS_PAGE_SIZE};
use crate::partials::{feed_link::feed_link, layout::Layout};
use crate::models::feed::{Feed, GetFeedsOptions};
use crate::partials::add_feed_form::add_feed_form;
use crate::partials::feed_list::feed_list;
use crate::partials::opml_import_form::opml_import_form;
use crate::partials::layout::Layout;
pub async fn get(State(pool): State<PgPool>, layout: Layout) -> Result<Response> {
let options = GetFeedsOptions::default();
let feeds = Feed::get_all(&pool, options.clone()).await?;
let len = feeds.len() as i64;
Ok(layout.render(html! {
h2 { "Feeds" }
div class="feeds" {
div class="feeds-list" {
@if len == 0 {
p id="no-feeds" { "No feeds found." }
} else {
ul id="feeds" {
@for feed in feeds {
li { (feed_link(&feed, false)) }
}
}
}
// TODO: pagination
@if len == options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE) {
button id="load-more-feeds" { "Load More" }
}
}
(feed_list(feeds, options))
div class="add-feed" {
h3 { "Add Feed" }
form action="/feed" method="post" class="feed-form" {
div class="form-grid" {
label for="url" { "URL: " }
input type="text" id="url" name="url" placeholder="https://example.com/feed.xml" required="true";
button type="submit" { "Add Feed" }
}
}
form action="/import/opml" method="post" enctype="multipart/form-data" class="feed-form" {
div class="form-grid" {
label for="opml" { "OPML: " }
input type="file" id="opml" name="opml" required="true" accept="text/x-opml,application/xml,text/xml";
button type="submit" { "Import Feeds" }
}
}
ul id="add-feed-messages" {}
(add_feed_form())
(opml_import_form())
}
}
}))

80
src/handlers/import.rs
View File

@ -14,8 +14,8 @@ use crate::actors::importer::{ImporterHandle, ImporterHandleMessage};
use crate::error::{Error, Result};
use crate::partials::entry_link::entry_link;
use crate::partials::feed_link::feed_link;
use crate::partials::opml_import_form::opml_import_form;
use crate::state::Imports;
use crate::turbo_stream::TurboStream;
use crate::uuid::Base62Uuid;
pub async fn opml(
@ -26,28 +26,27 @@ pub async fn opml(
if let Some(field) = multipart.next_field().await? {
let import_id = Base62Uuid::new();
let file_name = field.file_name().map(|s| s.to_string());
dbg!(&file_name);
let bytes = field.bytes().await?;
dbg!(bytes.len());
let receiver = importer.import(import_id.as_uuid(), file_name, bytes).await;
{
let mut imports = imports.lock().await;
imports.insert(import_id.as_uuid(), receiver);
}
let import_stream = format!("/import/{}/stream", import_id);
let import_stream = format!("connnect:/import/{}/stream", import_id);
return Ok((
StatusCode::CREATED,
TurboStream(
html! {
turbo-stream-source src=(import_stream) id="import-stream" {}
turbo-stream action="append" target="add-feed-messages" {
template {
li { "Uploading file..." }
(opml_import_form())
div hx-sse=(import_stream) {
ul class="stream-messages" hx-sse="swap:message" hx-swap="beforeend" {
li { "Uploading..."}
}
}
turbo-stream action="remove" target="no-feeds";
}
.into_string(),
),
)
.into_response());
}
@ -65,14 +64,11 @@ pub async fn stream(
.ok_or_else(|| Error::NotFound("import stream", id.as_uuid()))?;
let stream = BroadcastStream::new(receiver);
let import_html_id = format!("import-{}", id);
let stream = stream.map(move |msg| match msg {
Ok(ImporterHandleMessage::Import(Ok(_))) => Ok::<Event, String>(
Event::default().data(
html! {
turbo-stream action="append" target="add-feed-messages" {
template { li { "Importing...." } }
}
li { "Finished importing" }
}
.into_string(),
),
@ -82,11 +78,7 @@ pub async fn stream(
))) => Ok::<Event, String>(
Event::default().data(
html! {
turbo-stream action="append" target="add-feed-messages" {
template {
li { "Imported: " (entry_link(entry)) }
}
}
li { "Crawled entry: " (entry_link(entry)) }
}
.into_string(),
),
@ -96,17 +88,7 @@ pub async fn stream(
))) => Ok::<Event, String>(
Event::default().data(
html! {
turbo-stream action="remove" target="import-stream" {}
turbo-stream action="append" target="add-feed-messages" {
template {
li { "Finished import." }
}
}
turbo-stream action="prepend" target="feeds" {
template {
li id=(format!("feed-{}", feed.feed_id)) { (feed_link(&feed, false)) }
}
}
li { "Crawled feed: " (feed_link(&feed, false)) }
}
.into_string(),
),
@ -116,12 +98,8 @@ pub async fn stream(
))) => Ok::<Event, String>(
Event::default().data(
html! {
turbo-stream action="append" target="add-feed-messages" {
template {
li { span class="error" { (error) } }
}
}
}
.into_string(),
),
),
@ -130,10 +108,28 @@ pub async fn stream(
))) => Ok::<Event, String>(
Event::default().data(
html! {
turbo-stream action="append" target="add-feed-messages" {
template {
li { span class="error" { (error) } }
}
.into_string(),
),
),
Ok(ImporterHandleMessage::CrawlScheduler(CrawlSchedulerHandleMessage::Schedule(Err(
error,
)))) => Ok::<Event, String>(
Event::default().data(
html! {
li { span class="error" { (error) } }
}
.into_string(),
),
),
Ok(ImporterHandleMessage::CreateFeedError(url)) => Ok::<Event, String>(
Event::default().data(
html! {
li {
span class="error" {
"Could not create feed for url: " a href=(url) { (url) }
}
}
}
.into_string(),
@ -141,17 +137,13 @@ pub async fn stream(
),
Ok(ImporterHandleMessage::Import(Err(error))) => Ok(Event::default().data(
html! {
turbo-stream action="remove" target="import-stream" {}
turbo-stream action="append" target="add-feed-messages" {
template {
li { span class="error" { (error) } }
}
}
turbo-stream action="replace" target=(import_html_id) {
template {
li id=(import_html_id) { span class="error" { (error) } }
}
}
.into_string(),
)),
Ok(ImporterHandleMessage::AlreadyImported(url)) => Ok(Event::default().data(
html! {
li { "Already imported feed: " a href=(url) { (url) } }
}
.into_string(),
)),

9
src/handlers/log.rs
View File

@ -23,8 +23,9 @@ use crate::partials::layout::Layout;
pub async fn get(layout: Layout) -> Result<Response> {
let mem_buf = MEM_LOG.lock().unwrap();
Ok(layout.render(html! {
turbo-stream-source src="/log/stream" {}
pre id="log" { (PreEscaped(convert_escaped(from_utf8(mem_buf.as_slices().0).unwrap()).unwrap())) }
pre id="log" hx-sse="connect:/log/stream swap:message" hx-swap="beforeend" {
(PreEscaped(convert_escaped(from_utf8(mem_buf.as_slices().0).unwrap()).unwrap()))
}
}))
}
@ -35,12 +36,8 @@ pub async fn stream(
let log_stream = log_stream.map(|line| {
Ok(Event::default().data(
html! {
turbo-stream action="append" target="log" {
template {
(PreEscaped(convert_escaped(from_utf8(&line).unwrap()).unwrap()))
}
}
}
.into_string(),
))
});

1
src/lib.rs
View File

@ -7,7 +7,6 @@ pub mod log;
pub mod models;
pub mod partials;
pub mod state;
pub mod turbo_stream;
pub mod utils;
pub mod uuid;

2
src/models/feed.rs
View File

@ -71,7 +71,7 @@ pub struct Feed {
pub deleted_at: Option<DateTime<Utc>>,
}
#[derive(Debug, Deserialize, Validate)]
#[derive(Debug, Deserialize, Default, Validate)]
pub struct CreateFeed {
#[validate(length(max = 255))]
pub title: Option<String>,

13
src/partials/add_feed_form.rs Normal file
View File

@ -0,0 +1,13 @@
use maud::{html, Markup};
pub fn add_feed_form() -> Markup {
html! {
form hx-post="/feed" class="feed-form" {
div class="form-grid" {
label for="url" { "URL: " }
input type="text" id="url" name="url" placeholder="https://example.com/feed.xml" required="true";
button type="submit" { "Add Feed" }
}
}
}
}

25
src/partials/feed_list.rs Normal file
View File

@ -0,0 +1,25 @@
use maud::{html, Markup};
use crate::models::feed::{Feed, GetFeedsOptions, DEFAULT_FEEDS_PAGE_SIZE};
use crate::partials::feed_link::feed_link;
pub fn feed_list(feeds: Vec<Feed>, options: GetFeedsOptions) -> Markup {
let len = feeds.len() as i64;
html! {
div class="feeds-list" {
@if len == 0 {
p id="no-feeds" { "No feeds found." }
} else {
ul id="feeds" {
@for feed in feeds {
li { (feed_link(&feed, false)) }
}
}
}
// TODO: pagination
@if len == options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE) {
button id="load-more-feeds" { "Load More" }
}
}
}
}

7
src/partials/layout.rs
View File

@ -95,9 +95,8 @@ impl Layout {
meta charset="utf-8";
title { (self.title) }
// TODO: vendor this before going to prod
script type="module" {
r#"import * as Turbo from 'https://cdn.skypack.dev/@hotwired/turbo';"#
}
script src="https://unpkg.com/htmx.org@1.9.5" integrity="sha384-xcuj3WpfgjlKF+FXhSQFQ0ZNr39ln+hwjN3npfM9VBnUskLolQAcN80McRIVOPuO" crossorigin="anonymous" {}
script src="https://unpkg.com/htmx.org/dist/ext/sse.js" {}
@for js_bundle in js_bundles() {
script type="module" src=(js_bundle) {}
}
@ -105,7 +104,7 @@ impl Layout {
link rel="stylesheet" href=(css_bundle) {}
}
}
body {
body hx-booster="true" {
(header(&self.title))
(template)
}

3
src/partials/mod.rs
View File

@ -1,5 +1,8 @@
pub mod add_feed_form;
pub mod entry_link;
pub mod entry_list;
pub mod feed_link;
pub mod feed_list;
pub mod header;
pub mod layout;
pub mod opml_import_form;

24
src/partials/opml_import_form.rs Normal file
View File

@ -0,0 +1,24 @@
use maud::{html, Markup, PreEscaped};
pub fn opml_import_form() -> Markup {
html! {
form id="opml-import-form" hx-post="/import/opml" hx-encoding="multipart/form-data" class="feed-form" {
div class="form-grid" {
label for="opml" { "OPML: " }
input type="file" id="opml" name="opml" required="true" accept="text/x-opml,application/xml,text/xml";
button type="submit" { "Import Feeds" }
progress id="opml-upload-progress" max="100" value="0" hidden="true" {}
}
script {
(PreEscaped(r#"
htmx.on('#opml-import-form', 'htmx:xhr:progress', function (evt) {
htmx.find('#opml-upload-progress').setAttribute(
'value',
evt.detail.loaded / evt.detail.total * 100,
);
});
"#))
}
}
}
}

34
src/turbo_stream.rs
View File

@ -1,34 +0,0 @@
use axum::response::{IntoResponse, Response};
use axum::http::{header, HeaderValue};
use axum::body::{Bytes, Full};
/// A Turbo Stream HTML response.
///
/// See [the Turbo Streams specification](https://turbo.hotwire.dev/handbook/streams) for more
/// details.
///
/// Will automatically get `Content-Type: text/vnd.turbo-stream.html`.
#[derive(Clone, Copy, Debug)]
pub struct TurboStream<T>(pub T);
impl<T> IntoResponse for TurboStream<T>
where
T: Into<Full<Bytes>>,
{
fn into_response(self) -> Response {
(
[(
header::CONTENT_TYPE,
HeaderValue::from_static("text/vnd.turbo-stream.html"),
)],
self.0.into(),
)
.into_response()
}
}
impl<T> From<T> for TurboStream<T> {
fn from(inner: T) -> Self {
Self(inner)
}
}