thallada/crawlnicle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Begin adding pagination for feeds

Browse Source
This commit is contained in:
Tyler Hallada 2023-07-15 02:12:59 -04:00
parent ae95921966
commit 0dfde7cd31
5 changed files with 266 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
frontend/css/styles.css
View File

@ -103,7 +103,7 @@ div.feeds {
} }
} }
ul#feeds { .feeds-list {
grid-area: 'feeds'; grid-area: 'feeds';
list-style: none; list-style: none;
padding: 0; padding: 0;

2
src/handlers/api/feeds.rs
View File

@ -6,5 +6,5 @@ use crate::models::feed::Feed;
pub async fn get(State(pool): State<PgPool>) -> Result<Json<Vec<Feed>>, Error> { pub async fn get(State(pool): State<PgPool>) -> Result<Json<Vec<Feed>>, Error> {
// TODO: pagination // TODO: pagination
Ok(Json(Feed::get_all(&pool).await?)) Ok(Json(Feed::get_all(&pool, Default::default()).await?))
} }

4
src/handlers/feed.rs
View File

@ -14,7 +14,6 @@ use sqlx::PgPool;
use tokio_stream::wrappers::errors::BroadcastStreamRecvError; use tokio_stream::wrappers::errors::BroadcastStreamRecvError;
use tokio_stream::wrappers::BroadcastStream; use tokio_stream::wrappers::BroadcastStream;
use tokio_stream::StreamExt; use tokio_stream::StreamExt;
use url::Url;
use crate::actors::feed_crawler::{FeedCrawlerHandle, FeedCrawlerHandleMessage}; use crate::actors::feed_crawler::{FeedCrawlerHandle, FeedCrawlerHandleMessage};
use crate::config::Config; use crate::config::Config;
@ -145,8 +144,6 @@ pub async fn post(
AddFeedError::CreateFeedError(add_feed.url.clone(), err) AddFeedError::CreateFeedError(add_feed.url.clone(), err)
})?; })?;
let url: Url = Url::parse(&add_feed.url)
.map_err(|err| AddFeedError::InvalidUrl(add_feed.url.clone(), err))?;
let receiver = feed_crawler.crawl(feed.feed_id).await; let receiver = feed_crawler.crawl(feed.feed_id).await;
{ {
let mut crawls = crawls.lock().map_err(|_| { let mut crawls = crawls.lock().map_err(|_| {
@ -167,6 +164,7 @@ pub async fn post(
li id=(feed_id) { (feed_link(&feed, true)) } li id=(feed_id) { (feed_link(&feed, true)) }
} }
} }
turbo-stream action="remove" target="no-feeds";
} }
.into_string(), .into_string(),
), ),

17
src/handlers/feeds.rs
View File

@ -4,20 +4,31 @@ use maud::html;
use sqlx::PgPool; use sqlx::PgPool;
use crate::error::Result; use crate::error::Result;
use crate::models::feed::Feed; use crate::models::feed::{Feed, GetFeedsOptions, DEFAULT_FEEDS_PAGE_SIZE};
use crate::partials::{feed_link::feed_link, layout::Layout}; use crate::partials::{feed_link::feed_link, layout::Layout};
pub async fn get(State(pool): State<PgPool>, layout: Layout) -> Result<Response> { pub async fn get(State(pool): State<PgPool>, layout: Layout) -> Result<Response> {
// TODO: pagination let options = GetFeedsOptions::default();
let feeds = Feed::get_all(&pool).await?; let feeds = Feed::get_all(&pool, options.clone()).await?;
let len = feeds.len() as i64;
Ok(layout.render(html! { Ok(layout.render(html! {
h2 { "Feeds" } h2 { "Feeds" }
div class="feeds" { div class="feeds" {
div class="feeds-list" {
@if len == 0 {
p id="no-feeds" { "No feeds found." }
} else {
ul id="feeds" { ul id="feeds" {
@for feed in feeds { @for feed in feeds {
li { (feed_link(&feed, false)) } li { (feed_link(&feed, false)) }
} }
} }
}
// TODO: pagination
@if len == options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE) {
button id="load-more-feeds" { "Load More" }
}
}
div class="add-feed" { div class="add-feed" {
h3 { "Add Feed" } h3 { "Add Feed" }
form action="/feed" method="post" class="add-feed-form" { form action="/feed" method="post" class="add-feed-form" {

229
src/models/feed.rs
View File

@ -8,6 +8,8 @@ use validator::Validate;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
pub const DEFAULT_FEEDS_PAGE_SIZE: i64 = 50;
#[derive(Debug, Serialize, Deserialize, sqlx::Type, Clone, Copy)] #[derive(Debug, Serialize, Deserialize, sqlx::Type, Clone, Copy)]
#[sqlx(type_name = "feed_type", rename_all = "lowercase")] #[sqlx(type_name = "feed_type", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]
@ -105,6 +107,22 @@ pub struct UpdateFeed {
pub last_entry_published_at: Option<Option<DateTime<Utc>>>, pub last_entry_published_at: Option<Option<DateTime<Utc>>>,
} }
#[derive(Debug, Clone)]
pub enum GetFeedsSort {
Title,
CreatedAt,
LastCrawledAt,
LastEntryPublishedAt,
}
#[derive(Debug, Default, Clone)]
pub struct GetFeedsOptions {
pub sort: Option<GetFeedsSort>,
pub before: Option<DateTime<Utc>>,
pub after_title: Option<String>,
pub limit: Option<i64>,
}
impl Feed { impl Feed {
pub async fn get(pool: &PgPool, feed_id: Uuid) -> Result<Feed> { pub async fn get(pool: &PgPool, feed_id: Uuid) -> Result<Feed> {
sqlx::query_as!( sqlx::query_as!(
@ -137,7 +155,11 @@ impl Feed {
}) })
} }
pub async fn get_all(pool: &PgPool) -> sqlx::Result<Vec<Feed>> { pub async fn get_all(pool: &PgPool, options: GetFeedsOptions) -> sqlx::Result<Vec<Feed>> {
// TODO: make sure there are indices for all of these sort options
match options.sort.unwrap_or(GetFeedsSort::CreatedAt) {
GetFeedsSort::Title => {
if let Some(after_title) = options.after_title {
sqlx::query_as!( sqlx::query_as!(
Feed, Feed,
r#"select r#"select
@ -154,10 +176,213 @@ impl Feed {
updated_at, updated_at,
deleted_at deleted_at
from feed from feed
where deleted_at is null"# where deleted_at is null
and title > $1
order by title asc
limit $2
"#,
after_title,
options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE),
) )
.fetch_all(pool) .fetch_all(pool)
.await .await
} else {
sqlx::query_as!(
Feed,
r#"select
feed_id,
title,
url,
type as "feed_type: FeedType",
description,
crawl_interval_minutes,
last_crawl_error,
last_crawled_at,
last_entry_published_at,
created_at,
updated_at,
deleted_at
from feed
where deleted_at is null
order by title asc
limit $1
"#,
options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE),
)
.fetch_all(pool)
.await
}
}
GetFeedsSort::CreatedAt => {
if let Some(created_before) = options.before {
sqlx::query_as!(
Feed,
r#"select
feed_id,
title,
url,
type as "feed_type: FeedType",
description,
crawl_interval_minutes,
last_crawl_error,
last_crawled_at,
last_entry_published_at,
created_at,
updated_at,
deleted_at
from feed
where deleted_at is null
and created_at < $1
order by created_at desc
limit $2
"#,
created_before,
options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE),
)
.fetch_all(pool)
.await
} else {
sqlx::query_as!(
Feed,
r#"select
feed_id,
title,
url,
type as "feed_type: FeedType",
description,
crawl_interval_minutes,
last_crawl_error,
last_crawled_at,
last_entry_published_at,
created_at,
updated_at,
deleted_at
from feed
where deleted_at is null
order by created_at desc
limit $1
"#,
options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE),
)
.fetch_all(pool)
.await
}
}
GetFeedsSort::LastCrawledAt => {
if let Some(crawled_before) = options.before {
sqlx::query_as!(
Feed,
r#"select
feed_id,
title,
url,
type as "feed_type: FeedType",
description,
crawl_interval_minutes,
last_crawl_error,
last_crawled_at,
last_entry_published_at,
created_at,
updated_at,
deleted_at
from feed
where deleted_at is null
and last_crawled_at < $1
order by last_crawled_at desc
limit $2
"#,
crawled_before,
options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE),
)
.fetch_all(pool)
.await
} else {
sqlx::query_as!(
Feed,
r#"select
feed_id,
title,
url,
type as "feed_type: FeedType",
description,
crawl_interval_minutes,
last_crawl_error,
last_crawled_at,
last_entry_published_at,
created_at,
updated_at,
deleted_at
from feed
where deleted_at is null
order by last_crawled_at desc
limit $1
"#,
options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE),
)
.fetch_all(pool)
.await
}
}
GetFeedsSort::LastEntryPublishedAt => {
if let Some(published_before) = options.before {
sqlx::query_as!(
Feed,
r#"select
feed_id,
title,
url,
type as "feed_type: FeedType",
description,
crawl_interval_minutes,
last_crawl_error,
last_crawled_at,
last_entry_published_at,
created_at,
updated_at,
deleted_at
from feed
where deleted_at is null
and last_entry_published_at < $1
order by last_entry_published_at desc
limit $2
"#,
published_before,
options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE),
)
.fetch_all(pool)
.await
} else {
sqlx::query_as!(
Feed,
r#"select
feed_id,
title,
url,
type as "feed_type: FeedType",
description,
crawl_interval_minutes,
last_crawl_error,
last_crawled_at,
last_entry_published_at,
created_at,
updated_at,
deleted_at
from feed
where deleted_at is null
order by last_entry_published_at desc
limit $1
"#,
options.limit.unwrap_or(DEFAULT_FEEDS_PAGE_SIZE),
)
.fetch_all(pool)
.await
}
}
}
} }
pub async fn create(pool: &PgPool, payload: CreateFeed) -> Result<Feed> { pub async fn create(pool: &PgPool, payload: CreateFeed) -> Result<Feed> {