thallada/crawlnicle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix crawl_feed_interval_minutes update logic

Browse Source 
Was just going down to minimum for all feeds that weren't updating. I need to use the current timestamp if there is no new entry.
This commit is contained in:
Tyler Hallada 2024-01-31 22:25:05 -05:00
parent 4eee21caed
commit 2e5ff1337c
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/actors/feed_crawler.rs
View File

@ -200,7 +200,13 @@ impl FeedCrawler {
let last_entry_published_at = parsed_feed.entries.iter().filter_map(|e| e.published).max(); let last_entry_published_at = parsed_feed.entries.iter().filter_map(|e| e.published).max();
if let Some(prev_last_entry_published_at) = feed.last_entry_published_at { if let Some(prev_last_entry_published_at) = feed.last_entry_published_at {
if let Some(published_at) = last_entry_published_at { if let Some(published_at) = last_entry_published_at {
let time_since_last_entry = published_at - prev_last_entry_published_at; let time_since_last_entry = if published_at == prev_last_entry_published_at {
// No new entry since last crawl, compare current time to last publish instead
Utc::now() - prev_last_entry_published_at
} else {
// Compare new entry publish time to previous publish time
published_at - prev_last_entry_published_at
};
match time_since_last_entry match time_since_last_entry
.cmp(&Duration::minutes(feed.crawl_interval_minutes.into())) .cmp(&Duration::minutes(feed.crawl_interval_minutes.into()))
{ {