thallada/crawlnicle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Only crawl new entries

Browse Source
This commit is contained in:
Tyler Hallada 2023-07-14 21:27:23 -04:00
parent 923776d7a1
commit 0fa0cfc508
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/actors/feed_crawler.rs
View File

@ -135,9 +135,12 @@ impl FeedCrawler {
let entries = upsert_entries(&self.pool, payload)
.await
.map_err(|_| FeedCrawlerError::CreateFeedEntriesError(url.clone()))?;
info!("Created {} entries", entries.len());
let (new, updated) = entries
.into_iter()
.partition::<Vec<_>, _>(|entry| entry.updated_at.is_none());
info!(new = new.len(), updated = updated.len(), "saved entries");
for entry in entries {
for entry in new {
let entry_crawler = EntryCrawlerHandle::new(
self.pool.clone(),
self.client.clone(),