From 0fa0cfc508c3c1fe9f7a230a04cfa803a7ab412c Mon Sep 17 00:00:00 2001
From: Tyler Hallada <tyler@hallada.net>
Date: Fri, 14 Jul 2023 21:27:23 -0400
Subject: [PATCH] Only crawl new entries

---
 src/actors/feed_crawler.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/actors/feed_crawler.rs b/src/actors/feed_crawler.rs
index a387969..cd44080 100644
--- a/src/actors/feed_crawler.rs
+++ b/src/actors/feed_crawler.rs
@@ -135,9 +135,12 @@ impl FeedCrawler {
         let entries = upsert_entries(&self.pool, payload)
             .await
             .map_err(|_| FeedCrawlerError::CreateFeedEntriesError(url.clone()))?;
-        info!("Created {} entries", entries.len());
+        let (new, updated) = entries
+            .into_iter()
+            .partition::<Vec<_>, _>(|entry| entry.updated_at.is_none());
+        info!(new = new.len(), updated = updated.len(), "saved entries");
 
-        for entry in entries {
+        for entry in new {
             let entry_crawler = EntryCrawlerHandle::new(
                 self.pool.clone(),
                 self.client.clone(),