From a8424e830e2249a99f95435cf142ee41a2c5286d Mon Sep 17 00:00:00 2001
From: Tyler Hallada <tyler@hallada.net>
Date: Wed, 28 Jul 2021 23:54:39 -0400
Subject: [PATCH] Adds download_at to files, skips files that are impossible to
 download

---
 drop_all.pgsql                                | 13 ++++++++++++
 ...10729031521_add_downloaded_at_to_files.sql |  8 ++++++++
 src/main.rs                                   | 20 +++++++++++--------
 src/models/file.rs                            | 16 +++++++++++++++
 4 files changed, 49 insertions(+), 8 deletions(-)
 create mode 100644 drop_all.pgsql
 create mode 100644 migrations/20210729031521_add_downloaded_at_to_files.sql

diff --git a/drop_all.pgsql b/drop_all.pgsql
new file mode 100644
index 0000000..cf4fa48
--- /dev/null
+++ b/drop_all.pgsql
@@ -0,0 +1,13 @@
+/* !!! THIS DROPS ALL TABLES IN THE DATABASE WHICH DELETES ALL DATA IN THE DATABASE !!!
+ *
+ * ONLY RUN IN DEVELOPMENT!
+ */
+DROP TABLE _sqlx_migrations CASCADE;
+DROP TABLE games CASCADE;
+DROP TABLE mods CASCADE;
+DROP TABLE files CASCADE;
+DROP TABLE plugins CASCADE;
+DROP TABLE cells CASCADE;
+DROP TABLE worlds CASCADE;
+DROP TABLE plugin_cells CASCADE;
+DROP TABLE plugin_worlds CASCADE;
\ No newline at end of file
diff --git a/migrations/20210729031521_add_downloaded_at_to_files.sql b/migrations/20210729031521_add_downloaded_at_to_files.sql
new file mode 100644
index 0000000..305b4b6
--- /dev/null
+++ b/migrations/20210729031521_add_downloaded_at_to_files.sql
@@ -0,0 +1,8 @@
+ALTER TABLE "files" ADD COLUMN "downloaded_at" TIMESTAMP(3);
+
+/* Backfill existing columns the created_at timestamps.
+ *
+ * This is approximate since usually the file was downloaded shortly after the record was created.
+ * I mostly only care whether it is null or not null. All existing files need non-null values.
+ */
+UPDATE "files" SET "downloaded_at" = "created_at";
\ No newline at end of file
diff --git a/src/main.rs b/src/main.rs
index d704506..ffa693f 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -304,7 +304,7 @@ pub async fn main() -> Result<()> {
                         if reqwest_err.status() == Some(StatusCode::NOT_FOUND) {
                             warn!(
                                 status = ?reqwest_err.status(),
-                                "failed to get download link for file"
+                                "failed to get download link for file, skipping file"
                             );
                             file::update_has_download_link(&pool, db_file.id, false).await?;
                             continue;
@@ -312,14 +312,18 @@ pub async fn main() -> Result<()> {
                     }
                 }
                 let download_link_resp = download_link_resp?;
-                let mut tokio_file = download_link_resp.download_file(&client).await?;
-                info!(bytes = api_file.size, "download finished");
 
-                create_dir_all(format!(
-                    "plugins/{}/{}/{}",
-                    GAME_NAME, db_mod.nexus_mod_id, db_file.nexus_file_id
-                ))
-                .await?;
+                let mut tokio_file = match download_link_resp.download_file(&client).await {
+                    Ok(file) => {
+                        info!(bytes = api_file.size, "download finished");
+                        file::update_downloaded_at(&pool, db_file.id).await?;
+                        file
+                    }
+                    Err(err) => {
+                        warn!(error = %err, "failed all attempts at downloading file, skipping file");
+                        continue;
+                    }
+                };
 
                 let mut initial_bytes = [0; 8];
                 tokio_file.seek(SeekFrom::Start(0)).await?;
diff --git a/src/models/file.rs b/src/models/file.rs
index 494f5d9..b256b4f 100644
--- a/src/models/file.rs
+++ b/src/models/file.rs
@@ -18,6 +18,7 @@ pub struct File {
     pub has_download_link: bool,
     pub updated_at: NaiveDateTime,
     pub created_at: NaiveDateTime,
+    pub downloaded_at: Option<NaiveDateTime>,
 }
 
 #[instrument(level = "debug", skip(pool))]
@@ -91,3 +92,18 @@ pub async fn update_has_download_link(
     .await
     .context("Failed to update file")
 }
+
+#[instrument(level = "debug", skip(pool))]
+pub async fn update_downloaded_at(pool: &sqlx::Pool<sqlx::Postgres>, id: i32) -> Result<File> {
+    sqlx::query_as!(
+        File,
+        "UPDATE files
+            SET downloaded_at = now()
+            WHERE id = $1
+            RETURNING *",
+        id,
+    )
+    .fetch_one(pool)
+    .await
+    .context("Failed to update file")
+}