Adds download_at to files, skips files that are impossible to download

drop_all.pgsql

@@ -0,0 +1,13 @@
+/* !!! THIS DROPS ALL TABLES IN THE DATABASE WHICH DELETES ALL DATA IN THE DATABASE !!!
+ *
+ * ONLY RUN IN DEVELOPMENT!
+ */
+DROP TABLE _sqlx_migrations CASCADE;
+DROP TABLE games CASCADE;
+DROP TABLE mods CASCADE;
+DROP TABLE files CASCADE;
+DROP TABLE plugins CASCADE;
+DROP TABLE cells CASCADE;
+DROP TABLE worlds CASCADE;
+DROP TABLE plugin_cells CASCADE;
+DROP TABLE plugin_worlds CASCADE;

migrations/20210729031521_add_downloaded_at_to_files.sql

@@ -0,0 +1,8 @@
+ALTER TABLE "files" ADD COLUMN "downloaded_at" TIMESTAMP(3);
+
+/* Backfill existing columns the created_at timestamps.
+ *
+ * This is approximate since usually the file was downloaded shortly after the record was created.
+ * I mostly only care whether it is null or not null. All existing files need non-null values.
+ */
+UPDATE "files" SET "downloaded_at" = "created_at";

src/main.rs

@@ -304,7 +304,7 @@ pub async fn main() -> Result<()> {
                         if reqwest_err.status() == Some(StatusCode::NOT_FOUND) {
                             warn!(
                                 status = ?reqwest_err.status(),
-                                "failed to get download link for file"
+                                "failed to get download link for file, skipping file"
                             );
                             file::update_has_download_link(&pool, db_file.id, false).await?;
                             continue;
@@ -312,14 +312,18 @@ pub async fn main() -> Result<()> {
                     }
                 }
                 let download_link_resp = download_link_resp?;
-                let mut tokio_file = download_link_resp.download_file(&client).await?;
-                info!(bytes = api_file.size, "download finished");
 
-                create_dir_all(format!(
-                    "plugins/{}/{}/{}",
-                    GAME_NAME, db_mod.nexus_mod_id, db_file.nexus_file_id
-                ))
-                .await?;
+                let mut tokio_file = match download_link_resp.download_file(&client).await {
+                    Ok(file) => {
+                        info!(bytes = api_file.size, "download finished");
+                        file::update_downloaded_at(&pool, db_file.id).await?;
+                        file
+                    }
+                    Err(err) => {
+                        warn!(error = %err, "failed all attempts at downloading file, skipping file");
+                        continue;
+                    }
+                };
 
                 let mut initial_bytes = [0; 8];
                 tokio_file.seek(SeekFrom::Start(0)).await?;

src/models/file.rs

@@ -18,6 +18,7 @@ pub struct File {
     pub has_download_link: bool,
     pub updated_at: NaiveDateTime,
     pub created_at: NaiveDateTime,
+    pub downloaded_at: Option<NaiveDateTime>,
 }
 
 #[instrument(level = "debug", skip(pool))]
@@ -91,3 +92,18 @@ pub async fn update_has_download_link(
     .await
     .context("Failed to update file")
 }
+
+#[instrument(level = "debug", skip(pool))]
+pub async fn update_downloaded_at(pool: &sqlx::Pool<sqlx::Postgres>, id: i32) -> Result<File> {
+    sqlx::query_as!(
+        File,
+        "UPDATE files
+            SET downloaded_at = now()
+            WHERE id = $1
+            RETURNING *",
+        id,
+    )
+    .fetch_one(pool)
+    .await
+    .context("Failed to update file")
+}