From af5c11acc74fde674a054c9024b4de93e405653d Mon Sep 17 00:00:00 2001
From: Tyler Hallada <tyler@hallada.net>
Date: Mon, 12 Jul 2021 12:15:07 -0400
Subject: [PATCH] Tweak rate limit delay logic, log skipped files

---
 src/main.rs                    | 26 ++++++++++++++++----------
 src/nexus_api/download_link.rs |  2 +-
 src/nexus_api/files.rs         |  2 +-
 src/nexus_api/mod.rs           | 28 +++++++++++++++++-----------
 4 files changed, 35 insertions(+), 23 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 9c0001b..8270070 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -152,16 +152,24 @@ pub async fn main() -> Result<()> {
             let _mod_span = mod_span.enter();
             let files_resp = nexus_api::files::get(&client, db_mod.nexus_mod_id).await?;
 
-            if let Some(duration) = files_resp.wait {
-                debug!(?duration, "sleeping");
-                sleep(duration).await;
-            }
+            debug!(duration = ?files_resp.wait, "sleeping");
+            sleep(files_resp.wait).await;
 
             // Filter out replaced/deleted files (indicated by null category)
             let files = files_resp
                 .files()?
                 .into_iter()
-                .filter(|file| file.category.is_some());
+                .filter(|file| match file.category {
+                    Some(_) => true,
+                    None => {
+                        info!(
+                            name = file.file_name,
+                            id = file.file_id,
+                            "skipping file with no category"
+                        );
+                        false
+                    }
+                });
 
             for api_file in files {
                 let file_span =
@@ -326,16 +334,14 @@ pub async fn main() -> Result<()> {
                 }
 
                 plugins_archive.finish()?;
-                if let Some(duration) = download_link_resp.wait {
-                    debug!(?duration, "sleeping");
-                    sleep(duration).await;
-                }
+                debug!(duration = ?download_link_resp.wait, "sleeping");
+                sleep(download_link_resp.wait).await;
             }
         }
 
         page += 1;
         debug!(?page, ?has_next_page, "sleeping 1 second");
-        sleep(Duration::new(1, 0)).await;
+        sleep(Duration::from_secs(1)).await;
     }
 
     Ok(())
diff --git a/src/nexus_api/download_link.rs b/src/nexus_api/download_link.rs
index 488153a..fa983e9 100644
--- a/src/nexus_api/download_link.rs
+++ b/src/nexus_api/download_link.rs
@@ -11,7 +11,7 @@ use tracing::{info, instrument};
 use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT};
 
 pub struct DownloadLinkResponse {
-    pub wait: Option<Duration>,
+    pub wait: Duration,
     json: Value,
 }
 
diff --git a/src/nexus_api/files.rs b/src/nexus_api/files.rs
index 656038a..5066f1b 100644
--- a/src/nexus_api/files.rs
+++ b/src/nexus_api/files.rs
@@ -8,7 +8,7 @@ use tracing::{info, instrument};
 use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT};
 
 pub struct FilesResponse {
-    pub wait: Option<Duration>,
+    pub wait: Duration,
     json: Value,
 }
 
diff --git a/src/nexus_api/mod.rs b/src/nexus_api/mod.rs
index aa21fab..c5550b5 100644
--- a/src/nexus_api/mod.rs
+++ b/src/nexus_api/mod.rs
@@ -12,22 +12,28 @@ pub static GAME_NAME: &str = "skyrimspecialedition";
 pub const GAME_ID: u32 = 1704;
 pub static USER_AGENT: &str = "mod-mapper/0.1";
 
-pub fn rate_limit_wait_duration(res: &Response) -> Result<Option<std::time::Duration>> {
-    let daily_remaining = res
+pub fn rate_limit_wait_duration(res: &Response) -> Result<std::time::Duration> {
+    let daily_remaining: i32 = res
         .headers()
         .get("x-rl-daily-remaining")
-        .expect("No daily remaining in response headers");
-    let hourly_remaining = res
+        .expect("daily remaining in response headers")
+        .to_str()?
+        .parse()
+        .expect("daily remaining in response headers to be a number");
+    let hourly_remaining: i32 = res
         .headers()
         .get("x-rl-hourly-remaining")
-        .expect("No hourly limit in response headers");
+        .expect("hourly remaining in response headers")
+        .to_str()?
+        .parse()
+        .expect("hourly remaining in response headers to be a number");
     let hourly_reset = res
         .headers()
         .get("x-rl-hourly-reset")
-        .expect("No hourly reset in response headers");
-    info!(daily_remaining = ?daily_remaining, hourly_remaining = ?hourly_remaining, "rate limit check");
+        .expect("hourly reset in response headers");
+    info!(daily_remaining, hourly_remaining, "rate limit check");
 
-    if hourly_remaining == "0" {
+    if daily_remaining == 0 && hourly_remaining == 0 {
         let hourly_reset = hourly_reset.to_str()?.trim();
         let hourly_reset: DateTime<Utc> =
             (DateTime::parse_from_str(hourly_reset, "%Y-%m-%d %H:%M:%S %z")?
@@ -39,8 +45,8 @@ pub fn rate_limit_wait_duration(res: &Response) -> Result<Option<std::time::Dura
             duration = ?duration, "need to wait until rate-limit hourly reset"
         );
 
-        return Ok(Some(duration));
+        Ok(duration)
+    } else {
+        Ok(std::time::Duration::from_secs(1))
     }
-
-    Ok(None)
 }