Modularize scraping and api requests

src/nexus_api/download_link.rs

@@ -0,0 +1,70 @@
+use anyhow::{anyhow, Result};
+use futures::TryStreamExt;
+use reqwest::Client;
+use serde_json::Value;
+use std::{env, time::Duration};
+use tempfile::tempfile;
+use tokio::fs::File;
+use tokio_util::compat::FuturesAsyncReadCompatExt;
+
+use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT};
+
+pub struct DownloadLinkResponse {
+    pub wait: Option<Duration>,
+    json: Value,
+}
+
+pub async fn get(client: &Client, mod_id: i32, file_id: i64) -> Result<DownloadLinkResponse> {
+    let res = client
+        .get(format!(
+            "https://api.nexusmods.com/v1/games/{}/mods/{}/files/{}/download_link.json",
+            GAME_NAME, mod_id, file_id
+        ))
+        .header("accept", "application/json")
+        .header("apikey", env::var("NEXUS_API_KEY")?)
+        .header("user-agent", USER_AGENT)
+        .send()
+        .await?
+        .error_for_status()?;
+
+    let wait = rate_limit_wait_duration(&res)?;
+    let json = res.json::<Value>().await?;
+
+    Ok(DownloadLinkResponse { wait, json })
+}
+
+impl DownloadLinkResponse {
+    pub fn link<'a>(&'a self) -> Result<&'a str> {
+        let link = self
+            .json
+            .get(0)
+            .ok_or_else(|| anyhow!("Links array in API response is missing first element"))?
+            .get("URI")
+            .ok_or_else(|| anyhow!("Missing URI key in link in API response"))?
+            .as_str()
+            .ok_or_else(|| anyhow!("URI value in API response link is not a string"))?;
+        Ok(link)
+    }
+
+    pub async fn download_file(&self, client: &Client) -> Result<File> {
+        let mut tokio_file = File::from_std(tempfile()?);
+        let res = client
+            .get(self.link()?)
+            .header("apikey", env::var("NEXUS_API_KEY")?)
+            .header("user-agent", USER_AGENT)
+            .send()
+            .await?
+            .error_for_status()?;
+
+        // See: https://github.com/benkay86/async-applied/blob/master/reqwest-tokio-compat/src/main.rs
+        let mut byte_stream = res
+            .bytes_stream()
+            .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e))
+            .into_async_read()
+            .compat();
+
+        tokio::io::copy(&mut byte_stream, &mut tokio_file).await?;
+
+        return Ok(tokio_file);
+    }
+}

src/nexus_api/files.rs

@@ -0,0 +1,107 @@
+use anyhow::{anyhow, Result};
+use chrono::NaiveDateTime;
+use reqwest::Client;
+use serde_json::Value;
+use std::{env, time::Duration};
+
+use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT};
+
+pub struct FilesResponse {
+    pub wait: Option<Duration>,
+    json: Value,
+}
+
+pub struct ApiFile<'a> {
+    pub file_id: i64,
+    pub name: &'a str,
+    pub file_name: &'a str,
+    pub category: Option<&'a str>,
+    pub version: Option<&'a str>,
+    pub mod_version: Option<&'a str>,
+    pub uploaded_at: NaiveDateTime,
+}
+
+pub async fn get(client: &Client, nexus_mod_id: i32) -> Result<FilesResponse> {
+    let res = client
+        .get(format!(
+            "https://api.nexusmods.com/v1/games/{}/mods/{}/files.json",
+            GAME_NAME, nexus_mod_id
+        ))
+        .header("accept", "application/json")
+        .header("apikey", env::var("NEXUS_API_KEY")?)
+        .header("user-agent", USER_AGENT)
+        .send()
+        .await?
+        .error_for_status()?;
+
+    let wait = rate_limit_wait_duration(&res)?;
+    let json = res.json::<Value>().await?;
+
+    Ok(FilesResponse { wait, json })
+}
+
+impl FilesResponse {
+    pub fn files<'a>(&'a self) -> Result<Vec<ApiFile<'a>>> {
+        let files = self
+            .json
+            .get("files")
+            .ok_or_else(|| anyhow!("Missing files key in API response"))?
+            .as_array()
+            .ok_or_else(|| anyhow!("files value in API response is not an array"))?;
+        files
+            .into_iter()
+            .map(|file| {
+                let file_id = file
+                    .get("file_id")
+                    .ok_or_else(|| anyhow!("Missing file_id key in file in API response"))?
+                    .as_i64()
+                    .ok_or_else(|| anyhow!("file_id value in API response file is not a number"))?;
+                dbg!(file_id);
+                let name = file
+                    .get("name")
+                    .ok_or_else(|| anyhow!("Missing name key in file in API response"))?
+                    .as_str()
+                    .ok_or_else(|| anyhow!("name value in API response file is not a string"))?;
+                let file_name = file
+                    .get("file_name")
+                    .ok_or_else(|| anyhow!("Missing file_name key in file in API response"))?
+                    .as_str()
+                    .ok_or_else(|| {
+                        anyhow!("file_name value in API response file is not a string")
+                    })?;
+                let category = file
+                    .get("category_name")
+                    .ok_or_else(|| anyhow!("Missing category key in file in API response"))?
+                    .as_str();
+                let version = file
+                    .get("version")
+                    .ok_or_else(|| anyhow!("Missing version key in file in API response"))?
+                    .as_str();
+                let mod_version = file
+                    .get("mod_version")
+                    .ok_or_else(|| anyhow!("Missing mod_version key in file in API response"))?
+                    .as_str();
+                let uploaded_timestamp = file
+                    .get("uploaded_timestamp")
+                    .ok_or_else(|| {
+                        anyhow!("Missing uploaded_timestamp key in file in API response")
+                    })?
+                    .as_i64()
+                    .ok_or_else(|| {
+                        anyhow!("uploaded_timestamp value in API response file is not a number")
+                    })?;
+                let uploaded_at = NaiveDateTime::from_timestamp(uploaded_timestamp, 0);
+
+                Ok(ApiFile {
+                    file_id,
+                    name,
+                    file_name,
+                    category,
+                    version,
+                    mod_version,
+                    uploaded_at,
+                })
+            })
+            .collect()
+    }
+}

src/nexus_api/mod.rs

@@ -0,0 +1,44 @@
+use anyhow::Result;
+use chrono::DateTime;
+use chrono::Duration;
+use chrono::Utc;
+use reqwest::Response;
+
+pub mod download_link;
+pub mod files;
+
+pub static GAME_NAME: &str = "skyrimspecialedition";
+pub const GAME_ID: u32 = 1704;
+pub static USER_AGENT: &str = "mod-mapper/0.1";
+
+pub fn rate_limit_wait_duration(res: &Response) -> Result<Option<std::time::Duration>> {
+    let daily_remaining = res
+        .headers()
+        .get("x-rl-daily-remaining")
+        .expect("No daily remaining in response headers");
+    let hourly_remaining = res
+        .headers()
+        .get("x-rl-hourly-remaining")
+        .expect("No hourly limit in response headers");
+    let hourly_reset = res
+        .headers()
+        .get("x-rl-hourly-reset")
+        .expect("No hourly reset in response headers");
+    dbg!(daily_remaining);
+    dbg!(hourly_remaining);
+
+    if hourly_remaining == "0" {
+        let hourly_reset = hourly_reset.to_str()?.trim();
+        let hourly_reset: DateTime<Utc> =
+            (DateTime::parse_from_str(hourly_reset, "%Y-%m-%d %H:%M:%S %z")?
+                + Duration::seconds(5))
+            .into();
+        dbg!(hourly_reset);
+        let duration = (hourly_reset - Utc::now()).to_std()?;
+        dbg!(duration);
+
+        return Ok(Some(duration));
+    }
+
+    Ok(None)
+}