thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Retry connect failures, write plugins to disk instead of zip archive

Browse Source 
Writing to the zip was starting to take forever. It makes more sense to just use my big HD and then zip after I'm done downloading every file.
This commit is contained in:
Tyler Hallada
2021-07-26 19:31:25 -04:00
parent f62324d36c
commit 8a356ac7f5
6 changed files with 80 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
src/nexus_api/download_link.rs
View File

@@ -5,11 +5,10 @@ use serde_json::Value;
use std::{env, time::Duration};
use tempfile::tempfile;
use tokio::fs::File;
use tokio::time::sleep;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{info, instrument, warn};
use tracing::{info, instrument};
use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT};
use super::{rate_limit_wait_duration, warn_and_sleep, GAME_NAME, USER_AGENT};
pub struct DownloadLinkResponse {
pub wait: Duration,
@@ -28,13 +27,17 @@ pub async fn get(client: &Client, mod_id: i32, file_id: i64) -> Result<DownloadL
.header("apikey", env::var("NEXUS_API_KEY")?)
.header("user-agent", USER_AGENT)
.send()
.await?
.error_for_status()
.await
{
Ok(res) => res,
Ok(res) => match res.error_for_status() {
Ok(res) => res,
Err(err) => {
warn_and_sleep("download_link::get", anyhow!(err), attempt).await;
continue;
}
},
Err(err) => {
warn!(error = %err, attempt, "Failed to get download link for file, trying again after 1 second");
sleep(std::time::Duration::from_secs(1)).await;
warn_and_sleep("download_link::get", anyhow!(err), attempt).await;
continue;
}
};
@@ -69,13 +72,25 @@ impl DownloadLinkResponse {
pub async fn download_file(&self, client: &Client) -> Result<File> {
for attempt in 1..=3 {
let mut tokio_file = File::from_std(tempfile()?);
let res = client
let res = match client
.get(self.link()?)
.header("apikey", env::var("NEXUS_API_KEY")?)
.header("user-agent", USER_AGENT)
.send()
.await?
.error_for_status()?;
.await
{
Ok(res) => match res.error_for_status() {
Ok(res) => res,
Err(err) => {
warn_and_sleep("download_link::download_file", anyhow!(err), attempt).await;
continue;
}
},
Err(err) => {
warn_and_sleep("download_link::download_file", anyhow!(err), attempt).await;
continue;
}
};
info!(status = %res.status(), "downloading file from nexus");
// See: https://github.com/benkay86/async-applied/blob/master/reqwest-tokio-compat/src/main.rs
@@ -90,8 +105,7 @@ impl DownloadLinkResponse {
return Ok(tokio_file);
}
Err(err) => {
warn!(error = %err, attempt, "Failed to download file, trying again after 1 second");
sleep(std::time::Duration::from_secs(1)).await;
warn_and_sleep("download_link::download_file", anyhow!(err), attempt).await
}
}
}

19
src/nexus_api/files.rs
View File

@@ -3,10 +3,9 @@ use chrono::NaiveDateTime;
use reqwest::Client;
use serde_json::Value;
use std::{env, time::Duration};
use tokio::time::sleep;
use tracing::{info, instrument, warn};
use tracing::{info, instrument};
use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT};
use super::{rate_limit_wait_duration, warn_and_sleep, GAME_NAME, USER_AGENT};
pub struct FilesResponse {
pub wait: Duration,
@@ -37,13 +36,17 @@ pub async fn get(client: &Client, nexus_mod_id: i32) -> Result<FilesResponse> {
.header("apikey", env::var("NEXUS_API_KEY")?)
.header("user-agent", USER_AGENT)
.send()
.await?
.error_for_status()
.await
{
Ok(res) => res,
Ok(res) => match res.error_for_status() {
Ok(res) => res,
Err(err) => {
warn_and_sleep("files::get", anyhow!(err), attempt).await;
continue;
}
},
Err(err) => {
warn!(error = %err, attempt, "Failed to get files for mod, trying again after 1 second");
sleep(std::time::Duration::from_secs(1)).await;
warn_and_sleep("files::get", anyhow!(err), attempt).await;
continue;
}
};

19
src/nexus_api/metadata.rs
View File

@@ -2,11 +2,10 @@ use anyhow::{anyhow, Result};
use reqwest::Client;
use serde_json::Value;
use std::env;
use tokio::time::sleep;
use tracing::{info, instrument, warn};
use tracing::{info, instrument};
use super::files::ApiFile;
use super::USER_AGENT;
use super::{warn_and_sleep, USER_AGENT};
fn has_plugin(json: &Value) -> Result<bool> {
let node_type = json
@@ -53,13 +52,17 @@ pub async fn contains_plugin(client: &Client, api_file: &ApiFile<'_>) -> Result<
.header("apikey", env::var("NEXUS_API_KEY")?)
.header("user-agent", USER_AGENT)
.send()
.await?
.error_for_status()
.await
{
Ok(res) => res,
Ok(res) => match res.error_for_status() {
Ok(res) => res,
Err(err) => {
warn_and_sleep("metadata::contains_plugin", anyhow!(err), attempt).await;
continue;
}
},
Err(err) => {
warn!(error = %err, attempt, "Failed to get metadata for file, trying again after 1 second");
sleep(std::time::Duration::from_secs(1)).await;
warn_and_sleep("metadata::contains_plugin", anyhow!(err), attempt).await;
continue;
}
};

8
src/nexus_api/mod.rs
View File

@@ -3,7 +3,8 @@ use chrono::DateTime;
use chrono::Duration;
use chrono::Utc;
use reqwest::Response;
use tracing::info;
use tokio::time::sleep;
use tracing::{info, warn};
pub mod download_link;
pub mod files;
@@ -51,3 +52,8 @@ pub fn rate_limit_wait_duration(res: &Response) -> Result<std::time::Duration> {
Ok(std::time::Duration::from_secs(1))
}
}
async fn warn_and_sleep(request_name: &str, err: anyhow::Error, attempt: i32) {
warn!(error = %err, attempt, "{} request failed, trying again after 1 second", request_name);
sleep(std::time::Duration::from_secs(1)).await;
}