Add retry logic to API calls

Since apparently these can 500 occasionally.
This commit is contained in:
Tyler Hallada 2021-07-23 22:29:28 -04:00
parent 81f12da99c
commit 19126f4981
2 changed files with 59 additions and 32 deletions

View File

@ -5,6 +5,7 @@ use serde_json::Value;
use std::{env, time::Duration}; use std::{env, time::Duration};
use tempfile::tempfile; use tempfile::tempfile;
use tokio::fs::File; use tokio::fs::File;
use tokio::time::sleep;
use tokio_util::compat::FuturesAsyncReadCompatExt; use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{info, instrument, warn}; use tracing::{info, instrument, warn};
@ -17,7 +18,8 @@ pub struct DownloadLinkResponse {
#[instrument(skip(client))] #[instrument(skip(client))]
pub async fn get(client: &Client, mod_id: i32, file_id: i64) -> Result<DownloadLinkResponse> { pub async fn get(client: &Client, mod_id: i32, file_id: i64) -> Result<DownloadLinkResponse> {
let res = client for attempt in 1..=3 {
let res = match client
.get(format!( .get(format!(
"https://api.nexusmods.com/v1/games/{}/mods/{}/files/{}/download_link.json", "https://api.nexusmods.com/v1/games/{}/mods/{}/files/{}/download_link.json",
GAME_NAME, mod_id, file_id GAME_NAME, mod_id, file_id
@ -27,13 +29,25 @@ pub async fn get(client: &Client, mod_id: i32, file_id: i64) -> Result<DownloadL
.header("user-agent", USER_AGENT) .header("user-agent", USER_AGENT)
.send() .send()
.await? .await?
.error_for_status()?; .error_for_status()
{
Ok(res) => res,
Err(err) => {
warn!(error = %err, attempt, "Failed to get download link for file, trying again after 1 second");
sleep(std::time::Duration::from_secs(1)).await;
continue;
}
};
info!(status = %res.status(), "fetched file download link from API"); info!(status = %res.status(), "fetched file download link from API");
let wait = rate_limit_wait_duration(&res)?; let wait = rate_limit_wait_duration(&res)?;
let json = res.json::<Value>().await?; let json = res.json::<Value>().await?;
Ok(DownloadLinkResponse { wait, json }) return Ok(DownloadLinkResponse { wait, json });
}
Err(anyhow!(
"Failed to get download link for file in three attempts"
))
} }
impl DownloadLinkResponse { impl DownloadLinkResponse {
@ -76,7 +90,8 @@ impl DownloadLinkResponse {
return Ok(tokio_file); return Ok(tokio_file);
} }
Err(err) => { Err(err) => {
warn!(error = %err, attempt, "Failed to download file, trying again"); warn!(error = %err, attempt, "Failed to download file, trying again after 1 second");
sleep(std::time::Duration::from_secs(1)).await;
} }
} }
} }

View File

@ -3,7 +3,8 @@ use chrono::NaiveDateTime;
use reqwest::Client; use reqwest::Client;
use serde_json::Value; use serde_json::Value;
use std::{env, time::Duration}; use std::{env, time::Duration};
use tracing::{info, instrument}; use tokio::time::sleep;
use tracing::{info, instrument, warn};
use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT}; use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT};
@ -26,7 +27,8 @@ pub struct ApiFile<'a> {
#[instrument(skip(client))] #[instrument(skip(client))]
pub async fn get(client: &Client, nexus_mod_id: i32) -> Result<FilesResponse> { pub async fn get(client: &Client, nexus_mod_id: i32) -> Result<FilesResponse> {
let res = client for attempt in 1..=3 {
let res = match client
.get(format!( .get(format!(
"https://api.nexusmods.com/v1/games/{}/mods/{}/files.json", "https://api.nexusmods.com/v1/games/{}/mods/{}/files.json",
GAME_NAME, nexus_mod_id GAME_NAME, nexus_mod_id
@ -36,13 +38,23 @@ pub async fn get(client: &Client, nexus_mod_id: i32) -> Result<FilesResponse> {
.header("user-agent", USER_AGENT) .header("user-agent", USER_AGENT)
.send() .send()
.await? .await?
.error_for_status()?; .error_for_status()
{
Ok(res) => res,
Err(err) => {
warn!(error = %err, attempt, "Failed to get files for mod, trying again after 1 second");
sleep(std::time::Duration::from_secs(1)).await;
continue;
}
};
info!(status = %res.status(), "fetched files for mod from API"); info!(status = %res.status(), "fetched files for mod from API");
let wait = rate_limit_wait_duration(&res)?; let wait = rate_limit_wait_duration(&res)?;
let json = res.json::<Value>().await?; let json = res.json::<Value>().await?;
Ok(FilesResponse { wait, json }) return Ok(FilesResponse { wait, json });
}
Err(anyhow!("Failed to get files for mod in three attempts"))
} }
impl FilesResponse { impl FilesResponse {