Skip downloading files with no plugins

Can determine if file contains plugin by recursively walking the file metadata listing until a .esp/.esm/.esl is found.
This commit is contained in:
Tyler Hallada 2021-07-12 19:33:43 -04:00
parent fd8015024f
commit 93d4bfeca2
4 changed files with 81 additions and 1 deletions

View File

@ -189,7 +189,16 @@ pub async fn main() -> Result<()> {
) )
.await?; .await?;
// TODO: check the file metadata to see if there are any plugin files in the archive before bothering to download the file (checking metadata does not count against rate-limit) if let Some(contains_plugin) =
nexus_api::metadata::contains_plugin(&client, &api_file).await?
{
if !contains_plugin {
info!("file metadata does not contain a plugin, skip downloading");
continue;
}
} else {
warn!("file has no metadata link");
}
let download_link_resp = let download_link_resp =
nexus_api::download_link::get(&client, db_mod.nexus_mod_id, api_file.file_id) nexus_api::download_link::get(&client, db_mod.nexus_mod_id, api_file.file_id)

View File

@ -20,6 +20,7 @@ pub struct ApiFile<'a> {
pub version: Option<&'a str>, pub version: Option<&'a str>,
pub mod_version: Option<&'a str>, pub mod_version: Option<&'a str>,
pub size: i64, pub size: i64,
pub content_preview_link: Option<&'a str>,
pub uploaded_at: NaiveDateTime, pub uploaded_at: NaiveDateTime,
} }
@ -89,6 +90,10 @@ impl FilesResponse {
.get("size_in_bytes") .get("size_in_bytes")
.ok_or_else(|| anyhow!("Missing size_in_bytes key in file in API response"))? .ok_or_else(|| anyhow!("Missing size_in_bytes key in file in API response"))?
.as_i64(); .as_i64();
let content_preview_link = file
.get("content_preview_link")
.ok_or_else(|| anyhow!("Missing content_preview_link key in file in API response"))?
.as_str();
let size = if let Some(size) = size { let size = if let Some(size) = size {
size size
} else { } else {
@ -120,6 +125,7 @@ impl FilesResponse {
version, version,
mod_version, mod_version,
size, size,
content_preview_link,
uploaded_at, uploaded_at,
}) })
}) })

64
src/nexus_api/metadata.rs Normal file
View File

@ -0,0 +1,64 @@
use anyhow::{anyhow, Result};
use reqwest::Client;
use serde_json::Value;
use std::env;
use tracing::{info, instrument};
use super::files::ApiFile;
use super::USER_AGENT;
fn has_plugin(json: &Value) -> Result<bool> {
let node_type = json
.get("type")
.ok_or_else(|| anyhow!("Missing type key in metadata API response"))?
.as_str()
.ok_or_else(|| anyhow!("type value in metadata is not a string"))?;
if node_type == "file" {
let name = json
.get("name")
.ok_or_else(|| anyhow!("Missing name key in metadata API response"))?
.as_str()
.ok_or_else(|| anyhow!("name value in metadata is not a string"))?;
if name.ends_with(".esp") || name.ends_with(".esm") || name.ends_with(".esl") {
return Ok(true);
}
}
match json.get("children") {
None => Ok(false),
Some(children) => {
let children = children
.as_array()
.ok_or_else(|| anyhow!("children value in metadata is not an array"))?;
for child in children {
if has_plugin(child)? {
return Ok(true);
}
}
Ok(false)
}
}
}
#[instrument(skip(client, api_file), fields(metadata_link = api_file.content_preview_link.unwrap_or("null")))]
pub async fn contains_plugin(client: &Client, api_file: &ApiFile<'_>) -> Result<Option<bool>> {
if let Some(metadata_link) = api_file.content_preview_link {
let res = client
.get(metadata_link)
.header("accept", "application/json")
.header("apikey", env::var("NEXUS_API_KEY")?)
.header("user-agent", USER_AGENT)
.send()
.await?
.error_for_status()?;
info!(status = %res.status(), "fetched file metadata from API");
let json = res.json::<Value>().await?;
Ok(Some(has_plugin(&json)?))
} else {
Ok(None)
}
}

View File

@ -7,6 +7,7 @@ use tracing::info;
pub mod download_link; pub mod download_link;
pub mod files; pub mod files;
pub mod metadata;
pub static GAME_NAME: &str = "skyrimspecialedition"; pub static GAME_NAME: &str = "skyrimspecialedition";
pub const GAME_ID: u32 = 1704; pub const GAME_ID: u32 = 1704;