thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Start processing mod updates

Browse Source
This commit is contained in:
Tyler Hallada 2021-09-05 16:02:16 -04:00
parent 0989477b40
commit 804fac4eea
3 changed files with 70 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/main.rs
View File

@ -1,7 +1,8 @@
use anyhow::Result;
use argh::FromArgs;
use chrono::{NaiveDateTime, NaiveTime};
use dotenv::dotenv;
use humansize::{FileSize, file_size_opts};
use humansize::{file_size_opts, FileSize};
use models::file::File;
use models::game_mod::Mod;
use reqwest::StatusCode;
@ -160,14 +161,14 @@ async fn extract_with_unrar(
Err(err) => {
warn!(error = %err, "failed to extract with unrar");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
return Ok(())
return Ok(());
}
Ok(extract) => extract
Ok(extract) => extract,
};
if let Err(err) = extract.process() {
warn!(error = %err, "failed to extract with unrar");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
return Ok(())
return Ok(());
}
for file_path in plugin_file_paths.iter() {
@ -216,7 +217,7 @@ pub async fn main() -> Result<()> {
let scraped = mod_list_resp.scrape_mods()?;
has_next_page = scraped.has_next_page;
let processed_mods = game_mod::bulk_get_fully_processed_nexus_mod_ids(
let processed_mods = game_mod::bulk_get_last_updated_by_nexus_mod_ids(
&pool,
&scraped
.mods
@ -225,10 +226,22 @@ pub async fn main() -> Result<()> {
.collect::<Vec<i32>>(),
)
.await?;
let mods_to_create: Vec<UnsavedMod> = scraped
let mods_to_create_or_update: Vec<UnsavedMod> = scraped
.mods
.iter()
.filter(|scraped_mod| !processed_mods.contains(&scraped_mod.nexus_mod_id))
.filter(|scraped_mod| {
if let Some(processed_mod) = processed_mods
.iter()
.find(|processed_mod| processed_mod.nexus_mod_id == scraped_mod.nexus_mod_id)
{
if processed_mod.last_updated_files_at
> NaiveDateTime::new(scraped_mod.last_update, NaiveTime::from_hms(0, 0, 0))
{
return false;
}
}
true
})
.map(|scraped_mod| UnsavedMod {
name: scraped_mod.name,
nexus_mod_id: scraped_mod.nexus_mod_id,
@ -239,7 +252,7 @@ pub async fn main() -> Result<()> {
})
.collect();
let mods = game_mod::batched_insert(&pool, &mods_to_create).await?;
let mods = game_mod::batched_insert(&pool, &mods_to_create_or_update).await?;
for db_mod in mods {
let mod_span = info_span!("mod", name = ?&db_mod.name, id = &db_mod.nexus_mod_id);
@ -316,7 +329,9 @@ pub async fn main() -> Result<()> {
}
};
let humanized_size = api_file.size.file_size(file_size_opts::CONVENTIONAL)
let humanized_size = api_file
.size
.file_size(file_size_opts::CONVENTIONAL)
.expect("unable to create human-readable file size");
info!(size = %humanized_size, "decided to download file");
let download_link_resp =
@ -361,7 +376,7 @@ pub async fn main() -> Result<()> {
warn!(initial_bytes = ?initial_bytes, "unable to determine file type of archive, skipping file");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
continue;
},
}
};
info!(
mime_type = kind.mime_type(),
@ -375,7 +390,15 @@ pub async fn main() -> Result<()> {
// https://github.com/libarchive/libarchive/issues/373, https://github.com/libarchive/libarchive/issues/1426
tokio_file.seek(SeekFrom::Start(0)).await?;
let mut file = tokio_file.try_clone().await?.into_std().await;
match extract_with_unrar(&mut file, &pool, &db_file, &db_mod, checked_metadata).await {
match extract_with_unrar(
&mut file,
&pool,
&db_file,
&db_mod,
checked_metadata,
)
.await
{
Ok(_) => Ok(()),
Err(err) => {
// unrar failed to extract rar file (e.g. archive has unicode filenames)
@ -396,7 +419,8 @@ pub async fn main() -> Result<()> {
if err
.downcast_ref::<extractors::compress_tools::ExtractorError>()
.is_some()
&& (kind.mime_type() == "application/zip" || kind.mime_type() == "application/x-7z-compressed")
&& (kind.mime_type() == "application/zip"
|| kind.mime_type() == "application/x-7z-compressed")
{
// compress_tools or libarchive failed to extract zip/7z file (e.g. archive is deflate64 compressed)
// Attempt to uncompress the archive using `7z` unix command instead

20
src/models/game_mod.rs
View File

@ -45,21 +45,31 @@ pub async fn get_by_nexus_mod_id(
.context("Failed to get mod")
}
pub struct ModLastUpdatedFilesAt {
pub nexus_mod_id: i32,
pub last_updated_files_at: NaiveDateTime,
}
#[instrument(level = "debug", skip(pool))]
pub async fn bulk_get_fully_processed_nexus_mod_ids(
pub async fn bulk_get_last_updated_by_nexus_mod_ids(
pool: &sqlx::Pool<sqlx::Postgres>,
nexus_mod_ids: &[i32],
) -> Result<Vec<i32>> {
) -> Result<Vec<ModLastUpdatedFilesAt>> {
sqlx::query!(
"SELECT nexus_mod_id FROM mods
"SELECT nexus_mod_id, last_updated_files_at FROM mods
WHERE nexus_mod_id = ANY($1::int[])
AND last_updated_files_at IS NOT NULL",
nexus_mod_ids,
)
.map(|row| row.nexus_mod_id)
.map(|row| ModLastUpdatedFilesAt {
nexus_mod_id: row.nexus_mod_id,
last_updated_files_at: row
.last_updated_files_at
.expect("last_updated_files_at is null"),
})
.fetch_all(pool)
.await
.context("Failed to get fully processed , last_updated_files_at: () mods")
.context("Failed to bulk get last_updated_files_at by nexus_mod_ids")
}
#[instrument(level = "debug", skip(pool))]

23
src/nexus_scraper.rs
View File

@ -1,4 +1,5 @@
use anyhow::Result;
use chrono::NaiveDate;
use reqwest::Client;
use scraper::{Html, Selector};
use tracing::{info, instrument};
@ -8,12 +9,14 @@ use crate::nexus_api::GAME_ID;
pub struct ModListResponse {
html: Html,
}
pub struct ScrapedMod<'a> {
pub nexus_mod_id: i32,
pub name: &'a str,
pub category: Option<&'a str>,
pub author: &'a str,
pub desc: Option<&'a str>,
pub last_update: NaiveDate,
}
pub struct ModListScrape<'a> {
@ -25,7 +28,7 @@ pub struct ModListScrape<'a> {
pub async fn get_mod_list_page(client: &Client, page: usize) -> Result<ModListResponse> {
let res = client
.get(format!(
"https://www.nexusmods.com/Core/Libs/Common/Widgets/ModList?RH_ModList=nav:true,home:false,type:0,user_id:0,game_id:{},advfilt:true,include_adult:true,page_size:20,show_game_filter:false,open:false,page:{},sort_by:OLD_u_downloads",
"https://www.nexusmods.com/Core/Libs/Common/Widgets/ModList?RH_ModList=nav:true,home:false,type:0,user_id:0,game_id:{},advfilt:true,include_adult:true,page_size:20,show_game_filter:false,open:false,page:{},sort_by:lastupdate",
GAME_ID,
page
))
@ -52,6 +55,7 @@ impl ModListResponse {
Selector::parse("div.category a").expect("failed to parse CSS selector");
let author_select = Selector::parse("div.author a").expect("failed to parse CSS selector");
let desc_select = Selector::parse("p.desc").expect("failed to parse CSS selector");
let last_update_select = Selector::parse("div.date").expect("failed to parse CSS selector");
let next_page_select =
Selector::parse("div.pagination li.next").expect("failed to parse CSS selector");
@ -86,9 +90,7 @@ impl ModListResponse {
.select(&category_select)
.next()
.expect("Missing category link for mod");
let category = category_elem
.text()
.next();
let category = category_elem.text().next();
let author_elem = right
.select(&author_select)
.next()
@ -102,6 +104,18 @@ impl ModListResponse {
.next()
.expect("Missing desc elem for mod");
let desc = desc_elem.text().next();
let last_update_elem = right
.select(&last_update_select)
.next()
.expect("Missing last update elem for mod");
let last_update = last_update_elem
.text()
.skip(1)
.next()
.expect("Missing last update text for mod")
.trim();
let last_update = NaiveDate::parse_from_str(last_update, "%d %b %Y")
.expect("Cannot parse last update date");
ScrapedMod {
nexus_mod_id,
@ -109,6 +123,7 @@ impl ModListResponse {
category,
author,
desc,
last_update,
}
})
.collect();