Reset main.rs back w/ fix to listing RAR files
This commit is contained in:
parent
5d62fb7d61
commit
f62324d36c
211
src/main.rs
211
src/main.rs
@ -222,105 +222,98 @@ pub async fn main() -> Result<()> {
|
||||
while has_next_page {
|
||||
let page_span = info_span!("page", page);
|
||||
let _page_span = page_span.enter();
|
||||
// let mod_list_resp = nexus_scraper::get_mod_list_page(&client, page).await?;
|
||||
// let scraped = mod_list_resp.scrape_mods()?;
|
||||
let mod_list_resp = nexus_scraper::get_mod_list_page(&client, page).await?;
|
||||
let scraped = mod_list_resp.scrape_mods()?;
|
||||
|
||||
// TODO: delete
|
||||
has_next_page = false;
|
||||
// has_next_page = scraped.has_next_page;
|
||||
// let present_mods = game_mod::bulk_get_present_nexus_mod_ids(
|
||||
// &pool,
|
||||
// &scraped
|
||||
// .mods
|
||||
// .iter()
|
||||
// .map(|scraped_mod| scraped_mod.nexus_mod_id)
|
||||
// .collect::<Vec<i32>>(),
|
||||
// )
|
||||
// .await?;
|
||||
// let mods_to_create: Vec<UnsavedMod> = scraped
|
||||
// .mods
|
||||
// .iter()
|
||||
// .filter(|scraped_mod| !present_mods.contains(&scraped_mod.nexus_mod_id))
|
||||
// .map(|scraped_mod| UnsavedMod {
|
||||
// name: scraped_mod.name,
|
||||
// nexus_mod_id: scraped_mod.nexus_mod_id,
|
||||
// author: scraped_mod.author,
|
||||
// category: scraped_mod.category,
|
||||
// description: scraped_mod.desc,
|
||||
// game_id: game.id,
|
||||
// })
|
||||
// .collect();
|
||||
has_next_page = scraped.has_next_page;
|
||||
let present_mods = game_mod::bulk_get_present_nexus_mod_ids(
|
||||
&pool,
|
||||
&scraped
|
||||
.mods
|
||||
.iter()
|
||||
.map(|scraped_mod| scraped_mod.nexus_mod_id)
|
||||
.collect::<Vec<i32>>(),
|
||||
)
|
||||
.await?;
|
||||
let mods_to_create: Vec<UnsavedMod> = scraped
|
||||
.mods
|
||||
.iter()
|
||||
.filter(|scraped_mod| !present_mods.contains(&scraped_mod.nexus_mod_id))
|
||||
.map(|scraped_mod| UnsavedMod {
|
||||
name: scraped_mod.name,
|
||||
nexus_mod_id: scraped_mod.nexus_mod_id,
|
||||
author: scraped_mod.author,
|
||||
category: scraped_mod.category,
|
||||
description: scraped_mod.desc,
|
||||
game_id: game.id,
|
||||
})
|
||||
.collect();
|
||||
|
||||
// let mods = game_mod::batched_insert(&pool, &mods_to_create).await?;
|
||||
let mods = game_mod::batched_insert(&pool, &mods_to_create).await?;
|
||||
|
||||
for db_file in file::get_files_that_need_backfill(&pool).await? {
|
||||
let db_mod = game_mod::get(&pool, db_file.mod_id)
|
||||
.await?
|
||||
.expect("a mod to exist for file");
|
||||
for db_mod in mods {
|
||||
let mod_span = info_span!("mod", name = ?&db_mod.name, id = &db_mod.nexus_mod_id);
|
||||
let _mod_span = mod_span.enter();
|
||||
// let files_resp = nexus_api::files::get(&client, db_mod.nexus_mod_id).await?;
|
||||
let files_resp = nexus_api::files::get(&client, db_mod.nexus_mod_id).await?;
|
||||
|
||||
// debug!(duration = ?files_resp.wait, "sleeping");
|
||||
// sleep(files_resp.wait).await;
|
||||
debug!(duration = ?files_resp.wait, "sleeping");
|
||||
sleep(files_resp.wait).await;
|
||||
|
||||
// Filter out replaced/deleted files (indicated by null category) and archived files
|
||||
// let files = files_resp
|
||||
// .files()?
|
||||
// .into_iter()
|
||||
// .filter(|file| match file.category {
|
||||
// None => {
|
||||
// info!(
|
||||
// name = file.file_name,
|
||||
// id = file.file_id,
|
||||
// "skipping file with no category"
|
||||
// );
|
||||
// false
|
||||
// }
|
||||
// Some(category) if category == "ARCHIVED" => false,
|
||||
// Some(_) => true,
|
||||
// });
|
||||
let files = files_resp
|
||||
.files()?
|
||||
.into_iter()
|
||||
.filter(|file| match file.category {
|
||||
None => {
|
||||
info!(
|
||||
name = file.file_name,
|
||||
id = file.file_id,
|
||||
"skipping file with no category"
|
||||
);
|
||||
false
|
||||
}
|
||||
Some(category) if category == "ARCHIVED" => false,
|
||||
Some(_) => true,
|
||||
});
|
||||
|
||||
// for api_file in files {
|
||||
let file_span = info_span!("file", id = &db_file.nexus_file_id);
|
||||
for api_file in files {
|
||||
let file_span =
|
||||
info_span!("file", name = &api_file.file_name, id = &api_file.file_id);
|
||||
let _file_span = file_span.enter();
|
||||
// let db_file = file::insert(
|
||||
// &pool,
|
||||
// api_file.name,
|
||||
// api_file.file_name,
|
||||
// api_file.file_id as i32,
|
||||
// db_mod.id,
|
||||
// api_file.category,
|
||||
// api_file.version,
|
||||
// api_file.mod_version,
|
||||
// api_file.size,
|
||||
// api_file.uploaded_at,
|
||||
// )
|
||||
// .await?;
|
||||
|
||||
// let mut checked_metadata = false;
|
||||
// match nexus_api::metadata::contains_plugin(&client, &api_file).await {
|
||||
// Ok(contains_plugin) => {
|
||||
// if let Some(contains_plugin) = contains_plugin {
|
||||
// checked_metadata = true;
|
||||
// if !contains_plugin {
|
||||
// info!("file metadata does not contain a plugin, skip downloading");
|
||||
// continue;
|
||||
// }
|
||||
// } else {
|
||||
// warn!("file has no metadata link, continuing with download");
|
||||
// }
|
||||
// }
|
||||
// Err(err) => {
|
||||
// warn!(error = %err, "error retreiving metadata for file, continuing with download");
|
||||
// }
|
||||
// };
|
||||
|
||||
let download_link_resp = nexus_api::download_link::get(
|
||||
&client,
|
||||
db_mod.nexus_mod_id,
|
||||
db_file.nexus_file_id as i64,
|
||||
let db_file = file::insert(
|
||||
&pool,
|
||||
api_file.name,
|
||||
api_file.file_name,
|
||||
api_file.file_id as i32,
|
||||
db_mod.id,
|
||||
api_file.category,
|
||||
api_file.version,
|
||||
api_file.mod_version,
|
||||
api_file.size,
|
||||
api_file.uploaded_at,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut checked_metadata = false;
|
||||
match nexus_api::metadata::contains_plugin(&client, &api_file).await {
|
||||
Ok(contains_plugin) => {
|
||||
if let Some(contains_plugin) = contains_plugin {
|
||||
checked_metadata = true;
|
||||
if !contains_plugin {
|
||||
info!("file metadata does not contain a plugin, skip downloading");
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
warn!("file has no metadata link, continuing with download");
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(error = %err, "error retreiving metadata for file, continuing with download");
|
||||
}
|
||||
};
|
||||
|
||||
let download_link_resp =
|
||||
nexus_api::download_link::get(&client, db_mod.nexus_mod_id, api_file.file_id)
|
||||
.await;
|
||||
if let Err(err) = &download_link_resp {
|
||||
if let Some(reqwest_err) = err.downcast_ref::<reqwest::Error>() {
|
||||
@ -336,7 +329,7 @@ pub async fn main() -> Result<()> {
|
||||
}
|
||||
let download_link_resp = download_link_resp?;
|
||||
let mut tokio_file = download_link_resp.download_file(&client).await?;
|
||||
info!("download finished");
|
||||
info!(bytes = api_file.size, "download finished");
|
||||
|
||||
initialize_plugins_archive(db_mod.nexus_mod_id, db_file.nexus_file_id)?;
|
||||
let mut plugins_archive = ZipWriter::new_append(
|
||||
@ -363,9 +356,7 @@ pub async fn main() -> Result<()> {
|
||||
|
||||
match kind.mime_type() {
|
||||
"application/vnd.rar" => {
|
||||
info!(
|
||||
"downloaded archive is RAR archive, attempt to uncompress entire archive"
|
||||
);
|
||||
info!("downloaded archive is RAR archive, attempt to uncompress entire archive");
|
||||
// Use unrar to uncompress the entire .rar file to avoid bugs with compress_tools uncompressing certain .rar files:
|
||||
// https://github.com/libarchive/libarchive/issues/373, https://github.com/libarchive/libarchive/issues/1426
|
||||
tokio_file.seek(SeekFrom::Start(0)).await?;
|
||||
@ -376,7 +367,8 @@ pub async fn main() -> Result<()> {
|
||||
std::io::copy(&mut file, &mut temp_file)?;
|
||||
|
||||
let mut plugin_file_paths = Vec::new();
|
||||
let list = Archive::new(&temp_file_path.to_string_lossy().to_string())?.list();
|
||||
let list =
|
||||
Archive::new(&temp_file_path.to_string_lossy().to_string())?.list();
|
||||
if let Ok(list) = list {
|
||||
for entry in list {
|
||||
if let Ok(entry) = entry {
|
||||
@ -392,8 +384,13 @@ pub async fn main() -> Result<()> {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("failed to read archive, skipping file");
|
||||
if !checked_metadata {
|
||||
warn!("failed to read archive and server has no metadata, skipping file");
|
||||
continue;
|
||||
} else {
|
||||
error!("failed to read archive, but server had metadata");
|
||||
panic!("failed to read archive, but server had metadata");
|
||||
}
|
||||
}
|
||||
info!(
|
||||
num_plugin_files = plugin_file_paths.len(),
|
||||
@ -402,7 +399,8 @@ pub async fn main() -> Result<()> {
|
||||
|
||||
if plugin_file_paths.len() > 0 {
|
||||
info!("uncompressing downloaded archive");
|
||||
let extract = Archive::new(&temp_file_path.to_string_lossy().to_string())?
|
||||
let extract =
|
||||
Archive::new(&temp_file_path.to_string_lossy().to_string())?
|
||||
.extract_to(temp_dir.path().to_string_lossy().to_string());
|
||||
extract
|
||||
.expect("failed to extract")
|
||||
@ -414,7 +412,8 @@ pub async fn main() -> Result<()> {
|
||||
?file_path,
|
||||
"processing uncompressed file from downloaded archive"
|
||||
);
|
||||
let mut plugin_buf = std::fs::read(temp_dir.path().join(file_path))?;
|
||||
let mut plugin_buf =
|
||||
std::fs::read(temp_dir.path().join(file_path))?;
|
||||
process_plugin(
|
||||
&mut plugin_buf,
|
||||
&pool,
|
||||
@ -434,12 +433,17 @@ pub async fn main() -> Result<()> {
|
||||
let mut plugin_file_paths = Vec::new();
|
||||
|
||||
let archive_files = match list_archive_files(&file) {
|
||||
Ok(files) => files,
|
||||
Ok(files) => Ok(files),
|
||||
Err(err) => {
|
||||
warn!(error = %err, "failed to read archive, skipping file");
|
||||
if !checked_metadata {
|
||||
warn!(error = %err, "failed to read archive and server has no metadata, skipping file");
|
||||
continue;
|
||||
} else {
|
||||
error!(error = %err, "failed to read archive, but server had metadata");
|
||||
Err(err)
|
||||
}
|
||||
};
|
||||
}
|
||||
}?;
|
||||
for file_path in archive_files {
|
||||
if file_path.ends_with(".esp")
|
||||
|| file_path.ends_with(".esm")
|
||||
@ -485,11 +489,10 @@ pub async fn main() -> Result<()> {
|
||||
.status()?;
|
||||
|
||||
for file_path in plugin_file_paths.iter() {
|
||||
let plugin_span = info_span!("plugin", name = ?file_path);
|
||||
let plugin_span =
|
||||
info_span!("plugin", name = ?file_path);
|
||||
let _plugin_span = plugin_span.enter();
|
||||
info!(
|
||||
"processing uncompressed file from downloaded archive"
|
||||
);
|
||||
info!("processing uncompressed file from downloaded archive");
|
||||
let mut plugin_buf =
|
||||
std::fs::read(extracted_path.join(file_path))?;
|
||||
process_plugin(
|
||||
@ -525,7 +528,7 @@ pub async fn main() -> Result<()> {
|
||||
debug!(duration = ?download_link_resp.wait, "sleeping");
|
||||
sleep(download_link_resp.wait).await;
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
page += 1;
|
||||
debug!(?page, ?has_next_page, "sleeping 1 second");
|
||||
|
||||
@ -91,14 +91,3 @@ pub async fn update_has_download_link(
|
||||
.await
|
||||
.context("Failed to update file")
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(pool))]
|
||||
pub async fn get_files_that_need_backfill(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<Vec<File>> {
|
||||
sqlx::query_as!(
|
||||
File,
|
||||
"SELECT * FROM files WHERE created_at >= '2021-07-24 02:07:25' and file_name like '%.rar'",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.context("Failed to get files")
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user