thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Always uncompress entire rar archive

Browse Source 
To avoid libarchive bugs.
This commit is contained in:
Tyler Hallada 2021-07-12 13:49:02 -04:00
parent af5c11acc7
commit fd8015024f
1 changed files with 85 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
src/main.rs
View File

@ -13,7 +13,7 @@ use std::time::Duration;
use tempfile::tempdir; use tempfile::tempdir;
use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tokio::time::sleep; use tokio::time::sleep;
use tracing::{debug, error, info, info_span, warn}; use tracing::{debug, info, info_span, warn};
use unrar::Archive; use unrar::Archive;
use zip::write::{FileOptions, ZipWriter}; use zip::write::{FileOptions, ZipWriter};
@ -228,31 +228,93 @@ pub async fn main() -> Result<()> {
"inferred mime_type of downloaded archive" "inferred mime_type of downloaded archive"
); );
tokio_file.seek(SeekFrom::Start(0)).await?; match kind.mime_type() {
let mut file = tokio_file.try_clone().await?.into_std().await; "application/vnd.rar" => {
let mut plugin_file_paths = Vec::new(); info!("downloaded archive is RAR archive, attempt to uncompress entire archive");
// Use unrar to uncompress the entire .rar file to avoid bugs with compress_tools uncompressing certain .rar files:
// https://github.com/libarchive/libarchive/issues/373, https://github.com/libarchive/libarchive/issues/1426
tokio_file.seek(SeekFrom::Start(0)).await?;
let mut file = tokio_file.try_clone().await?.into_std().await;
let temp_dir = tempdir()?;
let temp_file_path = temp_dir.path().join("download.rar");
let mut temp_file = std::fs::File::create(&temp_file_path)?;
std::io::copy(&mut file, &mut temp_file)?;
for file_name in list_archive_files(&file)? { let mut plugin_file_paths = Vec::new();
if file_name.ends_with(".esp") let list =
|| file_name.ends_with(".esm") Archive::new(temp_file_path.to_string_lossy().to_string()).list();
|| file_name.ends_with(".esl") if let Ok(list) = list {
{ for entry in list {
plugin_file_paths.push(file_name); if let Ok(entry) = entry {
if entry.filename.ends_with(".esp")
|| entry.filename.ends_with(".esm")
|| entry.filename.ends_with(".esl")
{
plugin_file_paths.push(entry.filename);
}
}
}
}
info!(
num_plugin_files = plugin_file_paths.len(),
"listed plugins in downloaded archive"
);
if plugin_file_paths.len() > 0 {
info!("uncompressing downloaded archive");
let extract =
Archive::new(temp_file_path.to_string_lossy().to_string())
.extract_to(temp_dir.path().to_string_lossy().to_string());
extract
.expect("failed to extract")
.process()
.expect("failed to extract");
for file_name in plugin_file_paths.iter() {
info!(
?file_name,
"processing uncompressed file from downloaded archive"
);
let mut plugin_buf =
std::fs::read(temp_dir.path().join(file_name))?;
process_plugin(
&mut plugin_buf,
&pool,
&mut plugins_archive,
&db_file,
&db_mod,
file_name,
)
.await?;
}
}
temp_dir.close()?;
} }
} _ => {
info!( tokio_file.seek(SeekFrom::Start(0)).await?;
num_plugin_files = plugin_file_paths.len(), let mut file = tokio_file.try_clone().await?.into_std().await;
"listed plugins in downloaded archive" let mut plugin_file_paths = Vec::new();
);
for file_name in plugin_file_paths.iter() { for file_name in list_archive_files(&file)? {
let plugin_span = info_span!("plugin", name = ?file_name); if file_name.ends_with(".esp")
let _plugin_span = plugin_span.enter(); || file_name.ends_with(".esm")
file.seek(SeekFrom::Start(0))?; || file_name.ends_with(".esl")
info!("attempting to uncompress plugin file from downloaded archive"); {
let mut buf = Vec::default(); plugin_file_paths.push(file_name);
match uncompress_archive_file(&mut file, &mut buf, file_name) { }
Ok(_) => { }
info!(
num_plugin_files = plugin_file_paths.len(),
"listed plugins in downloaded archive"
);
for file_name in plugin_file_paths.iter() {
let plugin_span = info_span!("plugin", name = ?file_name);
let _plugin_span = plugin_span.enter();
file.seek(SeekFrom::Start(0))?;
let mut buf = Vec::default();
info!("uncompressing plugin file from downloaded archive");
uncompress_archive_file(&mut file, &mut buf, file_name)?;
process_plugin( process_plugin(
&mut buf, &mut buf,
&pool, &pool,
@ -263,73 +325,6 @@ pub async fn main() -> Result<()> {
) )
.await?; .await?;
} }
Err(error) => {
warn!(
?error,
"error occurred while attempting to uncompress archive file"
);
if kind.mime_type() == "application/x-rar-compressed"
|| kind.mime_type() == "application/vnd.rar"
{
info!("downloaded archive is RAR archive, attempt to uncompress entire archive instead");
// Use unrar to uncompress the entire .rar file to avoid a bug with compress_tools panicking when uncompressing
// certain .rar files: https://github.com/libarchive/libarchive/issues/373
tokio_file.seek(SeekFrom::Start(0)).await?;
let mut file = tokio_file.try_clone().await?.into_std().await;
let temp_dir = tempdir()?;
let temp_file_path = temp_dir.path().join("download.rar");
let mut temp_file = std::fs::File::create(&temp_file_path)?;
std::io::copy(&mut file, &mut temp_file)?;
let mut plugin_file_paths = Vec::new();
let list =
Archive::new(temp_file_path.to_string_lossy().to_string())
.list();
if let Ok(list) = list {
for entry in list {
if let Ok(entry) = entry {
if entry.filename.ends_with(".esp")
|| entry.filename.ends_with(".esm")
|| entry.filename.ends_with(".esl")
{
plugin_file_paths.push(entry.filename);
}
}
}
}
if plugin_file_paths.len() > 0 {
let extract =
Archive::new(temp_file_path.to_string_lossy().to_string())
.extract_to(
temp_dir.path().to_string_lossy().to_string(),
);
extract
.expect("failed to extract")
.process()
.expect("failed to extract");
for file_name in plugin_file_paths.iter() {
info!(
?file_name,
"processing uncompressed file from downloaded archive"
);
let mut plugin_buf =
std::fs::read(temp_dir.path().join(file_name))?;
process_plugin(
&mut plugin_buf,
&pool,
&mut plugins_archive,
&db_file,
&db_mod,
file_name,
)
.await?;
}
}
temp_dir.close()?;
}
error!(mime_type = ?kind.mime_type(), "downloaded archive is not RAR archive, skipping processing of this file");
}
} }
} }