thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Set columns to not null, remove backfill code

Browse Source
This commit is contained in:
Tyler Hallada 2022-01-18 21:50:09 -05:00
parent 337b673e02
commit e7fe7abfb7
3 changed files with 40 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
migrations/20220119022820_make_backfilled_mod_columns_non_null.sql Normal file
View File

@ -0,0 +1,3 @@
ALTER TABLE "mods" ALTER COLUMN "last_update_at" SET NOT NULL;
ALTER TABLE "mods" ALTER COLUMN "first_upload_at" SET NOT NULL;
ALTER TABLE "mods" ALTER COLUMN "author_id" SET NOT NULL;

69
src/main.rs
View File

@ -51,10 +51,6 @@ struct Args {
/// folder to output all cell data as json files
#[argh(option, short = 'c')]
cell_data: Option<String>,
/// backfill mods data from the Nexus API (temporary)
#[argh(switch, short = 'b')]
backfill_mods: bool,
}
async fn extract_with_compress_tools(
@ -217,24 +213,6 @@ pub async fn main() -> Result<()> {
let args: Args = argh::from_env();
let client = reqwest::Client::builder()
.timeout(REQUEST_TIMEOUT)
.connect_timeout(CONNECT_TIMEOUT)
.build()?;
if args.backfill_mods {
for game_mod in game_mod::bulk_get_need_backfill(&pool).await? {
let response = nexus_api::game_mod::get(&client, game_mod.nexus_mod_id).await?;
let mod_data = response.extract_data()?;
game_mod::update_from_api_response(&pool, &game_mod, &mod_data).await?;
info!(
id = game_mod.id,
nexus_mod_id = game_mod.nexus_mod_id,
"backfilled mod data from api"
);
}
}
if let Some(dump_edits) = args.dump_edits {
let mut cell_mod_edit_counts = HashMap::new();
for x in -77..75 {
@ -270,6 +248,11 @@ pub async fn main() -> Result<()> {
let game = game::insert(&pool, GAME_NAME, GAME_ID as i32).await?;
let client = reqwest::Client::builder()
.timeout(REQUEST_TIMEOUT)
.connect_timeout(CONNECT_TIMEOUT)
.build()?;
while has_next_page {
let page_span = info_span!("page", page);
let _page_span = page_span.enter();
@ -289,45 +272,45 @@ pub async fn main() -> Result<()> {
let mods_to_create_or_update: Vec<UnsavedMod> = scraped
.mods
.iter()
.filter(|scraped_mod| {
if let Some(processed_mod) = processed_mods
.iter()
.find(|processed_mod| processed_mod.nexus_mod_id == scraped_mod.nexus_mod_id)
{
if processed_mod.last_updated_files_at
> NaiveDateTime::new(
scraped_mod.last_update_at,
NaiveTime::from_hms(0, 0, 0),
)
{
return false;
}
}
true
})
.map(|scraped_mod| UnsavedMod {
name: scraped_mod.name,
nexus_mod_id: scraped_mod.nexus_mod_id,
author_name: scraped_mod.author_name,
author_id: Some(scraped_mod.author_id),
author_id: scraped_mod.author_id,
category_name: scraped_mod.category_name,
category_id: scraped_mod.category_id,
description: scraped_mod.desc,
thumbnail_link: scraped_mod.thumbnail_link,
game_id: game.id,
last_update_at: Some(NaiveDateTime::new(
last_update_at: NaiveDateTime::new(
scraped_mod.last_update_at,
NaiveTime::from_hms(0, 0, 0),
)),
first_upload_at: Some(NaiveDateTime::new(
),
first_upload_at: NaiveDateTime::new(
scraped_mod.first_upload_at,
NaiveTime::from_hms(0, 0, 0),
)),
),
})
.collect();
let mods = game_mod::batched_insert(&pool, &mods_to_create_or_update).await?;
let mods: Vec<Mod> =
mods.into_iter()
.filter(|updated_mod| {
if let Some(processed_mod) = processed_mods.iter().find(|processed_mod| {
processed_mod.nexus_mod_id == updated_mod.nexus_mod_id
}) {
if let Some(last_update_at) = updated_mod.last_update_at {
if processed_mod.last_updated_files_at > last_update_at {
return false;
}
}
}
true
})
.collect();
for db_mod in mods {
let mod_span = info_span!("mod", name = ?&db_mod.name, id = &db_mod.nexus_mod_id);
let _mod_span = mod_span.enter();

22
src/models/game_mod.rs
View File

@ -14,7 +14,7 @@ pub struct Mod {
pub name: String,
pub nexus_mod_id: i32,
pub author_name: String,
pub author_id: Option<i32>,
pub author_id: i32,
pub category_name: Option<String>,
pub category_id: Option<i32>,
pub description: Option<String>,
@ -22,8 +22,8 @@ pub struct Mod {
pub game_id: i32,
pub updated_at: NaiveDateTime,
pub created_at: NaiveDateTime,
pub last_update_at: Option<NaiveDateTime>,
pub first_upload_at: Option<NaiveDateTime>,
pub last_update_at: NaiveDateTime,
pub first_upload_at: NaiveDateTime,
pub last_updated_files_at: Option<NaiveDateTime>,
}
@ -32,14 +32,14 @@ pub struct UnsavedMod<'a> {
pub name: &'a str,
pub nexus_mod_id: i32,
pub author_name: &'a str,
pub author_id: Option<i32>,
pub author_id: i32,
pub category_name: Option<&'a str>,
pub category_id: Option<i32>,
pub description: Option<&'a str>,
pub thumbnail_link: Option<&'a str>,
pub game_id: i32,
pub last_update_at: Option<NaiveDateTime>,
pub first_upload_at: Option<NaiveDateTime>,
pub last_update_at: NaiveDateTime,
pub first_upload_at: NaiveDateTime,
}
#[instrument(level = "debug", skip(pool))]
@ -96,8 +96,8 @@ pub async fn insert(
description: Option<&str>,
thumbnail_link: Option<&str>,
game_id: i32,
last_update_at: Option<NaiveDateTime>,
first_upload_at: Option<NaiveDateTime>,
last_update_at: NaiveDateTime,
first_upload_at: NaiveDateTime,
) -> Result<Mod> {
sqlx::query_as!(
Mod,
@ -135,14 +135,14 @@ pub async fn batched_insert<'a>(
let mut names: Vec<&str> = vec![];
let mut nexus_mod_ids: Vec<i32> = vec![];
let mut author_names: Vec<&str> = vec![];
let mut author_ids: Vec<Option<i32>> = vec![];
let mut author_ids: Vec<i32> = vec![];
let mut category_names: Vec<Option<&str>> = vec![];
let mut category_ids: Vec<Option<i32>> = vec![];
let mut descriptions: Vec<Option<&str>> = vec![];
let mut thumbnail_links: Vec<Option<&str>> = vec![];
let mut game_ids: Vec<i32> = vec![];
let mut last_update_ats: Vec<Option<NaiveDateTime>> = vec![];
let mut first_upload_ats: Vec<Option<NaiveDateTime>> = vec![];
let mut last_update_ats: Vec<NaiveDateTime> = vec![];
let mut first_upload_ats: Vec<NaiveDateTime> = vec![];
batch.iter().for_each(|unsaved_mod| {
names.push(unsaved_mod.name);
nexus_mod_ids.push(unsaved_mod.nexus_mod_id);