diff --git a/README.md b/README.md index 3e2e501..bdad458 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ RUST_LOG=mod_mapper=debug [`sqlx_cli`](https://github.com/launchbadge/sqlx/tree/master/sqlx-cli) with `cargo install sqlx-cli --no-default-features --features postgres` 5. Run `sqlx migrate --source migrations run` which will run all the database migrations. -6. Get your personal Nexus API token from your profile settings and add it to +6. Get your personal Nexus API token from your profile settings and add it to the `.env` file: ``` @@ -46,12 +46,12 @@ NEXUS_API_KEY=... ``` 7. Build the release binary by running `cargo build --release`. -8. Run `./target/release/modmapper --backfill-is-game-cell` to pre-populate the - database with worlds and cells from the base game's Skyrim.esm. (This is so -that the base game cells can later be differentiated from cells in plugins that -also happen to be named Skyrim.esm and have cells that reference a world with -the same form ID as Tamriel.) -9. See `./target/release/modmapper -h` for further commands or run `./scripts/update.sh` to start populating the database with scraped mods and dumping the data to JSON files. +8. Run `./target/release/mod-mapper --backfill-is-base-game` to pre-populate the + database with worlds and cells from the base game's Skyrim.esm. (This is so + that the base game cells can later be differentiated from cells in plugins that + also happen to be named Skyrim.esm and have cells that reference a world with + the same form ID as Tamriel.) +9. See `./target/release/mod-mapper -h` for further commands or run `./scripts/update.sh` to start populating the database with scraped mods and dumping the data to JSON files. ## Sync and Backup Setup diff --git a/src/commands/backfills/is_base_game.rs b/src/commands/backfills/is_base_game.rs index 527e0b8..5a2ff80 100644 --- a/src/commands/backfills/is_base_game.rs +++ b/src/commands/backfills/is_base_game.rs @@ -60,7 +60,7 @@ pub async fn backfill_is_base_game(pool: &sqlx::Pool) -> Result< } }) .collect(); - let db_cells = cell::batched_insert(pool, &base_cells).await?; + let db_cells = cell::batched_insert(pool, &base_cells, true).await?; info!("Upserted {} Skyrim.esm base cells", db_cells.len()); // This works for exterior cells, but there's a bug with the unique index on cells that // creates duplicate interior cells. To fix that, I need to upgrade postgres to diff --git a/src/models/cell.rs b/src/models/cell.rs index fa59a1b..33264ed 100644 --- a/src/models/cell.rs +++ b/src/models/cell.rs @@ -80,6 +80,7 @@ pub async fn insert( pub async fn batched_insert<'a>( pool: &sqlx::Pool, cells: &[UnsavedCell<'a>], + allow_upserting_base_game_cells: bool, ) -> Result> { let mut saved_cells = vec![]; for batch in cells.chunks(BATCH_SIZE) { @@ -99,27 +100,58 @@ pub async fn batched_insert<'a>( is_persistents.push(unsaved_cell.is_persistent); is_base_games.push(unsaved_cell.is_base_game); }); - saved_cells.append( - // sqlx doesn't understand arrays of Options with the query_as! macro - &mut sqlx::query_as( - r#"INSERT INTO cells (form_id, master, x, y, world_id, is_persistent, is_base_game, created_at, updated_at) - SELECT *, now(), now() FROM UNNEST($1::int[], $2::text[], $3::int[], $4::int[], $5::int[], $6::bool[], $7::bool[]) - ON CONFLICT (form_id, master, world_id) DO UPDATE - SET (x, y, is_persistent, is_base_game, updated_at) = - (EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, EXCLUDED.is_base_game, now()) - RETURNING *"#, - ) - .bind(&form_ids) - .bind(&masters) - .bind(&xs) - .bind(&ys) - .bind(&world_ids) - .bind(&is_persistents) - .bind(&is_base_games) - .fetch_all(pool) - .await - .context("Failed to insert cells")?, - ); + if allow_upserting_base_game_cells { + saved_cells.append( + // sqlx doesn't understand arrays of Options with the query_as! macro + // NOTE: allows overwriting base game cells. This should only be run in the + // `is_base_game` backfill in order to seed the database with base game cells. + &mut sqlx::query_as( + r#"INSERT INTO cells (form_id, master, x, y, world_id, is_persistent, is_base_game, created_at, updated_at) + SELECT *, now(), now() FROM UNNEST($1::int[], $2::text[], $3::int[], $4::int[], $5::int[], $6::bool[], $7::bool[]) + ON CONFLICT (form_id, master, world_id) DO UPDATE + SET (x, y, is_persistent, is_base_game, updated_at) = + (EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, EXCLUDED.is_base_game, now()) + RETURNING *"#, + ) + .bind(&form_ids) + .bind(&masters) + .bind(&xs) + .bind(&ys) + .bind(&world_ids) + .bind(&is_persistents) + .bind(&is_base_games) + .fetch_all(pool) + .await + .context("Failed to insert cells")?, + ); + } else { + saved_cells.append( + // sqlx doesn't understand arrays of Options with the query_as! macro + // NOTE: excludes upserts on cells that have is_base_game = true since if we are trying + // to update base game cells that means a mod bundled the base game Skyrim.esm and we + // should ignore it. Additionally, overwriting `is_base_game` to false here will break dumping cell + // data since we rely on that field to find edits to Skyrim cells in `get_cell_data`. + &mut sqlx::query_as( + r#"INSERT INTO cells (form_id, master, x, y, world_id, is_persistent, is_base_game, created_at, updated_at) + SELECT *, now(), now() FROM UNNEST($1::int[], $2::text[], $3::int[], $4::int[], $5::int[], $6::bool[], $7::bool[]) + ON CONFLICT (form_id, master, world_id) DO UPDATE + SET (x, y, is_persistent, is_base_game, updated_at) = + (EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, EXCLUDED.is_base_game, now()) + WHERE NOT cells.is_base_game + RETURNING *"#, + ) + .bind(&form_ids) + .bind(&masters) + .bind(&xs) + .bind(&ys) + .bind(&world_ids) + .bind(&is_persistents) + .bind(&is_base_games) + .fetch_all(pool) + .await + .context("Failed to insert cells")?, + ); + } } Ok(saved_cells) } @@ -251,6 +283,5 @@ pub async fn get_cell_data( .fetch_one(pool) .await .context("Failed get cell data") - } } diff --git a/src/plugin_processor.rs b/src/plugin_processor.rs index 0feb5ae..283ae7f 100644 --- a/src/plugin_processor.rs +++ b/src/plugin_processor.rs @@ -128,7 +128,7 @@ pub async fn process_plugin( } }) .collect(); - let db_cells = cell::batched_insert(&pool, &cells).await?; + let db_cells = cell::batched_insert(&pool, &cells, false).await?; let plugin_cells: Vec = db_cells .iter() .zip(&plugin.cells)