thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prevent overwriting of base game cells is_base_game value

Browse Source 
This fixes a bug that was causing cell data to not get written since
November 9th, 2023. When batch inserting cells while processing plugins,
I allowed overwriting cells that had `is_base_game = true`. Since I
always set `is_base_game = false` for cell upserts from plugins, this
was causing the base game cells to revert to `is_base_game = false`. All
it took was one mod to bundle `Skyrim.esm` for this to happen.

This broke writing cell data since `get_cell_data` depends on the
`is_base_game` value to find edits to the Skyrim base game cells.

To prevent this in the future, batch inserts when processing plugins is
no longer allowed to update cells which have `is_base_game = true`. The
only time we allow upserting these rows is when running the
`is_base_game` backfill which initially seeds the database with the base
game cells.
This commit is contained in:
Tyler Hallada
2025-03-02 14:15:05 -05:00
parent a5135b30f4
commit a677325c4d
4 changed files with 62 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@@ -38,7 +38,7 @@ RUST_LOG=mod_mapper=debug
[`sqlx_cli`](https://github.com/launchbadge/sqlx/tree/master/sqlx-cli) with
`cargo install sqlx-cli --no-default-features --features postgres`
5. Run `sqlx migrate --source migrations run` which will run all the database migrations.
6. Get your personal Nexus API token from your profile settings and add it to
6. Get your personal Nexus API token from your profile settings and add it to
the `.env` file:
```
@@ -46,12 +46,12 @@ NEXUS_API_KEY=...
```
7. Build the release binary by running `cargo build --release`.
8. Run `./target/release/modmapper --backfill-is-game-cell` to pre-populate the
database with worlds and cells from the base game's Skyrim.esm. (This is so
that the base game cells can later be differentiated from cells in plugins that
also happen to be named Skyrim.esm and have cells that reference a world with
the same form ID as Tamriel.)
9. See `./target/release/modmapper -h` for further commands or run `./scripts/update.sh` to start populating the database with scraped mods and dumping the data to JSON files.
8. Run `./target/release/mod-mapper --backfill-is-base-game` to pre-populate the
database with worlds and cells from the base game's Skyrim.esm. (This is so
that the base game cells can later be differentiated from cells in plugins that
also happen to be named Skyrim.esm and have cells that reference a world with
the same form ID as Tamriel.)
9. See `./target/release/mod-mapper -h` for further commands or run `./scripts/update.sh` to start populating the database with scraped mods and dumping the data to JSON files.
## Sync and Backup Setup

2
src/commands/backfills/is_base_game.rs
View File

@@ -60,7 +60,7 @@ pub async fn backfill_is_base_game(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<
}
})
.collect();
let db_cells = cell::batched_insert(pool, &base_cells).await?;
let db_cells = cell::batched_insert(pool, &base_cells, true).await?;
info!("Upserted {} Skyrim.esm base cells", db_cells.len());
// This works for exterior cells, but there's a bug with the unique index on cells that
// creates duplicate interior cells. To fix that, I need to upgrade postgres to

75
src/models/cell.rs
View File

@@ -80,6 +80,7 @@ pub async fn insert(
pub async fn batched_insert<'a>(
pool: &sqlx::Pool<sqlx::Postgres>,
cells: &[UnsavedCell<'a>],
allow_upserting_base_game_cells: bool,
) -> Result<Vec<Cell>> {
let mut saved_cells = vec![];
for batch in cells.chunks(BATCH_SIZE) {
@@ -99,27 +100,58 @@ pub async fn batched_insert<'a>(
is_persistents.push(unsaved_cell.is_persistent);
is_base_games.push(unsaved_cell.is_base_game);
});
saved_cells.append(
// sqlx doesn't understand arrays of Options with the query_as! macro
&mut sqlx::query_as(
r#"INSERT INTO cells (form_id, master, x, y, world_id, is_persistent, is_base_game, created_at, updated_at)
SELECT *, now(), now() FROM UNNEST($1::int[], $2::text[], $3::int[], $4::int[], $5::int[], $6::bool[], $7::bool[])
ON CONFLICT (form_id, master, world_id) DO UPDATE
SET (x, y, is_persistent, is_base_game, updated_at) =
(EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, EXCLUDED.is_base_game, now())
RETURNING *"#,
)
.bind(&form_ids)
.bind(&masters)
.bind(&xs)
.bind(&ys)
.bind(&world_ids)
.bind(&is_persistents)
.bind(&is_base_games)
.fetch_all(pool)
.await
.context("Failed to insert cells")?,
);
if allow_upserting_base_game_cells {
saved_cells.append(
// sqlx doesn't understand arrays of Options with the query_as! macro
// NOTE: allows overwriting base game cells. This should only be run in the
// `is_base_game` backfill in order to seed the database with base game cells.
&mut sqlx::query_as(
r#"INSERT INTO cells (form_id, master, x, y, world_id, is_persistent, is_base_game, created_at, updated_at)
SELECT *, now(), now() FROM UNNEST($1::int[], $2::text[], $3::int[], $4::int[], $5::int[], $6::bool[], $7::bool[])
ON CONFLICT (form_id, master, world_id) DO UPDATE
SET (x, y, is_persistent, is_base_game, updated_at) =
(EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, EXCLUDED.is_base_game, now())
RETURNING *"#,
)
.bind(&form_ids)
.bind(&masters)
.bind(&xs)
.bind(&ys)
.bind(&world_ids)
.bind(&is_persistents)
.bind(&is_base_games)
.fetch_all(pool)
.await
.context("Failed to insert cells")?,
);
} else {
saved_cells.append(
// sqlx doesn't understand arrays of Options with the query_as! macro
// NOTE: excludes upserts on cells that have is_base_game = true since if we are trying
// to update base game cells that means a mod bundled the base game Skyrim.esm and we
// should ignore it. Additionally, overwriting `is_base_game` to false here will break dumping cell
// data since we rely on that field to find edits to Skyrim cells in `get_cell_data`.
&mut sqlx::query_as(
r#"INSERT INTO cells (form_id, master, x, y, world_id, is_persistent, is_base_game, created_at, updated_at)
SELECT *, now(), now() FROM UNNEST($1::int[], $2::text[], $3::int[], $4::int[], $5::int[], $6::bool[], $7::bool[])
ON CONFLICT (form_id, master, world_id) DO UPDATE
SET (x, y, is_persistent, is_base_game, updated_at) =
(EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, EXCLUDED.is_base_game, now())
WHERE NOT cells.is_base_game
RETURNING *"#,
)
.bind(&form_ids)
.bind(&masters)
.bind(&xs)
.bind(&ys)
.bind(&world_ids)
.bind(&is_persistents)
.bind(&is_base_games)
.fetch_all(pool)
.await
.context("Failed to insert cells")?,
);
}
}
Ok(saved_cells)
}
@@ -251,6 +283,5 @@ pub async fn get_cell_data(
.fetch_one(pool)
.await
.context("Failed get cell data")
}
}

2
src/plugin_processor.rs
View File

@@ -128,7 +128,7 @@ pub async fn process_plugin(
}
})
.collect();
let db_cells = cell::batched_insert(&pool, &cells).await?;
let db_cells = cell::batched_insert(&pool, &cells, false).await?;
let plugin_cells: Vec<UnsavedPluginCell> = db_cells
.iter()
.zip(&plugin.cells)