Prevent overwriting of base game cells is_base_game value

This fixes a bug that was causing cell data to not get written since
November 9th, 2023. When batch inserting cells while processing plugins,
I allowed overwriting cells that had `is_base_game = true`. Since I
always set `is_base_game = false` for cell upserts from plugins, this
was causing the base game cells to revert to `is_base_game = false`. All
it took was one mod to bundle `Skyrim.esm` for this to happen.

This broke writing cell data since `get_cell_data` depends on the
`is_base_game` value to find edits to the Skyrim base game cells.

To prevent this in the future, batch inserts when processing plugins is
no longer allowed to update cells which have `is_base_game = true`. The
only time we allow upserting these rows is when running the
`is_base_game` backfill which initially seeds the database with the base
game cells.

README.md

@@ -38,7 +38,7 @@ RUST_LOG=mod_mapper=debug
    [`sqlx_cli`](https://github.com/launchbadge/sqlx/tree/master/sqlx-cli) with
    `cargo install sqlx-cli --no-default-features --features postgres`
 5. Run `sqlx migrate --source migrations run` which will run all the database migrations.
-6. Get your personal Nexus API token from your profile settings and add it to 
+6. Get your personal Nexus API token from your profile settings and add it to
    the `.env` file:
 
 ```
@@ -46,12 +46,12 @@ NEXUS_API_KEY=...
 ```
 
 7. Build the release binary by running `cargo build --release`.
-8. Run `./target/release/modmapper --backfill-is-game-cell` to pre-populate the 
+8. Run `./target/release/mod-mapper --backfill-is-base-game` to pre-populate the
-   database with worlds and cells from the base game's Skyrim.esm. (This is so 
+   database with worlds and cells from the base game's Skyrim.esm. (This is so
-that the base game cells can later be differentiated from cells in plugins that 
+   that the base game cells can later be differentiated from cells in plugins that
-also happen to be named Skyrim.esm and have cells that reference a world with 
+   also happen to be named Skyrim.esm and have cells that reference a world with
-the same form ID as Tamriel.)
+   the same form ID as Tamriel.)
-9. See `./target/release/modmapper -h` for further commands or run `./scripts/update.sh` to start populating the database with scraped mods and dumping the data to JSON files.
+9. See `./target/release/mod-mapper -h` for further commands or run `./scripts/update.sh` to start populating the database with scraped mods and dumping the data to JSON files.
 
 ## Sync and Backup Setup
 

src/commands/backfills/is_base_game.rs

@@ -60,7 +60,7 @@ pub async fn backfill_is_base_game(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<
             }
         })
         .collect();
-    let db_cells = cell::batched_insert(pool, &base_cells).await?;
+    let db_cells = cell::batched_insert(pool, &base_cells, true).await?;
     info!("Upserted {} Skyrim.esm base cells", db_cells.len());
     // This works for exterior cells, but there's a bug with the unique index on cells that
     // creates duplicate interior cells. To fix that, I need to upgrade postgres to

src/models/cell.rs

@@ -80,6 +80,7 @@ pub async fn insert(
 pub async fn batched_insert<'a>(
     pool: &sqlx::Pool<sqlx::Postgres>,
     cells: &[UnsavedCell<'a>],
+    allow_upserting_base_game_cells: bool,
 ) -> Result<Vec<Cell>> {
     let mut saved_cells = vec![];
     for batch in cells.chunks(BATCH_SIZE) {
@@ -99,27 +100,58 @@ pub async fn batched_insert<'a>(
             is_persistents.push(unsaved_cell.is_persistent);
             is_base_games.push(unsaved_cell.is_base_game);
         });
-        saved_cells.append(
+        if allow_upserting_base_game_cells {
-            // sqlx doesn't understand arrays of Options with the query_as! macro
+            saved_cells.append(
-            &mut sqlx::query_as(
+                // sqlx doesn't understand arrays of Options with the query_as! macro
-                r#"INSERT INTO cells (form_id, master, x, y, world_id, is_persistent, is_base_game, created_at, updated_at)
+                // NOTE: allows overwriting base game cells. This should only be run in the
-                SELECT *, now(), now() FROM UNNEST($1::int[], $2::text[], $3::int[], $4::int[], $5::int[], $6::bool[], $7::bool[])
+                // `is_base_game` backfill in order to seed the database with base game cells.
-                ON CONFLICT (form_id, master, world_id) DO UPDATE
+                &mut sqlx::query_as(
-                SET (x, y, is_persistent, is_base_game, updated_at) =
+                    r#"INSERT INTO cells (form_id, master, x, y, world_id, is_persistent, is_base_game, created_at, updated_at)
-                (EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, EXCLUDED.is_base_game, now())
+                    SELECT *, now(), now() FROM UNNEST($1::int[], $2::text[], $3::int[], $4::int[], $5::int[], $6::bool[], $7::bool[])
-                RETURNING *"#,
+                    ON CONFLICT (form_id, master, world_id) DO UPDATE
-            )
+                    SET (x, y, is_persistent, is_base_game, updated_at) =
-            .bind(&form_ids)
+                    (EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, EXCLUDED.is_base_game, now())
-            .bind(&masters)
+                    RETURNING *"#,
-            .bind(&xs)
+                )
-            .bind(&ys)
+                .bind(&form_ids)
-            .bind(&world_ids)
+                .bind(&masters)
-            .bind(&is_persistents)
+                .bind(&xs)
-            .bind(&is_base_games)
+                .bind(&ys)
-            .fetch_all(pool)
+                .bind(&world_ids)
-            .await
+                .bind(&is_persistents)
-            .context("Failed to insert cells")?,
+                .bind(&is_base_games)
-        );
+                .fetch_all(pool)
+                .await
+                .context("Failed to insert cells")?,
+            );
+        } else {
+            saved_cells.append(
+                // sqlx doesn't understand arrays of Options with the query_as! macro
+                // NOTE: excludes upserts on cells that have is_base_game = true since if we are trying
+                // to update base game cells that means a mod bundled the base game Skyrim.esm and we
+                // should ignore it. Additionally, overwriting `is_base_game` to false here will break dumping cell
+                // data since we rely on that field to find edits to Skyrim cells in `get_cell_data`.
+                &mut sqlx::query_as(
+                    r#"INSERT INTO cells (form_id, master, x, y, world_id, is_persistent, is_base_game, created_at, updated_at)
+                    SELECT *, now(), now() FROM UNNEST($1::int[], $2::text[], $3::int[], $4::int[], $5::int[], $6::bool[], $7::bool[])
+                    ON CONFLICT (form_id, master, world_id) DO UPDATE
+                    SET (x, y, is_persistent, is_base_game, updated_at) =
+                    (EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, EXCLUDED.is_base_game, now())
+                    WHERE NOT cells.is_base_game
+                    RETURNING *"#,
+                )
+                .bind(&form_ids)
+                .bind(&masters)
+                .bind(&xs)
+                .bind(&ys)
+                .bind(&world_ids)
+                .bind(&is_persistents)
+                .bind(&is_base_games)
+                .fetch_all(pool)
+                .await
+                .context("Failed to insert cells")?,
+            );
+        }
     }
     Ok(saved_cells)
 }
@@ -251,6 +283,5 @@ pub async fn get_cell_data(
         .fetch_one(pool)
         .await
         .context("Failed get cell data")
-
     }
 }

src/plugin_processor.rs

@@ -128,7 +128,7 @@ pub async fn process_plugin(
                     }
                 })
                 .collect();
-            let db_cells = cell::batched_insert(&pool, &cells).await?;
+            let db_cells = cell::batched_insert(&pool, &cells, false).await?;
             let plugin_cells: Vec<UnsavedPluginCell> = db_cells
                 .iter()
                 .zip(&plugin.cells)