Serialize plugin hashes to strings

Avoids having dealing with JSON parsers on the frontend that loose precision and avoids having to convert integers to strings.

Cargo.lock

@@ -646,12 +646,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "hashbrown"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
-
 [[package]]
 name = "hashbrown"
 version = "0.11.2"
@@ -661,6 +655,12 @@ dependencies = [
  "ahash",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
 [[package]]
 name = "hashlink"
 version = "0.7.0"
@@ -808,12 +808,12 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "1.6.2"
+version = "1.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
+checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
 dependencies = [
  "autocfg",
- "hashbrown 0.9.1",
+ "hashbrown 0.12.3",
 ]
 
 [[package]]
@@ -845,9 +845,9 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
 
 [[package]]
 name = "js-sys"
-version = "0.3.51"
+version = "0.3.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062"
+checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -873,9 +873,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.94"
+version = "0.2.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"
+checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
 
 [[package]]
 name = "lock_api"
@@ -1172,9 +1172,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.7.2"
+version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
+checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
 
 [[package]]
 name = "opaque-debug"
@@ -1364,18 +1364,18 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.27"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
+checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.9"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
 dependencies = [
  "proc-macro2",
 ]
@@ -2004,13 +2004,13 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
 
 [[package]]
 name = "syn"
-version = "1.0.72"
+version = "1.0.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
+checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13"
 dependencies = [
  "proc-macro2",
  "quote",
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
@@ -2284,6 +2284,12 @@ dependencies = [
  "matches",
 ]
 
+[[package]]
+name = "unicode-ident"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"
+
 [[package]]
 name = "unicode-normalization"
 version = "0.1.17"
@@ -2305,12 +2311,6 @@ version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
 
-[[package]]
-name = "unicode-xid"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
-
 [[package]]
 name = "unicode_categories"
 version = "0.1.1"
@@ -2406,9 +2406,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.74"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
+checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d"
 dependencies = [
  "cfg-if",
  "serde",
@@ -2418,13 +2418,13 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.74"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
+checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f"
 dependencies = [
  "bumpalo",
- "lazy_static",
  "log",
+ "once_cell",
  "proc-macro2",
  "quote",
  "syn",
@@ -2445,9 +2445,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.74"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
+checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -2455,9 +2455,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.74"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
+checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2468,9 +2468,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.74"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"
+checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a"
 
 [[package]]
 name = "web-sys"

src/commands/dump_plugin_data.rs

@@ -5,23 +5,7 @@ use std::io::Write;
 use std::path::Path;
 use tracing::info;
 
-use crate::models::plugin;
+use crate::models::{plugin, format_radix};
-
-// From: https://stackoverflow.com/a/50278316/6620612
-fn format_radix(mut x: u64, radix: u32) -> String {
-    let mut result = vec![];
-    loop {
-        let m = x % radix as u64;
-        x /= radix as u64;
-
-        // will panic if you use a bad radix (< 2 or > 36).
-        result.push(std::char::from_digit(m as u32, radix).unwrap());
-        if x == 0 {
-            break;
-        }
-    }
-    result.into_iter().rev().collect()
-}
 
 pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
     let mut page: u32 = 1;
@@ -39,7 +23,8 @@ pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, upda
             let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36)));
             info!(page = page, hash = plugin.hash, "dumping plugin data to {}", path.display());
             let mut file = File::create(path)?;
-            write!(file, "{}", serde_json::to_string(&plugin)?)?;
+            let json_val = serde_json::to_string(&plugin)?;
+            write!(file, "{}", json_val)?;
             last_hash = Some(plugin.hash);
         }
         page += 1;

src/models/file.rs

@@ -1,8 +1,11 @@
 use anyhow::{Context, Result};
 use chrono::NaiveDateTime;
 use serde::{Deserialize, Serialize};
+use sqlx::types::Json;
 use tracing::instrument;
 
+use super::hash_to_string;
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct File {
     pub id: i32,
@@ -42,10 +45,17 @@ pub struct FileWithCells {
     pub has_plugin: bool,
     pub unable_to_extract_plugins: bool,
     pub cells: Option<serde_json::Value>,
-    pub plugins: Option<serde_json::Value>,
+    pub plugins: Option<Json<Vec<FilePlugin>>>,
     pub plugin_count: Option<i64>,
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct FilePlugin {
+    #[serde(serialize_with = "hash_to_string")]
+    pub hash: i64,
+    pub file_path: String,
+}
+
 #[derive(Debug)]
 pub struct UnsavedFile<'a> {
     pub name: &'a str,
@@ -211,10 +221,10 @@ pub async fn batched_get_with_cells(
     if let Some(updated_after) = updated_after {
         sqlx::query_as!(
             FileWithCells,
-            "SELECT
+            r#"SELECT
                 files.*,
                 COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
-                COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
+                COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS "plugins: Json<Vec<FilePlugin>>",
                 COUNT(plugins.*) AS plugin_count
             FROM files
             LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
@@ -223,7 +233,7 @@ pub async fn batched_get_with_cells(
             WHERE files.id > $2 AND files.updated_at > $5
             GROUP BY files.id
             ORDER BY files.id ASC
-            LIMIT $1",
+            LIMIT $1"#,
             page_size,
             last_id,
             master,
@@ -236,10 +246,10 @@ pub async fn batched_get_with_cells(
     } else {
         sqlx::query_as!(
             FileWithCells,
-            "SELECT
+            r#"SELECT
                 files.*,
                 COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
-                COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
+                COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS "plugins: Json<Vec<FilePlugin>>",
                 COUNT(plugins.*) AS plugin_count
             FROM files
             LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
@@ -248,7 +258,7 @@ pub async fn batched_get_with_cells(
             WHERE files.id > $2
             GROUP BY files.id
             ORDER BY files.id ASC
-            LIMIT $1",
+            LIMIT $1"#,
             page_size,
             last_id,
             master,

src/models/mod.rs

@@ -8,3 +8,29 @@ pub mod plugin_world;
 pub mod world;
 
 pub const BATCH_SIZE: usize = 50;
+
+use serde::Serializer;
+
+// From: https://stackoverflow.com/a/50278316/6620612
+pub fn format_radix(mut x: u64, radix: u32) -> String {
+    let mut result = vec![];
+    loop {
+        let m = x % radix as u64;
+        x /= radix as u64;
+
+        // will panic if you use a bad radix (< 2 or > 36).
+        result.push(std::char::from_digit(m as u32, radix).unwrap());
+        if x == 0 {
+            break;
+        }
+    }
+    result.into_iter().rev().collect()
+}
+
+// Because JSON parsers are dumb and loose precision on i64s, serialize them to strings instead
+pub fn hash_to_string<S>(hash: &i64, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    serializer.serialize_str(&format_radix(*hash as u64, 36))
+}

src/models/plugin.rs

@@ -2,12 +2,16 @@ use anyhow::{Context, Result};
 use chrono::NaiveDateTime;
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
+use sqlx::types::Json;
 use tracing::instrument;
 
+use super::hash_to_string;
+
 #[derive(Debug, Serialize, Deserialize, FromRow)]
 pub struct Plugin {
     pub id: i32,
     pub name: String,
+    #[serde(serialize_with = "hash_to_string")]
     pub hash: i64,
     pub file_id: i32,
     pub mod_id: i32,
@@ -39,8 +43,9 @@ pub struct UnsavedPlugin<'a> {
 
 #[derive(Debug, Serialize, Deserialize, FromRow)]
 pub struct PluginsByHashWithMods {
+    #[serde(serialize_with = "hash_to_string")]
     pub hash: i64,
-    pub plugins: Option<serde_json::Value>,
+    pub plugins: Option<Json<Vec<Plugin>>>,
     pub files: Option<serde_json::Value>,
     pub mods: Option<serde_json::Value>,
     pub cells: Option<serde_json::Value>,
@@ -90,9 +95,9 @@ pub async fn batched_get_by_hash_with_mods(
     if let Some(updated_after) = updated_after {
         sqlx::query_as!(
             PluginsByHashWithMods,
-            "SELECT
+            r#"SELECT
                 plugins.hash,
-                json_agg(DISTINCT plugins.*) as plugins,
+                json_agg(DISTINCT plugins.*) as "plugins: Json<Vec<Plugin>>",
                 json_agg(DISTINCT files.*) as files,
                 json_agg(DISTINCT mods.*) as mods,
                 COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells
@@ -104,7 +109,7 @@ pub async fn batched_get_by_hash_with_mods(
             WHERE plugins.hash > $2 AND plugins.updated_at > $5
             GROUP BY plugins.hash
             ORDER BY plugins.hash ASC
-            LIMIT $1",
+            LIMIT $1"#,
             page_size,
             last_hash,
             master,
@@ -117,9 +122,9 @@ pub async fn batched_get_by_hash_with_mods(
     } else {
         sqlx::query_as!(
             PluginsByHashWithMods,
-            "SELECT
+            r#"SELECT
                 plugins.hash,
-                json_agg(DISTINCT plugins.*) as plugins,
+                json_agg(DISTINCT plugins.*) as "plugins: Json<Vec<Plugin>>",
                 json_agg(DISTINCT files.*) as files,
                 json_agg(DISTINCT mods.*) as mods,
                 COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells
@@ -131,7 +136,7 @@ pub async fn batched_get_by_hash_with_mods(
             WHERE plugins.hash > $2
             GROUP BY plugins.hash
             ORDER BY plugins.hash ASC
-            LIMIT $1",
+            LIMIT $1"#,
             page_size,
             last_hash,
             master,