Serialize plugin hashes to strings

Avoids having dealing with JSON parsers on the frontend that loose precision and avoids having to convert integers to strings.
This commit is contained in:
Tyler Hallada 2022-08-08 23:37:27 -04:00
parent 33a29df2cb
commit b80edb49fa
5 changed files with 98 additions and 72 deletions

80
Cargo.lock generated
View File

@ -646,12 +646,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "hashbrown"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
[[package]]
name = "hashbrown"
version = "0.11.2"
@ -661,6 +655,12 @@ dependencies = [
"ahash",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashlink"
version = "0.7.0"
@ -808,12 +808,12 @@ dependencies = [
[[package]]
name = "indexmap"
version = "1.6.2"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
dependencies = [
"autocfg",
"hashbrown 0.9.1",
"hashbrown 0.12.3",
]
[[package]]
@ -845,9 +845,9 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
[[package]]
name = "js-sys"
version = "0.3.51"
version = "0.3.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062"
checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2"
dependencies = [
"wasm-bindgen",
]
@ -873,9 +873,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.94"
version = "0.2.127"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"
checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
[[package]]
name = "lock_api"
@ -1172,9 +1172,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.7.2"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
[[package]]
name = "opaque-debug"
@ -1364,18 +1364,18 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
[[package]]
name = "proc-macro2"
version = "1.0.27"
version = "1.0.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
dependencies = [
"unicode-xid",
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.9"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
dependencies = [
"proc-macro2",
]
@ -2004,13 +2004,13 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
[[package]]
name = "syn"
version = "1.0.72"
version = "1.0.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
"unicode-ident",
]
[[package]]
@ -2284,6 +2284,12 @@ dependencies = [
"matches",
]
[[package]]
name = "unicode-ident"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"
[[package]]
name = "unicode-normalization"
version = "0.1.17"
@ -2305,12 +2311,6 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
[[package]]
name = "unicode-xid"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "unicode_categories"
version = "0.1.1"
@ -2406,9 +2406,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.74"
version = "0.2.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d"
dependencies = [
"cfg-if",
"serde",
@ -2418,13 +2418,13 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.74"
version = "0.2.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
@ -2445,9 +2445,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.74"
version = "0.2.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -2455,9 +2455,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.74"
version = "0.2.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da"
dependencies = [
"proc-macro2",
"quote",
@ -2468,9 +2468,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.74"
version = "0.2.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"
checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a"
[[package]]
name = "web-sys"

View File

@ -5,23 +5,7 @@ use std::io::Write;
use std::path::Path;
use tracing::info;
use crate::models::plugin;
// From: https://stackoverflow.com/a/50278316/6620612
fn format_radix(mut x: u64, radix: u32) -> String {
let mut result = vec![];
loop {
let m = x % radix as u64;
x /= radix as u64;
// will panic if you use a bad radix (< 2 or > 36).
result.push(std::char::from_digit(m as u32, radix).unwrap());
if x == 0 {
break;
}
}
result.into_iter().rev().collect()
}
use crate::models::{plugin, format_radix};
pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
let mut page: u32 = 1;
@ -39,7 +23,8 @@ pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, upda
let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36)));
info!(page = page, hash = plugin.hash, "dumping plugin data to {}", path.display());
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&plugin)?)?;
let json_val = serde_json::to_string(&plugin)?;
write!(file, "{}", json_val)?;
last_hash = Some(plugin.hash);
}
page += 1;

View File

@ -1,8 +1,11 @@
use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use sqlx::types::Json;
use tracing::instrument;
use super::hash_to_string;
#[derive(Debug, Serialize, Deserialize)]
pub struct File {
pub id: i32,
@ -42,10 +45,17 @@ pub struct FileWithCells {
pub has_plugin: bool,
pub unable_to_extract_plugins: bool,
pub cells: Option<serde_json::Value>,
pub plugins: Option<serde_json::Value>,
pub plugins: Option<Json<Vec<FilePlugin>>>,
pub plugin_count: Option<i64>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct FilePlugin {
#[serde(serialize_with = "hash_to_string")]
pub hash: i64,
pub file_path: String,
}
#[derive(Debug)]
pub struct UnsavedFile<'a> {
pub name: &'a str,
@ -211,10 +221,10 @@ pub async fn batched_get_with_cells(
if let Some(updated_after) = updated_after {
sqlx::query_as!(
FileWithCells,
"SELECT
r#"SELECT
files.*,
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS "plugins: Json<Vec<FilePlugin>>",
COUNT(plugins.*) AS plugin_count
FROM files
LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
@ -223,7 +233,7 @@ pub async fn batched_get_with_cells(
WHERE files.id > $2 AND files.updated_at > $5
GROUP BY files.id
ORDER BY files.id ASC
LIMIT $1",
LIMIT $1"#,
page_size,
last_id,
master,
@ -236,10 +246,10 @@ pub async fn batched_get_with_cells(
} else {
sqlx::query_as!(
FileWithCells,
"SELECT
r#"SELECT
files.*,
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS "plugins: Json<Vec<FilePlugin>>",
COUNT(plugins.*) AS plugin_count
FROM files
LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
@ -248,7 +258,7 @@ pub async fn batched_get_with_cells(
WHERE files.id > $2
GROUP BY files.id
ORDER BY files.id ASC
LIMIT $1",
LIMIT $1"#,
page_size,
last_id,
master,

View File

@ -8,3 +8,29 @@ pub mod plugin_world;
pub mod world;
pub const BATCH_SIZE: usize = 50;
use serde::Serializer;
// From: https://stackoverflow.com/a/50278316/6620612
pub fn format_radix(mut x: u64, radix: u32) -> String {
let mut result = vec![];
loop {
let m = x % radix as u64;
x /= radix as u64;
// will panic if you use a bad radix (< 2 or > 36).
result.push(std::char::from_digit(m as u32, radix).unwrap());
if x == 0 {
break;
}
}
result.into_iter().rev().collect()
}
// Because JSON parsers are dumb and loose precision on i64s, serialize them to strings instead
pub fn hash_to_string<S>(hash: &i64, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&format_radix(*hash as u64, 36))
}

View File

@ -2,12 +2,16 @@ use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use sqlx::FromRow;
use sqlx::types::Json;
use tracing::instrument;
use super::hash_to_string;
#[derive(Debug, Serialize, Deserialize, FromRow)]
pub struct Plugin {
pub id: i32,
pub name: String,
#[serde(serialize_with = "hash_to_string")]
pub hash: i64,
pub file_id: i32,
pub mod_id: i32,
@ -39,8 +43,9 @@ pub struct UnsavedPlugin<'a> {
#[derive(Debug, Serialize, Deserialize, FromRow)]
pub struct PluginsByHashWithMods {
#[serde(serialize_with = "hash_to_string")]
pub hash: i64,
pub plugins: Option<serde_json::Value>,
pub plugins: Option<Json<Vec<Plugin>>>,
pub files: Option<serde_json::Value>,
pub mods: Option<serde_json::Value>,
pub cells: Option<serde_json::Value>,
@ -90,9 +95,9 @@ pub async fn batched_get_by_hash_with_mods(
if let Some(updated_after) = updated_after {
sqlx::query_as!(
PluginsByHashWithMods,
"SELECT
r#"SELECT
plugins.hash,
json_agg(DISTINCT plugins.*) as plugins,
json_agg(DISTINCT plugins.*) as "plugins: Json<Vec<Plugin>>",
json_agg(DISTINCT files.*) as files,
json_agg(DISTINCT mods.*) as mods,
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells
@ -104,7 +109,7 @@ pub async fn batched_get_by_hash_with_mods(
WHERE plugins.hash > $2 AND plugins.updated_at > $5
GROUP BY plugins.hash
ORDER BY plugins.hash ASC
LIMIT $1",
LIMIT $1"#,
page_size,
last_hash,
master,
@ -117,9 +122,9 @@ pub async fn batched_get_by_hash_with_mods(
} else {
sqlx::query_as!(
PluginsByHashWithMods,
"SELECT
r#"SELECT
plugins.hash,
json_agg(DISTINCT plugins.*) as plugins,
json_agg(DISTINCT plugins.*) as "plugins: Json<Vec<Plugin>>",
json_agg(DISTINCT files.*) as files,
json_agg(DISTINCT mods.*) as mods,
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells
@ -131,7 +136,7 @@ pub async fn batched_get_by_hash_with_mods(
WHERE plugins.hash > $2
GROUP BY plugins.hash
ORDER BY plugins.hash ASC
LIMIT $1",
LIMIT $1"#,
page_size,
last_hash,
master,