Serialize plugin hashes to strings
Avoids having dealing with JSON parsers on the frontend that loose precision and avoids having to convert integers to strings.
This commit is contained in:
parent
33a29df2cb
commit
b80edb49fa
80
Cargo.lock
generated
80
Cargo.lock
generated
@ -646,12 +646,6 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.11.2"
|
||||
@ -661,6 +655,12 @@ dependencies = [
|
||||
"ahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.7.0"
|
||||
@ -808,12 +808,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "1.6.2"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
|
||||
checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"hashbrown 0.9.1",
|
||||
"hashbrown 0.12.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -845,9 +845,9 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.51"
|
||||
version = "0.3.59"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062"
|
||||
checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2"
|
||||
dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
@ -873,9 +873,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.94"
|
||||
version = "0.2.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"
|
||||
checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
@ -1172,9 +1172,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.7.2"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
|
||||
checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
@ -1364,18 +1364,18 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.27"
|
||||
version = "1.0.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
|
||||
checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.9"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
|
||||
checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@ -2004,13 +2004,13 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.72"
|
||||
version = "1.0.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
|
||||
checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-xid",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2284,6 +2284,12 @@ dependencies = [
|
||||
"matches",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.17"
|
||||
@ -2305,12 +2311,6 @@ version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
|
||||
|
||||
[[package]]
|
||||
name = "unicode_categories"
|
||||
version = "0.1.1"
|
||||
@ -2406,9 +2406,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.74"
|
||||
version = "0.2.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
|
||||
checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"serde",
|
||||
@ -2418,13 +2418,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-backend"
|
||||
version = "0.2.74"
|
||||
version = "0.2.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
|
||||
checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
@ -2445,9 +2445,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.74"
|
||||
version = "0.2.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
|
||||
checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
@ -2455,9 +2455,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.74"
|
||||
version = "0.2.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
|
||||
checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@ -2468,9 +2468,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.74"
|
||||
version = "0.2.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"
|
||||
checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a"
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
|
||||
@ -5,23 +5,7 @@ use std::io::Write;
|
||||
use std::path::Path;
|
||||
use tracing::info;
|
||||
|
||||
use crate::models::plugin;
|
||||
|
||||
// From: https://stackoverflow.com/a/50278316/6620612
|
||||
fn format_radix(mut x: u64, radix: u32) -> String {
|
||||
let mut result = vec![];
|
||||
loop {
|
||||
let m = x % radix as u64;
|
||||
x /= radix as u64;
|
||||
|
||||
// will panic if you use a bad radix (< 2 or > 36).
|
||||
result.push(std::char::from_digit(m as u32, radix).unwrap());
|
||||
if x == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
result.into_iter().rev().collect()
|
||||
}
|
||||
use crate::models::{plugin, format_radix};
|
||||
|
||||
pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
|
||||
let mut page: u32 = 1;
|
||||
@ -39,7 +23,8 @@ pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, upda
|
||||
let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36)));
|
||||
info!(page = page, hash = plugin.hash, "dumping plugin data to {}", path.display());
|
||||
let mut file = File::create(path)?;
|
||||
write!(file, "{}", serde_json::to_string(&plugin)?)?;
|
||||
let json_val = serde_json::to_string(&plugin)?;
|
||||
write!(file, "{}", json_val)?;
|
||||
last_hash = Some(plugin.hash);
|
||||
}
|
||||
page += 1;
|
||||
|
||||
@ -1,8 +1,11 @@
|
||||
use anyhow::{Context, Result};
|
||||
use chrono::NaiveDateTime;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::types::Json;
|
||||
use tracing::instrument;
|
||||
|
||||
use super::hash_to_string;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct File {
|
||||
pub id: i32,
|
||||
@ -42,10 +45,17 @@ pub struct FileWithCells {
|
||||
pub has_plugin: bool,
|
||||
pub unable_to_extract_plugins: bool,
|
||||
pub cells: Option<serde_json::Value>,
|
||||
pub plugins: Option<serde_json::Value>,
|
||||
pub plugins: Option<Json<Vec<FilePlugin>>>,
|
||||
pub plugin_count: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct FilePlugin {
|
||||
#[serde(serialize_with = "hash_to_string")]
|
||||
pub hash: i64,
|
||||
pub file_path: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct UnsavedFile<'a> {
|
||||
pub name: &'a str,
|
||||
@ -211,10 +221,10 @@ pub async fn batched_get_with_cells(
|
||||
if let Some(updated_after) = updated_after {
|
||||
sqlx::query_as!(
|
||||
FileWithCells,
|
||||
"SELECT
|
||||
r#"SELECT
|
||||
files.*,
|
||||
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
|
||||
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
|
||||
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS "plugins: Json<Vec<FilePlugin>>",
|
||||
COUNT(plugins.*) AS plugin_count
|
||||
FROM files
|
||||
LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
|
||||
@ -223,7 +233,7 @@ pub async fn batched_get_with_cells(
|
||||
WHERE files.id > $2 AND files.updated_at > $5
|
||||
GROUP BY files.id
|
||||
ORDER BY files.id ASC
|
||||
LIMIT $1",
|
||||
LIMIT $1"#,
|
||||
page_size,
|
||||
last_id,
|
||||
master,
|
||||
@ -236,10 +246,10 @@ pub async fn batched_get_with_cells(
|
||||
} else {
|
||||
sqlx::query_as!(
|
||||
FileWithCells,
|
||||
"SELECT
|
||||
r#"SELECT
|
||||
files.*,
|
||||
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
|
||||
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
|
||||
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS "plugins: Json<Vec<FilePlugin>>",
|
||||
COUNT(plugins.*) AS plugin_count
|
||||
FROM files
|
||||
LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
|
||||
@ -248,7 +258,7 @@ pub async fn batched_get_with_cells(
|
||||
WHERE files.id > $2
|
||||
GROUP BY files.id
|
||||
ORDER BY files.id ASC
|
||||
LIMIT $1",
|
||||
LIMIT $1"#,
|
||||
page_size,
|
||||
last_id,
|
||||
master,
|
||||
|
||||
@ -8,3 +8,29 @@ pub mod plugin_world;
|
||||
pub mod world;
|
||||
|
||||
pub const BATCH_SIZE: usize = 50;
|
||||
|
||||
use serde::Serializer;
|
||||
|
||||
// From: https://stackoverflow.com/a/50278316/6620612
|
||||
pub fn format_radix(mut x: u64, radix: u32) -> String {
|
||||
let mut result = vec![];
|
||||
loop {
|
||||
let m = x % radix as u64;
|
||||
x /= radix as u64;
|
||||
|
||||
// will panic if you use a bad radix (< 2 or > 36).
|
||||
result.push(std::char::from_digit(m as u32, radix).unwrap());
|
||||
if x == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
result.into_iter().rev().collect()
|
||||
}
|
||||
|
||||
// Because JSON parsers are dumb and loose precision on i64s, serialize them to strings instead
|
||||
pub fn hash_to_string<S>(hash: &i64, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str(&format_radix(*hash as u64, 36))
|
||||
}
|
||||
|
||||
@ -2,12 +2,16 @@ use anyhow::{Context, Result};
|
||||
use chrono::NaiveDateTime;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::FromRow;
|
||||
use sqlx::types::Json;
|
||||
use tracing::instrument;
|
||||
|
||||
use super::hash_to_string;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, FromRow)]
|
||||
pub struct Plugin {
|
||||
pub id: i32,
|
||||
pub name: String,
|
||||
#[serde(serialize_with = "hash_to_string")]
|
||||
pub hash: i64,
|
||||
pub file_id: i32,
|
||||
pub mod_id: i32,
|
||||
@ -39,8 +43,9 @@ pub struct UnsavedPlugin<'a> {
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, FromRow)]
|
||||
pub struct PluginsByHashWithMods {
|
||||
#[serde(serialize_with = "hash_to_string")]
|
||||
pub hash: i64,
|
||||
pub plugins: Option<serde_json::Value>,
|
||||
pub plugins: Option<Json<Vec<Plugin>>>,
|
||||
pub files: Option<serde_json::Value>,
|
||||
pub mods: Option<serde_json::Value>,
|
||||
pub cells: Option<serde_json::Value>,
|
||||
@ -90,9 +95,9 @@ pub async fn batched_get_by_hash_with_mods(
|
||||
if let Some(updated_after) = updated_after {
|
||||
sqlx::query_as!(
|
||||
PluginsByHashWithMods,
|
||||
"SELECT
|
||||
r#"SELECT
|
||||
plugins.hash,
|
||||
json_agg(DISTINCT plugins.*) as plugins,
|
||||
json_agg(DISTINCT plugins.*) as "plugins: Json<Vec<Plugin>>",
|
||||
json_agg(DISTINCT files.*) as files,
|
||||
json_agg(DISTINCT mods.*) as mods,
|
||||
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells
|
||||
@ -104,7 +109,7 @@ pub async fn batched_get_by_hash_with_mods(
|
||||
WHERE plugins.hash > $2 AND plugins.updated_at > $5
|
||||
GROUP BY plugins.hash
|
||||
ORDER BY plugins.hash ASC
|
||||
LIMIT $1",
|
||||
LIMIT $1"#,
|
||||
page_size,
|
||||
last_hash,
|
||||
master,
|
||||
@ -117,9 +122,9 @@ pub async fn batched_get_by_hash_with_mods(
|
||||
} else {
|
||||
sqlx::query_as!(
|
||||
PluginsByHashWithMods,
|
||||
"SELECT
|
||||
r#"SELECT
|
||||
plugins.hash,
|
||||
json_agg(DISTINCT plugins.*) as plugins,
|
||||
json_agg(DISTINCT plugins.*) as "plugins: Json<Vec<Plugin>>",
|
||||
json_agg(DISTINCT files.*) as files,
|
||||
json_agg(DISTINCT mods.*) as mods,
|
||||
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells
|
||||
@ -131,7 +136,7 @@ pub async fn batched_get_by_hash_with_mods(
|
||||
WHERE plugins.hash > $2
|
||||
GROUP BY plugins.hash
|
||||
ORDER BY plugins.hash ASC
|
||||
LIMIT $1",
|
||||
LIMIT $1"#,
|
||||
page_size,
|
||||
last_hash,
|
||||
master,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user