thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Serialize plugin hashes to strings

Browse Source 
Avoids having dealing with JSON parsers on the frontend that loose precision and avoids having to convert integers to strings.
This commit is contained in:
Tyler Hallada
2022-08-08 23:37:27 -04:00
parent 33a29df2cb
commit b80edb49fa
5 changed files with 98 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/commands/dump_plugin_data.rs
View File

@@ -5,23 +5,7 @@ use std::io::Write;
use std::path::Path;
use tracing::info;
use crate::models::plugin;
// From: https://stackoverflow.com/a/50278316/6620612
fn format_radix(mut x: u64, radix: u32) -> String {
let mut result = vec![];
loop {
let m = x % radix as u64;
x /= radix as u64;
// will panic if you use a bad radix (< 2 or > 36).
result.push(std::char::from_digit(m as u32, radix).unwrap());
if x == 0 {
break;
}
}
result.into_iter().rev().collect()
}
use crate::models::{plugin, format_radix};
pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
let mut page: u32 = 1;
@@ -39,7 +23,8 @@ pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, upda
let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36)));
info!(page = page, hash = plugin.hash, "dumping plugin data to {}", path.display());
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&plugin)?)?;
let json_val = serde_json::to_string(&plugin)?;
write!(file, "{}", json_val)?;
last_hash = Some(plugin.hash);
}
page += 1;

24
src/models/file.rs
View File

@@ -1,8 +1,11 @@
use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use sqlx::types::Json;
use tracing::instrument;
use super::hash_to_string;
#[derive(Debug, Serialize, Deserialize)]
pub struct File {
pub id: i32,
@@ -42,10 +45,17 @@ pub struct FileWithCells {
pub has_plugin: bool,
pub unable_to_extract_plugins: bool,
pub cells: Option<serde_json::Value>,
pub plugins: Option<serde_json::Value>,
pub plugins: Option<Json<Vec<FilePlugin>>>,
pub plugin_count: Option<i64>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct FilePlugin {
#[serde(serialize_with = "hash_to_string")]
pub hash: i64,
pub file_path: String,
}
#[derive(Debug)]
pub struct UnsavedFile<'a> {
pub name: &'a str,
@@ -211,10 +221,10 @@ pub async fn batched_get_with_cells(
if let Some(updated_after) = updated_after {
sqlx::query_as!(
FileWithCells,
"SELECT
r#"SELECT
files.*,
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS "plugins: Json<Vec<FilePlugin>>",
COUNT(plugins.*) AS plugin_count
FROM files
LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
@@ -223,7 +233,7 @@ pub async fn batched_get_with_cells(
WHERE files.id > $2 AND files.updated_at > $5
GROUP BY files.id
ORDER BY files.id ASC
LIMIT $1",
LIMIT $1"#,
page_size,
last_id,
master,
@@ -236,10 +246,10 @@ pub async fn batched_get_with_cells(
} else {
sqlx::query_as!(
FileWithCells,
"SELECT
r#"SELECT
files.*,
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_path', plugins.file_path)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS "plugins: Json<Vec<FilePlugin>>",
COUNT(plugins.*) AS plugin_count
FROM files
LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
@@ -248,7 +258,7 @@ pub async fn batched_get_with_cells(
WHERE files.id > $2
GROUP BY files.id
ORDER BY files.id ASC
LIMIT $1",
LIMIT $1"#,
page_size,
last_id,
master,

26
src/models/mod.rs
View File

@@ -8,3 +8,29 @@ pub mod plugin_world;
pub mod world;
pub const BATCH_SIZE: usize = 50;
use serde::Serializer;
// From: https://stackoverflow.com/a/50278316/6620612
pub fn format_radix(mut x: u64, radix: u32) -> String {
let mut result = vec![];
loop {
let m = x % radix as u64;
x /= radix as u64;
// will panic if you use a bad radix (< 2 or > 36).
result.push(std::char::from_digit(m as u32, radix).unwrap());
if x == 0 {
break;
}
}
result.into_iter().rev().collect()
}
// Because JSON parsers are dumb and loose precision on i64s, serialize them to strings instead
pub fn hash_to_string<S>(hash: &i64, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&format_radix(*hash as u64, 36))
}

19
src/models/plugin.rs
View File

@@ -2,12 +2,16 @@ use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use sqlx::FromRow;
use sqlx::types::Json;
use tracing::instrument;
use super::hash_to_string;
#[derive(Debug, Serialize, Deserialize, FromRow)]
pub struct Plugin {
pub id: i32,
pub name: String,
#[serde(serialize_with = "hash_to_string")]
pub hash: i64,
pub file_id: i32,
pub mod_id: i32,
@@ -39,8 +43,9 @@ pub struct UnsavedPlugin<'a> {
#[derive(Debug, Serialize, Deserialize, FromRow)]
pub struct PluginsByHashWithMods {
#[serde(serialize_with = "hash_to_string")]
pub hash: i64,
pub plugins: Option<serde_json::Value>,
pub plugins: Option<Json<Vec<Plugin>>>,
pub files: Option<serde_json::Value>,
pub mods: Option<serde_json::Value>,
pub cells: Option<serde_json::Value>,
@@ -90,9 +95,9 @@ pub async fn batched_get_by_hash_with_mods(
if let Some(updated_after) = updated_after {
sqlx::query_as!(
PluginsByHashWithMods,
"SELECT
r#"SELECT
plugins.hash,
json_agg(DISTINCT plugins.*) as plugins,
json_agg(DISTINCT plugins.*) as "plugins: Json<Vec<Plugin>>",
json_agg(DISTINCT files.*) as files,
json_agg(DISTINCT mods.*) as mods,
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells
@@ -104,7 +109,7 @@ pub async fn batched_get_by_hash_with_mods(
WHERE plugins.hash > $2 AND plugins.updated_at > $5
GROUP BY plugins.hash
ORDER BY plugins.hash ASC
LIMIT $1",
LIMIT $1"#,
page_size,
last_hash,
master,
@@ -117,9 +122,9 @@ pub async fn batched_get_by_hash_with_mods(
} else {
sqlx::query_as!(
PluginsByHashWithMods,
"SELECT
r#"SELECT
plugins.hash,
json_agg(DISTINCT plugins.*) as plugins,
json_agg(DISTINCT plugins.*) as "plugins: Json<Vec<Plugin>>",
json_agg(DISTINCT files.*) as files,
json_agg(DISTINCT mods.*) as mods,
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells
@@ -131,7 +136,7 @@ pub async fn batched_get_by_hash_with_mods(
WHERE plugins.hash > $2
GROUP BY plugins.hash
ORDER BY plugins.hash ASC
LIMIT $1",
LIMIT $1"#,
page_size,
last_hash,
master,