Recreate pg connection pool in command to prevent slowdown

This commit is contained in:
Tyler Hallada 2023-11-15 18:07:26 -05:00
parent c931eeca86
commit 8cb9bfa4ca
2 changed files with 18 additions and 4 deletions

View File

@ -1,23 +1,37 @@
use crate::models::cell::{self, CellFileEditCount}; use crate::models::cell::{self, CellFileEditCount};
use anyhow::Result; use anyhow::Result;
use chrono::{Duration, NaiveDateTime}; use chrono::{Duration, NaiveDateTime};
use std::collections::HashMap; use sqlx::postgres::PgPoolOptions;
use std::{collections::HashMap, env};
use tokio::fs::File; use tokio::fs::File;
use tokio::io::AsyncWriteExt; use tokio::io::AsyncWriteExt;
use tracing::{debug, info}; use tracing::{debug, info};
pub async fn dump_cell_edit_counts_over_time( pub async fn dump_cell_edit_counts_over_time(
pool: &sqlx::Pool<sqlx::Postgres>,
start_date: NaiveDateTime, start_date: NaiveDateTime,
end_date: NaiveDateTime, end_date: NaiveDateTime,
path: &str, path: &str,
) -> Result<()> { ) -> Result<()> {
let mut pool = PgPoolOptions::new()
.max_connections(5)
.connect(&env::var("DATABASE_URL")?)
.await?;
let mut i = 0;
let mut current_date = start_date; let mut current_date = start_date;
while current_date <= end_date { while current_date <= end_date {
if i % 5 == 0 {
// There's a weird issue that slows down this query after 5 iterations. Recreating the
// connection pool seems to fix it. I don't know why.
info!("reconnecting to database");
pool = PgPoolOptions::new()
.max_connections(5)
.connect(&env::var("DATABASE_URL")?)
.await?;
}
let next_date = current_date + Duration::weeks(1); let next_date = current_date + Duration::weeks(1);
let mut cell_file_edit_counts = HashMap::new(); let mut cell_file_edit_counts = HashMap::new();
let counts = let counts =
cell::count_file_edits_in_time_range(pool, "Skyrim.esm", 1, current_date, next_date) cell::count_file_edits_in_time_range(&pool, "Skyrim.esm", 1, current_date, next_date)
.await?; .await?;
for x in -77..75 { for x in -77..75 {
for y in -50..44 { for y in -50..44 {
@ -42,6 +56,7 @@ pub async fn dump_cell_edit_counts_over_time(
file.write_all(serde_json::to_string(&cell_file_edit_counts)?.as_bytes()).await?; file.write_all(serde_json::to_string(&cell_file_edit_counts)?.as_bytes()).await?;
current_date = next_date; current_date = next_date;
i += 1;
} }
Ok(()) Ok(())
} }

View File

@ -113,7 +113,6 @@ pub async fn main() -> Result<()> {
} }
if let Some(path) = args.dump_edits_over_time { if let Some(path) = args.dump_edits_over_time {
return dump_cell_edit_counts_over_time( return dump_cell_edit_counts_over_time(
&pool,
NaiveDate::from_ymd_opt(2011, 11, 11).unwrap().and_hms_opt(0, 0, 0).unwrap(), NaiveDate::from_ymd_opt(2011, 11, 11).unwrap().and_hms_opt(0, 0, 0).unwrap(),
Utc::now().naive_utc(), Utc::now().naive_utc(),
&path, &path,