thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update deps, rustfmt, clippy fixes

Browse Source
This commit is contained in:
Tyler Hallada 2023-04-20 00:47:14 -04:00
parent 03cf20029e
commit 01444cd928
22 changed files with 583 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

413
Cargo.lock generated
View File

@ -31,6 +31,18 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "ahash"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
dependencies = [
"cfg-if",
"getrandom 0.2.7",
"once_cell",
"version_check",
]
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "0.7.19" version = "0.7.19"
@ -84,7 +96,7 @@ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -234,9 +246,9 @@ dependencies = [
[[package]] [[package]]
name = "compress-tools" name = "compress-tools"
version = "0.13.0" version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d4318a3d43a20f0eacfa42b5492eb9cbab5875bb5bb5d1feb07d673a0b72777" checksum = "537b7d54356f29ce8e3fc8fdf89ab1ad752ac22987ff4cb9841e2bcf55785134"
dependencies = [ dependencies = [
"derive_more", "derive_more",
"libc", "libc",
@ -346,19 +358,19 @@ dependencies = [
[[package]] [[package]]
name = "cssparser" name = "cssparser"
version = "0.27.2" version = "0.29.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" checksum = "f93d03419cb5950ccfd3daf3ff1c7a36ace64609a1a8746d493df1ca0afde0fa"
dependencies = [ dependencies = [
"cssparser-macros", "cssparser-macros",
"dtoa-short", "dtoa-short",
"itoa 0.4.8", "itoa",
"matches", "matches",
"phf 0.8.0", "phf 0.10.1",
"proc-macro2", "proc-macro2",
"quote", "quote",
"smallvec", "smallvec",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -368,7 +380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
dependencies = [ dependencies = [
"quote", "quote",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -381,7 +393,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"rustc_version", "rustc_version",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -475,6 +487,27 @@ dependencies = [
"num-traits 0.1.43", "num-traits 0.1.43",
] ]
[[package]]
name = "errno"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
dependencies = [
"errno-dragonfly",
"libc",
"windows-sys 0.48.0",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]] [[package]]
name = "event-listener" name = "event-listener"
version = "2.5.3" version = "2.5.3"
@ -619,7 +652,7 @@ checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -727,7 +760,7 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [ dependencies = [
"ahash", "ahash 0.7.6",
] ]
[[package]] [[package]]
@ -757,6 +790,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "hermit-abi"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
[[package]] [[package]]
name = "hex" name = "hex"
version = "0.4.3" version = "0.4.3"
@ -792,7 +831,7 @@ dependencies = [
"markup5ever", "markup5ever",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -803,7 +842,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
dependencies = [ dependencies = [
"bytes", "bytes",
"fnv", "fnv",
"itoa 1.0.3", "itoa",
] ]
[[package]] [[package]]
@ -831,9 +870,12 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
[[package]] [[package]]
name = "humansize" name = "humansize"
version = "1.1.1" version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026" checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
dependencies = [
"libm",
]
[[package]] [[package]]
name = "hyper" name = "hyper"
@ -850,7 +892,7 @@ dependencies = [
"http-body", "http-body",
"httparse", "httparse",
"httpdate", "httpdate",
"itoa 1.0.3", "itoa",
"pin-project-lite", "pin-project-lite",
"socket2", "socket2",
"tokio", "tokio",
@ -907,9 +949,9 @@ dependencies = [
[[package]] [[package]]
name = "infer" name = "infer"
version = "0.9.0" version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f178e61cdbfe084aa75a2f4f7a25a5bb09701a47ae1753608f194b15783c937a" checksum = "f551f8c3a39f68f986517db0d1759de85881894fdc7db798bd2a9df9cb04b7fc"
[[package]] [[package]]
name = "instant" name = "instant"
@ -920,6 +962,17 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "io-lifetimes"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220"
dependencies = [
"hermit-abi 0.3.1",
"libc",
"windows-sys 0.48.0",
]
[[package]] [[package]]
name = "ipnet" name = "ipnet"
version = "2.5.0" version = "2.5.0"
@ -935,12 +988,6 @@ dependencies = [
"either", "either",
] ]
[[package]]
name = "itoa"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]] [[package]]
name = "itoa" name = "itoa"
version = "1.0.3" version = "1.0.3"
@ -986,9 +1033,21 @@ dependencies = [
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.134" version = "0.2.141"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
[[package]]
name = "libm"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
[[package]]
name = "linux-raw-sys"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
@ -1080,7 +1139,7 @@ dependencies = [
"libc", "libc",
"log", "log",
"wasi 0.11.0+wasi-snapshot-preview1", "wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys", "windows-sys 0.36.1",
] ]
[[package]] [[package]]
@ -1259,7 +1318,7 @@ version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
dependencies = [ dependencies = [
"hermit-abi", "hermit-abi 0.1.19",
"libc", "libc",
] ]
@ -1307,7 +1366,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -1359,7 +1418,7 @@ dependencies = [
"cfg-if", "cfg-if",
"instant", "instant",
"libc", "libc",
"redox_syscall", "redox_syscall 0.2.16",
"smallvec", "smallvec",
"winapi", "winapi",
] ]
@ -1372,9 +1431,9 @@ checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"libc", "libc",
"redox_syscall", "redox_syscall 0.2.16",
"smallvec", "smallvec",
"windows-sys", "windows-sys 0.36.1",
] ]
[[package]] [[package]]
@ -1418,9 +1477,7 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
dependencies = [ dependencies = [
"phf_macros",
"phf_shared 0.8.0", "phf_shared 0.8.0",
"proc-macro-hack",
] ]
[[package]] [[package]]
@ -1429,7 +1486,9 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"
dependencies = [ dependencies = [
"phf_macros",
"phf_shared 0.10.0", "phf_shared 0.10.0",
"proc-macro-hack",
] ]
[[package]] [[package]]
@ -1474,16 +1533,16 @@ dependencies = [
[[package]] [[package]]
name = "phf_macros" name = "phf_macros"
version = "0.8.0" version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0"
dependencies = [ dependencies = [
"phf_generator 0.8.0", "phf_generator 0.10.0",
"phf_shared 0.8.0", "phf_shared 0.10.0",
"proc-macro-hack", "proc-macro-hack",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -1542,18 +1601,18 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.46" version = "1.0.56"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
dependencies = [ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.21" version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
] ]
@ -1691,6 +1750,15 @@ dependencies = [
"bitflags", "bitflags",
] ]
[[package]]
name = "redox_syscall"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
"bitflags",
]
[[package]] [[package]]
name = "redox_users" name = "redox_users"
version = "0.4.3" version = "0.4.3"
@ -1698,7 +1766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [ dependencies = [
"getrandom 0.2.7", "getrandom 0.2.7",
"redox_syscall", "redox_syscall 0.2.16",
"thiserror", "thiserror",
] ]
@ -1719,15 +1787,6 @@ version = "0.6.27"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
[[package]]
name = "remove_dir_all"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "reqwest" name = "reqwest"
version = "0.11.12" version = "0.11.12"
@ -1781,6 +1840,20 @@ dependencies = [
"semver", "semver",
] ]
[[package]]
name = "rustix"
version = "0.37.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659"
dependencies = [
"bitflags",
"errno",
"io-lifetimes",
"libc",
"linux-raw-sys",
"windows-sys 0.48.0",
]
[[package]] [[package]]
name = "ryu" name = "ryu"
version = "1.0.11" version = "1.0.11"
@ -1803,7 +1876,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2"
dependencies = [ dependencies = [
"lazy_static", "lazy_static",
"windows-sys", "windows-sys 0.36.1",
] ]
[[package]] [[package]]
@ -1814,15 +1887,16 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]] [[package]]
name = "scraper" name = "scraper"
version = "0.13.0" version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5684396b456f3eb69ceeb34d1b5cb1a2f6acf7ca4452131efa3ba0ee2c2d0a70" checksum = "59e25654b5e9fd557a67dbaab5a5d36b8c448d0561beb4c041b6dbb902eddfa6"
dependencies = [ dependencies = [
"ahash 0.8.3",
"cssparser", "cssparser",
"ego-tree", "ego-tree",
"getopts", "getopts",
"html5ever", "html5ever",
"matches", "once_cell",
"selectors", "selectors",
"smallvec", "smallvec",
"tendril", "tendril",
@ -1859,22 +1933,20 @@ dependencies = [
[[package]] [[package]]
name = "selectors" name = "selectors"
version = "0.22.0" version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" checksum = "0c37578180969d00692904465fb7f6b3d50b9a2b952b87c23d0e2e5cb5013416"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"cssparser", "cssparser",
"derive_more", "derive_more",
"fxhash", "fxhash",
"log", "log",
"matches",
"phf 0.8.0", "phf 0.8.0",
"phf_codegen 0.8.0", "phf_codegen 0.8.0",
"precomputed-hash", "precomputed-hash",
"servo_arc", "servo_arc",
"smallvec", "smallvec",
"thin-slice",
] ]
[[package]] [[package]]
@ -1900,7 +1972,7 @@ checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -1909,7 +1981,7 @@ version = "1.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
dependencies = [ dependencies = [
"itoa 1.0.3", "itoa",
"ryu", "ryu",
"serde", "serde",
] ]
@ -1921,16 +1993,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
dependencies = [ dependencies = [
"form_urlencoded", "form_urlencoded",
"itoa 1.0.3", "itoa",
"ryu", "ryu",
"serde", "serde",
] ]
[[package]] [[package]]
name = "servo_arc" name = "servo_arc"
version = "0.1.1" version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" checksum = "d52aa42f8fdf0fed91e5ce7f23d8138441002fa31dca008acf47e6fd4721f741"
dependencies = [ dependencies = [
"nodrop", "nodrop",
"stable_deref_trait", "stable_deref_trait",
@ -2007,15 +2079,15 @@ dependencies = [
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.9.0" version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
[[package]] [[package]]
name = "socket2" name = "socket2"
version = "0.4.7" version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
dependencies = [ dependencies = [
"libc", "libc",
"winapi", "winapi",
@ -2048,7 +2120,7 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcbc16ddba161afc99e14d1713a453747a2b07fc097d2009f4c300ec99286105" checksum = "dcbc16ddba161afc99e14d1713a453747a2b07fc097d2009f4c300ec99286105"
dependencies = [ dependencies = [
"ahash", "ahash 0.7.6",
"atoi", "atoi",
"base64", "base64",
"bitflags", "bitflags",
@ -2070,7 +2142,7 @@ dependencies = [
"hkdf", "hkdf",
"hmac", "hmac",
"indexmap", "indexmap",
"itoa 1.0.3", "itoa",
"libc", "libc",
"log", "log",
"md-5", "md-5",
@ -2109,7 +2181,7 @@ dependencies = [
"sha2", "sha2",
"sqlx-core", "sqlx-core",
"sqlx-rt", "sqlx-rt",
"syn", "syn 1.0.101",
"url", "url",
] ]
@ -2190,6 +2262,17 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "syn"
version = "2.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]] [[package]]
name = "tap" name = "tap"
version = "1.0.1" version = "1.0.1"
@ -2198,16 +2281,15 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]] [[package]]
name = "tempfile" name = "tempfile"
version = "3.3.0" version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"fastrand", "fastrand",
"libc", "redox_syscall 0.3.5",
"redox_syscall", "rustix",
"remove_dir_all", "windows-sys 0.45.0",
"winapi",
] ]
[[package]] [[package]]
@ -2221,12 +2303,6 @@ dependencies = [
"utf-8", "utf-8",
] ]
[[package]]
name = "thin-slice"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.37" version = "1.0.37"
@ -2244,7 +2320,7 @@ checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -2273,7 +2349,7 @@ version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b"
dependencies = [ dependencies = [
"itoa 1.0.3", "itoa",
"libc", "libc",
"num_threads", "num_threads",
"time-macros", "time-macros",
@ -2302,14 +2378,13 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.21.2" version = "1.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"bytes", "bytes",
"libc", "libc",
"memchr",
"mio", "mio",
"num_cpus", "num_cpus",
"parking_lot 0.12.1", "parking_lot 0.12.1",
@ -2317,18 +2392,18 @@ dependencies = [
"signal-hook-registry", "signal-hook-registry",
"socket2", "socket2",
"tokio-macros", "tokio-macros",
"winapi", "windows-sys 0.45.0",
] ]
[[package]] [[package]]
name = "tokio-macros" name = "tokio-macros"
version = "1.8.0" version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.15",
] ]
[[package]] [[package]]
@ -2404,7 +2479,7 @@ checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
] ]
[[package]] [[package]]
@ -2612,7 +2687,7 @@ dependencies = [
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -2646,7 +2721,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.101",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -2721,43 +2796,175 @@ version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
dependencies = [ dependencies = [
"windows_aarch64_msvc", "windows_aarch64_msvc 0.36.1",
"windows_i686_gnu", "windows_i686_gnu 0.36.1",
"windows_i686_msvc", "windows_i686_msvc 0.36.1",
"windows_x86_64_gnu", "windows_x86_64_gnu 0.36.1",
"windows_x86_64_msvc", "windows_x86_64_msvc 0.36.1",
] ]
[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
"windows-targets 0.42.2",
]
[[package]]
name = "windows-sys"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
dependencies = [
"windows-targets 0.48.0",
]
[[package]]
name = "windows-targets"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
dependencies = [
"windows_aarch64_gnullvm 0.42.2",
"windows_aarch64_msvc 0.42.2",
"windows_i686_gnu 0.42.2",
"windows_i686_msvc 0.42.2",
"windows_x86_64_gnu 0.42.2",
"windows_x86_64_gnullvm 0.42.2",
"windows_x86_64_msvc 0.42.2",
]
[[package]]
name = "windows-targets"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
dependencies = [
"windows_aarch64_gnullvm 0.48.0",
"windows_aarch64_msvc 0.48.0",
"windows_i686_gnu 0.48.0",
"windows_i686_msvc 0.48.0",
"windows_x86_64_gnu 0.48.0",
"windows_x86_64_gnullvm 0.48.0",
"windows_x86_64_msvc 0.48.0",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
version = "0.36.1" version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
version = "0.36.1" version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
[[package]]
name = "windows_i686_gnu"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
version = "0.36.1" version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
[[package]]
name = "windows_i686_msvc"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
version = "0.36.1" version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
version = "0.36.1" version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
[[package]] [[package]]
name = "winreg" name = "winreg"
version = "0.10.1" version = "0.10.1"

14
Cargo.toml
View File

@ -11,20 +11,20 @@ description = "Saves Skyrim mod cell edits to a database"
anyhow = "1.0" anyhow = "1.0"
argh = "0.1" argh = "0.1"
chrono = { version = "0.4", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] }
compress-tools = "0.13" compress-tools = "0.14"
dotenv = "0.15" dotenv = "0.15"
futures = "0.3" futures = "0.3"
humansize = "1.1" humansize = "2.1"
infer = { version = "0.9", default-features = false } infer = { version = "0.13", default-features = false }
reqwest = { version = "0.11", features = ["json", "stream"] } reqwest = { version = "0.11", features = ["json", "stream"] }
scraper = "0.13" scraper = "0.16"
seahash = "4.1" seahash = "4.1"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "postgres", "migrate", "chrono", "json"] } sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "postgres", "migrate", "chrono", "json"] }
skyrim-cell-dump = "0.4" skyrim-cell-dump = "0.4"
tempfile = "3.3" tempfile = "3.5"
tokio = { version = "1.21", features = ["full"] } tokio = { version = "1.27", features = ["full"] }
tokio-util = { version = "0.7", features = ["compat"] } tokio-util = { version = "0.7", features = ["compat"] }
tracing = "0.1" tracing = "0.1"
tracing-appender = "0.2" tracing-appender = "0.2"
@ -32,4 +32,4 @@ tracing-subscriber = "0.3"
# Need this unicode fix: https://github.com/muja/unrar.rs/commit/3af9a6015dc89c1329a2fe5d6f4a7f69ded8ba1d # Need this unicode fix: https://github.com/muja/unrar.rs/commit/3af9a6015dc89c1329a2fe5d6f4a7f69ded8ba1d
unrar = { git = "https://github.com/muja/unrar.rs.git" } unrar = { git = "https://github.com/muja/unrar.rs.git" }
walkdir = "2" walkdir = "2"
zip = "0.6" zip = "0.6"

5
src/commands/backfills/is_translation.rs
View File

@ -3,8 +3,8 @@ use std::time::Duration;
use tokio::time::sleep; use tokio::time::sleep;
use tracing::{debug, info, info_span}; use tracing::{debug, info, info_span};
use crate::nexus_scraper;
use crate::nexus_api::SSE_GAME_ID; use crate::nexus_api::SSE_GAME_ID;
use crate::nexus_scraper;
const REQUEST_TIMEOUT: Duration = Duration::from_secs(7200); // 2 hours const REQUEST_TIMEOUT: Duration = Duration::from_secs(7200); // 2 hours
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
@ -25,7 +25,8 @@ pub async fn backfill_is_translation(pool: &sqlx::Pool<sqlx::Postgres>) -> Resul
while has_next_page { while has_next_page {
let page_span = info_span!("page", page); let page_span = info_span!("page", page);
let _page_span = page_span.enter(); let _page_span = page_span.enter();
let mod_list_resp = nexus_scraper::get_mod_list_page(&client, page, SSE_GAME_ID, true).await?; let mod_list_resp =
nexus_scraper::get_mod_list_page(&client, page, SSE_GAME_ID, true).await?;
let scraped = mod_list_resp.scrape_mods()?; let scraped = mod_list_resp.scrape_mods()?;
let scraped_ids: Vec<i32> = scraped.mods.iter().map(|m| m.nexus_mod_id).collect(); let scraped_ids: Vec<i32> = scraped.mods.iter().map(|m| m.nexus_mod_id).collect();

14
src/commands/dump_cell_data.rs
View File

@ -9,16 +9,22 @@ use crate::models::cell;
pub async fn dump_cell_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str) -> Result<()> { pub async fn dump_cell_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str) -> Result<()> {
for x in -77..75 { for x in -77..75 {
for y in -50..44 { for y in -50..44 {
if let Ok(data) = cell::get_cell_data(&pool, "Skyrim.esm", 1, x, y).await { if let Ok(data) = cell::get_cell_data(pool, "Skyrim.esm", 1, x, y).await {
let path = format!("{}/{}", &dir, x); let path = format!("{}/{}", &dir, x);
let path = Path::new(&path); let path = Path::new(&path);
create_dir_all(&path)?; create_dir_all(path)?;
let path = path.join(format!("{}.json", y)); let path = path.join(format!("{}.json", y));
info!(x = x, y = y, form_id = data.form_id, "dumping cell data to {}", path.display()); info!(
x = x,
y = y,
form_id = data.form_id,
"dumping cell data to {}",
path.display()
);
let mut file = File::create(path)?; let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&data)?)?; write!(file, "{}", serde_json::to_string(&data)?)?;
} }
} }
} }
return Ok(()); Ok(())
} }

10
src/commands/dump_cell_edit_counts.rs
View File

@ -10,14 +10,18 @@ pub async fn dump_cell_edit_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str
let mut cell_mod_edit_counts = HashMap::new(); let mut cell_mod_edit_counts = HashMap::new();
for x in -77..75 { for x in -77..75 {
for y in -50..44 { for y in -50..44 {
if let Some(count) = cell::count_mod_edits(&pool, "Skyrim.esm", 1, x, y).await? { if let Some(count) = cell::count_mod_edits(pool, "Skyrim.esm", 1, x, y).await? {
info!(x = x, y = y, count = count, "read cell edit count"); info!(x = x, y = y, count = count, "read cell edit count");
cell_mod_edit_counts.insert(format!("{},{}", x, y), count); cell_mod_edit_counts.insert(format!("{},{}", x, y), count);
} }
} }
} }
info!("writing {} cell edit counts to {}", cell_mod_edit_counts.len(), path); info!(
"writing {} cell edit counts to {}",
cell_mod_edit_counts.len(),
path
);
let mut file = File::create(path)?; let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&cell_mod_edit_counts)?)?; write!(file, "{}", serde_json::to_string(&cell_mod_edit_counts)?)?;
return Ok(()); Ok(())
} }

20
src/commands/dump_file_data.rs
View File

@ -7,26 +7,36 @@ use tracing::info;
use crate::models::file; use crate::models::file;
pub async fn dump_file_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> { pub async fn dump_file_data(
pool: &sqlx::Pool<sqlx::Postgres>,
dir: &str,
updated_after: Option<NaiveDateTime>,
) -> Result<()> {
let mut page = 1; let mut page = 1;
let page_size = 20; let page_size = 20;
let mut last_id = None; let mut last_id = None;
loop { loop {
let files = let files =
file::batched_get_with_cells(&pool, page_size, last_id, "Skyrim.esm", 1, updated_after).await?; file::batched_get_with_cells(pool, page_size, last_id, "Skyrim.esm", 1, updated_after)
.await?;
if files.is_empty() { if files.is_empty() {
break; break;
} }
for file_with_cells in files { for file_with_cells in files {
let path = Path::new(&dir); let path = Path::new(&dir);
std::fs::create_dir_all(&path)?; std::fs::create_dir_all(path)?;
let path = path.join(format!("{}.json", file_with_cells.nexus_file_id)); let path = path.join(format!("{}.json", file_with_cells.nexus_file_id));
info!(page = page, nexus_file_id = file_with_cells.nexus_file_id, "dumping file data to {}", path.display()); info!(
page = page,
nexus_file_id = file_with_cells.nexus_file_id,
"dumping file data to {}",
path.display()
);
let mut file = File::create(path)?; let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&file_with_cells)?)?; write!(file, "{}", serde_json::to_string(&file_with_cells)?)?;
last_id = Some(file_with_cells.id); last_id = Some(file_with_cells.id);
} }
page += 1; page += 1;
} }
return Ok(()); Ok(())
} }

4
src/commands/dump_games.rs
View File

@ -6,9 +6,9 @@ use tracing::info;
use crate::models::game; use crate::models::game;
pub async fn dump_games(pool: &sqlx::Pool<sqlx::Postgres>, path: &str) -> Result<()> { pub async fn dump_games(pool: &sqlx::Pool<sqlx::Postgres>, path: &str) -> Result<()> {
let games = game::get_all(&pool).await?; let games = game::get_all(pool).await?;
info!("writing {} games to {}", games.len(), path); info!("writing {} games to {}", games.len(), path);
let mut file = File::create(path)?; let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&games)?)?; write!(file, "{}", serde_json::to_string(&games)?)?;
return Ok(()); Ok(())
} }

11
src/commands/dump_mod_cell_counts.rs
View File

@ -13,12 +13,17 @@ pub async fn dump_mod_cell_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str)
let mut counts = HashMap::new(); let mut counts = HashMap::new();
loop { loop {
let mod_cell_counts = let mod_cell_counts =
game_mod::batched_get_cell_counts(&pool, page_size, last_id, "Skyrim.esm", 1).await?; game_mod::batched_get_cell_counts(pool, page_size, last_id, "Skyrim.esm", 1).await?;
if mod_cell_counts.is_empty() { if mod_cell_counts.is_empty() {
break; break;
} }
for mod_cell_count in mod_cell_counts { for mod_cell_count in mod_cell_counts {
info!(page = page, nexus_mod_id = mod_cell_count.nexus_mod_id, count = mod_cell_count.cells.unwrap_or(0), "read mod cell count"); info!(
page = page,
nexus_mod_id = mod_cell_count.nexus_mod_id,
count = mod_cell_count.cells.unwrap_or(0),
"read mod cell count"
);
counts.insert(mod_cell_count.nexus_mod_id, mod_cell_count.cells); counts.insert(mod_cell_count.nexus_mod_id, mod_cell_count.cells);
last_id = Some(mod_cell_count.nexus_mod_id); last_id = Some(mod_cell_count.nexus_mod_id);
} }
@ -27,5 +32,5 @@ pub async fn dump_mod_cell_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str)
info!("writing {} mod cell counts to {}", counts.len(), path); info!("writing {} mod cell counts to {}", counts.len(), path);
let mut file = File::create(path)?; let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&counts)?)?; write!(file, "{}", serde_json::to_string(&counts)?)?;
return Ok(()); Ok(())
} }

38
src/commands/dump_mod_data.rs
View File

@ -9,27 +9,51 @@ use tracing::info;
use crate::models::game; use crate::models::game;
use crate::models::game_mod; use crate::models::game_mod;
pub async fn dump_mod_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> { pub async fn dump_mod_data(
pool: &sqlx::Pool<sqlx::Postgres>,
dir: &str,
updated_after: Option<NaiveDateTime>,
) -> Result<()> {
let mut page = 1; let mut page = 1;
let page_size = 20; let page_size = 20;
let mut last_id = None; let mut last_id = None;
let game_id_to_name: HashMap<_, _> = game::get_all(&pool).await?.into_iter().map(|game| (game.id, game.name)).collect(); let game_id_to_name: HashMap<_, _> = game::get_all(pool)
.await?
.into_iter()
.map(|game| (game.id, game.name))
.collect();
loop { loop {
let mods = let mods = game_mod::batched_get_with_cells_and_files(
game_mod::batched_get_with_cells_and_files(&pool, page_size, last_id, "Skyrim.esm", 1, updated_after).await?; pool,
page_size,
last_id,
"Skyrim.esm",
1,
updated_after,
)
.await?;
if mods.is_empty() { if mods.is_empty() {
break; break;
} }
for mod_with_cells in mods { for mod_with_cells in mods {
let path = Path::new(&dir).join(game_id_to_name.get(&mod_with_cells.game_id).expect("valid mod.game_id")); let path = Path::new(&dir).join(
game_id_to_name
.get(&mod_with_cells.game_id)
.expect("valid mod.game_id"),
);
std::fs::create_dir_all(&path)?; std::fs::create_dir_all(&path)?;
let path = path.join(format!("{}.json", mod_with_cells.nexus_mod_id)); let path = path.join(format!("{}.json", mod_with_cells.nexus_mod_id));
info!(page = page, nexus_mod_id = mod_with_cells.nexus_mod_id, "dumping mod data to {}", path.display()); info!(
page = page,
nexus_mod_id = mod_with_cells.nexus_mod_id,
"dumping mod data to {}",
path.display()
);
let mut file = File::create(path)?; let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&mod_with_cells)?)?; write!(file, "{}", serde_json::to_string(&mod_with_cells)?)?;
last_id = Some(mod_with_cells.id); last_id = Some(mod_with_cells.id);
} }
page += 1; page += 1;
} }
return Ok(()); Ok(())
} }

24
src/commands/dump_mod_search_index.rs
View File

@ -13,19 +13,27 @@ struct ModForSearchIdTranslated {
id: i32, id: i32,
} }
pub async fn dump_mod_search_index(pool: &sqlx::Pool<sqlx::Postgres>, game: &str, path: &str) -> Result<()> { pub async fn dump_mod_search_index(
pool: &sqlx::Pool<sqlx::Postgres>,
game: &str,
path: &str,
) -> Result<()> {
let mut page = 1; let mut page = 1;
let mut search_index = vec![]; let mut search_index = vec![];
let page_size = 20; let page_size = 20;
let mut last_id = None; let mut last_id = None;
let game_id = game::get_id_by_name(&pool, game).await?; let game_id = game::get_id_by_name(pool, game).await?;
loop { loop {
let mods = game_mod::batched_get_for_search(&pool, game_id, page_size, last_id).await?; let mods = game_mod::batched_get_for_search(pool, game_id, page_size, last_id).await?;
if mods.is_empty() { if mods.is_empty() {
break; break;
} }
for mod_for_search in mods { for mod_for_search in mods {
info!(page = page, nexus_mod_id = mod_for_search.nexus_mod_id, "read mod name for search index"); info!(
page = page,
nexus_mod_id = mod_for_search.nexus_mod_id,
"read mod name for search index"
);
search_index.push(ModForSearchIdTranslated { search_index.push(ModForSearchIdTranslated {
name: mod_for_search.name, name: mod_for_search.name,
id: mod_for_search.nexus_mod_id, id: mod_for_search.nexus_mod_id,
@ -34,8 +42,12 @@ pub async fn dump_mod_search_index(pool: &sqlx::Pool<sqlx::Postgres>, game: &str
} }
page += 1; page += 1;
} }
info!("writing {} mod names for search index to {}", search_index.len(), path); info!(
"writing {} mod names for search index to {}",
search_index.len(),
path
);
let mut file = File::create(path)?; let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&search_index)?)?; write!(file, "{}", serde_json::to_string(&search_index)?)?;
return Ok(()); Ok(())
} }

28
src/commands/dump_plugin_data.rs
View File

@ -5,23 +5,39 @@ use std::io::Write;
use std::path::Path; use std::path::Path;
use tracing::info; use tracing::info;
use crate::models::{plugin, format_radix}; use crate::models::{format_radix, plugin};
pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> { pub async fn dump_plugin_data(
pool: &sqlx::Pool<sqlx::Postgres>,
dir: &str,
updated_after: Option<NaiveDateTime>,
) -> Result<()> {
let mut page: u32 = 1; let mut page: u32 = 1;
let page_size = 20; let page_size = 20;
let mut last_hash = None; let mut last_hash = None;
loop { loop {
let plugins = let plugins = plugin::batched_get_by_hash_with_mods(
plugin::batched_get_by_hash_with_mods(pool, page_size, last_hash, "Skyrim.esm", 1, updated_after).await?; pool,
page_size,
last_hash,
"Skyrim.esm",
1,
updated_after,
)
.await?;
if plugins.is_empty() { if plugins.is_empty() {
break; break;
} }
for plugin in plugins { for plugin in plugins {
let path = Path::new(&dir); let path = Path::new(&dir);
create_dir_all(&path)?; create_dir_all(path)?;
let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36))); let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36)));
info!(page = page, hash = plugin.hash, "dumping plugin data to {}", path.display()); info!(
page = page,
hash = plugin.hash,
"dumping plugin data to {}",
path.display()
);
let mut file = File::create(path)?; let mut file = File::create(path)?;
let json_val = serde_json::to_string(&plugin)?; let json_val = serde_json::to_string(&plugin)?;
write!(file, "{}", json_val)?; write!(file, "{}", json_val)?;

8
src/commands/mod.rs
View File

@ -2,21 +2,21 @@ pub mod backfills;
pub mod download_tiles; pub mod download_tiles;
pub mod dump_cell_data; pub mod dump_cell_data;
pub mod dump_cell_edit_counts; pub mod dump_cell_edit_counts;
pub mod dump_file_data;
pub mod dump_games;
pub mod dump_mod_cell_counts; pub mod dump_mod_cell_counts;
pub mod dump_mod_data; pub mod dump_mod_data;
pub mod dump_mod_search_index; pub mod dump_mod_search_index;
pub mod dump_plugin_data; pub mod dump_plugin_data;
pub mod dump_file_data;
pub mod dump_games;
pub mod update; pub mod update;
pub use download_tiles::download_tiles; pub use download_tiles::download_tiles;
pub use dump_cell_data::dump_cell_data; pub use dump_cell_data::dump_cell_data;
pub use dump_cell_edit_counts::dump_cell_edit_counts; pub use dump_cell_edit_counts::dump_cell_edit_counts;
pub use dump_file_data::dump_file_data;
pub use dump_games::dump_games;
pub use dump_mod_cell_counts::dump_mod_cell_counts; pub use dump_mod_cell_counts::dump_mod_cell_counts;
pub use dump_mod_data::dump_mod_data; pub use dump_mod_data::dump_mod_data;
pub use dump_mod_search_index::dump_mod_search_index; pub use dump_mod_search_index::dump_mod_search_index;
pub use dump_plugin_data::dump_plugin_data; pub use dump_plugin_data::dump_plugin_data;
pub use dump_file_data::dump_file_data;
pub use dump_games::dump_games;
pub use update::update; pub use update::update;

78
src/commands/update.rs
View File

@ -1,6 +1,6 @@
use anyhow::Result; use anyhow::Result;
use chrono::{NaiveDateTime, NaiveTime}; use chrono::{NaiveDateTime, NaiveTime};
use humansize::{file_size_opts, FileSize}; use humansize::{format_size_i, DECIMAL};
use reqwest::StatusCode; use reqwest::StatusCode;
use std::collections::HashSet; use std::collections::HashSet;
use std::io::SeekFrom; use std::io::SeekFrom;
@ -36,7 +36,7 @@ pub async fn update(
.build()?; .build()?;
let game_id = get_game_id(game_name).expect("valid game name"); let game_id = get_game_id(game_name).expect("valid game name");
let game = game::insert(&pool, game_name, game_id).await?; let game = game::insert(pool, game_name, game_id).await?;
while has_next_page { while has_next_page {
if !full && pages_with_no_updates >= 50 { if !full && pages_with_no_updates >= 50 {
@ -46,13 +46,18 @@ pub async fn update(
let page_span = info_span!("page", page, game_name, include_translations); let page_span = info_span!("page", page, game_name, include_translations);
let _page_span = page_span.enter(); let _page_span = page_span.enter();
let mod_list_resp = let mod_list_resp = nexus_scraper::get_mod_list_page(
nexus_scraper::get_mod_list_page(&client, page, game.nexus_game_id, include_translations).await?; &client,
page,
game.nexus_game_id,
include_translations,
)
.await?;
let scraped = mod_list_resp.scrape_mods()?; let scraped = mod_list_resp.scrape_mods()?;
has_next_page = scraped.has_next_page; has_next_page = scraped.has_next_page;
let processed_mods = game_mod::bulk_get_last_updated_by_nexus_mod_ids( let processed_mods = game_mod::bulk_get_last_updated_by_nexus_mod_ids(
&pool, pool,
game.id, game.id,
&scraped &scraped
.mods .mods
@ -101,7 +106,7 @@ pub async fn update(
}) })
.collect(); .collect();
let mods = game_mod::batched_insert(&pool, &mods_to_create_or_update).await?; let mods = game_mod::batched_insert(pool, &mods_to_create_or_update).await?;
if mods.is_empty() { if mods.is_empty() {
pages_with_no_updates += 1; pages_with_no_updates += 1;
@ -112,7 +117,8 @@ pub async fn update(
for db_mod in mods { for db_mod in mods {
let mod_span = info_span!("mod", name = ?&db_mod.name, id = &db_mod.nexus_mod_id); let mod_span = info_span!("mod", name = ?&db_mod.name, id = &db_mod.nexus_mod_id);
let _mod_span = mod_span.enter(); let _mod_span = mod_span.enter();
let files_resp = nexus_api::files::get(&client, game_name, db_mod.nexus_mod_id).await?; let files_resp =
nexus_api::files::get(&client, game_name, db_mod.nexus_mod_id).await?;
debug!(duration = ?files_resp.wait, "sleeping"); debug!(duration = ?files_resp.wait, "sleeping");
sleep(files_resp.wait).await; sleep(files_resp.wait).await;
@ -135,7 +141,7 @@ pub async fn update(
}); });
let processed_file_ids: HashSet<i32> = let processed_file_ids: HashSet<i32> =
file::get_processed_nexus_file_ids_by_mod_id(&pool, db_mod.id) file::get_processed_nexus_file_ids_by_mod_id(pool, db_mod.id)
.await? .await?
.into_iter() .into_iter()
.collect(); .collect();
@ -150,7 +156,7 @@ pub async fn update(
continue; continue;
} }
let db_file = file::insert( let db_file = file::insert(
&pool, pool,
&file::UnsavedFile { &file::UnsavedFile {
name: api_file.name, name: api_file.name,
file_name: api_file.file_name, file_name: api_file.file_name,
@ -174,7 +180,7 @@ pub async fn update(
info!( info!(
"file metadata does not contain a plugin, skip downloading" "file metadata does not contain a plugin, skip downloading"
); );
file::update_has_plugin(&pool, db_file.id, false).await?; file::update_has_plugin(pool, db_file.id, false).await?;
continue; continue;
} }
} else { } else {
@ -186,10 +192,7 @@ pub async fn update(
} }
}; };
let humanized_size = api_file let humanized_size = format_size_i(api_file.size, DECIMAL);
.size
.file_size(file_size_opts::CONVENTIONAL)
.expect("unable to create human-readable file size");
info!(size = %humanized_size, "decided to download file"); info!(size = %humanized_size, "decided to download file");
let download_link_resp = nexus_api::download_link::get( let download_link_resp = nexus_api::download_link::get(
&client, &client,
@ -205,7 +208,7 @@ pub async fn update(
status = ?reqwest_err.status(), status = ?reqwest_err.status(),
"failed to get download link for file, skipping file" "failed to get download link for file, skipping file"
); );
file::update_has_download_link(&pool, db_file.id, false).await?; file::update_has_download_link(pool, db_file.id, false).await?;
continue; continue;
} }
} }
@ -215,7 +218,7 @@ pub async fn update(
let mut tokio_file = match download_link_resp.download_file(&client).await { let mut tokio_file = match download_link_resp.download_file(&client).await {
Ok(file) => { Ok(file) => {
info!(bytes = api_file.size, "download finished"); info!(bytes = api_file.size, "download finished");
file::update_downloaded_at(&pool, db_file.id).await?; file::update_downloaded_at(pool, db_file.id).await?;
file file
} }
Err(err) => { Err(err) => {
@ -228,14 +231,14 @@ pub async fn update(
tokio_file.seek(SeekFrom::Start(0)).await?; tokio_file.seek(SeekFrom::Start(0)).await?;
if let Err(err) = tokio_file.read_exact(&mut initial_bytes).await { if let Err(err) = tokio_file.read_exact(&mut initial_bytes).await {
warn!(error = %err, "failed to read initial bytes, skipping file"); warn!(error = %err, "failed to read initial bytes, skipping file");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?; file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
continue; continue;
} }
let kind = match infer::get(&initial_bytes) { let kind = match infer::get(&initial_bytes) {
Some(kind) => kind, Some(kind) => kind,
None => { None => {
warn!(initial_bytes = ?initial_bytes, "unable to determine file type of archive, skipping file"); warn!(initial_bytes = ?initial_bytes, "unable to determine file type of archive, skipping file");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?; file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
continue; continue;
} }
}; };
@ -253,7 +256,7 @@ pub async fn update(
let mut file = tokio_file.try_clone().await?.into_std().await; let mut file = tokio_file.try_clone().await?.into_std().await;
match extract_with_unrar( match extract_with_unrar(
&mut file, &mut file,
&pool, pool,
&db_file, &db_file,
&db_mod, &db_mod,
game_name, game_name,
@ -266,7 +269,15 @@ pub async fn update(
// unrar failed to extract rar file (e.g. archive has unicode filenames) // unrar failed to extract rar file (e.g. archive has unicode filenames)
// Attempt to uncompress the archive using `7z` unix command instead // Attempt to uncompress the archive using `7z` unix command instead
warn!(error = %err, "failed to extract file with unrar, extracting whole archive with 7z instead"); warn!(error = %err, "failed to extract file with unrar, extracting whole archive with 7z instead");
extract_with_7zip(&mut file, &pool, &db_file, &db_mod, game_name, checked_metadata).await extract_with_7zip(
&mut file,
pool,
&db_file,
&db_mod,
game_name,
checked_metadata,
)
.await
} }
}?; }?;
} }
@ -274,8 +285,10 @@ pub async fn update(
tokio_file.seek(SeekFrom::Start(0)).await?; tokio_file.seek(SeekFrom::Start(0)).await?;
let mut file = tokio_file.try_clone().await?.into_std().await; let mut file = tokio_file.try_clone().await?.into_std().await;
match extract_with_compress_tools(&mut file, &pool, &db_file, &db_mod, game_name) match extract_with_compress_tools(
.await &mut file, pool, &db_file, &db_mod, game_name,
)
.await
{ {
Ok(_) => Ok(()), Ok(_) => Ok(()),
Err(err) => { Err(err) => {
@ -289,11 +302,24 @@ pub async fn update(
// compress_tools or libarchive failed to extract zip/7z file (e.g. archive is deflate64 compressed) // compress_tools or libarchive failed to extract zip/7z file (e.g. archive is deflate64 compressed)
// Attempt to uncompress the archive using `7z` unix command instead // Attempt to uncompress the archive using `7z` unix command instead
warn!(error = %err, "failed to extract file with compress_tools, extracting whole archive with 7z instead"); warn!(error = %err, "failed to extract file with compress_tools, extracting whole archive with 7z instead");
extract_with_7zip(&mut file, &pool, &db_file, &db_mod, game_name, checked_metadata).await extract_with_7zip(
} else if kind.mime_type() == "application/vnd.microsoft.portable-executable" { &mut file,
pool,
&db_file,
&db_mod,
game_name,
checked_metadata,
)
.await
} else if kind.mime_type()
== "application/vnd.microsoft.portable-executable"
{
// we tried to extract this .exe file, but it's not an archive so there's nothing we can do // we tried to extract this .exe file, but it's not an archive so there's nothing we can do
warn!("archive is an .exe file that cannot be extracted, skipping file"); warn!("archive is an .exe file that cannot be extracted, skipping file");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?; file::update_unable_to_extract_plugins(
pool, db_file.id, true,
)
.await?;
continue; continue;
} else { } else {
Err(err) Err(err)
@ -307,7 +333,7 @@ pub async fn update(
sleep(download_link_resp.wait).await; sleep(download_link_resp.wait).await;
} }
game_mod::update_last_updated_files_at(&pool, db_mod.id).await?; game_mod::update_last_updated_files_at(pool, db_mod.id).await?;
} }
page += 1; page += 1;

14
src/extractors/compress_tools.rs
View File

@ -57,7 +57,7 @@ impl<'a> Extractor<'a> {
self.file.seek(SeekFrom::Start(0))?; self.file.seek(SeekFrom::Start(0))?;
let mut buf = Vec::default(); let mut buf = Vec::default();
info!("uncompressing plugin file from downloaded archive"); info!("uncompressing plugin file from downloaded archive");
uncompress_archive_file(&mut self.file, &mut buf, &file_path)?; uncompress_archive_file(&mut self.file, &mut buf, file_path)?;
Ok(buf) Ok(buf)
} }
} }
@ -95,8 +95,16 @@ pub async fn extract_with_compress_tools(
let (file_path, mut plugin_buf) = plugin?; let (file_path, mut plugin_buf) = plugin?;
let plugin_span = info_span!("plugin", name = ?file_path); let plugin_span = info_span!("plugin", name = ?file_path);
let _plugin_span = plugin_span.enter(); let _plugin_span = plugin_span.enter();
let safe_file_path = file_path.replace("\\", "/"); let safe_file_path = file_path.replace('\\', "/");
process_plugin(&mut plugin_buf, &pool, &db_file, &db_mod, &safe_file_path, game_name).await?; process_plugin(
&mut plugin_buf,
pool,
db_file,
db_mod,
&safe_file_path,
game_name,
)
.await?;
} }
Ok(()) Ok(())
} }

18
src/extractors/seven_zip.rs
View File

@ -5,8 +5,8 @@ use tempfile::tempdir;
use tracing::{info, info_span, warn}; use tracing::{info, info_span, warn};
use walkdir::WalkDir; use walkdir::WalkDir;
use crate::models::{file, file::File};
use crate::models::game_mod::Mod; use crate::models::game_mod::Mod;
use crate::models::{file, file::File};
use crate::plugin_processor::process_plugin; use crate::plugin_processor::process_plugin;
pub async fn extract_with_7zip( pub async fn extract_with_7zip(
@ -26,16 +26,16 @@ pub async fn extract_with_7zip(
let extracted_path = temp_dir.path().join("extracted"); let extracted_path = temp_dir.path().join("extracted");
let status = Command::new("7z") let status = Command::new("7z")
.args(&[ .args([
"x", "x",
&format!("-o{}", &extracted_path.to_string_lossy()), &format!("-o{}", &extracted_path.to_string_lossy()),
&temp_file_path.to_string_lossy().to_string(), &temp_file_path.to_string_lossy(),
]) ])
.status()?; .status()?;
if !status.success() && !checked_metadata { if !status.success() && !checked_metadata {
warn!("failed to extract archive and server has no metadata, skipping file"); warn!("failed to extract archive and server has no metadata, skipping file");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?; file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
return Ok(()); return Ok(());
} }
@ -43,7 +43,9 @@ pub async fn extract_with_7zip(
.contents_first(true) .contents_first(true)
.into_iter() .into_iter()
.filter_entry(|e| { .filter_entry(|e| {
if e.file_type().is_dir() { return false } if e.file_type().is_dir() {
return false;
}
if let Some(extension) = e.path().extension() { if let Some(extension) = e.path().extension() {
extension == "esp" || extension == "esm" || extension == "esl" extension == "esp" || extension == "esm" || extension == "esl"
} else { } else {
@ -59,9 +61,9 @@ pub async fn extract_with_7zip(
let mut plugin_buf = std::fs::read(extracted_path.join(file_path))?; let mut plugin_buf = std::fs::read(extracted_path.join(file_path))?;
process_plugin( process_plugin(
&mut plugin_buf, &mut plugin_buf,
&pool, pool,
&db_file, db_file,
&db_mod, db_mod,
&file_path.to_string_lossy(), &file_path.to_string_lossy(),
game_name, game_name,
) )

12
src/extractors/unrar.rs
View File

@ -37,7 +37,7 @@ pub async fn extract_with_unrar(
Err(_) => { Err(_) => {
if !checked_metadata { if !checked_metadata {
warn!("failed to read archive and server has no metadata, skipping file"); warn!("failed to read archive and server has no metadata, skipping file");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?; file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
return Ok(()); return Ok(());
} else { } else {
error!("failed to read archive, but server had metadata"); error!("failed to read archive, but server had metadata");
@ -58,14 +58,14 @@ pub async fn extract_with_unrar(
let mut extract = match extract { let mut extract = match extract {
Err(err) => { Err(err) => {
warn!(error = %err, "failed to extract with unrar"); warn!(error = %err, "failed to extract with unrar");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?; file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
return Ok(()); return Ok(());
} }
Ok(extract) => extract, Ok(extract) => extract,
}; };
if let Err(err) = extract.process() { if let Err(err) = extract.process() {
warn!(error = %err, "failed to extract with unrar"); warn!(error = %err, "failed to extract with unrar");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?; file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
return Ok(()); return Ok(());
} }
@ -77,9 +77,9 @@ pub async fn extract_with_unrar(
let mut plugin_buf = std::fs::read(temp_dir.path().join(file_path))?; let mut plugin_buf = std::fs::read(temp_dir.path().join(file_path))?;
process_plugin( process_plugin(
&mut plugin_buf, &mut plugin_buf,
&pool, pool,
&db_file, db_file,
&db_mod, db_mod,
&file_path.to_string_lossy(), &file_path.to_string_lossy(),
game_name, game_name,
) )

11
src/main.rs
View File

@ -14,7 +14,8 @@ mod plugin_processor;
use commands::{ use commands::{
backfills::backfill_is_translation, download_tiles, dump_cell_data, dump_cell_edit_counts, backfills::backfill_is_translation, download_tiles, dump_cell_data, dump_cell_edit_counts,
dump_mod_cell_counts, dump_mod_data, dump_mod_search_index, dump_plugin_data, dump_file_data, dump_games, update, dump_file_data, dump_games, dump_mod_cell_counts, dump_mod_data, dump_mod_search_index,
dump_plugin_data, update,
}; };
#[derive(FromArgs)] #[derive(FromArgs)]
@ -24,7 +25,11 @@ struct Args {
/// the page number to start scraping for mods on nexus mods /// the page number to start scraping for mods on nexus mods
page: usize, page: usize,
#[argh(option, short = 'g', default = "String::from(\"skyrimspecialedition\")")] #[argh(
option,
short = 'g',
default = "String::from(\"skyrimspecialedition\")"
)]
/// name of nexus game to scrape (e.g. "skyrim" or "skyrimspecialedition") /// name of nexus game to scrape (e.g. "skyrim" or "skyrimspecialedition")
game: String, game: String,
@ -121,5 +126,5 @@ pub async fn main() -> Result<()> {
return backfill_is_translation(&pool).await; return backfill_is_translation(&pool).await;
} }
return update(&pool, args.page, &args.game, args.full).await; update(&pool, args.page, &args.game, args.full).await
} }

2
src/models/file.rs
View File

@ -268,4 +268,4 @@ pub async fn batched_get_with_cells(
.await .await
.context("Failed to batch get with cells") .context("Failed to batch get with cells")
} }
} }

34
src/models/game.rs
View File

@ -34,28 +34,18 @@ pub async fn insert(
} }
#[instrument(level = "debug", skip(pool))] #[instrument(level = "debug", skip(pool))]
pub async fn get_all( pub async fn get_all(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<Vec<Game>> {
pool: &sqlx::Pool<sqlx::Postgres>, sqlx::query_as!(Game, "SELECT * FROM games")
) -> Result<Vec<Game>> { .fetch_all(pool)
sqlx::query_as!( .await
Game, .context("Failed to fetch games")
"SELECT * FROM games"
)
.fetch_all(pool)
.await
.context("Failed to fetch games")
} }
#[instrument(level = "debug", skip(pool))] #[instrument(level = "debug", skip(pool))]
pub async fn get_id_by_name( pub async fn get_id_by_name(pool: &sqlx::Pool<sqlx::Postgres>, name: &str) -> Result<i32> {
pool: &sqlx::Pool<sqlx::Postgres>, sqlx::query_scalar!("SELECT id FROM games WHERE name = $1", name)
name: &str, .fetch_one(pool)
) -> Result<i32> { .await
sqlx::query_scalar!( .context("Failed to fetch game id by name")
"SELECT id FROM games WHERE name = $1", }
name
)
.fetch_one(pool)
.await
.context("Failed to fetch game id by name")
}

75
src/models/game_mod.rs
View File

@ -474,42 +474,45 @@ pub async fn batched_get_with_cells_and_files(
.await .await
.context("Failed to batch get mod files")?; .context("Failed to batch get mod files")?;
Ok(mods.into_iter().map(|m| { Ok(mods
let id = m.id; .into_iter()
ModWithCellsAndFiles { .map(|m| {
id: m.id, let id = m.id;
name: m.name, ModWithCellsAndFiles {
nexus_mod_id: m.nexus_mod_id, id: m.id,
author_name: m.author_name, name: m.name,
author_id: m.author_id, nexus_mod_id: m.nexus_mod_id,
category_name: m.category_name, author_name: m.author_name,
category_id: m.category_id, author_id: m.author_id,
description: m.description, category_name: m.category_name,
thumbnail_link: m.thumbnail_link, category_id: m.category_id,
game_id: m.game_id, description: m.description,
is_translation: m.is_translation, thumbnail_link: m.thumbnail_link,
updated_at: m.updated_at, game_id: m.game_id,
created_at: m.created_at, is_translation: m.is_translation,
last_update_at: m.last_update_at, updated_at: m.updated_at,
first_upload_at: m.first_upload_at, created_at: m.created_at,
last_updated_files_at: m.last_updated_files_at, last_update_at: m.last_update_at,
cells: mod_cells first_upload_at: m.first_upload_at,
.iter() last_updated_files_at: m.last_updated_files_at,
.find(|c| c.mod_id == id) cells: mod_cells
.map(|c| c.cells.clone()) .iter()
.unwrap_or_else(|| Some(serde_json::Value::Array(vec![]))), .find(|c| c.mod_id == id)
files: mod_files .map(|c| c.cells.clone())
.iter() .unwrap_or_else(|| Some(serde_json::Value::Array(vec![]))),
.find(|f| f.mod_id == id) files: mod_files
.map(|f| f.files.clone()) .iter()
.unwrap_or_else(|| Some(serde_json::Value::Array(vec![]))), .find(|f| f.mod_id == id)
plugin_count: plugins_count .map(|f| f.files.clone())
.iter() .unwrap_or_else(|| Some(serde_json::Value::Array(vec![]))),
.find(|p| p.mod_id == id) plugin_count: plugins_count
.map(|p| p.plugin_count) .iter()
.unwrap_or(Some(0)), .find(|p| p.mod_id == id)
} .map(|p| p.plugin_count)
}).collect()) .unwrap_or(Some(0)),
}
})
.collect())
} }
#[instrument(level = "debug", skip(pool))] #[instrument(level = "debug", skip(pool))]

2
src/models/plugin.rs
View File

@ -1,8 +1,8 @@
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use chrono::NaiveDateTime; use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use sqlx::FromRow;
use sqlx::types::Json; use sqlx::types::Json;
use sqlx::FromRow;
use tracing::instrument; use tracing::instrument;
use super::hash_to_string; use super::hash_to_string;

7
src/nexus_api/download_link.rs
View File

@ -16,7 +16,12 @@ pub struct DownloadLinkResponse {
} }
#[instrument(skip(client))] #[instrument(skip(client))]
pub async fn get(client: &Client, game_name: &str, mod_id: i32, file_id: i64) -> Result<DownloadLinkResponse> { pub async fn get(
client: &Client,
game_name: &str,
mod_id: i32,
file_id: i64,
) -> Result<DownloadLinkResponse> {
for attempt in 1..=3 { for attempt in 1..=3 {
let res = match client let res = match client
.get(format!( .get(format!(