Update deps, rustfmt, clippy fixes
This commit is contained in:
parent
03cf20029e
commit
01444cd928
413
Cargo.lock
generated
413
Cargo.lock
generated
@ -31,6 +31,18 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"getrandom 0.2.7",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.19"
|
||||
@ -84,7 +96,7 @@ dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -234,9 +246,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "compress-tools"
|
||||
version = "0.13.0"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d4318a3d43a20f0eacfa42b5492eb9cbab5875bb5bb5d1feb07d673a0b72777"
|
||||
checksum = "537b7d54356f29ce8e3fc8fdf89ab1ad752ac22987ff4cb9841e2bcf55785134"
|
||||
dependencies = [
|
||||
"derive_more",
|
||||
"libc",
|
||||
@ -346,19 +358,19 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cssparser"
|
||||
version = "0.27.2"
|
||||
version = "0.29.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
|
||||
checksum = "f93d03419cb5950ccfd3daf3ff1c7a36ace64609a1a8746d493df1ca0afde0fa"
|
||||
dependencies = [
|
||||
"cssparser-macros",
|
||||
"dtoa-short",
|
||||
"itoa 0.4.8",
|
||||
"itoa",
|
||||
"matches",
|
||||
"phf 0.8.0",
|
||||
"phf 0.10.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"smallvec",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -368,7 +380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -381,7 +393,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustc_version",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -475,6 +487,27 @@ dependencies = [
|
||||
"num-traits 0.1.43",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
|
||||
dependencies = [
|
||||
"errno-dragonfly",
|
||||
"libc",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno-dragonfly"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "event-listener"
|
||||
version = "2.5.3"
|
||||
@ -619,7 +652,7 @@ checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -727,7 +760,7 @@ version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"ahash 0.7.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -757,6 +790,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
@ -792,7 +831,7 @@ dependencies = [
|
||||
"markup5ever",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -803,7 +842,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
"itoa 1.0.3",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -831,9 +870,12 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
|
||||
|
||||
[[package]]
|
||||
name = "humansize"
|
||||
version = "1.1.1"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026"
|
||||
checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
|
||||
dependencies = [
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
@ -850,7 +892,7 @@ dependencies = [
|
||||
"http-body",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa 1.0.3",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"tokio",
|
||||
@ -907,9 +949,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "infer"
|
||||
version = "0.9.0"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f178e61cdbfe084aa75a2f4f7a25a5bb09701a47ae1753608f194b15783c937a"
|
||||
checksum = "f551f8c3a39f68f986517db0d1759de85881894fdc7db798bd2a9df9cb04b7fc"
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
@ -920,6 +962,17 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "io-lifetimes"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220"
|
||||
dependencies = [
|
||||
"hermit-abi 0.3.1",
|
||||
"libc",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.5.0"
|
||||
@ -935,12 +988,6 @@ dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.3"
|
||||
@ -986,9 +1033,21 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.134"
|
||||
version = "0.2.141"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb"
|
||||
checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
@ -1080,7 +1139,7 @@ dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
"windows-sys",
|
||||
"windows-sys 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1259,7 +1318,7 @@ version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"hermit-abi 0.1.19",
|
||||
"libc",
|
||||
]
|
||||
|
||||
@ -1307,7 +1366,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1359,7 +1418,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"redox_syscall 0.2.16",
|
||||
"smallvec",
|
||||
"winapi",
|
||||
]
|
||||
@ -1372,9 +1431,9 @@ checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"redox_syscall 0.2.16",
|
||||
"smallvec",
|
||||
"windows-sys",
|
||||
"windows-sys 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1418,9 +1477,7 @@ version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.8.0",
|
||||
"proc-macro-hack",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1429,7 +1486,9 @@ version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.10.0",
|
||||
"proc-macro-hack",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1474,16 +1533,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.8.0"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"
|
||||
checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0"
|
||||
dependencies = [
|
||||
"phf_generator 0.8.0",
|
||||
"phf_shared 0.8.0",
|
||||
"phf_generator 0.10.0",
|
||||
"phf_shared 0.10.0",
|
||||
"proc-macro-hack",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1542,18 +1601,18 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.46"
|
||||
version = "1.0.56"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b"
|
||||
checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.21"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
|
||||
checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@ -1691,6 +1750,15 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.4.3"
|
||||
@ -1698,7 +1766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
|
||||
dependencies = [
|
||||
"getrandom 0.2.7",
|
||||
"redox_syscall",
|
||||
"redox_syscall 0.2.16",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
@ -1719,15 +1787,6 @@ version = "0.6.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.11.12"
|
||||
@ -1781,6 +1840,20 @@ dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.37.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"io-lifetimes",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.11"
|
||||
@ -1803,7 +1876,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"windows-sys",
|
||||
"windows-sys 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1814,15 +1887,16 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "scraper"
|
||||
version = "0.13.0"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5684396b456f3eb69ceeb34d1b5cb1a2f6acf7ca4452131efa3ba0ee2c2d0a70"
|
||||
checksum = "59e25654b5e9fd557a67dbaab5a5d36b8c448d0561beb4c041b6dbb902eddfa6"
|
||||
dependencies = [
|
||||
"ahash 0.8.3",
|
||||
"cssparser",
|
||||
"ego-tree",
|
||||
"getopts",
|
||||
"html5ever",
|
||||
"matches",
|
||||
"once_cell",
|
||||
"selectors",
|
||||
"smallvec",
|
||||
"tendril",
|
||||
@ -1859,22 +1933,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.22.0"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
|
||||
checksum = "0c37578180969d00692904465fb7f6b3d50b9a2b952b87c23d0e2e5cb5013416"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"fxhash",
|
||||
"log",
|
||||
"matches",
|
||||
"phf 0.8.0",
|
||||
"phf_codegen 0.8.0",
|
||||
"precomputed-hash",
|
||||
"servo_arc",
|
||||
"smallvec",
|
||||
"thin-slice",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1900,7 +1972,7 @@ checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1909,7 +1981,7 @@ version = "1.0.85"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
|
||||
dependencies = [
|
||||
"itoa 1.0.3",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
@ -1921,16 +1993,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"itoa 1.0.3",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "servo_arc"
|
||||
version = "0.1.1"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"
|
||||
checksum = "d52aa42f8fdf0fed91e5ce7f23d8138441002fa31dca008acf47e6fd4721f741"
|
||||
dependencies = [
|
||||
"nodrop",
|
||||
"stable_deref_trait",
|
||||
@ -2007,15 +2079,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.9.0"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
|
||||
checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.4.7"
|
||||
version = "0.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
|
||||
checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
@ -2048,7 +2120,7 @@ version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcbc16ddba161afc99e14d1713a453747a2b07fc097d2009f4c300ec99286105"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"ahash 0.7.6",
|
||||
"atoi",
|
||||
"base64",
|
||||
"bitflags",
|
||||
@ -2070,7 +2142,7 @@ dependencies = [
|
||||
"hkdf",
|
||||
"hmac",
|
||||
"indexmap",
|
||||
"itoa 1.0.3",
|
||||
"itoa",
|
||||
"libc",
|
||||
"log",
|
||||
"md-5",
|
||||
@ -2109,7 +2181,7 @@ dependencies = [
|
||||
"sha2",
|
||||
"sqlx-core",
|
||||
"sqlx-rt",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
"url",
|
||||
]
|
||||
|
||||
@ -2190,6 +2262,17 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tap"
|
||||
version = "1.0.1"
|
||||
@ -2198,16 +2281,15 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.3.0"
|
||||
version = "3.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
|
||||
checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"fastrand",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"remove_dir_all",
|
||||
"winapi",
|
||||
"redox_syscall 0.3.5",
|
||||
"rustix",
|
||||
"windows-sys 0.45.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2221,12 +2303,6 @@ dependencies = [
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thin-slice"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.37"
|
||||
@ -2244,7 +2320,7 @@ checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2273,7 +2349,7 @@ version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b"
|
||||
dependencies = [
|
||||
"itoa 1.0.3",
|
||||
"itoa",
|
||||
"libc",
|
||||
"num_threads",
|
||||
"time-macros",
|
||||
@ -2302,14 +2378,13 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.21.2"
|
||||
version = "1.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099"
|
||||
checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"bytes",
|
||||
"libc",
|
||||
"memchr",
|
||||
"mio",
|
||||
"num_cpus",
|
||||
"parking_lot 0.12.1",
|
||||
@ -2317,18 +2392,18 @@ dependencies = [
|
||||
"signal-hook-registry",
|
||||
"socket2",
|
||||
"tokio-macros",
|
||||
"winapi",
|
||||
"windows-sys 0.45.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "1.8.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
|
||||
checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.15",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2404,7 +2479,7 @@ checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2612,7 +2687,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@ -2646,7 +2721,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 1.0.101",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
@ -2721,43 +2796,175 @@ version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
|
||||
dependencies = [
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_msvc",
|
||||
"windows_aarch64_msvc 0.36.1",
|
||||
"windows_i686_gnu 0.36.1",
|
||||
"windows_i686_msvc 0.36.1",
|
||||
"windows_x86_64_gnu 0.36.1",
|
||||
"windows_x86_64_msvc 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
|
||||
dependencies = [
|
||||
"windows-targets 0.42.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
|
||||
dependencies = [
|
||||
"windows-targets 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm 0.42.2",
|
||||
"windows_aarch64_msvc 0.42.2",
|
||||
"windows_i686_gnu 0.42.2",
|
||||
"windows_i686_msvc 0.42.2",
|
||||
"windows_x86_64_gnu 0.42.2",
|
||||
"windows_x86_64_gnullvm 0.42.2",
|
||||
"windows_x86_64_msvc 0.42.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm 0.48.0",
|
||||
"windows_aarch64_msvc 0.48.0",
|
||||
"windows_i686_gnu 0.48.0",
|
||||
"windows_i686_msvc 0.48.0",
|
||||
"windows_x86_64_gnu 0.48.0",
|
||||
"windows_x86_64_gnullvm 0.48.0",
|
||||
"windows_x86_64_msvc 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.10.1"
|
||||
|
||||
12
Cargo.toml
12
Cargo.toml
@ -11,20 +11,20 @@ description = "Saves Skyrim mod cell edits to a database"
|
||||
anyhow = "1.0"
|
||||
argh = "0.1"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
compress-tools = "0.13"
|
||||
compress-tools = "0.14"
|
||||
dotenv = "0.15"
|
||||
futures = "0.3"
|
||||
humansize = "1.1"
|
||||
infer = { version = "0.9", default-features = false }
|
||||
humansize = "2.1"
|
||||
infer = { version = "0.13", default-features = false }
|
||||
reqwest = { version = "0.11", features = ["json", "stream"] }
|
||||
scraper = "0.13"
|
||||
scraper = "0.16"
|
||||
seahash = "4.1"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "postgres", "migrate", "chrono", "json"] }
|
||||
skyrim-cell-dump = "0.4"
|
||||
tempfile = "3.3"
|
||||
tokio = { version = "1.21", features = ["full"] }
|
||||
tempfile = "3.5"
|
||||
tokio = { version = "1.27", features = ["full"] }
|
||||
tokio-util = { version = "0.7", features = ["compat"] }
|
||||
tracing = "0.1"
|
||||
tracing-appender = "0.2"
|
||||
|
||||
@ -3,8 +3,8 @@ use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use tracing::{debug, info, info_span};
|
||||
|
||||
use crate::nexus_scraper;
|
||||
use crate::nexus_api::SSE_GAME_ID;
|
||||
use crate::nexus_scraper;
|
||||
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(7200); // 2 hours
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
@ -25,7 +25,8 @@ pub async fn backfill_is_translation(pool: &sqlx::Pool<sqlx::Postgres>) -> Resul
|
||||
while has_next_page {
|
||||
let page_span = info_span!("page", page);
|
||||
let _page_span = page_span.enter();
|
||||
let mod_list_resp = nexus_scraper::get_mod_list_page(&client, page, SSE_GAME_ID, true).await?;
|
||||
let mod_list_resp =
|
||||
nexus_scraper::get_mod_list_page(&client, page, SSE_GAME_ID, true).await?;
|
||||
let scraped = mod_list_resp.scrape_mods()?;
|
||||
let scraped_ids: Vec<i32> = scraped.mods.iter().map(|m| m.nexus_mod_id).collect();
|
||||
|
||||
|
||||
@ -9,16 +9,22 @@ use crate::models::cell;
|
||||
pub async fn dump_cell_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str) -> Result<()> {
|
||||
for x in -77..75 {
|
||||
for y in -50..44 {
|
||||
if let Ok(data) = cell::get_cell_data(&pool, "Skyrim.esm", 1, x, y).await {
|
||||
if let Ok(data) = cell::get_cell_data(pool, "Skyrim.esm", 1, x, y).await {
|
||||
let path = format!("{}/{}", &dir, x);
|
||||
let path = Path::new(&path);
|
||||
create_dir_all(&path)?;
|
||||
create_dir_all(path)?;
|
||||
let path = path.join(format!("{}.json", y));
|
||||
info!(x = x, y = y, form_id = data.form_id, "dumping cell data to {}", path.display());
|
||||
info!(
|
||||
x = x,
|
||||
y = y,
|
||||
form_id = data.form_id,
|
||||
"dumping cell data to {}",
|
||||
path.display()
|
||||
);
|
||||
let mut file = File::create(path)?;
|
||||
write!(file, "{}", serde_json::to_string(&data)?)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -10,14 +10,18 @@ pub async fn dump_cell_edit_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str
|
||||
let mut cell_mod_edit_counts = HashMap::new();
|
||||
for x in -77..75 {
|
||||
for y in -50..44 {
|
||||
if let Some(count) = cell::count_mod_edits(&pool, "Skyrim.esm", 1, x, y).await? {
|
||||
if let Some(count) = cell::count_mod_edits(pool, "Skyrim.esm", 1, x, y).await? {
|
||||
info!(x = x, y = y, count = count, "read cell edit count");
|
||||
cell_mod_edit_counts.insert(format!("{},{}", x, y), count);
|
||||
}
|
||||
}
|
||||
}
|
||||
info!("writing {} cell edit counts to {}", cell_mod_edit_counts.len(), path);
|
||||
info!(
|
||||
"writing {} cell edit counts to {}",
|
||||
cell_mod_edit_counts.len(),
|
||||
path
|
||||
);
|
||||
let mut file = File::create(path)?;
|
||||
write!(file, "{}", serde_json::to_string(&cell_mod_edit_counts)?)?;
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -7,26 +7,36 @@ use tracing::info;
|
||||
|
||||
use crate::models::file;
|
||||
|
||||
pub async fn dump_file_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
|
||||
pub async fn dump_file_data(
|
||||
pool: &sqlx::Pool<sqlx::Postgres>,
|
||||
dir: &str,
|
||||
updated_after: Option<NaiveDateTime>,
|
||||
) -> Result<()> {
|
||||
let mut page = 1;
|
||||
let page_size = 20;
|
||||
let mut last_id = None;
|
||||
loop {
|
||||
let files =
|
||||
file::batched_get_with_cells(&pool, page_size, last_id, "Skyrim.esm", 1, updated_after).await?;
|
||||
file::batched_get_with_cells(pool, page_size, last_id, "Skyrim.esm", 1, updated_after)
|
||||
.await?;
|
||||
if files.is_empty() {
|
||||
break;
|
||||
}
|
||||
for file_with_cells in files {
|
||||
let path = Path::new(&dir);
|
||||
std::fs::create_dir_all(&path)?;
|
||||
std::fs::create_dir_all(path)?;
|
||||
let path = path.join(format!("{}.json", file_with_cells.nexus_file_id));
|
||||
info!(page = page, nexus_file_id = file_with_cells.nexus_file_id, "dumping file data to {}", path.display());
|
||||
info!(
|
||||
page = page,
|
||||
nexus_file_id = file_with_cells.nexus_file_id,
|
||||
"dumping file data to {}",
|
||||
path.display()
|
||||
);
|
||||
let mut file = File::create(path)?;
|
||||
write!(file, "{}", serde_json::to_string(&file_with_cells)?)?;
|
||||
last_id = Some(file_with_cells.id);
|
||||
}
|
||||
page += 1;
|
||||
}
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -6,9 +6,9 @@ use tracing::info;
|
||||
use crate::models::game;
|
||||
|
||||
pub async fn dump_games(pool: &sqlx::Pool<sqlx::Postgres>, path: &str) -> Result<()> {
|
||||
let games = game::get_all(&pool).await?;
|
||||
let games = game::get_all(pool).await?;
|
||||
info!("writing {} games to {}", games.len(), path);
|
||||
let mut file = File::create(path)?;
|
||||
write!(file, "{}", serde_json::to_string(&games)?)?;
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -13,12 +13,17 @@ pub async fn dump_mod_cell_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str)
|
||||
let mut counts = HashMap::new();
|
||||
loop {
|
||||
let mod_cell_counts =
|
||||
game_mod::batched_get_cell_counts(&pool, page_size, last_id, "Skyrim.esm", 1).await?;
|
||||
game_mod::batched_get_cell_counts(pool, page_size, last_id, "Skyrim.esm", 1).await?;
|
||||
if mod_cell_counts.is_empty() {
|
||||
break;
|
||||
}
|
||||
for mod_cell_count in mod_cell_counts {
|
||||
info!(page = page, nexus_mod_id = mod_cell_count.nexus_mod_id, count = mod_cell_count.cells.unwrap_or(0), "read mod cell count");
|
||||
info!(
|
||||
page = page,
|
||||
nexus_mod_id = mod_cell_count.nexus_mod_id,
|
||||
count = mod_cell_count.cells.unwrap_or(0),
|
||||
"read mod cell count"
|
||||
);
|
||||
counts.insert(mod_cell_count.nexus_mod_id, mod_cell_count.cells);
|
||||
last_id = Some(mod_cell_count.nexus_mod_id);
|
||||
}
|
||||
@ -27,5 +32,5 @@ pub async fn dump_mod_cell_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str)
|
||||
info!("writing {} mod cell counts to {}", counts.len(), path);
|
||||
let mut file = File::create(path)?;
|
||||
write!(file, "{}", serde_json::to_string(&counts)?)?;
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -9,27 +9,51 @@ use tracing::info;
|
||||
use crate::models::game;
|
||||
use crate::models::game_mod;
|
||||
|
||||
pub async fn dump_mod_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
|
||||
pub async fn dump_mod_data(
|
||||
pool: &sqlx::Pool<sqlx::Postgres>,
|
||||
dir: &str,
|
||||
updated_after: Option<NaiveDateTime>,
|
||||
) -> Result<()> {
|
||||
let mut page = 1;
|
||||
let page_size = 20;
|
||||
let mut last_id = None;
|
||||
let game_id_to_name: HashMap<_, _> = game::get_all(&pool).await?.into_iter().map(|game| (game.id, game.name)).collect();
|
||||
let game_id_to_name: HashMap<_, _> = game::get_all(pool)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|game| (game.id, game.name))
|
||||
.collect();
|
||||
loop {
|
||||
let mods =
|
||||
game_mod::batched_get_with_cells_and_files(&pool, page_size, last_id, "Skyrim.esm", 1, updated_after).await?;
|
||||
let mods = game_mod::batched_get_with_cells_and_files(
|
||||
pool,
|
||||
page_size,
|
||||
last_id,
|
||||
"Skyrim.esm",
|
||||
1,
|
||||
updated_after,
|
||||
)
|
||||
.await?;
|
||||
if mods.is_empty() {
|
||||
break;
|
||||
}
|
||||
for mod_with_cells in mods {
|
||||
let path = Path::new(&dir).join(game_id_to_name.get(&mod_with_cells.game_id).expect("valid mod.game_id"));
|
||||
let path = Path::new(&dir).join(
|
||||
game_id_to_name
|
||||
.get(&mod_with_cells.game_id)
|
||||
.expect("valid mod.game_id"),
|
||||
);
|
||||
std::fs::create_dir_all(&path)?;
|
||||
let path = path.join(format!("{}.json", mod_with_cells.nexus_mod_id));
|
||||
info!(page = page, nexus_mod_id = mod_with_cells.nexus_mod_id, "dumping mod data to {}", path.display());
|
||||
info!(
|
||||
page = page,
|
||||
nexus_mod_id = mod_with_cells.nexus_mod_id,
|
||||
"dumping mod data to {}",
|
||||
path.display()
|
||||
);
|
||||
let mut file = File::create(path)?;
|
||||
write!(file, "{}", serde_json::to_string(&mod_with_cells)?)?;
|
||||
last_id = Some(mod_with_cells.id);
|
||||
}
|
||||
page += 1;
|
||||
}
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -13,19 +13,27 @@ struct ModForSearchIdTranslated {
|
||||
id: i32,
|
||||
}
|
||||
|
||||
pub async fn dump_mod_search_index(pool: &sqlx::Pool<sqlx::Postgres>, game: &str, path: &str) -> Result<()> {
|
||||
pub async fn dump_mod_search_index(
|
||||
pool: &sqlx::Pool<sqlx::Postgres>,
|
||||
game: &str,
|
||||
path: &str,
|
||||
) -> Result<()> {
|
||||
let mut page = 1;
|
||||
let mut search_index = vec![];
|
||||
let page_size = 20;
|
||||
let mut last_id = None;
|
||||
let game_id = game::get_id_by_name(&pool, game).await?;
|
||||
let game_id = game::get_id_by_name(pool, game).await?;
|
||||
loop {
|
||||
let mods = game_mod::batched_get_for_search(&pool, game_id, page_size, last_id).await?;
|
||||
let mods = game_mod::batched_get_for_search(pool, game_id, page_size, last_id).await?;
|
||||
if mods.is_empty() {
|
||||
break;
|
||||
}
|
||||
for mod_for_search in mods {
|
||||
info!(page = page, nexus_mod_id = mod_for_search.nexus_mod_id, "read mod name for search index");
|
||||
info!(
|
||||
page = page,
|
||||
nexus_mod_id = mod_for_search.nexus_mod_id,
|
||||
"read mod name for search index"
|
||||
);
|
||||
search_index.push(ModForSearchIdTranslated {
|
||||
name: mod_for_search.name,
|
||||
id: mod_for_search.nexus_mod_id,
|
||||
@ -34,8 +42,12 @@ pub async fn dump_mod_search_index(pool: &sqlx::Pool<sqlx::Postgres>, game: &str
|
||||
}
|
||||
page += 1;
|
||||
}
|
||||
info!("writing {} mod names for search index to {}", search_index.len(), path);
|
||||
info!(
|
||||
"writing {} mod names for search index to {}",
|
||||
search_index.len(),
|
||||
path
|
||||
);
|
||||
let mut file = File::create(path)?;
|
||||
write!(file, "{}", serde_json::to_string(&search_index)?)?;
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -5,23 +5,39 @@ use std::io::Write;
|
||||
use std::path::Path;
|
||||
use tracing::info;
|
||||
|
||||
use crate::models::{plugin, format_radix};
|
||||
use crate::models::{format_radix, plugin};
|
||||
|
||||
pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
|
||||
pub async fn dump_plugin_data(
|
||||
pool: &sqlx::Pool<sqlx::Postgres>,
|
||||
dir: &str,
|
||||
updated_after: Option<NaiveDateTime>,
|
||||
) -> Result<()> {
|
||||
let mut page: u32 = 1;
|
||||
let page_size = 20;
|
||||
let mut last_hash = None;
|
||||
loop {
|
||||
let plugins =
|
||||
plugin::batched_get_by_hash_with_mods(pool, page_size, last_hash, "Skyrim.esm", 1, updated_after).await?;
|
||||
let plugins = plugin::batched_get_by_hash_with_mods(
|
||||
pool,
|
||||
page_size,
|
||||
last_hash,
|
||||
"Skyrim.esm",
|
||||
1,
|
||||
updated_after,
|
||||
)
|
||||
.await?;
|
||||
if plugins.is_empty() {
|
||||
break;
|
||||
}
|
||||
for plugin in plugins {
|
||||
let path = Path::new(&dir);
|
||||
create_dir_all(&path)?;
|
||||
create_dir_all(path)?;
|
||||
let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36)));
|
||||
info!(page = page, hash = plugin.hash, "dumping plugin data to {}", path.display());
|
||||
info!(
|
||||
page = page,
|
||||
hash = plugin.hash,
|
||||
"dumping plugin data to {}",
|
||||
path.display()
|
||||
);
|
||||
let mut file = File::create(path)?;
|
||||
let json_val = serde_json::to_string(&plugin)?;
|
||||
write!(file, "{}", json_val)?;
|
||||
|
||||
@ -2,21 +2,21 @@ pub mod backfills;
|
||||
pub mod download_tiles;
|
||||
pub mod dump_cell_data;
|
||||
pub mod dump_cell_edit_counts;
|
||||
pub mod dump_file_data;
|
||||
pub mod dump_games;
|
||||
pub mod dump_mod_cell_counts;
|
||||
pub mod dump_mod_data;
|
||||
pub mod dump_mod_search_index;
|
||||
pub mod dump_plugin_data;
|
||||
pub mod dump_file_data;
|
||||
pub mod dump_games;
|
||||
pub mod update;
|
||||
|
||||
pub use download_tiles::download_tiles;
|
||||
pub use dump_cell_data::dump_cell_data;
|
||||
pub use dump_cell_edit_counts::dump_cell_edit_counts;
|
||||
pub use dump_file_data::dump_file_data;
|
||||
pub use dump_games::dump_games;
|
||||
pub use dump_mod_cell_counts::dump_mod_cell_counts;
|
||||
pub use dump_mod_data::dump_mod_data;
|
||||
pub use dump_mod_search_index::dump_mod_search_index;
|
||||
pub use dump_plugin_data::dump_plugin_data;
|
||||
pub use dump_file_data::dump_file_data;
|
||||
pub use dump_games::dump_games;
|
||||
pub use update::update;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
use anyhow::Result;
|
||||
use chrono::{NaiveDateTime, NaiveTime};
|
||||
use humansize::{file_size_opts, FileSize};
|
||||
use humansize::{format_size_i, DECIMAL};
|
||||
use reqwest::StatusCode;
|
||||
use std::collections::HashSet;
|
||||
use std::io::SeekFrom;
|
||||
@ -36,7 +36,7 @@ pub async fn update(
|
||||
.build()?;
|
||||
|
||||
let game_id = get_game_id(game_name).expect("valid game name");
|
||||
let game = game::insert(&pool, game_name, game_id).await?;
|
||||
let game = game::insert(pool, game_name, game_id).await?;
|
||||
|
||||
while has_next_page {
|
||||
if !full && pages_with_no_updates >= 50 {
|
||||
@ -46,13 +46,18 @@ pub async fn update(
|
||||
|
||||
let page_span = info_span!("page", page, game_name, include_translations);
|
||||
let _page_span = page_span.enter();
|
||||
let mod_list_resp =
|
||||
nexus_scraper::get_mod_list_page(&client, page, game.nexus_game_id, include_translations).await?;
|
||||
let mod_list_resp = nexus_scraper::get_mod_list_page(
|
||||
&client,
|
||||
page,
|
||||
game.nexus_game_id,
|
||||
include_translations,
|
||||
)
|
||||
.await?;
|
||||
let scraped = mod_list_resp.scrape_mods()?;
|
||||
|
||||
has_next_page = scraped.has_next_page;
|
||||
let processed_mods = game_mod::bulk_get_last_updated_by_nexus_mod_ids(
|
||||
&pool,
|
||||
pool,
|
||||
game.id,
|
||||
&scraped
|
||||
.mods
|
||||
@ -101,7 +106,7 @@ pub async fn update(
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mods = game_mod::batched_insert(&pool, &mods_to_create_or_update).await?;
|
||||
let mods = game_mod::batched_insert(pool, &mods_to_create_or_update).await?;
|
||||
|
||||
if mods.is_empty() {
|
||||
pages_with_no_updates += 1;
|
||||
@ -112,7 +117,8 @@ pub async fn update(
|
||||
for db_mod in mods {
|
||||
let mod_span = info_span!("mod", name = ?&db_mod.name, id = &db_mod.nexus_mod_id);
|
||||
let _mod_span = mod_span.enter();
|
||||
let files_resp = nexus_api::files::get(&client, game_name, db_mod.nexus_mod_id).await?;
|
||||
let files_resp =
|
||||
nexus_api::files::get(&client, game_name, db_mod.nexus_mod_id).await?;
|
||||
|
||||
debug!(duration = ?files_resp.wait, "sleeping");
|
||||
sleep(files_resp.wait).await;
|
||||
@ -135,7 +141,7 @@ pub async fn update(
|
||||
});
|
||||
|
||||
let processed_file_ids: HashSet<i32> =
|
||||
file::get_processed_nexus_file_ids_by_mod_id(&pool, db_mod.id)
|
||||
file::get_processed_nexus_file_ids_by_mod_id(pool, db_mod.id)
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect();
|
||||
@ -150,7 +156,7 @@ pub async fn update(
|
||||
continue;
|
||||
}
|
||||
let db_file = file::insert(
|
||||
&pool,
|
||||
pool,
|
||||
&file::UnsavedFile {
|
||||
name: api_file.name,
|
||||
file_name: api_file.file_name,
|
||||
@ -174,7 +180,7 @@ pub async fn update(
|
||||
info!(
|
||||
"file metadata does not contain a plugin, skip downloading"
|
||||
);
|
||||
file::update_has_plugin(&pool, db_file.id, false).await?;
|
||||
file::update_has_plugin(pool, db_file.id, false).await?;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
@ -186,10 +192,7 @@ pub async fn update(
|
||||
}
|
||||
};
|
||||
|
||||
let humanized_size = api_file
|
||||
.size
|
||||
.file_size(file_size_opts::CONVENTIONAL)
|
||||
.expect("unable to create human-readable file size");
|
||||
let humanized_size = format_size_i(api_file.size, DECIMAL);
|
||||
info!(size = %humanized_size, "decided to download file");
|
||||
let download_link_resp = nexus_api::download_link::get(
|
||||
&client,
|
||||
@ -205,7 +208,7 @@ pub async fn update(
|
||||
status = ?reqwest_err.status(),
|
||||
"failed to get download link for file, skipping file"
|
||||
);
|
||||
file::update_has_download_link(&pool, db_file.id, false).await?;
|
||||
file::update_has_download_link(pool, db_file.id, false).await?;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@ -215,7 +218,7 @@ pub async fn update(
|
||||
let mut tokio_file = match download_link_resp.download_file(&client).await {
|
||||
Ok(file) => {
|
||||
info!(bytes = api_file.size, "download finished");
|
||||
file::update_downloaded_at(&pool, db_file.id).await?;
|
||||
file::update_downloaded_at(pool, db_file.id).await?;
|
||||
file
|
||||
}
|
||||
Err(err) => {
|
||||
@ -228,14 +231,14 @@ pub async fn update(
|
||||
tokio_file.seek(SeekFrom::Start(0)).await?;
|
||||
if let Err(err) = tokio_file.read_exact(&mut initial_bytes).await {
|
||||
warn!(error = %err, "failed to read initial bytes, skipping file");
|
||||
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
|
||||
file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
|
||||
continue;
|
||||
}
|
||||
let kind = match infer::get(&initial_bytes) {
|
||||
Some(kind) => kind,
|
||||
None => {
|
||||
warn!(initial_bytes = ?initial_bytes, "unable to determine file type of archive, skipping file");
|
||||
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
|
||||
file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@ -253,7 +256,7 @@ pub async fn update(
|
||||
let mut file = tokio_file.try_clone().await?.into_std().await;
|
||||
match extract_with_unrar(
|
||||
&mut file,
|
||||
&pool,
|
||||
pool,
|
||||
&db_file,
|
||||
&db_mod,
|
||||
game_name,
|
||||
@ -266,7 +269,15 @@ pub async fn update(
|
||||
// unrar failed to extract rar file (e.g. archive has unicode filenames)
|
||||
// Attempt to uncompress the archive using `7z` unix command instead
|
||||
warn!(error = %err, "failed to extract file with unrar, extracting whole archive with 7z instead");
|
||||
extract_with_7zip(&mut file, &pool, &db_file, &db_mod, game_name, checked_metadata).await
|
||||
extract_with_7zip(
|
||||
&mut file,
|
||||
pool,
|
||||
&db_file,
|
||||
&db_mod,
|
||||
game_name,
|
||||
checked_metadata,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}?;
|
||||
}
|
||||
@ -274,8 +285,10 @@ pub async fn update(
|
||||
tokio_file.seek(SeekFrom::Start(0)).await?;
|
||||
let mut file = tokio_file.try_clone().await?.into_std().await;
|
||||
|
||||
match extract_with_compress_tools(&mut file, &pool, &db_file, &db_mod, game_name)
|
||||
.await
|
||||
match extract_with_compress_tools(
|
||||
&mut file, pool, &db_file, &db_mod, game_name,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
@ -289,11 +302,24 @@ pub async fn update(
|
||||
// compress_tools or libarchive failed to extract zip/7z file (e.g. archive is deflate64 compressed)
|
||||
// Attempt to uncompress the archive using `7z` unix command instead
|
||||
warn!(error = %err, "failed to extract file with compress_tools, extracting whole archive with 7z instead");
|
||||
extract_with_7zip(&mut file, &pool, &db_file, &db_mod, game_name, checked_metadata).await
|
||||
} else if kind.mime_type() == "application/vnd.microsoft.portable-executable" {
|
||||
extract_with_7zip(
|
||||
&mut file,
|
||||
pool,
|
||||
&db_file,
|
||||
&db_mod,
|
||||
game_name,
|
||||
checked_metadata,
|
||||
)
|
||||
.await
|
||||
} else if kind.mime_type()
|
||||
== "application/vnd.microsoft.portable-executable"
|
||||
{
|
||||
// we tried to extract this .exe file, but it's not an archive so there's nothing we can do
|
||||
warn!("archive is an .exe file that cannot be extracted, skipping file");
|
||||
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
|
||||
file::update_unable_to_extract_plugins(
|
||||
pool, db_file.id, true,
|
||||
)
|
||||
.await?;
|
||||
continue;
|
||||
} else {
|
||||
Err(err)
|
||||
@ -307,7 +333,7 @@ pub async fn update(
|
||||
sleep(download_link_resp.wait).await;
|
||||
}
|
||||
|
||||
game_mod::update_last_updated_files_at(&pool, db_mod.id).await?;
|
||||
game_mod::update_last_updated_files_at(pool, db_mod.id).await?;
|
||||
}
|
||||
|
||||
page += 1;
|
||||
|
||||
@ -57,7 +57,7 @@ impl<'a> Extractor<'a> {
|
||||
self.file.seek(SeekFrom::Start(0))?;
|
||||
let mut buf = Vec::default();
|
||||
info!("uncompressing plugin file from downloaded archive");
|
||||
uncompress_archive_file(&mut self.file, &mut buf, &file_path)?;
|
||||
uncompress_archive_file(&mut self.file, &mut buf, file_path)?;
|
||||
Ok(buf)
|
||||
}
|
||||
}
|
||||
@ -95,8 +95,16 @@ pub async fn extract_with_compress_tools(
|
||||
let (file_path, mut plugin_buf) = plugin?;
|
||||
let plugin_span = info_span!("plugin", name = ?file_path);
|
||||
let _plugin_span = plugin_span.enter();
|
||||
let safe_file_path = file_path.replace("\\", "/");
|
||||
process_plugin(&mut plugin_buf, &pool, &db_file, &db_mod, &safe_file_path, game_name).await?;
|
||||
let safe_file_path = file_path.replace('\\', "/");
|
||||
process_plugin(
|
||||
&mut plugin_buf,
|
||||
pool,
|
||||
db_file,
|
||||
db_mod,
|
||||
&safe_file_path,
|
||||
game_name,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -5,8 +5,8 @@ use tempfile::tempdir;
|
||||
use tracing::{info, info_span, warn};
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use crate::models::{file, file::File};
|
||||
use crate::models::game_mod::Mod;
|
||||
use crate::models::{file, file::File};
|
||||
use crate::plugin_processor::process_plugin;
|
||||
|
||||
pub async fn extract_with_7zip(
|
||||
@ -26,16 +26,16 @@ pub async fn extract_with_7zip(
|
||||
let extracted_path = temp_dir.path().join("extracted");
|
||||
|
||||
let status = Command::new("7z")
|
||||
.args(&[
|
||||
.args([
|
||||
"x",
|
||||
&format!("-o{}", &extracted_path.to_string_lossy()),
|
||||
&temp_file_path.to_string_lossy().to_string(),
|
||||
&temp_file_path.to_string_lossy(),
|
||||
])
|
||||
.status()?;
|
||||
|
||||
if !status.success() && !checked_metadata {
|
||||
warn!("failed to extract archive and server has no metadata, skipping file");
|
||||
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
|
||||
file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@ -43,7 +43,9 @@ pub async fn extract_with_7zip(
|
||||
.contents_first(true)
|
||||
.into_iter()
|
||||
.filter_entry(|e| {
|
||||
if e.file_type().is_dir() { return false }
|
||||
if e.file_type().is_dir() {
|
||||
return false;
|
||||
}
|
||||
if let Some(extension) = e.path().extension() {
|
||||
extension == "esp" || extension == "esm" || extension == "esl"
|
||||
} else {
|
||||
@ -59,9 +61,9 @@ pub async fn extract_with_7zip(
|
||||
let mut plugin_buf = std::fs::read(extracted_path.join(file_path))?;
|
||||
process_plugin(
|
||||
&mut plugin_buf,
|
||||
&pool,
|
||||
&db_file,
|
||||
&db_mod,
|
||||
pool,
|
||||
db_file,
|
||||
db_mod,
|
||||
&file_path.to_string_lossy(),
|
||||
game_name,
|
||||
)
|
||||
|
||||
@ -37,7 +37,7 @@ pub async fn extract_with_unrar(
|
||||
Err(_) => {
|
||||
if !checked_metadata {
|
||||
warn!("failed to read archive and server has no metadata, skipping file");
|
||||
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
|
||||
file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
|
||||
return Ok(());
|
||||
} else {
|
||||
error!("failed to read archive, but server had metadata");
|
||||
@ -58,14 +58,14 @@ pub async fn extract_with_unrar(
|
||||
let mut extract = match extract {
|
||||
Err(err) => {
|
||||
warn!(error = %err, "failed to extract with unrar");
|
||||
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
|
||||
file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
|
||||
return Ok(());
|
||||
}
|
||||
Ok(extract) => extract,
|
||||
};
|
||||
if let Err(err) = extract.process() {
|
||||
warn!(error = %err, "failed to extract with unrar");
|
||||
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
|
||||
file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@ -77,9 +77,9 @@ pub async fn extract_with_unrar(
|
||||
let mut plugin_buf = std::fs::read(temp_dir.path().join(file_path))?;
|
||||
process_plugin(
|
||||
&mut plugin_buf,
|
||||
&pool,
|
||||
&db_file,
|
||||
&db_mod,
|
||||
pool,
|
||||
db_file,
|
||||
db_mod,
|
||||
&file_path.to_string_lossy(),
|
||||
game_name,
|
||||
)
|
||||
|
||||
11
src/main.rs
11
src/main.rs
@ -14,7 +14,8 @@ mod plugin_processor;
|
||||
|
||||
use commands::{
|
||||
backfills::backfill_is_translation, download_tiles, dump_cell_data, dump_cell_edit_counts,
|
||||
dump_mod_cell_counts, dump_mod_data, dump_mod_search_index, dump_plugin_data, dump_file_data, dump_games, update,
|
||||
dump_file_data, dump_games, dump_mod_cell_counts, dump_mod_data, dump_mod_search_index,
|
||||
dump_plugin_data, update,
|
||||
};
|
||||
|
||||
#[derive(FromArgs)]
|
||||
@ -24,7 +25,11 @@ struct Args {
|
||||
/// the page number to start scraping for mods on nexus mods
|
||||
page: usize,
|
||||
|
||||
#[argh(option, short = 'g', default = "String::from(\"skyrimspecialedition\")")]
|
||||
#[argh(
|
||||
option,
|
||||
short = 'g',
|
||||
default = "String::from(\"skyrimspecialedition\")"
|
||||
)]
|
||||
/// name of nexus game to scrape (e.g. "skyrim" or "skyrimspecialedition")
|
||||
game: String,
|
||||
|
||||
@ -121,5 +126,5 @@ pub async fn main() -> Result<()> {
|
||||
return backfill_is_translation(&pool).await;
|
||||
}
|
||||
|
||||
return update(&pool, args.page, &args.game, args.full).await;
|
||||
update(&pool, args.page, &args.game, args.full).await
|
||||
}
|
||||
|
||||
@ -34,28 +34,18 @@ pub async fn insert(
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(pool))]
|
||||
pub async fn get_all(
|
||||
pool: &sqlx::Pool<sqlx::Postgres>,
|
||||
) -> Result<Vec<Game>> {
|
||||
sqlx::query_as!(
|
||||
Game,
|
||||
"SELECT * FROM games"
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.context("Failed to fetch games")
|
||||
pub async fn get_all(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<Vec<Game>> {
|
||||
sqlx::query_as!(Game, "SELECT * FROM games")
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.context("Failed to fetch games")
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(pool))]
|
||||
pub async fn get_id_by_name(
|
||||
pool: &sqlx::Pool<sqlx::Postgres>,
|
||||
name: &str,
|
||||
) -> Result<i32> {
|
||||
sqlx::query_scalar!(
|
||||
"SELECT id FROM games WHERE name = $1",
|
||||
name
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.context("Failed to fetch game id by name")
|
||||
pub async fn get_id_by_name(pool: &sqlx::Pool<sqlx::Postgres>, name: &str) -> Result<i32> {
|
||||
sqlx::query_scalar!("SELECT id FROM games WHERE name = $1", name)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.context("Failed to fetch game id by name")
|
||||
}
|
||||
|
||||
|
||||
@ -474,42 +474,45 @@ pub async fn batched_get_with_cells_and_files(
|
||||
.await
|
||||
.context("Failed to batch get mod files")?;
|
||||
|
||||
Ok(mods.into_iter().map(|m| {
|
||||
let id = m.id;
|
||||
ModWithCellsAndFiles {
|
||||
id: m.id,
|
||||
name: m.name,
|
||||
nexus_mod_id: m.nexus_mod_id,
|
||||
author_name: m.author_name,
|
||||
author_id: m.author_id,
|
||||
category_name: m.category_name,
|
||||
category_id: m.category_id,
|
||||
description: m.description,
|
||||
thumbnail_link: m.thumbnail_link,
|
||||
game_id: m.game_id,
|
||||
is_translation: m.is_translation,
|
||||
updated_at: m.updated_at,
|
||||
created_at: m.created_at,
|
||||
last_update_at: m.last_update_at,
|
||||
first_upload_at: m.first_upload_at,
|
||||
last_updated_files_at: m.last_updated_files_at,
|
||||
cells: mod_cells
|
||||
.iter()
|
||||
.find(|c| c.mod_id == id)
|
||||
.map(|c| c.cells.clone())
|
||||
.unwrap_or_else(|| Some(serde_json::Value::Array(vec![]))),
|
||||
files: mod_files
|
||||
.iter()
|
||||
.find(|f| f.mod_id == id)
|
||||
.map(|f| f.files.clone())
|
||||
.unwrap_or_else(|| Some(serde_json::Value::Array(vec![]))),
|
||||
plugin_count: plugins_count
|
||||
.iter()
|
||||
.find(|p| p.mod_id == id)
|
||||
.map(|p| p.plugin_count)
|
||||
.unwrap_or(Some(0)),
|
||||
}
|
||||
}).collect())
|
||||
Ok(mods
|
||||
.into_iter()
|
||||
.map(|m| {
|
||||
let id = m.id;
|
||||
ModWithCellsAndFiles {
|
||||
id: m.id,
|
||||
name: m.name,
|
||||
nexus_mod_id: m.nexus_mod_id,
|
||||
author_name: m.author_name,
|
||||
author_id: m.author_id,
|
||||
category_name: m.category_name,
|
||||
category_id: m.category_id,
|
||||
description: m.description,
|
||||
thumbnail_link: m.thumbnail_link,
|
||||
game_id: m.game_id,
|
||||
is_translation: m.is_translation,
|
||||
updated_at: m.updated_at,
|
||||
created_at: m.created_at,
|
||||
last_update_at: m.last_update_at,
|
||||
first_upload_at: m.first_upload_at,
|
||||
last_updated_files_at: m.last_updated_files_at,
|
||||
cells: mod_cells
|
||||
.iter()
|
||||
.find(|c| c.mod_id == id)
|
||||
.map(|c| c.cells.clone())
|
||||
.unwrap_or_else(|| Some(serde_json::Value::Array(vec![]))),
|
||||
files: mod_files
|
||||
.iter()
|
||||
.find(|f| f.mod_id == id)
|
||||
.map(|f| f.files.clone())
|
||||
.unwrap_or_else(|| Some(serde_json::Value::Array(vec![]))),
|
||||
plugin_count: plugins_count
|
||||
.iter()
|
||||
.find(|p| p.mod_id == id)
|
||||
.map(|p| p.plugin_count)
|
||||
.unwrap_or(Some(0)),
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(pool))]
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
use anyhow::{Context, Result};
|
||||
use chrono::NaiveDateTime;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::FromRow;
|
||||
use sqlx::types::Json;
|
||||
use sqlx::FromRow;
|
||||
use tracing::instrument;
|
||||
|
||||
use super::hash_to_string;
|
||||
|
||||
@ -16,7 +16,12 @@ pub struct DownloadLinkResponse {
|
||||
}
|
||||
|
||||
#[instrument(skip(client))]
|
||||
pub async fn get(client: &Client, game_name: &str, mod_id: i32, file_id: i64) -> Result<DownloadLinkResponse> {
|
||||
pub async fn get(
|
||||
client: &Client,
|
||||
game_name: &str,
|
||||
mod_id: i32,
|
||||
file_id: i64,
|
||||
) -> Result<DownloadLinkResponse> {
|
||||
for attempt in 1..=3 {
|
||||
let res = match client
|
||||
.get(format!(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user