crawlnicle/src/utils.rs
Tyler Hallada 3f028c3088 Store entry html content outside DB in file storage
Since the HTML content can get quite big and can have embeded images.
2023-07-05 23:45:49 -04:00

15 lines
375 B
Rust

use url::Url;
pub fn get_domain(url: &str) -> Option<String> {
Url::parse(url)
.ok()
.and_then(|url| url.host_str().map(|s| s.to_string()))
.map(|domain| {
if domain.starts_with("www.") && domain.matches('.').count() > 1 {
domain[4..].to_string()
} else {
domain
}
})
}