|
|
@ -51,7 +51,7 @@ static CLIENT: Lazy<Client> = Lazy::new(|| {
|
|
|
|
|
|
|
|
|
|
|
|
// Reuse the client between requests
|
|
|
|
// Reuse the client between requests
|
|
|
|
let client = get_reqwest_client_builder()
|
|
|
|
let client = get_reqwest_client_builder()
|
|
|
|
.cookie_provider(cookie_store.clone())
|
|
|
|
.cookie_provider(Arc::clone(&cookie_store))
|
|
|
|
.timeout(Duration::from_secs(CONFIG.icon_download_timeout()))
|
|
|
|
.timeout(Duration::from_secs(CONFIG.icon_download_timeout()))
|
|
|
|
.default_headers(default_headers.clone());
|
|
|
|
.default_headers(default_headers.clone());
|
|
|
|
|
|
|
|
|
|
|
@ -77,7 +77,7 @@ static ICON_SIZE_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?x)(\d+)\D*(\d+
|
|
|
|
static ICON_BLACKLIST_REGEX: Lazy<dashmap::DashMap<String, Regex>> = Lazy::new(dashmap::DashMap::new);
|
|
|
|
static ICON_BLACKLIST_REGEX: Lazy<dashmap::DashMap<String, Regex>> = Lazy::new(dashmap::DashMap::new);
|
|
|
|
|
|
|
|
|
|
|
|
async fn icon_redirect(domain: &str, template: &str) -> Option<Redirect> {
|
|
|
|
async fn icon_redirect(domain: &str, template: &str) -> Option<Redirect> {
|
|
|
|
if !is_valid_domain(domain).await {
|
|
|
|
if !is_valid_domain(domain) {
|
|
|
|
warn!("Invalid domain: {}", domain);
|
|
|
|
warn!("Invalid domain: {}", domain);
|
|
|
|
return None;
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -123,7 +123,7 @@ async fn icon_google(domain: String) -> Option<Redirect> {
|
|
|
|
async fn icon_internal(domain: String) -> Cached<(ContentType, Vec<u8>)> {
|
|
|
|
async fn icon_internal(domain: String) -> Cached<(ContentType, Vec<u8>)> {
|
|
|
|
const FALLBACK_ICON: &[u8] = include_bytes!("../static/images/fallback-icon.png");
|
|
|
|
const FALLBACK_ICON: &[u8] = include_bytes!("../static/images/fallback-icon.png");
|
|
|
|
|
|
|
|
|
|
|
|
if !is_valid_domain(&domain).await {
|
|
|
|
if !is_valid_domain(&domain) {
|
|
|
|
warn!("Invalid domain: {}", domain);
|
|
|
|
warn!("Invalid domain: {}", domain);
|
|
|
|
return Cached::ttl(
|
|
|
|
return Cached::ttl(
|
|
|
|
(ContentType::new("image", "png"), FALLBACK_ICON.to_vec()),
|
|
|
|
(ContentType::new("image", "png"), FALLBACK_ICON.to_vec()),
|
|
|
@ -144,7 +144,7 @@ async fn icon_internal(domain: String) -> Cached<(ContentType, Vec<u8>)> {
|
|
|
|
///
|
|
|
|
///
|
|
|
|
/// This does some manual checks and makes use of Url to do some basic checking.
|
|
|
|
/// This does some manual checks and makes use of Url to do some basic checking.
|
|
|
|
/// domains can't be larger then 63 characters (not counting multiple subdomains) according to the RFC's, but we limit the total size to 255.
|
|
|
|
/// domains can't be larger then 63 characters (not counting multiple subdomains) according to the RFC's, but we limit the total size to 255.
|
|
|
|
async fn is_valid_domain(domain: &str) -> bool {
|
|
|
|
fn is_valid_domain(domain: &str) -> bool {
|
|
|
|
const ALLOWED_CHARS: &str = "_-.";
|
|
|
|
const ALLOWED_CHARS: &str = "_-.";
|
|
|
|
|
|
|
|
|
|
|
|
// If parsing the domain fails using Url, it will not work with reqwest.
|
|
|
|
// If parsing the domain fails using Url, it will not work with reqwest.
|
|
|
@ -278,6 +278,7 @@ mod tests {
|
|
|
|
|
|
|
|
|
|
|
|
use cached::proc_macro::cached;
|
|
|
|
use cached::proc_macro::cached;
|
|
|
|
#[cached(key = "String", convert = r#"{ domain.to_string() }"#, size = 16, time = 60)]
|
|
|
|
#[cached(key = "String", convert = r#"{ domain.to_string() }"#, size = 16, time = 60)]
|
|
|
|
|
|
|
|
#[allow(clippy::unused_async)] // This is needed because cached causes a false-positive here.
|
|
|
|
async fn is_domain_blacklisted(domain: &str) -> bool {
|
|
|
|
async fn is_domain_blacklisted(domain: &str) -> bool {
|
|
|
|
if CONFIG.icon_blacklist_non_global_ips() {
|
|
|
|
if CONFIG.icon_blacklist_non_global_ips() {
|
|
|
|
if let Ok(s) = lookup_host((domain, 0)).await {
|
|
|
|
if let Ok(s) = lookup_host((domain, 0)).await {
|
|
|
@ -304,7 +305,7 @@ async fn is_domain_blacklisted(domain: &str) -> bool {
|
|
|
|
// Generate the regex to store in too the Lazy Static HashMap.
|
|
|
|
// Generate the regex to store in too the Lazy Static HashMap.
|
|
|
|
let blacklist_regex = Regex::new(&blacklist).unwrap();
|
|
|
|
let blacklist_regex = Regex::new(&blacklist).unwrap();
|
|
|
|
let is_match = blacklist_regex.is_match(domain);
|
|
|
|
let is_match = blacklist_regex.is_match(domain);
|
|
|
|
ICON_BLACKLIST_REGEX.insert(blacklist.to_string(), blacklist_regex);
|
|
|
|
ICON_BLACKLIST_REGEX.insert(blacklist.clone(), blacklist_regex);
|
|
|
|
|
|
|
|
|
|
|
|
is_match
|
|
|
|
is_match
|
|
|
|
};
|
|
|
|
};
|
|
|
@ -326,7 +327,7 @@ async fn get_icon(domain: &str) -> Option<(Vec<u8>, String)> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if let Some(icon) = get_cached_icon(&path).await {
|
|
|
|
if let Some(icon) = get_cached_icon(&path).await {
|
|
|
|
let icon_type = match get_icon_type(&icon).await {
|
|
|
|
let icon_type = match get_icon_type(&icon) {
|
|
|
|
Some(x) => x,
|
|
|
|
Some(x) => x,
|
|
|
|
_ => "x-icon",
|
|
|
|
_ => "x-icon",
|
|
|
|
};
|
|
|
|
};
|
|
|
@ -416,7 +417,7 @@ impl Icon {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn get_favicons_node(
|
|
|
|
fn get_favicons_node(
|
|
|
|
dom: InfallibleTokenizer<StringReader<'_>, FaviconEmitter>,
|
|
|
|
dom: InfallibleTokenizer<StringReader<'_>, FaviconEmitter>,
|
|
|
|
icons: &mut Vec<Icon>,
|
|
|
|
icons: &mut Vec<Icon>,
|
|
|
|
url: &url::Url,
|
|
|
|
url: &url::Url,
|
|
|
@ -468,7 +469,7 @@ async fn get_favicons_node(
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
""
|
|
|
|
""
|
|
|
|
};
|
|
|
|
};
|
|
|
|
let priority = get_icon_priority(full_href.as_str(), sizes).await;
|
|
|
|
let priority = get_icon_priority(full_href.as_str(), sizes);
|
|
|
|
icons.push(Icon::new(priority, full_href.to_string()));
|
|
|
|
icons.push(Icon::new(priority, full_href.to_string()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
};
|
|
|
@ -512,7 +513,7 @@ async fn get_icon_url(domain: &str) -> Result<IconUrlResult, Error> {
|
|
|
|
tld = domain_parts.next_back().unwrap(),
|
|
|
|
tld = domain_parts.next_back().unwrap(),
|
|
|
|
base = domain_parts.next_back().unwrap()
|
|
|
|
base = domain_parts.next_back().unwrap()
|
|
|
|
);
|
|
|
|
);
|
|
|
|
if is_valid_domain(&base_domain).await {
|
|
|
|
if is_valid_domain(&base_domain) {
|
|
|
|
let sslbase = format!("https://{base_domain}");
|
|
|
|
let sslbase = format!("https://{base_domain}");
|
|
|
|
let httpbase = format!("http://{base_domain}");
|
|
|
|
let httpbase = format!("http://{base_domain}");
|
|
|
|
debug!("[get_icon_url]: Trying without subdomains '{base_domain}'");
|
|
|
|
debug!("[get_icon_url]: Trying without subdomains '{base_domain}'");
|
|
|
@ -523,7 +524,7 @@ async fn get_icon_url(domain: &str) -> Result<IconUrlResult, Error> {
|
|
|
|
// When the domain is not an IP, and has less then 2 dots, try to add www. infront of it.
|
|
|
|
// When the domain is not an IP, and has less then 2 dots, try to add www. infront of it.
|
|
|
|
} else if is_ip.is_err() && domain.matches('.').count() < 2 {
|
|
|
|
} else if is_ip.is_err() && domain.matches('.').count() < 2 {
|
|
|
|
let www_domain = format!("www.{domain}");
|
|
|
|
let www_domain = format!("www.{domain}");
|
|
|
|
if is_valid_domain(&www_domain).await {
|
|
|
|
if is_valid_domain(&www_domain) {
|
|
|
|
let sslwww = format!("https://{www_domain}");
|
|
|
|
let sslwww = format!("https://{www_domain}");
|
|
|
|
let httpwww = format!("http://{www_domain}");
|
|
|
|
let httpwww = format!("http://{www_domain}");
|
|
|
|
debug!("[get_icon_url]: Trying with www. prefix '{www_domain}'");
|
|
|
|
debug!("[get_icon_url]: Trying with www. prefix '{www_domain}'");
|
|
|
@ -555,7 +556,7 @@ async fn get_icon_url(domain: &str) -> Result<IconUrlResult, Error> {
|
|
|
|
let limited_reader = stream_to_bytes_limit(content, 384 * 1024).await?.to_vec();
|
|
|
|
let limited_reader = stream_to_bytes_limit(content, 384 * 1024).await?.to_vec();
|
|
|
|
|
|
|
|
|
|
|
|
let dom = Tokenizer::new_with_emitter(limited_reader.to_reader(), FaviconEmitter::default()).infallible();
|
|
|
|
let dom = Tokenizer::new_with_emitter(limited_reader.to_reader(), FaviconEmitter::default()).infallible();
|
|
|
|
get_favicons_node(dom, &mut iconlist, &url).await;
|
|
|
|
get_favicons_node(dom, &mut iconlist, &url);
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
// Add the default favicon.ico to the list with just the given domain
|
|
|
|
// Add the default favicon.ico to the list with just the given domain
|
|
|
|
iconlist.push(Icon::new(35, format!("{ssldomain}/favicon.ico")));
|
|
|
|
iconlist.push(Icon::new(35, format!("{ssldomain}/favicon.ico")));
|
|
|
@ -603,12 +604,12 @@ async fn get_page_with_referer(url: &str, referer: &str) -> Result<Response, Err
|
|
|
|
///
|
|
|
|
///
|
|
|
|
/// # Example
|
|
|
|
/// # Example
|
|
|
|
/// ```
|
|
|
|
/// ```
|
|
|
|
/// priority1 = get_icon_priority("http://example.com/path/to/a/favicon.png", "32x32").await;
|
|
|
|
/// priority1 = get_icon_priority("http://example.com/path/to/a/favicon.png", "32x32");
|
|
|
|
/// priority2 = get_icon_priority("https://example.com/path/to/a/favicon.ico", "").await;
|
|
|
|
/// priority2 = get_icon_priority("https://example.com/path/to/a/favicon.ico", "");
|
|
|
|
/// ```
|
|
|
|
/// ```
|
|
|
|
async fn get_icon_priority(href: &str, sizes: &str) -> u8 {
|
|
|
|
fn get_icon_priority(href: &str, sizes: &str) -> u8 {
|
|
|
|
// Check if there is a dimension set
|
|
|
|
// Check if there is a dimension set
|
|
|
|
let (width, height) = parse_sizes(sizes).await;
|
|
|
|
let (width, height) = parse_sizes(sizes);
|
|
|
|
|
|
|
|
|
|
|
|
// Check if there is a size given
|
|
|
|
// Check if there is a size given
|
|
|
|
if width != 0 && height != 0 {
|
|
|
|
if width != 0 && height != 0 {
|
|
|
@ -650,11 +651,11 @@ async fn get_icon_priority(href: &str, sizes: &str) -> u8 {
|
|
|
|
///
|
|
|
|
///
|
|
|
|
/// # Example
|
|
|
|
/// # Example
|
|
|
|
/// ```
|
|
|
|
/// ```
|
|
|
|
/// let (width, height) = parse_sizes("64x64").await; // (64, 64)
|
|
|
|
/// let (width, height) = parse_sizes("64x64"); // (64, 64)
|
|
|
|
/// let (width, height) = parse_sizes("x128x128").await; // (128, 128)
|
|
|
|
/// let (width, height) = parse_sizes("x128x128"); // (128, 128)
|
|
|
|
/// let (width, height) = parse_sizes("32").await; // (0, 0)
|
|
|
|
/// let (width, height) = parse_sizes("32"); // (0, 0)
|
|
|
|
/// ```
|
|
|
|
/// ```
|
|
|
|
async fn parse_sizes(sizes: &str) -> (u16, u16) {
|
|
|
|
fn parse_sizes(sizes: &str) -> (u16, u16) {
|
|
|
|
let mut width: u16 = 0;
|
|
|
|
let mut width: u16 = 0;
|
|
|
|
let mut height: u16 = 0;
|
|
|
|
let mut height: u16 = 0;
|
|
|
|
|
|
|
|
|
|
|
@ -698,7 +699,7 @@ async fn download_icon(domain: &str) -> Result<(Bytes, Option<&str>), Error> {
|
|
|
|
// Also check if the size is atleast 67 bytes, which seems to be the smallest png i could create
|
|
|
|
// Also check if the size is atleast 67 bytes, which seems to be the smallest png i could create
|
|
|
|
if body.len() >= 67 {
|
|
|
|
if body.len() >= 67 {
|
|
|
|
// Check if the icon type is allowed, else try an icon from the list.
|
|
|
|
// Check if the icon type is allowed, else try an icon from the list.
|
|
|
|
icon_type = get_icon_type(&body).await;
|
|
|
|
icon_type = get_icon_type(&body);
|
|
|
|
if icon_type.is_none() {
|
|
|
|
if icon_type.is_none() {
|
|
|
|
debug!("Icon from {} data:image uri, is not a valid image type", domain);
|
|
|
|
debug!("Icon from {} data:image uri, is not a valid image type", domain);
|
|
|
|
continue;
|
|
|
|
continue;
|
|
|
@ -716,7 +717,7 @@ async fn download_icon(domain: &str) -> Result<(Bytes, Option<&str>), Error> {
|
|
|
|
buffer = stream_to_bytes_limit(res, 5120 * 1024).await?; // 5120KB/5MB for each icon max (Same as icons.bitwarden.net)
|
|
|
|
buffer = stream_to_bytes_limit(res, 5120 * 1024).await?; // 5120KB/5MB for each icon max (Same as icons.bitwarden.net)
|
|
|
|
|
|
|
|
|
|
|
|
// Check if the icon type is allowed, else try an icon from the list.
|
|
|
|
// Check if the icon type is allowed, else try an icon from the list.
|
|
|
|
icon_type = get_icon_type(&buffer).await;
|
|
|
|
icon_type = get_icon_type(&buffer);
|
|
|
|
if icon_type.is_none() {
|
|
|
|
if icon_type.is_none() {
|
|
|
|
buffer.clear();
|
|
|
|
buffer.clear();
|
|
|
|
debug!("Icon from {}, is not a valid image type", icon.href);
|
|
|
|
debug!("Icon from {}, is not a valid image type", icon.href);
|
|
|
@ -751,7 +752,7 @@ async fn save_icon(path: &str, icon: &[u8]) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn get_icon_type(bytes: &[u8]) -> Option<&'static str> {
|
|
|
|
fn get_icon_type(bytes: &[u8]) -> Option<&'static str> {
|
|
|
|
match bytes {
|
|
|
|
match bytes {
|
|
|
|
[137, 80, 78, 71, ..] => Some("png"),
|
|
|
|
[137, 80, 78, 71, ..] => Some("png"),
|
|
|
|
[0, 0, 1, 0, ..] => Some("x-icon"),
|
|
|
|
[0, 0, 1, 0, ..] => Some("x-icon"),
|
|
|
|