@ -1,6 +1,6 @@
use std ::{
net ::IpAddr ,
sync ::Arc ,
sync ::{ Arc , Mutex } ,
time ::{ Duration , SystemTime } ,
} ;
@ -16,14 +16,13 @@ use rocket::{http::ContentType, response::Redirect, Route};
use tokio ::{
fs ::{ create_dir_all , remove_file , symlink_metadata , File } ,
io ::{ AsyncReadExt , AsyncWriteExt } ,
net ::lookup_host ,
} ;
use html5gum ::{ Emitter , HtmlString , InfallibleTokenizer , Readable , StringReader , Tokenizer } ;
use crate ::{
error ::Error ,
util ::{ get_reqwest_client_builder , Cached ,
util ::{ get_reqwest_client_builder , Cached , CustomDnsResolver , CustomResolverError },
CONFIG ,
} ;
@ -49,48 +48,32 @@ static CLIENT: Lazy<Client> = Lazy::new(|| {
let icon_download_timeout = Duration ::from_secs ( CONFIG . icon_download_timeout ( ) ) ;
let pool_idle_timeout = Duration ::from_secs ( 10 ) ;
// Reuse the client between requests
let client = get_reqwest_client_builder ( )
get_reqwest_client_builder ( )
. cookie_provider ( Arc ::clone ( & cookie_store ) )
. timeout ( icon_download_timeout )
. pool_max_idle_per_host ( 5 ) // Configure the Hyper Pool to only have max 5 idle connections
. pool_idle_timeout ( pool_idle_timeout ) // Configure the Hyper Pool to timeout after 10 seconds
. hickory_dns ( true )
. default_headers ( default_headers . clone ( ) ) ;
match client . build ( ) {
Ok ( client ) = > client ,
Err ( e ) = > {
error ! ( "Possible trust-dns error, trying with trust-dns disabled: '{e}'" ) ;
get_reqwest_client_builder ( )
. cookie_provider ( cookie_store )
. timeout ( icon_download_timeout )
. pool_max_idle_per_host ( 5 ) // Configure the Hyper Pool to only have max 5 idle connections
. pool_idle_timeout ( pool_idle_timeout ) // Configure the Hyper Pool to timeout after 10 seconds
. hickory_dns ( false )
. default_headers ( default_headers )
. build ( )
. expect ( "Failed to build client" )
}
}
. dns_resolver ( CustomDnsResolver ::instance ( ) )
. default_headers ( default_headers . clone ( ) )
. build ( )
. expect ( "Failed to build client" )
} ) ;
// Build Regex only once since this takes a lot of time.
static ICON_SIZE_REGEX : Lazy < Regex > = Lazy ::new ( | | Regex ::new ( r"(?x)(\d+)\D*(\d+)" ) . unwrap ( ) ) ;
// Special HashMap which holds the user defined Regex to speedup matching the regex.
static ICON_BLACKLIST_REGEX : Lazy < dashmap ::DashMap < String , Regex > > = Lazy ::new ( dashmap ::DashMap ::new ) ;
async fn icon_redirect ( domain : & str , template : & str ) -> Option < Redirect > {
#[ get( " /<domain>/icon.png " ) ]
fn icon_external ( domain : & str ) -> Option < Redirect > {
if ! is_valid_domain ( domain ) {
warn ! ( "Invalid domain: {}" , domain ) ;
return None ;
}
if check_domain_blacklist_reason( domain ) . await . is_some ( ) {
if is_domain_blacklisted ( domain ) {
return None ;
}
let url = template . replace ( "{}" , domain ) ;
let url = CONFIG. _icon_service_url ( ) . replace ( "{}" , domain ) ;
match CONFIG . icon_redirect_code ( ) {
301 = > Some ( Redirect ::moved ( url ) ) , // legacy permanent redirect
302 = > Some ( Redirect ::found ( url ) ) , // legacy temporary redirect
@ -103,11 +86,6 @@ async fn icon_redirect(domain: &str, template: &str) -> Option<Redirect> {
}
}
#[ get( " /<domain>/icon.png " ) ]
async fn icon_external ( domain : & str ) -> Option < Redirect > {
icon_redirect ( domain , & CONFIG . _icon_service_url ( ) ) . await
}
#[ get( " /<domain>/icon.png " ) ]
async fn icon_internal ( domain : & str ) -> Cached < ( ContentType , Vec < u8 > ) > {
const FALLBACK_ICON : & [ u8 ] = include_bytes! ( "../static/images/fallback-icon.png" ) ;
@ -166,153 +144,28 @@ fn is_valid_domain(domain: &str) -> bool {
true
}
/// TODO: This is extracted from IpAddr::is_global, which is unstable:
/// https://doc.rust-lang.org/nightly/std/net/enum.IpAddr.html#method.is_global
/// Remove once https://github.com/rust-lang/rust/issues/27709 is merged
#[ allow(clippy::nonminimal_bool) ]
#[ cfg(not(feature = " unstable " )) ]
fn is_global ( ip : IpAddr ) -> bool {
match ip {
IpAddr ::V4 ( ip ) = > {
// check if this address is 192.0.0.9 or 192.0.0.10. These addresses are the only two
// globally routable addresses in the 192.0.0.0/24 range.
if u32 ::from ( ip ) = = 0xc0000009 | | u32 ::from ( ip ) = = 0xc000000a {
return true ;
}
! ip . is_private ( )
& & ! ip . is_loopback ( )
& & ! ip . is_link_local ( )
& & ! ip . is_broadcast ( )
& & ! ip . is_documentation ( )
& & ! ( ip . octets ( ) [ 0 ] = = 100 & & ( ip . octets ( ) [ 1 ] & 0b1100_0000 = = 0b0100_0000 ) )
& & ! ( ip . octets ( ) [ 0 ] = = 192 & & ip . octets ( ) [ 1 ] = = 0 & & ip . octets ( ) [ 2 ] = = 0 )
& & ! ( ip . octets ( ) [ 0 ] & 240 = = 240 & & ! ip . is_broadcast ( ) )
& & ! ( ip . octets ( ) [ 0 ] = = 198 & & ( ip . octets ( ) [ 1 ] & 0xfe ) = = 18 )
// Make sure the address is not in 0.0.0.0/8
& & ip . octets ( ) [ 0 ] ! = 0
}
IpAddr ::V6 ( ip ) = > {
if ip . is_multicast ( ) & & ip . segments ( ) [ 0 ] & 0x000f = = 14 {
true
} else {
! ip . is_multicast ( )
& & ! ip . is_loopback ( )
& & ! ( ( ip . segments ( ) [ 0 ] & 0xffc0 ) = = 0xfe80 )
& & ! ( ( ip . segments ( ) [ 0 ] & 0xfe00 ) = = 0xfc00 )
& & ! ip . is_unspecified ( )
& & ! ( ( ip . segments ( ) [ 0 ] = = 0x2001 ) & & ( ip . segments ( ) [ 1 ] = = 0xdb8 ) )
}
}
}
}
#[ cfg(feature = " unstable " ) ]
fn is_global ( ip : IpAddr ) -> bool {
ip . is_global ( )
}
/// These are some tests to check that the implementations match
/// The IPv4 can be all checked in 5 mins or so and they are correct as of nightly 2020-07-11
/// The IPV6 can't be checked in a reasonable time, so we check about ten billion random ones, so far correct
/// Note that the is_global implementation is subject to change as new IP RFCs are created
///
/// To run while showing progress output:
/// cargo test --features sqlite,unstable -- --nocapture --ignored
#[ cfg(test) ]
#[ cfg(feature = " unstable " ) ]
mod tests {
use super ::* ;
#[ test ]
#[ ignore ]
fn test_ipv4_global ( ) {
for a in 0 .. u8 ::MAX {
println! ( "Iter: {}/255" , a ) ;
for b in 0 .. u8 ::MAX {
for c in 0 .. u8 ::MAX {
for d in 0 .. u8 ::MAX {
let ip = IpAddr ::V4 ( std ::net ::Ipv4Addr ::new ( a , b , c , d ) ) ;
assert_eq! ( ip . is_global ( ) , is_global ( ip ) )
}
}
}
}
}
#[ test ]
#[ ignore ]
fn test_ipv6_global ( ) {
use ring ::rand ::{ SecureRandom , SystemRandom } ;
let mut v = [ 0 u8 ; 16 ] ;
let rand = SystemRandom ::new ( ) ;
for i in 0 .. 1_000 {
println! ( "Iter: {}/1_000" , i ) ;
for _ in 0 .. 10_000_000 {
rand . fill ( & mut v ) . expect ( "Error generating random values" ) ;
let ip = IpAddr ::V6 ( std ::net ::Ipv6Addr ::new (
( v [ 14 ] as u16 ) < < 8 | v [ 15 ] as u16 ,
( v [ 12 ] as u16 ) < < 8 | v [ 13 ] as u16 ,
( v [ 10 ] as u16 ) < < 8 | v [ 11 ] as u16 ,
( v [ 8 ] as u16 ) < < 8 | v [ 9 ] as u16 ,
( v [ 6 ] as u16 ) < < 8 | v [ 7 ] as u16 ,
( v [ 4 ] as u16 ) < < 8 | v [ 5 ] as u16 ,
( v [ 2 ] as u16 ) < < 8 | v [ 3 ] as u16 ,
( v [ 0 ] as u16 ) < < 8 | v [ 1 ] as u16 ,
) ) ;
assert_eq! ( ip . is_global ( ) , is_global ( ip ) )
}
}
}
}
#[ derive(Clone) ]
enum DomainBlacklistReason {
Regex ,
IP ,
}
use cached ::proc_macro ::cached ;
#[ cached(key = " String " , convert = r# " { domain.to_string() } " #, size = 16, time = 60) ]
async fn check_domain_blacklist_reason ( domain : & str ) -> Option < DomainBlacklistReason > {
// First check the blacklist regex if there is a match.
// This prevents the blocked domain(s) from being leaked via a DNS lookup.
if let Some ( blacklist ) = CONFIG . icon_blacklist_regex ( ) {
// Use the pre-generate Regex stored in a Lazy HashMap if there's one, else generate it.
let is_match = if let Some ( regex ) = ICON_BLACKLIST_REGEX . get ( & blacklist ) {
regex . is_match ( domain )
} else {
// Clear the current list if the previous key doesn't exists.
// To prevent growing of the HashMap after someone has changed it via the admin interface.
if ICON_BLACKLIST_REGEX . len ( ) > = 1 {
ICON_BLACKLIST_REGEX . clear ( ) ;
}
// Generate the regex to store in too the Lazy Static HashMap.
let blacklist_regex = Regex ::new ( & blacklist ) . unwrap ( ) ;
let is_match = blacklist_regex . is_match ( domain ) ;
ICON_BLACKLIST_REGEX . insert ( blacklist . clone ( ) , blacklist_regex ) ;
pub fn is_domain_blacklisted ( domain : & str ) -> bool {
let Some ( config_blacklist ) = CONFIG . icon_blacklist_regex ( ) else {
return false ;
} ;
is_match
} ;
// Compiled domain blacklist
static COMPILED_BLACKLIST : Mutex < Option < ( String , Regex ) > > = Mutex ::new ( None ) ;
let mut guard = COMPILED_BLACKLIST . lock ( ) . unwrap ( ) ;
if is_match {
debug ! ( "Blacklisted domain: {} matched ICON_BLACKLIST_REGEX" , domain ) ;
return Some ( DomainBlacklistReason ::Regex ) ;
// If the stored regex is up to date, use it
if let Some ( ( value , regex ) ) = & * guard {
if value = = & config_blacklist {
return regex . is_match ( domain ) ;
}
}
if CONFIG . icon_blacklist_non_global_ips ( ) {
if let Ok ( s ) = lookup_host ( ( domain , 0 ) ) . await {
for addr in s {
if ! is_global ( addr . ip ( ) ) {
debug ! ( "IP {} for domain '{}' is not a global IP!" , addr . ip ( ) , domain ) ;
return Some ( DomainBlacklistReason ::IP ) ;
}
}
}
}
// If we don't have a regex stored, or it's not up to date, recreate it
let regex = Regex ::new ( & config_blacklist ) . unwrap ( ) ;
let is_match = regex . is_match ( domain ) ;
* guard = Some ( ( config_blacklist , regex ) ) ;
None
is_match
}
async fn get_icon ( domain : & str ) -> Option < ( Vec < u8 > , String ) > {
@ -342,6 +195,13 @@ async fn get_icon(domain: &str) -> Option<(Vec<u8>, String)> {
Some ( ( icon . to_vec ( ) , icon_type . unwrap_or ( "x-icon" ) . to_string ( ) ) )
}
Err ( e ) = > {
// If this error comes from the resolver, this means this is a blacklisted domain
// or non global IP, don't save the miss file in this case to avoid leaking it
if let Some ( error ) = CustomResolverError ::downcast_ref ( & e ) {
warn ! ( "{error}" ) ;
return None ;
}
warn ! ( "Unable to download icon: {:?}" , e ) ;
let miss_indicator = path + ".miss" ;
save_icon ( & miss_indicator , & [ ] ) . await ;
@ -491,42 +351,48 @@ async fn get_icon_url(domain: &str) -> Result<IconUrlResult, Error> {
let ssldomain = format! ( "https://{domain}" ) ;
let httpdomain = format! ( "http://{domain}" ) ;
// First check the domain as given during the request for both HTTPS and HTTP .
let resp = match get_page ( & ssldomain ) . or_else ( | _ | get_page ( & httpdomain ) ) . await {
Ok( c ) = > Ok ( c ) ,
Err ( e ) = > {
let mut sub_resp = Err ( e ) ;
// When the domain is not an IP, and has more then one dot, remove all subdomains.
let is_ip = domain . parse ::< IpAddr > ( ) ;
if is_ip . is_err ( ) & & domain . matches ( '.' ) . count ( ) > 1 {
let mut domain_parts = domain . split ( '.' ) ;
let base_domain = format! (
"{base}.{tld}" ,
tld = domain_parts . next_back ( ) . unwrap ( ) ,
base = domain_parts . next_back ( ) . unwrap ( )
) ;
if is_valid_domain ( & base_domain ) {
let sslbase = format! ( "https://{base_domain}" ) ;
let httpbase = format! ( "http://{base_domain}" ) ;
debug ! ( "[get_icon_url]: Trying without subdomains '{base_domain}'" ) ;
sub_resp = get_page ( & sslbase ) . or_else ( | _ | get_page ( & httpbase ) ) . await ;
}
// First check the domain as given during the request for HTTPS.
let resp = match get_page ( & ssldomain ) . await {
Err( e ) if CustomResolverError ::downcast_ref ( & e ) . is_none ( ) = > {
// If we get an error that is not caused by the blacklist, we retry with HTTP
match get_page ( & httpdomain ) . await {
mut sub_resp @ Err ( _ ) = > {
// When the domain is not an IP, and has more then one dot, remove all subdomains.
let is_ip = domain . parse ::< IpAddr > ( ) ;
if is_ip . is_err ( ) & & domain . matches ( '.' ) . count ( ) > 1 {
let mut domain_parts = domain . split ( '.' ) ;
let base_domain = format! (
"{base}.{tld}" ,
tld = domain_parts . next_back ( ) . unwrap ( ) ,
base = domain_parts . next_back ( ) . unwrap ( )
) ;
if is_valid_domain ( & base_domain ) {
let sslbase = format! ( "https://{base_domain}" ) ;
let httpbase = format! ( "http://{base_domain}" ) ;
debug ! ( "[get_icon_url]: Trying without subdomains '{base_domain}'" ) ;
sub_resp = get_page ( & sslbase ) . or_else ( | _ | get_page ( & httpbase ) ) . await ;
}
// When the domain is not an IP, and has less then 2 dots, try to add www. infront of it.
} else if is_ip . is_err ( ) & & domain . matches ( '.' ) . count ( ) < 2 {
let www_domain = format! ( "www.{domain}" ) ;
if is_valid_domain ( & www_domain ) {
let sslwww = format! ( "https://{www_domain}" ) ;
let httpwww = format! ( "http://{www_domain}" ) ;
debug ! ( "[get_icon_url]: Trying with www. prefix '{www_domain}'" ) ;
// When the domain is not an IP, and has less then 2 dots, try to add www. infront of it.
} else if is_ip . is_err ( ) & & domain . matches ( '.' ) . count ( ) < 2 {
let www_domain = format! ( "www.{domain}" ) ;
if is_valid_domain ( & www_domain ) {
let sslwww = format! ( "https://{www_domain}" ) ;
let httpwww = format! ( "http://{www_domain}" ) ;
debug ! ( "[get_icon_url]: Trying with www. prefix '{www_domain}'" ) ;
sub_resp = get_page ( & sslwww ) . or_else ( | _ | get_page ( & httpwww ) ) . await ;
sub_resp = get_page ( & sslwww ) . or_else ( | _ | get_page ( & httpwww ) ) . await ;
}
}
sub_resp
}
res = > res ,
}
sub_resp
}
// If we get a result or a blacklist error, just continue
res = > res ,
} ;
// Create the iconlist
@ -573,21 +439,12 @@ async fn get_page(url: &str) -> Result<Response, Error> {
}
async fn get_page_with_referer ( url : & str , referer : & str ) -> Result < Response , Error > {
match check_domain_blacklist_reason ( url ::Url ::parse ( url ) . unwrap ( ) . host_str ( ) . unwrap_or_default ( ) ) . await {
Some ( DomainBlacklistReason ::Regex ) = > warn ! ( "Favicon '{}' is from a blacklisted domain!" , url ) ,
Some ( DomainBlacklistReason ::IP ) = > warn ! ( "Favicon '{}' is hosted on a non-global IP!" , url ) ,
None = > ( ) ,
}
let mut client = CLIENT . get ( url ) ;
if ! referer . is_empty ( ) {
client = client . header ( "Referer" , referer )
}
match client . send ( ) . await {
Ok ( c ) = > c . error_for_status ( ) . map_err ( Into ::into ) ,
Err ( e ) = > err_silent ! ( format! ( "{e}" ) ) ,
}
Ok ( client . send ( ) . await ? . error_for_status ( ) ? )
}
/// Returns a Integer with the priority of the type of the icon which to prefer.
@ -670,12 +527,6 @@ fn parse_sizes(sizes: &str) -> (u16, u16) {
}
async fn download_icon ( domain : & str ) -> Result < ( Bytes , Option < & str > ) , Error > {
match check_domain_blacklist_reason ( domain ) . await {
Some ( DomainBlacklistReason ::Regex ) = > err_silent ! ( "Domain is blacklisted" , domain ) ,
Some ( DomainBlacklistReason ::IP ) = > err_silent ! ( "Host resolves to a non-global IP" , domain ) ,
None = > ( ) ,
}
let icon_result = get_icon_url ( domain ) . await ? ;
let mut buffer = Bytes ::new ( ) ;
@ -711,22 +562,19 @@ async fn download_icon(domain: &str) -> Result<(Bytes, Option<&str>), Error> {
_ = > debug ! ( "Extracted icon from data:image uri is invalid" ) ,
} ;
} else {
match get_page_with_referer ( & icon . href , & icon_result . referer ) . await {
Ok ( res ) = > {
buffer = stream_to_bytes_limit ( res , 5120 * 1024 ) . await ? ; // 5120KB/5MB for each icon max (Same as icons.bitwarden.net)
// Check if the icon type is allowed, else try an icon from the list.
icon_type = get_icon_type ( & buffer ) ;
if icon_type . is_none ( ) {
buffer . clear ( ) ;
debug ! ( "Icon from {}, is not a valid image type" , icon . href ) ;
continue ;
}
info ! ( "Downloaded icon from {}" , icon . href ) ;
break ;
}
Err ( e ) = > debug ! ( "{:?}" , e ) ,
} ;
let res = get_page_with_referer ( & icon . href , & icon_result . referer ) . await ? ;
buffer = stream_to_bytes_limit ( res , 5120 * 1024 ) . await ? ; // 5120KB/5MB for each icon max (Same as icons.bitwarden.net)
// Check if the icon type is allowed, else try an icon from the list.
icon_type = get_icon_type ( & buffer ) ;
if icon_type . is_none ( ) {
buffer . clear ( ) ;
debug ! ( "Icon from {}, is not a valid image type" , icon . href ) ;
continue ;
}
info ! ( "Downloaded icon from {}" , icon . href ) ;
break ;
}
}
Daniel García
Mathijs van Veluw
Stefan Melmuk