Upstream/bitwarden-server
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2024-11-21 12:05:42 +01:00
Code Issues Projects Releases Wiki Activity

Rewrite Icon fetching (#3023)

Browse Source 
* Rewrite Icon fetching

* Move validation to IconUri, Uri, or UriBuilder

* `dotnet format` 🤖

* PR suggestions

* Add not null compiler hint

* Add twitter to test case

* Move Uri manipulation to UriService

* Implement MockedHttpClient

Presents better, fluent handling of message matching and response
building.

* Add redirect handling tests

* Add testing to models

* More aggressively dispose content in icon link

* Format 🤖

* Update icon lockfile

* Convert to cloned stream for HttpResponseBuilder

Content was being disposed when HttResponseMessage was being disposed.
This avoids losing our reference to our content and allows multiple
usages of the same `MockedHttpMessageResponse`

* Move services to extension

Extension is shared by testing and allows access to services from
our service tests

* Remove unused `using`

* Prefer awaiting asyncs for better exception handling

* `dotnet format` 🤖

* Await async

* Update tests to use test TLD and ip ranges

* Remove unused interfaces

* Make assignments static when possible

* Prefer invariant comparer to downcasing

* Prefer injecting interface services to implementations

* Prefer comparer set in HashSet initialization

* Allow SVG icons

* Filter out icons with unknown formats

* Seek to beginning of MemoryStream after writing it

* More appropriate to not return icon if it's invalid

* Add svg icon test
This commit is contained in:
Matt Gibson 2023-08-08 15:29:40 -04:00 committed by GitHub
parent ca368466ce
commit 4377c7a897
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 1685 additions and 522 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Icons/Controllers/IconsController.cs
View File

@ -81,7 +81,7 @@ public class IconsController : Controller
}
else
{
icon = result.Icon;
icon = result;
}
// Only cache not found and smaller images (<= 50kb)

100
src/Icons/Models/DomainIcons.cs Normal file
View File

@ -0,0 +1,100 @@
#nullable enable
using System.Collections;
using AngleSharp.Html.Parser;
using Bit.Icons.Extensions;
using Bit.Icons.Services;
namespace Bit.Icons.Models;
public class DomainIcons : IEnumerable<Icon>
{
private readonly ILogger<IIconFetchingService> _logger;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IUriService _uriService;
private readonly List<Icon> _icons = new();
public string Domain { get; }
public Icon this[int i]
{
get
{
return _icons[i];
}
}
public IEnumerator<Icon> GetEnumerator() => ((IEnumerable<Icon>)_icons).GetEnumerator();
IEnumerator IEnumerable.GetEnumerator() => ((IEnumerable)_icons).GetEnumerator();
private DomainIcons(string domain, ILogger<IIconFetchingService> logger, IHttpClientFactory httpClientFactory, IUriService uriService)
{
_logger = logger;
_httpClientFactory = httpClientFactory;
_uriService = uriService;
Domain = domain;
}
public static async Task<DomainIcons> FetchAsync(string domain, ILogger<IIconFetchingService> logger, IHttpClientFactory httpClientFactory, IHtmlParser parser, IUriService uriService)
{
var pageIcons = new DomainIcons(domain, logger, httpClientFactory, uriService);
await pageIcons.FetchIconsAsync(parser);
return pageIcons;
}
private async Task FetchIconsAsync(IHtmlParser parser)
{
if (!Uri.TryCreate($"https://{Domain}", UriKind.Absolute, out var uri))
{
_logger.LogWarning("Bad domain: {domain}.", Domain);
return;
}
var host = uri.Host;
// first try https
using (var response = await IconHttpRequest.FetchAsync(uri, _logger, _httpClientFactory, _uriService))
{
if (response.IsSuccessStatusCode)
{
_icons.AddRange(await response.RetrieveIconsAsync(uri, Domain, parser));
return;
}
}
// then try http
uri = uri.ChangeScheme("http");
using (var response = await IconHttpRequest.FetchAsync(uri, _logger, _httpClientFactory, _uriService))
{
if (response.IsSuccessStatusCode)
{
_icons.AddRange(await response.RetrieveIconsAsync(uri, Domain, parser));
return;
}
}
var dotCount = Domain.Count(c => c == '.');
// Then try base domain
if (dotCount > 1 && DomainName.TryParseBaseDomain(Domain, out var baseDomain) &&
Uri.TryCreate($"https://{baseDomain}", UriKind.Absolute, out uri))
{
using var response = await IconHttpRequest.FetchAsync(uri, _logger, _httpClientFactory, _uriService);
if (response.IsSuccessStatusCode)
{
_icons.AddRange(await response.RetrieveIconsAsync(uri, Domain, parser));
return;
}
}
// Then try www
if (dotCount < 2 && Uri.TryCreate($"https://www.{host}", UriKind.Absolute, out uri))
{
using var response = await IconHttpRequest.FetchAsync(uri, _logger, _httpClientFactory, _uriService);
if (response.IsSuccessStatusCode)
{
_icons.AddRange(await response.RetrieveIconsAsync(uri, Domain, parser));
return;
}
}
}
}

110
src/Icons/Models/IconHttpRequest.cs Normal file
View File

@ -0,0 +1,110 @@
#nullable enable
using System.Net;
using Bit.Icons.Extensions;
using Bit.Icons.Services;
namespace Bit.Icons.Models;
public class IconHttpRequest
{
private const int _maxRedirects = 2;
private static readonly HttpStatusCode[] _redirectStatusCodes = new HttpStatusCode[] { HttpStatusCode.Redirect, HttpStatusCode.MovedPermanently, HttpStatusCode.RedirectKeepVerb, HttpStatusCode.SeeOther };
private readonly ILogger<IIconFetchingService> _logger;
private readonly HttpClient _httpClient;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IUriService _uriService;
private readonly int _redirectsCount;
private readonly Uri _uri;
private static HttpResponseMessage NotFound => new(HttpStatusCode.NotFound);
private IconHttpRequest(Uri uri, ILogger<IIconFetchingService> logger, IHttpClientFactory httpClientFactory, IUriService uriService, int redirectsCount)
{
_logger = logger;
_httpClientFactory = httpClientFactory;
_httpClient = _httpClientFactory.CreateClient("Icons");
_uriService = uriService;
_redirectsCount = redirectsCount;
_uri = uri;
}
public static async Task<IconHttpResponse> FetchAsync(Uri uri, ILogger<IIconFetchingService> logger, IHttpClientFactory httpClientFactory, IUriService uriService)
{
var pageIcons = new IconHttpRequest(uri, logger, httpClientFactory, uriService, 0);
var httpResponse = await pageIcons.FetchAsync();
return new IconHttpResponse(httpResponse, logger, httpClientFactory, uriService);
}
private async Task<HttpResponseMessage> FetchAsync()
{
if (!_uriService.TryGetUri(_uri, out var iconUri) || !iconUri!.IsValid)
{
return NotFound;
}
var response = await GetAsync(iconUri);
if (response.IsSuccessStatusCode)
{
return response;
}
using var responseForRedirect = response;
return await FollowRedirectsAsync(responseForRedirect, iconUri);
}
private async Task<HttpResponseMessage> GetAsync(IconUri iconUri)
{
using var message = new HttpRequestMessage();
message.RequestUri = iconUri.InnerUri;
message.Headers.Host = iconUri.Host;
message.Method = HttpMethod.Get;
try
{
return await _httpClient.SendAsync(message);
}
catch
{
return NotFound;
}
}
private async Task<HttpResponseMessage> FollowRedirectsAsync(HttpResponseMessage response, IconUri originalIconUri)
{
if (_redirectsCount >= _maxRedirects || response.Headers.Location == null ||
!_redirectStatusCodes.Contains(response.StatusCode))
{
return NotFound;
}
using var responseForRedirect = response;
var redirectUri = DetermineRedirectUri(responseForRedirect.Headers.Location, originalIconUri);
return await new IconHttpRequest(redirectUri, _logger, _httpClientFactory, _uriService, _redirectsCount + 1).FetchAsync();
}
private static Uri DetermineRedirectUri(Uri responseUri, IconUri originalIconUri)
{
if (responseUri.IsAbsoluteUri)
{
if (!responseUri.IsHypertext())
{
return responseUri.ChangeScheme("https");
}
return responseUri;
}
else
{
return new UriBuilder
{
Scheme = originalIconUri.Scheme,
Host = originalIconUri.Host,
Path = responseUri.ToString()
}.Uri;
}
}
}

72
src/Icons/Models/IconHttpResponse.cs Normal file
View File

@ -0,0 +1,72 @@
#nullable enable
using System.Net;
using AngleSharp.Html.Parser;
using Bit.Icons.Services;
namespace Bit.Icons.Models;
public class IconHttpResponse : IDisposable
{
private const int _maxIconLinksProcessed = 200;
private const int _maxRetrievedIcons = 10;
private readonly HttpResponseMessage _response;
private readonly ILogger<IIconFetchingService> _logger;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IUriService _uriService;
public HttpStatusCode StatusCode => _response.StatusCode;
public bool IsSuccessStatusCode => _response.IsSuccessStatusCode;
public string? ContentType => _response.Content.Headers.ContentType?.MediaType;
public HttpContent Content => _response.Content;
public IconHttpResponse(HttpResponseMessage response, ILogger<IIconFetchingService> logger, IHttpClientFactory httpClientFactory, IUriService uriService)
{
_response = response;
_logger = logger;
_httpClientFactory = httpClientFactory;
_uriService = uriService;
}
public async Task<IEnumerable<Icon>> RetrieveIconsAsync(Uri requestUri, string domain, IHtmlParser parser)
{
using var htmlStream = await _response.Content.ReadAsStreamAsync();
var head = await parser.ParseHeadAsync(htmlStream);
if (head == null)
{
_logger.LogWarning("No DocumentElement for {domain}.", domain);
return Array.Empty<Icon>();
}
// Make sure uri uses domain name, not ip
var uri = _response.RequestMessage?.RequestUri;
if (uri == null || IPAddress.TryParse(_response.RequestMessage!.RequestUri!.Host, out var _))
{
uri = requestUri;
}
var baseUrl = head.QuerySelector("base[href]")?.Attributes["href"]?.Value;
if (string.IsNullOrWhiteSpace(baseUrl))
{
baseUrl = "/";
}
var links = head.QuerySelectorAll("link[href]")
?.Take(_maxIconLinksProcessed)
.Select(l => new IconLink(l, uri, baseUrl))
.Where(l => l.IsUsable())
.OrderBy(l => l.Priority)
.Take(_maxRetrievedIcons)
.ToArray() ?? Array.Empty<IconLink>();
var results = await Task.WhenAll(links.Select(l => l.FetchAsync(_logger, _httpClientFactory, _uriService)));
return results.Where(r => r != null).Select(r => r!);
}
public void Dispose()
{
_response.Dispose();
}
}

220
src/Icons/Models/IconLink.cs Normal file
View File

@ -0,0 +1,220 @@
#nullable enable
using System.Text;
using AngleSharp.Dom;
using Bit.Icons.Extensions;
using Bit.Icons.Services;
namespace Bit.Icons.Models;
public class IconLink
{
private static readonly HashSet<string> _iconRels = new(StringComparer.InvariantCultureIgnoreCase) { "icon", "apple-touch-icon", "shortcut icon" };
private static readonly HashSet<string> _blocklistedRels = new(StringComparer.InvariantCultureIgnoreCase) { "preload", "image_src", "preconnect", "canonical", "alternate", "stylesheet" };
private static readonly HashSet<string> _iconExtensions = new(StringComparer.InvariantCultureIgnoreCase) { ".ico", ".png", ".jpg", ".jpeg" };
private const string _pngMediaType = "image/png";
private static readonly byte[] _pngHeader = new byte[] { 137, 80, 78, 71 };
private static readonly byte[] _webpHeader = Encoding.UTF8.GetBytes("RIFF");
private const string _icoMediaType = "image/x-icon";
private const string _icoAltMediaType = "image/vnd.microsoft.icon";
private static readonly byte[] _icoHeader = new byte[] { 00, 00, 01, 00 };
private const string _jpegMediaType = "image/jpeg";
private static readonly byte[] _jpegHeader = new byte[] { 255, 216, 255 };
private const string _svgXmlMediaType = "image/svg+xml";
private static readonly HashSet<string> _allowedMediaTypes = new(StringComparer.InvariantCultureIgnoreCase)
{
_pngMediaType,
_icoMediaType,
_icoAltMediaType,
_jpegMediaType,
_svgXmlMediaType,
};
private bool _useUriDirectly = false;
private bool _validated = false;
private int? _width;
private int? _height;
public IAttr? Href { get; }
public IAttr? Rel { get; }
public IAttr? Type { get; }
public IAttr? Sizes { get; }
public Uri ParentUri { get; }
public string BaseUrlPath { get; }
public int Priority
{
get
{
if (_width == null || _width != _height)
{
return 200;
}
return _width switch
{
32 => 1,
64 => 2,
>= 24 and <= 128 => 3,
16 => 4,
_ => 100,
};
}
}
public IconLink(Uri parentPage)
{
_useUriDirectly = true;
_validated = true;
ParentUri = parentPage;
BaseUrlPath = parentPage.PathAndQuery;
}
public IconLink(IElement element, Uri parentPage, string baseUrlPath)
{
Href = element.Attributes["href"];
ParentUri = parentPage;
BaseUrlPath = baseUrlPath;
Rel = element.Attributes["rel"];
Type = element.Attributes["type"];
Sizes = element.Attributes["sizes"];
if (!string.IsNullOrWhiteSpace(Sizes?.Value))
{
var sizeParts = Sizes.Value.Split('x');
if (sizeParts.Length == 2 && int.TryParse(sizeParts[0].Trim(), out var width) &&
int.TryParse(sizeParts[1].Trim(), out var height))
{
_width = width;
_height = height;
}
}
}
public bool IsUsable()
{
if (string.IsNullOrWhiteSpace(Href?.Value))
{
return false;
}
if (Rel != null && _iconRels.Contains(Rel.Value))
{
_validated = true;
}
if (Rel == null || !_blocklistedRels.Contains(Rel.Value))
{
try
{
var extension = Path.GetExtension(Href.Value);
if (_iconExtensions.Contains(extension))
{
_validated = true;
}
}
catch (ArgumentException) { }
}
return _validated;
}
/// <summary>
/// Fetches the icon from the Href. Will always fail unless first validated with IsUsable().
/// </summary>
public async Task<Icon?> FetchAsync(ILogger<IIconFetchingService> logger, IHttpClientFactory httpClientFactory, IUriService uriService)
{
if (!_validated)
{
return null;
}
var uri = BuildUri();
if (uri == null)
{
return null;
}
using var response = await IconHttpRequest.FetchAsync(uri, logger, httpClientFactory, uriService);
if (!response.IsSuccessStatusCode)
{
return null;
}
var format = response.Content.Headers.ContentType?.MediaType;
var bytes = await response.Content.ReadAsByteArrayAsync();
response.Content.Dispose();
if (format == null || !_allowedMediaTypes.Contains(format))
{
format = DetermineImageFormatFromFile(bytes);
}
if (format == null || !_allowedMediaTypes.Contains(format))
{
return null;
}
return new Icon { Image = bytes, Format = format };
}
private Uri? BuildUri()
{
if (_useUriDirectly)
{
return ParentUri;
}
if (Href == null)
{
return null;
}
if (Href.Value.StartsWith("//") && Uri.TryCreate($"{ParentUri.Scheme}://{Href.Value[2..]}", UriKind.Absolute, out var uri))
{
return uri;
}
if (Uri.TryCreate(Href.Value, UriKind.Relative, out uri))
{
return new UriBuilder()
{
Scheme = ParentUri.Scheme,
Host = ParentUri.Host,
}.Uri.ConcatPath(BaseUrlPath, uri.OriginalString);
}
if (Uri.TryCreate(Href.Value, UriKind.Absolute, out uri))
{
return uri;
}
return null;
}
private static bool HeaderMatch(byte[] imageBytes, byte[] header)
{
return imageBytes.Length >= header.Length && header.SequenceEqual(imageBytes.Take(header.Length));
}
private static string DetermineImageFormatFromFile(byte[] imageBytes)
{
if (HeaderMatch(imageBytes, _icoHeader))
{
return _icoMediaType;
}
else if (HeaderMatch(imageBytes, _pngHeader) || HeaderMatch(imageBytes, _webpHeader))
{
return _pngMediaType;
}
else if (HeaderMatch(imageBytes, _jpegHeader))
{
return _jpegMediaType;
}
else
{
return string.Empty;
}
}
}

65
src/Icons/Models/IconResult.cs
View File

@ -1,65 +0,0 @@
namespace Bit.Icons.Models;
public class IconResult
{
public IconResult(string href, string sizes)
{
Path = href;
if (!string.IsNullOrWhiteSpace(sizes))
{
var sizeParts = sizes.Split('x');
if (sizeParts.Length == 2 && int.TryParse(sizeParts[0].Trim(), out var width) &&
int.TryParse(sizeParts[1].Trim(), out var height))
{
DefinedWidth = width;
DefinedHeight = height;
if (width == height)
{
if (width == 32)
{
Priority = 1;
}
else if (width == 64)
{
Priority = 2;
}
else if (width >= 24 && width <= 128)
{
Priority = 3;
}
else if (width == 16)
{
Priority = 4;
}
else
{
Priority = 100;
}
}
}
}
if (Priority == 0)
{
Priority = 200;
}
}
public IconResult(Uri uri, byte[] bytes, string format)
{
Path = uri.ToString();
Icon = new Icon
{
Image = bytes,
Format = format
};
Priority = 10;
}
public string Path { get; set; }
public int? DefinedWidth { get; set; }
public int? DefinedHeight { get; set; }
public Icon Icon { get; set; }
public int Priority { get; set; }
}

52
src/Icons/Models/IconUri.cs Normal file
View File

@ -0,0 +1,52 @@
#nullable enable
using System.Net;
using Bit.Icons.Extensions;
namespace Bit.Icons.Models;
public class IconUri
{
private readonly IPAddress _ip;
public string Host { get; }
public Uri InnerUri { get; }
public string Scheme => InnerUri.Scheme;
public bool IsValid
{
get
{
// Prevent direct access to any ip
if (IPAddress.TryParse(Host, out _))
{
return false;
}
// Prevent non-http(s) and non-default ports
if ((InnerUri.Scheme != "http" && InnerUri.Scheme != "https") || !InnerUri.IsDefaultPort)
{
return false;
}
// Prevent local hosts (localhost, bobs-pc, etc) and IP addresses
if (!Host.Contains('.') || _ip.IsInternal())
{
return false;
}
return true;
}
}
/// <summary>
/// Represents an ip-validated Uri for use in grabbing an icon.
/// </summary>
/// <param name="uriString"></param>
/// <param name="ip"></param>
public IconUri(Uri uri, IPAddress ip)
{
_ip = ip;
InnerUri = uri.ChangeHost(_ip.ToString());
Host = uri.Host;
}
}

6
src/Icons/Services/IIconFetchingService.cs
View File

@ -1,8 +1,10 @@
using Bit.Icons.Models;
#nullable enable
using Bit.Icons.Models;
namespace Bit.Icons.Services;
public interface IIconFetchingService
{
Task<IconResult> GetIconAsync(string domain);
Task<Icon?> GetIconAsync(string domain);
}

12
src/Icons/Services/IUriService.cs Normal file
View File

@ -0,0 +1,12 @@
#nullable enable
using Bit.Icons.Models;
namespace Bit.Icons.Services;
public interface IUriService
{
bool TryGetUri(string stringUri, out IconUri? iconUri);
bool TryGetUri(Uri uri, out IconUri? iconUri);
bool TryGetRedirect(HttpResponseMessage response, IconUri originalUri, out IconUri? iconUri);
}

458
src/Icons/Services/IconFetchingService.cs
View File

@ -1,449 +1,47 @@
using System.Net;
using System.Text;
#nullable enable
using AngleSharp.Html.Parser;
using Bit.Icons.Extensions;
using Bit.Icons.Models;
namespace Bit.Icons.Services;
public class IconFetchingService : IIconFetchingService
{
private readonly HashSet<string> _iconRels =
new HashSet<string> { "icon", "apple-touch-icon", "shortcut icon" };
private readonly HashSet<string> _blacklistedRels =
new HashSet<string> { "preload", "image_src", "preconnect", "canonical", "alternate", "stylesheet" };
private readonly HashSet<string> _iconExtensions =
new HashSet<string> { ".ico", ".png", ".jpg", ".jpeg" };
private readonly string _pngMediaType = "image/png";
private readonly byte[] _pngHeader = new byte[] { 137, 80, 78, 71 };
private readonly byte[] _webpHeader = Encoding.UTF8.GetBytes("RIFF");
private readonly string _icoMediaType = "image/x-icon";
private readonly string _icoAltMediaType = "image/vnd.microsoft.icon";
private readonly byte[] _icoHeader = new byte[] { 00, 00, 01, 00 };
private readonly string _jpegMediaType = "image/jpeg";
private readonly byte[] _jpegHeader = new byte[] { 255, 216, 255 };
private readonly HashSet<string> _allowedMediaTypes;
private readonly HttpClient _httpClient;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<IIconFetchingService> _logger;
private readonly IHtmlParser _parser;
private readonly IUriService _uriService;
public IconFetchingService(ILogger<IIconFetchingService> logger)
public IconFetchingService(ILogger<IIconFetchingService> logger, IHttpClientFactory httpClientFactory, IHtmlParser parser, IUriService uriService)
{
_logger = logger;
_allowedMediaTypes = new HashSet<string>
_httpClientFactory = httpClientFactory;
_parser = parser;
_uriService = uriService;
}
public async Task<Icon?> GetIconAsync(string domain)
{
var domainIcons = await DomainIcons.FetchAsync(domain, _logger, _httpClientFactory, _parser, _uriService);
var result = domainIcons.Where(result => result != null).FirstOrDefault();
return result ?? await GetFaviconAsync(domain);
}
private async Task<Icon?> GetFaviconAsync(string domain)
{
// Fall back to favicon
var faviconUriBuilder = new UriBuilder
{
_pngMediaType,
_icoMediaType,
_icoAltMediaType,
_jpegMediaType
Scheme = "https",
Host = domain,
Path = "/favicon.ico"
};
_httpClient = new HttpClient(new HttpClientHandler
if (faviconUriBuilder.TryBuild(out var faviconUri))
{
AllowAutoRedirect = false,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
});
_httpClient.Timeout = TimeSpan.FromSeconds(20);
_httpClient.MaxResponseContentBufferSize = 5000000; // 5 MB
}
public async Task<IconResult> GetIconAsync(string domain)
{
if (IPAddress.TryParse(domain, out _))
{
_logger.LogWarning("IP address: {0}.", domain);
return null;
return await new IconLink(faviconUri!).FetchAsync(_logger, _httpClientFactory, _uriService);
}
if (!Uri.TryCreate($"https://{domain}", UriKind.Absolute, out var parsedHttpsUri))
{
_logger.LogWarning("Bad domain: {0}.", domain);
return null;
}
var uri = parsedHttpsUri;
var response = await GetAndFollowAsync(uri, 2);
if ((response == null || !response.IsSuccessStatusCode) &&
Uri.TryCreate($"http://{parsedHttpsUri.Host}", UriKind.Absolute, out var parsedHttpUri))
{
Cleanup(response);
uri = parsedHttpUri;
response = await GetAndFollowAsync(uri, 2);
if (response == null || !response.IsSuccessStatusCode)
{
var dotCount = domain.Count(c => c == '.');
if (dotCount > 1 && DomainName.TryParseBaseDomain(domain, out var baseDomain) &&
Uri.TryCreate($"https://{baseDomain}", UriKind.Absolute, out var parsedBaseUri))
{
Cleanup(response);
uri = parsedBaseUri;
response = await GetAndFollowAsync(uri, 2);
}
else if (dotCount < 2 &&
Uri.TryCreate($"https://www.{parsedHttpsUri.Host}", UriKind.Absolute, out var parsedWwwUri))
{
Cleanup(response);
uri = parsedWwwUri;
response = await GetAndFollowAsync(uri, 2);
}
}
}
if (response?.Content == null || !response.IsSuccessStatusCode)
{
_logger.LogWarning("Couldn't load a website for {0}: {1}.", domain,
response?.StatusCode.ToString() ?? "null");
Cleanup(response);
return null;
}
var parser = new HtmlParser();
using (response)
using (var htmlStream = await response.Content.ReadAsStreamAsync())
using (var document = await parser.ParseDocumentAsync(htmlStream))
{
uri = response.RequestMessage.RequestUri;
if (document.DocumentElement == null)
{
_logger.LogWarning("No DocumentElement for {0}.", domain);
return null;
}
var baseUrl = "/";
var baseUrlNode = document.QuerySelector("head base[href]");
if (baseUrlNode != null)
{
var hrefAttr = baseUrlNode.Attributes["href"];
if (!string.IsNullOrWhiteSpace(hrefAttr?.Value))
{
baseUrl = hrefAttr.Value;
}
baseUrlNode = null;
hrefAttr = null;
}
var icons = new List<IconResult>();
var links = document.QuerySelectorAll("head link[href]");
if (links != null)
{
foreach (var link in links.Take(200))
{
var hrefAttr = link.Attributes["href"];
if (string.IsNullOrWhiteSpace(hrefAttr?.Value))
{
continue;
}
var relAttr = link.Attributes["rel"];
var sizesAttr = link.Attributes["sizes"];
if (relAttr != null && _iconRels.Contains(relAttr.Value.ToLower()))
{
icons.Add(new IconResult(hrefAttr.Value, sizesAttr?.Value));
}
else if (relAttr == null || !_blacklistedRels.Contains(relAttr.Value.ToLower()))
{
try
{
var extension = Path.GetExtension(hrefAttr.Value);
if (_iconExtensions.Contains(extension.ToLower()))
{
icons.Add(new IconResult(hrefAttr.Value, sizesAttr?.Value));
}
}
catch (ArgumentException) { }
}
sizesAttr = null;
relAttr = null;
hrefAttr = null;
}
links = null;
}
var iconResultTasks = new List<Task>();
foreach (var icon in icons.OrderBy(i => i.Priority).Take(10))
{
Uri iconUri = null;
if (icon.Path.StartsWith("//") && Uri.TryCreate($"{GetScheme(uri)}://{icon.Path.Substring(2)}",
UriKind.Absolute, out var slashUri))
{
iconUri = slashUri;
}
else if (Uri.TryCreate(icon.Path, UriKind.Relative, out var relUri))
{
iconUri = ResolveUri($"{GetScheme(uri)}://{uri.Host}", baseUrl, relUri.OriginalString);
}
else if (Uri.TryCreate(icon.Path, UriKind.Absolute, out var absUri))
{
iconUri = absUri;
}
if (iconUri != null)
{
var task = GetIconAsync(iconUri).ContinueWith(async (r) =>
{
var result = await r;
if (result != null)
{
icon.Path = iconUri.ToString();
icon.Icon = result.Icon;
}
});
iconResultTasks.Add(task);
}
}
await Task.WhenAll(iconResultTasks);
if (!icons.Any(i => i.Icon != null))
{
var faviconUri = ResolveUri($"{GetScheme(uri)}://{uri.Host}", "favicon.ico");
var result = await GetIconAsync(faviconUri);
if (result != null)
{
icons.Add(result);
}
else
{
_logger.LogWarning("No favicon.ico found for {0}.", uri.Host);
return null;
}
}
return icons.Where(i => i.Icon != null).OrderBy(i => i.Priority).First();
}
}
private async Task<IconResult> GetIconAsync(Uri uri)
{
using (var response = await GetAndFollowAsync(uri, 2))
{
if (response?.Content?.Headers == null || !response.IsSuccessStatusCode)
{
response?.Content?.Dispose();
return null;
}
var format = response.Content.Headers?.ContentType?.MediaType;
var bytes = await response.Content.ReadAsByteArrayAsync();
response.Content.Dispose();
if (format == null || !_allowedMediaTypes.Contains(format))
{
if (HeaderMatch(bytes, _icoHeader))
{
format = _icoMediaType;
}
else if (HeaderMatch(bytes, _pngHeader) || HeaderMatch(bytes, _webpHeader))
{
format = _pngMediaType;
}
else if (HeaderMatch(bytes, _jpegHeader))
{
format = _jpegMediaType;
}
else
{
return null;
}
}
return new IconResult(uri, bytes, format);
}
}
private async Task<HttpResponseMessage> GetAndFollowAsync(Uri uri, int maxRedirectCount)
{
var response = await GetAsync(uri);
if (response == null)
{
return null;
}
return await FollowRedirectsAsync(response, maxRedirectCount);
}
private async Task<HttpResponseMessage> GetAsync(Uri uri)
{
if (uri == null)
{
return null;
}
// Prevent non-http(s) and non-default ports
if ((uri.Scheme != "http" && uri.Scheme != "https") || !uri.IsDefaultPort)
{
return null;
}
// Prevent local hosts (localhost, bobs-pc, etc) and IP addresses
if (!uri.Host.Contains(".") || IPAddress.TryParse(uri.Host, out _))
{
return null;
}
// Resolve host to make sure it is not an internal/private IP address
try
{
var hostEntry = Dns.GetHostEntry(uri.Host);
if (hostEntry?.AddressList.Any(ip => IsInternal(ip)) ?? true)
{
return null;
}
}
catch
{
return null;
}
using (var message = new HttpRequestMessage())
{
message.RequestUri = uri;
message.Method = HttpMethod.Get;
// Let's add some headers to look like we're coming from a web browser request. Some websites
// will block our request without these.
message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
"(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299");
message.Headers.Add("Accept-Language", "en-US,en;q=0.8");
message.Headers.Add("Cache-Control", "no-cache");
message.Headers.Add("Pragma", "no-cache");
message.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;" +
"q=0.9,image/webp,image/apng,*/*;q=0.8");
try
{
return await _httpClient.SendAsync(message);
}
catch
{
return null;
}
}
}
private async Task<HttpResponseMessage> FollowRedirectsAsync(HttpResponseMessage response,
int maxFollowCount, int followCount = 0)
{
if (response == null || response.IsSuccessStatusCode || followCount > maxFollowCount)
{
return response;
}
if (!(response.StatusCode == HttpStatusCode.Redirect ||
response.StatusCode == HttpStatusCode.MovedPermanently ||
response.StatusCode == HttpStatusCode.RedirectKeepVerb ||
response.StatusCode == HttpStatusCode.SeeOther) ||
response.Headers.Location == null)
{
Cleanup(response);
return null;
}
Uri location = null;
if (response.Headers.Location.IsAbsoluteUri)
{
if (response.Headers.Location.Scheme != "http" && response.Headers.Location.Scheme != "https")
{
if (Uri.TryCreate($"https://{response.Headers.Location.OriginalString}",
UriKind.Absolute, out var newUri))
{
location = newUri;
}
}
else
{
location = response.Headers.Location;
}
}
else
{
var requestUri = response.RequestMessage.RequestUri;
location = ResolveUri($"{GetScheme(requestUri)}://{requestUri.Host}",
response.Headers.Location.OriginalString);
}
Cleanup(response);
var newResponse = await GetAsync(location);
if (newResponse != null)
{
followCount++;
var redirectedResponse = await FollowRedirectsAsync(newResponse, maxFollowCount, followCount);
if (redirectedResponse != null)
{
if (redirectedResponse != newResponse)
{
Cleanup(newResponse);
}
return redirectedResponse;
}
}
return null;
}
private bool HeaderMatch(byte[] imageBytes, byte[] header)
{
return imageBytes.Length >= header.Length && header.SequenceEqual(imageBytes.Take(header.Length));
}
private Uri ResolveUri(string baseUrl, params string[] paths)
{
var url = baseUrl;
foreach (var path in paths)
{
if (Uri.TryCreate(new Uri(url), path, out var r))
{
url = r.ToString();
}
}
return new Uri(url);
}
private void Cleanup(IDisposable obj)
{
obj?.Dispose();
obj = null;
}
private string GetScheme(Uri uri)
{
return uri != null && uri.Scheme == "http" ? "http" : "https";
}
public static bool IsInternal(IPAddress ip)
{
if (IPAddress.IsLoopback(ip))
{
return true;
}
var ipString = ip.ToString();
if (ipString == "::1" || ipString == "::" || ipString.StartsWith("::ffff:"))
{
return true;
}
// IPv6
if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6)
{
return ipString.StartsWith("fc") || ipString.StartsWith("fd") ||
ipString.StartsWith("fe") || ipString.StartsWith("ff");
}
// IPv4
var bytes = ip.GetAddressBytes();
return (bytes[0]) switch
{
0 => true,
10 => true,
127 => true,
169 => bytes[1] == 254, // Cloud environments, such as AWS
172 => bytes[1] < 32 && bytes[1] >= 16,
192 => bytes[1] == 168,
_ => false,
};
}
}

109
src/Icons/Services/UriService.cs Normal file
View File

@ -0,0 +1,109 @@
#nullable enable
using System.Net;
using System.Net.Sockets;
using Bit.Icons.Extensions;
using Bit.Icons.Models;
namespace Bit.Icons.Services;
public class UriService : IUriService
{
public IconUri GetUri(string inputUri)
{
var uri = new Uri(inputUri);
return new IconUri(uri, DetermineIp(uri));
}
public bool TryGetUri(string stringUri, out IconUri? iconUri)
{
if (!Uri.TryCreate(stringUri, UriKind.Absolute, out var uri))
{
iconUri = null;
return false;
}
return TryGetUri(uri, out iconUri);
}
public IconUri GetUri(Uri uri)
{
return new IconUri(uri, DetermineIp(uri));
}
public bool TryGetUri(Uri uri, out IconUri? iconUri)
{
try
{
iconUri = GetUri(uri);
return true;
}
catch (Exception)
{
iconUri = null;
return false;
}
}
public IconUri GetRedirect(HttpResponseMessage response, IconUri originalUri)
{
if (response.Headers.Location == null)
{
throw new Exception("No redirect location found.");
}
var redirectUri = DetermineRedirectUri(response.Headers.Location, originalUri);
return new IconUri(redirectUri, DetermineIp(redirectUri));
}
public bool TryGetRedirect(HttpResponseMessage response, IconUri originalUri, out IconUri? iconUri)
{
try
{
iconUri = GetRedirect(response, originalUri);
return true;
}
catch (Exception)
{
iconUri = null;
return false;
}
}
private static Uri DetermineRedirectUri(Uri responseUri, IconUri originalIconUri)
{
if (responseUri.IsAbsoluteUri)
{
if (!responseUri.IsHypertext())
{
return responseUri.ChangeScheme("https");
}
return responseUri;
}
else
{
return new UriBuilder
{
Scheme = originalIconUri.Scheme,
Host = originalIconUri.Host,
Path = responseUri.ToString()
}.Uri;
}
}
private static IPAddress DetermineIp(Uri uri)
{
if (IPAddress.TryParse(uri.Host, out var ip))
{
return ip;
}
var hostEntry = Dns.GetHostEntry(uri.Host);
ip = hostEntry.AddressList.FirstOrDefault(ip => ip.AddressFamily == AddressFamily.InterNetwork || ip.IsIPv4MappedToIPv6)?.MapToIPv4();
if (ip == null)
{
throw new Exception($"Unable to determine IP for {uri.Host}");
}
return ip;
}
}

11
src/Icons/Startup.cs
View File

@ -1,7 +1,7 @@
using System.Globalization;
using Bit.Core.Settings;
using Bit.Core.Utilities;
using Bit.Icons.Services;
using Bit.Icons.Extensions;
using Bit.SharedWeb.Utilities;
using Microsoft.Net.Http.Headers;
@ -30,6 +30,12 @@ public class Startup
ConfigurationBinder.Bind(Configuration.GetSection("IconsSettings"), iconsSettings);
services.AddSingleton(s => iconsSettings);
// Http client
services.ConfigureHttpClients();
// Add HtmlParser
services.AddHtmlParsing();
// Cache
services.AddMemoryCache(options =>
{
@ -37,8 +43,7 @@ public class Startup
});
// Services
services.AddSingleton<IDomainMappingService, DomainMappingService>();
services.AddSingleton<IIconFetchingService, IconFetchingService>();
services.AddServices();
// Mvc
services.AddMvc();

42
src/Icons/Util/IPAddressExtension.cs Normal file
View File

@ -0,0 +1,42 @@
#nullable enable
using System.Net;
namespace Bit.Icons.Extensions;
public static class IPAddressExtension
{
public static bool IsInternal(this IPAddress ip)
{
if (IPAddress.IsLoopback(ip))
{
return true;
}
var ipString = ip.ToString();
if (ipString == "::1" || ipString == "::" || ipString.StartsWith("::ffff:"))
{
return true;
}
// IPv6
if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6)
{
return ipString.StartsWith("fc") || ipString.StartsWith("fd") ||
ipString.StartsWith("fe") || ipString.StartsWith("ff");
}
// IPv4
var bytes = ip.GetAddressBytes();
return (bytes[0]) switch
{
0 => true,
10 => true,
127 => true,
169 => bytes[1] == 254, // Cloud environments, such as AWS
172 => bytes[1] < 32 && bytes[1] >= 16,
192 => bytes[1] == 168,
_ => false,
};
}
}

44
src/Icons/Util/ServiceCollectionExtension.cs Normal file
View File

@ -0,0 +1,44 @@
# nullable enable
using System.Net;
using AngleSharp.Html.Parser;
using Bit.Icons.Services;
namespace Bit.Icons.Extensions;
public static class ServiceCollectionExtension
{
public static void ConfigureHttpClients(this IServiceCollection services)
{
services.AddHttpClient("Icons", client =>
{
client.Timeout = TimeSpan.FromSeconds(20);
client.MaxResponseContentBufferSize = 5000000; // 5 MB
// Let's add some headers to look like we're coming from a web browser request. Some websites
// will block our request without these.
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
"(KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36");
client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.8");
client.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
client.DefaultRequestHeaders.Add("Pragma", "no-cache");
client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;" +
"q=0.9,image/webp,image/apng,*/*;q=0.8");
}).ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler
{
AllowAutoRedirect = false,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
});
}
public static void AddHtmlParsing(this IServiceCollection services)
{
services.AddSingleton<IHtmlParser, HtmlParser>();
}
public static void AddServices(this IServiceCollection services)
{
services.AddSingleton<IUriService, UriService>();
services.AddSingleton<IDomainMappingService, DomainMappingService>();
services.AddSingleton<IIconFetchingService, IconFetchingService>();
}
}

20
src/Icons/Util/UriBuilderExtension.cs Normal file
View File

@ -0,0 +1,20 @@
#nullable enable
namespace Bit.Icons.Extensions;
public static class UriBuilderExtension
{
public static bool TryBuild(this UriBuilder builder, out Uri? uri)
{
try
{
uri = builder.Uri;
return true;
}
catch (UriFormatException)
{
uri = null;
return false;
}
}
}

41
src/Icons/Util/UriExtension.cs Normal file
View File

@ -0,0 +1,41 @@

#nullable enable
namespace Bit.Icons.Extensions;
public static class UriExtension
{
public static bool IsHypertext(this Uri uri)
{
return uri.Scheme == "http" || uri.Scheme == "https";
}
public static Uri ChangeScheme(this Uri uri, string scheme)
{
return new UriBuilder(scheme, uri.Host) { Path = uri.PathAndQuery }.Uri;
}
public static Uri ChangeHost(this Uri uri, string host)
{
return new UriBuilder(uri) { Host = host }.Uri;
}
public static Uri ConcatPath(this Uri uri, params string[] paths)
=> uri.ConcatPath(paths.AsEnumerable());
public static Uri ConcatPath(this Uri uri, IEnumerable<string> paths)
{
if (!paths.Any())
{
return uri;
}
if (Uri.TryCreate(uri, paths.First(), out var newUri))
{
return newUri.ConcatPath(paths.Skip(1));
}
else
{
return uri;
}
}
}

59
test/Common/Helpers/HtmlBuilder.cs Normal file
View File

@ -0,0 +1,59 @@
using System.Text;
namespace Bit.Test.Common.Helpers;
public class HtmlBuilder
{
private string _topLevelNode;
private readonly StringBuilder _builder = new();
public HtmlBuilder(string topLevelNode = "html")
{
_topLevelNode = CoerceTopLevelNode(topLevelNode);
}
public HtmlBuilder Append(string node)
{
_builder.Append(node);
return this;
}
public HtmlBuilder Append(HtmlBuilder builder)
{
_builder.Append(builder.ToString());
return this;
}
public HtmlBuilder WithAttribute(string name, string value)
{
_topLevelNode = $"{_topLevelNode} {name}=\"{value}\"";
return this;
}
public override string ToString()
{
_builder.Insert(0, $"<{_topLevelNode}>");
_builder.Append($"</{_topLevelNode}>");
return _builder.ToString();
}
private static string CoerceTopLevelNode(string topLevelNode)
{
var result = topLevelNode;
if (topLevelNode.StartsWith("<"))
{
result = topLevelNode[1..];
}
if (topLevelNode.EndsWith(">"))
{
result = result[..^1];
}
if (topLevelNode.IndexOf(">") != -1)
{
throw new ArgumentException("Top level nodes cannot contain '>' characters.");
}
return result;
}
}

104
test/Common/MockedHttpClient/HttpRequestMatcher.cs Normal file
View File

@ -0,0 +1,104 @@
#nullable enable
using System.Net;
namespace Bit.Test.Common.MockedHttpClient;
public class HttpRequestMatcher : IHttpRequestMatcher
{
private readonly Func<HttpRequestMessage, bool> _matcher;
private HttpRequestMatcher? _childMatcher;
private MockedHttpResponse _mockedResponse = new(HttpStatusCode.OK);
private bool _responseSpecified = false;
public int NumberOfMatches { get; private set; }
/// <summary>
/// Returns whether or not the provided request can be handled by this matcher chain.
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public bool Matches(HttpRequestMessage request) => _matcher(request) && (_childMatcher == null || _childMatcher.Matches(request));
public HttpRequestMatcher(HttpMethod method)
{
_matcher = request => request.Method == method;
}
public HttpRequestMatcher(string uri)
{
_matcher = request => request.RequestUri == new Uri(uri);
}
public HttpRequestMatcher(Uri uri)
{
_matcher = request => request.RequestUri == uri;
}
public HttpRequestMatcher(HttpMethod method, string uri)
{
_matcher = request => request.Method == method && request.RequestUri == new Uri(uri);
}
public HttpRequestMatcher(Func<HttpRequestMessage, bool> matcher)
{
_matcher = matcher;
}
public HttpRequestMatcher WithHeader(string name, string value)
{
return AddChild(request => request.Headers.TryGetValues(name, out var values) && values.Contains(value));
}
public HttpRequestMatcher WithQueryParameters(Dictionary<string, string> requiredQueryParameters) =>
WithQueryParameters(requiredQueryParameters.Select(x => $"{x.Key}={x.Value}").ToArray());
public HttpRequestMatcher WithQueryParameters(string name, string value) =>
WithQueryParameters($"{name}={value}");
public HttpRequestMatcher WithQueryParameters(params string[] queryKeyValues)
{
bool matcher(HttpRequestMessage request)
{
var query = request.RequestUri?.Query;
if (query == null)
{
return false;
}
return queryKeyValues.All(queryKeyValue => query.Contains(queryKeyValue));
}
return AddChild(matcher);
}
/// <summary>
/// Configure how this matcher should respond to matching HttpRequestMessages.
/// Note, after specifying a response, you can no longer further specify match criteria.
/// </summary>
/// <param name="statusCode"></param>
/// <returns></returns>
public MockedHttpResponse RespondWith(HttpStatusCode statusCode)
{
_responseSpecified = true;
_mockedResponse = new MockedHttpResponse(statusCode);
return _mockedResponse;
}
/// <summary>
/// Called to produce an HttpResponseMessage for the given request. This is probably something you want to leave alone
/// </summary>
/// <param name="request"></param>
public async Task<HttpResponseMessage> RespondToAsync(HttpRequestMessage request)
{
NumberOfMatches++;
return await (_childMatcher == null ? _mockedResponse.RespondToAsync(request) : _childMatcher.RespondToAsync(request));
}
private HttpRequestMatcher AddChild(Func<HttpRequestMessage, bool> matcher)
{
if (_responseSpecified)
{
throw new Exception("Cannot continue to configure a matcher after a response has been specified");
}
_childMatcher = new HttpRequestMatcher(matcher);
return _childMatcher;
}
}

84
test/Common/MockedHttpClient/HttpResponseBuilder.cs Normal file
View File

@ -0,0 +1,84 @@
using System.Net;
namespace Bit.Test.Common.MockedHttpClient;
public class HttpResponseBuilder : IDisposable
{
private bool _disposedValue;
public HttpStatusCode StatusCode { get; set; }
public IEnumerable<KeyValuePair<string, string>> Headers { get; set; } = new List<KeyValuePair<string, string>>();
public IEnumerable<string> HeadersToRemove { get; set; } = new List<string>();
public HttpContent Content { get; set; }
public async Task<HttpResponseMessage> ToHttpResponseAsync()
{
var copiedContentStream = new MemoryStream();
await Content.CopyToAsync(copiedContentStream); // This is important, otherwise the content stream will be disposed when the response is disposed.
copiedContentStream.Seek(0, SeekOrigin.Begin);
var message = new HttpResponseMessage(StatusCode)
{
Content = new StreamContent(copiedContentStream),
};
foreach (var header in Headers)
{
message.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
return message;
}
public HttpResponseBuilder WithStatusCode(HttpStatusCode statusCode)
{
return new()
{
StatusCode = statusCode,
Headers = Headers,
HeadersToRemove = HeadersToRemove,
Content = Content,
};
}
public HttpResponseBuilder WithHeader(string name, string value)
{
return new()
{
StatusCode = StatusCode,
Headers = Headers.Append(new KeyValuePair<string, string>(name, value)),
HeadersToRemove = HeadersToRemove,
Content = Content,
};
}
public HttpResponseBuilder WithContent(HttpContent content)
{
return new()
{
StatusCode = StatusCode,
Headers = Headers,
HeadersToRemove = HeadersToRemove,
Content = content,
};
}
protected virtual void Dispose(bool disposing)
{
if (!_disposedValue)
{
if (disposing)
{
Content?.Dispose();
}
_disposedValue = true;
}
}
public void Dispose()
{
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}

10
test/Common/MockedHttpClient/IHttpRequestMatcher.cs Normal file
View File

@ -0,0 +1,10 @@
#nullable enable
namespace Bit.Test.Common.MockedHttpClient;
public interface IHttpRequestMatcher
{
int NumberOfMatches { get; }
bool Matches(HttpRequestMessage request);
Task<HttpResponseMessage> RespondToAsync(HttpRequestMessage request);
}

7
test/Common/MockedHttpClient/IMockedHttpResponse.cs Normal file
View File

@ -0,0 +1,7 @@
namespace Bit.Test.Common.MockedHttpClient;
public interface IMockedHttpResponse
{
int NumberOfResponses { get; }
Task<HttpResponseMessage> RespondToAsync(HttpRequestMessage request);
}

113
test/Common/MockedHttpClient/MockedHttpMessageHandler.cs Normal file
View File

@ -0,0 +1,113 @@
#nullable enable
using System.Net;
namespace Bit.Test.Common.MockedHttpClient;
public class MockedHttpMessageHandler : HttpMessageHandler
{
private readonly List<IHttpRequestMatcher> _matchers = new();
/// <summary>
/// The fallback handler to use when the request does not match any of the provided matchers.
/// </summary>
/// <returns>A Matcher that responds with 404 Not Found</returns>
public MockedHttpResponse Fallback { get; set; } = new(HttpStatusCode.NotFound);
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var matcher = _matchers.FirstOrDefault(x => x.Matches(request));
if (matcher == null)
{
return await Fallback.RespondToAsync(request);
}
return await matcher.RespondToAsync(request);
}
/// <summary>
/// Instantiates a new HttpRequestMessage matcher that will handle requests in fitting with the returned matcher. Configuration can be chained.
/// </summary>
/// <param name="requestMatcher"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public T When<T>(T requestMatcher) where T : IHttpRequestMatcher
{
_matchers.Add(requestMatcher);
return requestMatcher;
}
/// <summary>
/// Instantiates a new HttpRequestMessage matcher that will handle requests in fitting with the returned matcher. Configuration can be chained.
/// </summary>
/// <param name="requestMatcher"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public HttpRequestMatcher When(string uri)
{
var matcher = new HttpRequestMatcher(uri);
_matchers.Add(matcher);
return matcher;
}
/// <summary>
/// Instantiates a new HttpRequestMessage matcher that will handle requests in fitting with the returned matcher. Configuration can be chained.
/// </summary>
/// <param name="requestMatcher"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public HttpRequestMatcher When(Uri uri)
{
var matcher = new HttpRequestMatcher(uri);
_matchers.Add(matcher);
return matcher;
}
/// <summary>
/// Instantiates a new HttpRequestMessage matcher that will handle requests in fitting with the returned matcher. Configuration can be chained.
/// </summary>
/// <param name="requestMatcher"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public HttpRequestMatcher When(HttpMethod method)
{
var matcher = new HttpRequestMatcher(method);
_matchers.Add(matcher);
return matcher;
}
/// <summary>
/// Instantiates a new HttpRequestMessage matcher that will handle requests in fitting with the returned matcher. Configuration can be chained.
/// </summary>
/// <param name="requestMatcher"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public HttpRequestMatcher When(HttpMethod method, string uri)
{
var matcher = new HttpRequestMatcher(method, uri);
_matchers.Add(matcher);
return matcher;
}
/// <summary>
/// Instantiates a new HttpRequestMessage matcher that will handle requests in fitting with the returned matcher. Configuration can be chained.
/// </summary>
/// <param name="requestMatcher"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public HttpRequestMatcher When(Func<HttpRequestMessage, bool> matcher)
{
var requestMatcher = new HttpRequestMatcher(matcher);
_matchers.Add(requestMatcher);
return requestMatcher;
}
/// <summary>
/// Converts the MockedHttpMessageHandler to a HttpClient that can be used in your tests after setup.
/// </summary>
/// <returns></returns>
public HttpClient ToHttpClient()
{
return new HttpClient(this);
}
}

68
test/Common/MockedHttpClient/MockedHttpResponse.cs Normal file
View File

@ -0,0 +1,68 @@
using System.Net;
using System.Net.Http.Headers;
using System.Text;
namespace Bit.Test.Common.MockedHttpClient;
public class MockedHttpResponse : IMockedHttpResponse
{
private MockedHttpResponse _childResponse;
private readonly Func<HttpRequestMessage, HttpResponseBuilder, HttpResponseBuilder> _responder;
public int NumberOfResponses { get; private set; }
public MockedHttpResponse(HttpStatusCode statusCode)
{
_responder = (_, builder) => builder.WithStatusCode(statusCode);
}
private MockedHttpResponse(Func<HttpRequestMessage, HttpResponseBuilder, HttpResponseBuilder> responder)
{
_responder = responder;
}
public MockedHttpResponse WithStatusCode(HttpStatusCode statusCode)
{
return AddChild((_, builder) => builder.WithStatusCode(statusCode));
}
public MockedHttpResponse WithHeader(string name, string value)
{
return AddChild((_, builder) => builder.WithHeader(name, value));
}
public MockedHttpResponse WithHeaders(params KeyValuePair<string, string>[] headers)
{
return AddChild((_, builder) => headers.Aggregate(builder, (b, header) => b.WithHeader(header.Key, header.Value)));
}
public MockedHttpResponse WithContent(string mediaType, string content)
{
return WithContent(new StringContent(content, Encoding.UTF8, mediaType));
}
public MockedHttpResponse WithContent(string mediaType, byte[] content)
{
return WithContent(new ByteArrayContent(content) { Headers = { ContentType = new MediaTypeHeaderValue(mediaType) } });
}
public MockedHttpResponse WithContent(HttpContent content)
{
return AddChild((_, builder) => builder.WithContent(content));
}
public async Task<HttpResponseMessage> RespondToAsync(HttpRequestMessage request)
{
return await RespondToAsync(request, new HttpResponseBuilder());
}
private async Task<HttpResponseMessage> RespondToAsync(HttpRequestMessage request, HttpResponseBuilder currentBuilder)
{
NumberOfResponses++;
var nextBuilder = _responder(request, currentBuilder);
return await (_childResponse == null ? nextBuilder.ToHttpResponseAsync() : _childResponse.RespondToAsync(request, nextBuilder));
}
private MockedHttpResponse AddChild(Func<HttpRequestMessage, HttpResponseBuilder, HttpResponseBuilder> responder)
{
_childResponse = new MockedHttpResponse(responder);
return _childResponse;
}
}

1
test/Icons.Test/Icons.Test.csproj
View File

@ -20,6 +20,7 @@
<ItemGroup>
<ProjectReference Include="..\..\src\Icons\Icons.csproj" />
<ProjectReference Include="..\Common\Common.csproj" />
</ItemGroup>
</Project>

38
test/Icons.Test/Models/IconHttpRequestTests.cs Normal file
View File

@ -0,0 +1,38 @@
using System.Net;
using Bit.Icons.Models;
using Bit.Icons.Services;
using Bit.Test.Common.MockedHttpClient;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Net.Http.Headers;
using NSubstitute;
using Xunit;
namespace Bit.Icons.Test.Models;
public class IconHttpRequestTests
{
[Fact]
public async Task FetchAsync_FollowsTwoRedirectsAsync()
{
var handler = new MockedHttpMessageHandler();
var request = handler
.Fallback
.WithStatusCode(HttpStatusCode.Redirect)
.WithContent("text/html", "<html><head><title>Redirect 2</title></head><body><a href=\"https://icon.test\">Redirect 3</a></body></html>")
.WithHeader(HeaderNames.Location, "https://icon.test");
var clientFactory = Substitute.For<IHttpClientFactory>();
clientFactory.CreateClient("Icons").Returns(handler.ToHttpClient());
var uriService = Substitute.For<IUriService>();
uriService.TryGetUri(Arg.Any<Uri>(), out Arg.Any<IconUri>()).Returns(x =>
{
x[1] = new IconUri(new Uri("https://icon.test"), IPAddress.Parse("192.0.2.1"));
return true;
});
var result = await IconHttpRequest.FetchAsync(new Uri("https://icon.test"), NullLogger<IIconFetchingService>.Instance, clientFactory, uriService);
Assert.Equal(3, request.NumberOfResponses); // Initial + 2 redirects
}
}

101
test/Icons.Test/Models/IconHttpResponseTests.cs Normal file
View File

@ -0,0 +1,101 @@
using System.Net;
using AngleSharp.Html.Parser;
using Bit.Icons.Models;
using Bit.Icons.Services;
using Bit.Test.Common.Helpers;
using Bit.Test.Common.MockedHttpClient;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using Xunit;
namespace Bit.Icons.Test.Models;
public class IconHttpResponseTests
{
private readonly IUriService _mockedUriService;
private static readonly IHtmlParser _parser = new HtmlParser();
public IconHttpResponseTests()
{
_mockedUriService = Substitute.For<IUriService>();
_mockedUriService.TryGetUri(Arg.Any<Uri>(), out Arg.Any<IconUri>()).Returns(x =>
{
x[1] = new IconUri(new Uri("https://icon.test"), IPAddress.Parse("192.0.2.1"));
return true;
});
}
[Fact]
public async Task RetrieveIconsAsync_Processes200LinksAsync()
{
var htmlBuilder = new HtmlBuilder();
var headBuilder = new HtmlBuilder("head");
for (var i = 0; i < 200; i++)
{
headBuilder.Append(UnusableLinkNode());
}
headBuilder.Append(UsableLinkNode());
htmlBuilder.Append(headBuilder);
var response = GetHttpResponseMessage(htmlBuilder.ToString());
var sut = CurriedIconHttpResponse()(response);
var result = await sut.RetrieveIconsAsync(new Uri("https://icon.test"), "icon.test", _parser);
Assert.Empty(result);
}
[Fact]
public async Task RetrieveIconsAsync_Processes10IconsAsync()
{
var htmlBuilder = new HtmlBuilder();
var headBuilder = new HtmlBuilder("head");
for (var i = 0; i < 11; i++)
{
headBuilder.Append(UsableLinkNode());
}
htmlBuilder.Append(headBuilder);
var response = GetHttpResponseMessage(htmlBuilder.ToString());
var sut = CurriedIconHttpResponse()(response);
var result = await sut.RetrieveIconsAsync(new Uri("https://icon.test"), "icon.test", _parser);
Assert.Equal(10, result.Count());
}
private static string UsableLinkNode()
{
return "<link rel=\"icon\" href=\"https://icon.test/favicon.ico\" />";
}
private static string UnusableLinkNode()
{
// Empty href links are not usable
return "<link rel=\"icon\" href=\"\" />";
}
private static HttpResponseMessage GetHttpResponseMessage(string content)
{
return new HttpResponseMessage(HttpStatusCode.OK)
{
RequestMessage = new HttpRequestMessage(HttpMethod.Get, "https://icon.test"),
Content = new StringContent(content)
};
}
private Func<HttpResponseMessage, IconHttpResponse> CurriedIconHttpResponse()
{
return (HttpResponseMessage response) => new IconHttpResponse(response, NullLogger<IIconFetchingService>.Instance, UsableIconHttpClientFactory(), _mockedUriService);
}
private static IHttpClientFactory UsableIconHttpClientFactory()
{
var substitute = Substitute.For<IHttpClientFactory>();
var handler = new MockedHttpMessageHandler();
handler.Fallback
.WithStatusCode(HttpStatusCode.OK)
.WithContent("image/png", new byte[] { 137, 80, 78, 71 });
substitute.CreateClient("Icons").Returns(handler.ToHttpClient());
return substitute;
}
}

85
test/Icons.Test/Models/IconLinkTests.cs Normal file
View File

@ -0,0 +1,85 @@
using System.Net;
using AngleSharp.Dom;
using Bit.Icons.Models;
using Bit.Icons.Services;
using Microsoft.Extensions.Logging;
using NSubstitute;
using Xunit;
namespace Bit.Icons.Test.Models;
public class IconLinkTests
{
private readonly IElement _element;
private readonly Uri _uri = new("https://icon.test");
private readonly ILogger<IIconFetchingService> _logger = Substitute.For<ILogger<IIconFetchingService>>();
private readonly IHttpClientFactory _httpClientFactory;
private readonly IUriService _uriService;
private readonly string _baseUrlPath = "/";
public IconLinkTests()
{
_element = Substitute.For<IElement>();
_httpClientFactory = Substitute.For<IHttpClientFactory>();
_uriService = Substitute.For<IUriService>();
_uriService.TryGetUri(Arg.Any<Uri>(), out Arg.Any<IconUri>()).Returns(x =>
{
x[1] = new IconUri(new Uri("https://icon.test"), IPAddress.Parse("192.0.2.1"));
return true;
});
}
[Fact]
public void WithNoHref_IsNotUsable()
{
_element.GetAttribute("href").Returns(string.Empty);
var result = new IconLink(_element, _uri, _baseUrlPath).IsUsable();
Assert.False(result);
}
[Theory]
[InlineData(null, false)]
[InlineData("", false)]
[InlineData(" ", false)]
[InlineData("unusable", false)]
[InlineData("ico", true)]
public void WithNoRel_IsUsable(string extension, bool expectedResult)
{
SetAttributeValue("href", $"/favicon.{extension}");
var result = new IconLink(_element, _uri, _baseUrlPath).IsUsable();
Assert.Equal(expectedResult, result);
}
[Theory]
[InlineData("icon", true)]
[InlineData("stylesheet", false)]
public void WithRel_IsUsable(string rel, bool expectedResult)
{
SetAttributeValue("href", "/favicon.ico");
SetAttributeValue("rel", rel);
var result = new IconLink(_element, _uri, _baseUrlPath).IsUsable();
Assert.Equal(expectedResult, result);
}
[Fact]
public void FetchAsync_Unvalidated_ReturnsNull()
{
var result = new IconLink(_element, _uri, _baseUrlPath).FetchAsync(_logger, _httpClientFactory, _uriService);
Assert.Null(result.Result);
}
private void SetAttributeValue(string attribute, string value)
{
var attr = Substitute.For<IAttr>();
attr.Value.Returns(value);
_element.Attributes[attribute].Returns(attr);
}
}

22
test/Icons.Test/Models/IconUriTests.cs Normal file
View File

@ -0,0 +1,22 @@
using System.Net;
using Bit.Icons.Models;
using Xunit;
namespace Bit.Icons.Test.Models;
public class IconUriTests
{
[Theory]
[InlineData("https://icon.test", "1.1.1.1", true)]
[InlineData("https://icon.test:4443", "1.1.1.1", false)] // Non standard port
[InlineData("http://test", "1.1.1.1", false)] // top level domain
[InlineData("https://icon.test", "127.0.0.1", false)] // IP is internal
[InlineData("https://icon.test", "::1", false)] // IP is internal
[InlineData("https://1.1.1.1", "::1", false)] // host is IP
public void IsValid(string uri, string ip, bool expectedResult)
{
var result = new IconUri(new Uri(uri), IPAddress.Parse(ip)).IsValid;
Assert.Equal(expectedResult, result);
}
}

29
test/Icons.Test/Services/IconFetchingServiceTests.cs
View File

@ -1,25 +1,25 @@
using Bit.Icons.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Xunit;
namespace Bit.Icons.Test.Services;
public class IconFetchingServiceTests
public class IconFetchingServiceTests : ServiceTestBase<IconFetchingService>
{
[Theory]
[InlineData("www.twitter.com")] // https site
[InlineData("www.google.com")] // https site
[InlineData("neverssl.com")] // http site
[InlineData("ameritrade.com")]
[InlineData("neopets.com")] // uses favicon.ico
[InlineData("hopin.com")] // uses svg+xml format
[InlineData("ameritrade.com")] // redirects to tdameritrace.com
[InlineData("icloud.com")]
[InlineData("bofa.com", Skip = "Broken in pipeline for .NET 6. Tracking link: https://bitwarden.atlassian.net/browse/PS-982")]
public async Task GetIconAsync_Success(string domain)
{
var sut = new IconFetchingService(GetLogger());
var sut = BuildSut();
var result = await sut.GetIconAsync(domain);
Assert.NotNull(result);
Assert.NotNull(result.Icon);
}
[Theory]
@ -28,23 +28,12 @@ public class IconFetchingServiceTests
[InlineData("localhost")]
public async Task GetIconAsync_ReturnsNull(string domain)
{
var sut = new IconFetchingService(GetLogger());
var sut = BuildSut();
var result = await sut.GetIconAsync(domain);
Assert.Null(result);
}
private static ILogger<IconFetchingService> GetLogger()
{
var services = new ServiceCollection();
services.AddLogging(b =>
{
b.ClearProviders();
b.AddDebug();
});
var provider = services.BuildServiceProvider();
return provider.GetRequiredService<ILogger<IconFetchingService>>();
}
private IconFetchingService BuildSut() =>
GetService<IconFetchingService>();
}

41
test/Icons.Test/Services/ServiceTestBase.cs Normal file
View File

@ -0,0 +1,41 @@
using Bit.Icons.Extensions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace Bit.Icons.Test.Services;
public class ServiceTestBase
{
internal ServiceCollection _services = new();
internal ServiceProvider _provider;
public ServiceTestBase()
{
_services = new ServiceCollection();
_services.AddLogging(b =>
{
b.ClearProviders();
b.AddDebug();
});
_services.ConfigureHttpClients();
_services.AddHtmlParsing();
_services.AddServices();
_provider = _services.BuildServiceProvider();
}
public T GetService<T>() =>
_provider.GetRequiredService<T>();
}
public class ServiceTestBase<TSut> : ServiceTestBase where TSut : class
{
public ServiceTestBase() : base()
{
_services.AddTransient<TSut>();
_provider = _services.BuildServiceProvider();
}
public TSut Sut => GetService<TSut>();
}

81
test/Icons.Test/packages.lock.json
View File

@ -73,6 +73,33 @@
"StackExchange.Redis": "2.5.43"
}
},
"AutoFixture": {
"type": "Transitive",
"resolved": "4.17.0",
"contentHash": "efMRCG3Epc4QDELwdmQGf6/caQUleRXPRCnLAq5gLMpTuOTcOQWV12vEJ8qo678Rj97/TjjxHYu/34rGkXdVAA==",
"dependencies": {
"Fare": "[2.1.1, 3.0.0)",
"System.ComponentModel.Annotations": "4.3.0"
}
},
"AutoFixture.AutoNSubstitute": {
"type": "Transitive",
"resolved": "4.17.0",
"contentHash": "iWsRiDQ7T8s6F4mvYbSvPTq0GDtxJD6D+E1Fu9gVbHUvJiNikC1yIDNTH+3tQF7RK864HH/3R8ETj9m2X8UXvg==",
"dependencies": {
"AutoFixture": "4.17.0",
"NSubstitute": "[2.0.3, 5.0.0)"
}
},
"AutoFixture.Xunit2": {
"type": "Transitive",
"resolved": "4.17.0",
"contentHash": "lrURL/LhJLPkn2tSPUEW8Wscr5LoV2Mr8A+ikn5gwkofex3o7qWUsBswlLw+KCA7EOpeqwZOldp3k91zDF+48Q==",
"dependencies": {
"AutoFixture": "4.17.0",
"xunit.extensibility.core": "[2.2.0, 3.0.0)"
}
},
"AutoMapper": {
"type": "Transitive",
"resolved": "12.0.1",
@ -246,6 +273,14 @@
"Microsoft.Win32.Registry": "5.0.0"
}
},
"Fare": {
"type": "Transitive",
"resolved": "2.1.1",
"contentHash": "HaI8puqA66YU7/9cK4Sgbs1taUTP1Ssa4QT2PIzqJ7GvAbN1QgkjbRsjH+FSbMh1MJdvS0CIwQNLtFT+KF6KpA==",
"dependencies": {
"NETStandard.Library": "1.6.1"
}
},
"Fido2": {
"type": "Transitive",
"resolved": "3.0.1",
@ -326,6 +361,15 @@
"IdentityModel": "4.4.0"
}
},
"Kralizek.AutoFixture.Extensions.MockHttp": {
"type": "Transitive",
"resolved": "1.2.0",
"contentHash": "6zmks7/5mVczazv910N7V2EdiU6B+rY61lwdgVO0o2iZuTI6KI3T+Hgkrjv0eGOKYucq2OMC+gnAc5Ej2ajoTQ==",
"dependencies": {
"AutoFixture": "4.11.0",
"RichardSzalay.MockHttp": "6.0.0"
}
},
"LaunchDarkly.Cache": {
"type": "Transitive",
"resolved": "1.0.2",
@ -1148,6 +1192,11 @@
"System.Diagnostics.DiagnosticSource": "4.7.1"
}
},
"RichardSzalay.MockHttp": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "bStGNqIX/MGYtML7K3EzdsE/k5HGVAcg7XgN23TQXGXqxNC9fvYFR94fA0sGM5hAT36R+BBGet6ZDQxXL/IPxg=="
},
"runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl": {
"type": "Transitive",
"resolved": "4.3.2",
@ -1568,6 +1617,24 @@
"System.Runtime": "4.3.0"
}
},
"System.ComponentModel.Annotations": {
"type": "Transitive",
"resolved": "4.3.0",
"contentHash": "SY2RLItHt43rd8J9D8M8e8NM4m+9WLN2uUd9G0n1I4hj/7w+v3pzK6ZBjexlG1/2xvLKQsqir3UGVSyBTXMLWA==",
"dependencies": {
"System.Collections": "4.3.0",
"System.ComponentModel": "4.3.0",
"System.Globalization": "4.3.0",
"System.Linq": "4.3.0",
"System.Reflection": "4.3.0",
"System.Reflection.Extensions": "4.3.0",
"System.Resources.ResourceManager": "4.3.0",
"System.Runtime": "4.3.0",
"System.Runtime.Extensions": "4.3.0",
"System.Text.RegularExpressions": "4.3.0",
"System.Threading": "4.3.0"
}
},
"System.ComponentModel.Primitives": {
"type": "Transitive",
"resolved": "4.3.0",
@ -2770,6 +2837,18 @@
"NETStandard.Library": "1.6.1"
}
},
"common": {
"type": "Project",
"dependencies": {
"AutoFixture.AutoNSubstitute": "[4.17.0, )",
"AutoFixture.Xunit2": "[4.17.0, )",
"Core": "[2023.5.1, )",
"Kralizek.AutoFixture.Extensions.MockHttp": "[1.2.0, )",
"Microsoft.NET.Test.Sdk": "[17.1.0, )",
"NSubstitute": "[4.3.0, )",
"xunit": "[2.4.1, )"
}
},
"core": {
"type": "Project",
"dependencies": {
@ -2850,4 +2929,4 @@
}
}
}
}
}