Upstream/bitwarden-server
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-02-02 23:41:21 +01:00
Code Issues Projects Releases Wiki Activity

new icon fetching service. remove besticon dep.

Browse Source
This commit is contained in:
Kyle Spearrin 2018-05-22 12:49:34 -04:00
parent 60bb4d466c
commit 0e4ffc7d7f
11 changed files with 357 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
src/Icons/Controllers/IconsController.cs
View File

@ -1,8 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using Bit.Icons.Models;
using Bit.Icons.Services;
@ -14,37 +10,20 @@ namespace Bit.Icons.Controllers
[Route("")]
public class IconsController : Controller
{
private static readonly HttpClient _httpClient = new HttpClient(new HttpClientHandler
{
AllowAutoRedirect = false,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
});
private static string _pngMediaType = "image/png";
private static byte[] _pngHeader = new byte[] { 137, 80, 78, 71 };
private static string _icoMediaType = "image/x-icon";
private static string _icoAltMediaType = "image/vnd.microsoft.icon";
private static byte[] _icoHeader = new byte[] { 00, 00, 01, 00 };
private static string _jpegMediaType = "image/jpeg";
private static byte[] _jpegHeader = new byte[] { 255, 216, 255 };
private static string _octetMediaType = "application/octet-stream";
private static readonly HashSet<string> _allowedMediaTypes = new HashSet<string>{
_pngMediaType,
_icoMediaType,
_icoAltMediaType,
_jpegMediaType,
_octetMediaType
};
private readonly IMemoryCache _memoryCache;
private readonly IDomainMappingService _domainMappingService;
private readonly IIconFetchingService _iconFetchingService;
private readonly IconsSettings _iconsSettings;
public IconsController(
IMemoryCache memoryCache,
IDomainMappingService domainMappingService,
IIconFetchingService iconFetchingService,
IconsSettings iconsSettings)
{
_memoryCache = memoryCache;
_domainMappingService = domainMappingService;
_iconFetchingService = iconFetchingService;
_iconsSettings = iconsSettings;
}
@ -66,46 +45,16 @@ namespace Bit.Icons.Controllers
var mappedDomain = _domainMappingService.MapDomain(uri.Host);
if(!_memoryCache.TryGetValue(mappedDomain, out Icon icon))
{
var iconUrl = new Uri($"{_iconsSettings.BestIconBaseUrl}/icon" +
$"?url={mappedDomain}&size=16..32..256&fallback_icon_url=" +
$"https://raw.githubusercontent.com/bitwarden/web/master/src/images/fa-globe.png");
var response = await _httpClient.GetAsync(iconUrl);
response = await FollowRedirectsAsync(response, 1);
if(!response.IsSuccessStatusCode ||
!_allowedMediaTypes.Contains(response.Content.Headers.ContentType.MediaType))
var result = await _iconFetchingService.GetIconAsync(mappedDomain);
if(result == null)
{
return new NotFoundResult();
}
var image = await response.Content.ReadAsByteArrayAsync();
icon = new Icon
{
Image = image,
Format = response.Content.Headers.ContentType.MediaType
};
if(icon.Format == _octetMediaType)
{
if(HeaderMatch(icon, _icoHeader))
{
icon.Format = _icoMediaType;
}
else if(HeaderMatch(icon, _pngHeader))
{
icon.Format = _pngMediaType;
}
else if(HeaderMatch(icon, _jpegHeader))
{
icon.Format = _jpegMediaType;
}
else
{
return new NotFoundResult();
}
}
icon = result.Icon;
// Only cache smaller images (<= 50kb)
if(image.Length <= 50012)
if(icon.Image.Length <= 50012)
{
_memoryCache.Set(mappedDomain, icon, new MemoryCacheEntryOptions
{
@ -116,47 +65,5 @@ namespace Bit.Icons.Controllers
return new FileContentResult(icon.Image, icon.Format);
}
private async Task<HttpResponseMessage> FollowRedirectsAsync(HttpResponseMessage response, int followCount)
{
if(response.IsSuccessStatusCode || followCount > 2)
{
return response;
}
if((response.StatusCode == HttpStatusCode.Redirect || response.StatusCode == HttpStatusCode.MovedPermanently) &&
response.Headers.Contains("Location"))
{
var locationHeader = response.Headers.GetValues("Location").FirstOrDefault();
if(!string.IsNullOrWhiteSpace(locationHeader) &&
Uri.TryCreate(locationHeader, UriKind.Absolute, out Uri location))
{
var message = new HttpRequestMessage
{
RequestUri = location,
Method = HttpMethod.Get
};
// Let's add some headers to look like we're coming from a web browser request. Some websites
// will block our request without these.
message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36");
message.Headers.Add("Accept-Language", "en-US,en;q=0.8");
message.Headers.Add("Cache-Control", "no-cache");
message.Headers.Add("Pragma", "no-cache");
message.Headers.Add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8");
response = await _httpClient.SendAsync(message);
response = await FollowRedirectsAsync(response, followCount++);
}
}
return response;
}
private bool HeaderMatch(Icon icon, byte[] header)
{
return icon.Image.Length >= header.Length && header.SequenceEqual(icon.Image.Take(header.Length));
}
}
}

8
src/Icons/Dockerfile
View File

@ -2,17 +2,9 @@ FROM microsoft/aspnetcore:2.0.6
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
unzip \
gosu \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /tmp
COPY iconserver.sha256 .
RUN curl -L -o iconserver.zip https://github.com/mat/besticon/releases/download/v3.6.0/iconserver_linux_amd64.zip \
&& sha256sum -c iconserver.sha256 \
&& unzip iconserver.zip -d /etc/iconserver \
&& rm iconserver.*
ENV ASPNETCORE_URLS http://+:5000
WORKDIR /app
EXPOSE 5000

1
src/Icons/Icons.csproj
View File

@ -9,6 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.8.2" />
<PackageReference Include="Microsoft.AspNetCore.All" Version="2.0.6" />
</ItemGroup>

1
src/Icons/IconsSettings.cs
View File

@ -2,7 +2,6 @@
{
public class IconsSettings
{
public virtual string BestIconBaseUrl { get; set; }
public virtual int CacheHours { get; set; }
public virtual long? CacheSizeLimit { get; set; }
}

70
src/Icons/Models/IconResult.cs Normal file
View File

@ -0,0 +1,70 @@
using System;
using HtmlAgilityPack;
namespace Bit.Icons.Models
{
public class IconResult
{
public IconResult(string href, HtmlNode node)
{
Path = href;
var sizesAttr = node.Attributes["sizes"];
if(!string.IsNullOrWhiteSpace(sizesAttr?.Value))
{
var sizeParts = sizesAttr.Value.Split('x');
if(sizeParts.Length == 2 && int.TryParse(sizeParts[0].Trim(), out var width) &&
int.TryParse(sizeParts[1].Trim(), out var height))
{
DefinedWidth = width;
DefinedHeight = height;
if(width == height)
{
if(width == 32)
{
Priority = 1;
}
else if(width == 64)
{
Priority = 2;
}
else if(width >= 24 && width <= 128)
{
Priority = 3;
}
else if(width == 16)
{
Priority = 4;
}
else
{
Priority = 100;
}
}
}
}
if(Priority == 0)
{
Priority = 200;
}
}
public IconResult(Uri uri, byte[] bytes, string format)
{
Path = uri.ToString();
Icon = new Icon
{
Image = bytes,
Format = format
};
Priority = 10;
}
public string Path { get; set; }
public int? DefinedWidth { get; set; }
public int? DefinedHeight { get; set; }
public Icon Icon { get; set; }
public int Priority { get; set; }
}
}

10
src/Icons/Services/IIconFetchingService.cs Normal file
View File

@ -0,0 +1,10 @@
using System.Threading.Tasks;
using Bit.Icons.Models;
namespace Bit.Icons.Services
{
public interface IIconFetchingService
{
Task<IconResult> GetIconAsync(string domain);
}
}

267
src/Icons/Services/IconFetchingService.cs Normal file
View File

@ -0,0 +1,267 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using Bit.Icons.Models;
using HtmlAgilityPack;
namespace Bit.Icons.Services
{
public class IconFetchingService : IIconFetchingService
{
private static HashSet<string> _iconRels = new HashSet<string> { "icon", "apple-touch-icon", "shortcut icon" };
private static HashSet<string> _iconExtensions = new HashSet<string> { ".ico", ".png", ".jpg", ".jpeg" };
private static readonly HttpClient _httpClient = new HttpClient(new HttpClientHandler
{
AllowAutoRedirect = false,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
});
private static string _pngMediaType = "image/png";
private static byte[] _pngHeader = new byte[] { 137, 80, 78, 71 };
private static string _icoMediaType = "image/x-icon";
private static string _icoAltMediaType = "image/vnd.microsoft.icon";
private static byte[] _icoHeader = new byte[] { 00, 00, 01, 00 };
private static string _jpegMediaType = "image/jpeg";
private static byte[] _jpegHeader = new byte[] { 255, 216, 255 };
private static string _octetMediaType = "application/octet-stream";
private static readonly HashSet<string> _allowedMediaTypes = new HashSet<string>{
_pngMediaType,
_icoMediaType,
_icoAltMediaType,
_jpegMediaType,
_octetMediaType
};
public async Task<IconResult> GetIconAsync(string domain)
{
var uri = new Uri($"http://{domain}");
var response = await GetAndFollowAsync(uri, 2);
if(response == null || !response.IsSuccessStatusCode)
{
uri = new Uri($"https://{domain}");
response = await GetAndFollowAsync(uri, 2);
}
if(response?.Content == null || !response.IsSuccessStatusCode)
{
return null;
}
if(response.Content.Headers?.ContentType?.MediaType != "text/html")
{
return null;
}
uri = response.RequestMessage.RequestUri;
var html = await response.Content.ReadAsStringAsync();
var doc = new HtmlDocument();
doc.LoadHtml(html);
if(doc.DocumentNode == null)
{
return null;
}
var icons = new List<IconResult>();
var links = doc.DocumentNode.SelectNodes(@"//link[@href]");
if(links != null)
{
foreach(var link in links)
{
var hrefAttr = link.Attributes["href"];
if(string.IsNullOrWhiteSpace(hrefAttr?.Value))
{
continue;
}
var relAttr = link.Attributes["rel"];
if(relAttr != null && _iconRels.Contains(relAttr.Value))
{
icons.Add(new IconResult(hrefAttr.Value, link));
}
else
{
var extension = Path.GetExtension(hrefAttr.Value);
if(_iconExtensions.Contains(extension))
{
icons.Add(new IconResult(hrefAttr.Value, link));
}
}
}
}
var iconResultTasks = new List<Task>();
foreach(var icon in icons)
{
Uri iconUri = null;
if(Uri.TryCreate(icon.Path, UriKind.Relative, out Uri relUri))
{
iconUri = new Uri($"{uri.Scheme}://{uri.Host}/{relUri.OriginalString}");
}
else if(Uri.TryCreate(icon.Path, UriKind.Absolute, out Uri absUri))
{
iconUri = absUri;
}
if(iconUri != null)
{
var task = GetIconAsync(iconUri).ContinueWith(async (r) =>
{
var result = await r;
if(result != null)
{
icon.Path = iconUri.ToString();
icon.Icon = result.Icon;
}
});
iconResultTasks.Add(task);
}
}
await Task.WhenAll(iconResultTasks);
if(!icons.Any(i => i.Icon != null))
{
var faviconUri = new Uri($"{uri.Scheme}://{uri.Host}/favicon.ico");
var result = await GetIconAsync(faviconUri);
if(result != null)
{
icons.Add(result);
}
else
{
return null;
}
}
return icons.Where(i => i.Icon != null).OrderBy(i => i.Priority).First();
}
private async Task<IconResult> GetIconAsync(Uri uri)
{
var response = await GetAndFollowAsync(uri, 2);
if(response?.Content?.Headers == null || !response.IsSuccessStatusCode)
{
return null;
}
var format = response.Content.Headers?.ContentType?.MediaType;
if(format == null || !_allowedMediaTypes.Contains(format))
{
return null;
}
var bytes = await response.Content.ReadAsByteArrayAsync();
if(format == _octetMediaType)
{
if(HeaderMatch(bytes, _icoHeader))
{
format = _icoMediaType;
}
else if(HeaderMatch(bytes, _pngHeader))
{
format = _pngMediaType;
}
else if(HeaderMatch(bytes, _jpegHeader))
{
format = _jpegMediaType;
}
else
{
return null;
}
}
return new IconResult(uri, bytes, format);
}
private async Task<HttpResponseMessage> GetAndFollowAsync(Uri uri, int maxRedirectCount)
{
var response = await GetAsync(uri);
if(response == null)
{
return null;
}
return await FollowRedirectsAsync(response, maxRedirectCount);
}
private async Task<HttpResponseMessage> GetAsync(Uri uri)
{
var message = new HttpRequestMessage
{
RequestUri = uri,
Method = HttpMethod.Get
};
// Let's add some headers to look like we're coming from a web browser request. Some websites
// will block our request without these.
message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36");
message.Headers.Add("Accept-Language", "en-US,en;q=0.8");
message.Headers.Add("Cache-Control", "no-cache");
message.Headers.Add("Pragma", "no-cache");
message.Headers.Add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8");
try
{
return await _httpClient.SendAsync(message);
}
catch
{
return null;
}
}
private async Task<HttpResponseMessage> FollowRedirectsAsync(HttpResponseMessage response,
int maxFollowCount, int followCount = 0)
{
if(response.IsSuccessStatusCode || followCount > maxFollowCount)
{
return response;
}
if(!(response.StatusCode == HttpStatusCode.Redirect ||
response.StatusCode == HttpStatusCode.MovedPermanently ||
response.StatusCode == HttpStatusCode.RedirectKeepVerb) ||
!response.Headers.Contains("Location"))
{
return null;
}
var locationHeader = response.Headers.GetValues("Location").FirstOrDefault();
if(!string.IsNullOrWhiteSpace(locationHeader))
{
if(!Uri.TryCreate(locationHeader, UriKind.Absolute, out Uri location))
{
if(Uri.TryCreate(locationHeader, UriKind.Relative, out Uri relLocation))
{
var requestUri = response.RequestMessage.RequestUri;
location = new Uri($"{requestUri.Scheme}://{requestUri.Host}/{relLocation.OriginalString}");
}
else
{
return null;
}
}
var newResponse = await GetAsync(location);
if(newResponse != null)
{
var redirectedResponse = await FollowRedirectsAsync(newResponse, maxFollowCount, followCount++);
if(redirectedResponse != null)
{
return redirectedResponse;
}
}
}
return null;
}
private bool HeaderMatch(byte[] imageBytes, byte[] header)
{
return imageBytes.Length >= header.Length && header.SequenceEqual(imageBytes.Take(header.Length));
}
}
}

1
src/Icons/Startup.cs
View File

@ -35,6 +35,7 @@ namespace Bit.Icons
// Services
services.AddSingleton<IDomainMappingService, DomainMappingService>();
services.AddSingleton<IIconFetchingService, IconFetchingService>();
// Mvc
services.AddMvc();

1
src/Icons/appsettings.json
View File

@ -13,7 +13,6 @@
}
},
"iconsSettings": {
"bestIconBaseUrl": "https://besticon-demo.herokuapp.com",
"cacheHours": 24,
"cacheSizeLimit": null
}

5
src/Icons/entrypoint.sh
View File

@ -55,7 +55,4 @@ fi
# The rest...
chown -R $USERNAME:$GROUPNAME /app
chown -R $USERNAME:$GROUPNAME /etc/iconserver
gosu $USERNAME:$GROUPNAME /etc/iconserver/iconserver &
gosu $USERNAME:$GROUPNAME dotnet /app/Icons.dll iconsSettings:bestIconBaseUrl=http://localhost:8080
gosu $USERNAME:$GROUPNAME dotnet /app/Icons.dll

1
src/Icons/iconserver.sha256
View File

@ -1 +0,0 @@
2fc9b9b34d4c4cba0cd90fe4d7ecf93e493c17ef240c1f4b757c9a293461f877 *iconserver.zip