From cfc76878154c5b292b729e2909bec1848f5cc23c Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 25 Mar 2021 10:20:38 -0500 Subject: [PATCH] Upload to Azure strorage blobs (#296) * Implemen AzureStorageService handes uploading files to azure blob * Correct one-shot size * Add azureStorage.service abstraction * Rename azure upload method * Prefer abstractions in DI * Abstract file upload to a single service handling uploads * Fallback to legacy upload method * Linter fix * Limit legacy upload to 404 error --- src/abstractions/api.service.ts | 10 +- src/abstractions/fileUpload.service.ts | 7 + .../components/send/add-edit.component.ts | 5 - src/enums/fileUploadType.ts | 4 + src/misc/utils.ts | 48 +++-- .../response/sendFileUploadDataResponse.ts | 17 ++ src/services/api.service.ts | 21 +- src/services/azureFileUpload.service.ts | 192 ++++++++++++++++++ src/services/bitwardenFileUpload.service.ts | 29 +++ src/services/fileUpload.service.ts | 45 ++++ src/services/send.service.ts | 50 +++-- 11 files changed, 388 insertions(+), 40 deletions(-) create mode 100644 src/abstractions/fileUpload.service.ts create mode 100644 src/enums/fileUploadType.ts create mode 100644 src/models/response/sendFileUploadDataResponse.ts create mode 100644 src/services/azureFileUpload.service.ts create mode 100644 src/services/bitwardenFileUpload.service.ts create mode 100644 src/services/fileUpload.service.ts diff --git a/src/abstractions/api.service.ts b/src/abstractions/api.service.ts index 4de63ddd10..f1f5983c8a 100644 --- a/src/abstractions/api.service.ts +++ b/src/abstractions/api.service.ts @@ -107,6 +107,7 @@ import { ProfileResponse } from '../models/response/profileResponse'; import { SelectionReadOnlyResponse } from '../models/response/selectionReadOnlyResponse'; import { SendAccessResponse } from '../models/response/sendAccessResponse'; import { SendFileDownloadDataResponse } from '../models/response/sendFileDownloadDataResponse'; +import { SendFileUploadDataResponse } from '../models/response/sendFileUploadDataResponse'; import { SendResponse } from '../models/response/sendResponse'; import { SubscriptionResponse } from '../models/response/subscriptionResponse'; import { SyncResponse } from '../models/response/syncResponse'; @@ -177,11 +178,18 @@ export abstract class ApiService { postSendAccess: (id: string, request: SendAccessRequest, apiUrl?: string) => Promise; getSends: () => Promise>; postSend: (request: SendRequest) => Promise; - postSendFile: (data: FormData) => Promise; + postFileTypeSend: (request: SendRequest) => Promise; + postSendFile: (sendId: string, fileId: string, data: FormData) => Promise; + /** + * @deprecated Mar 25 2021: This method has been deprecated in favor of direct uploads. + * This method still exists for backward compatibility with old server versions. + */ + postSendFileLegacy: (data: FormData) => Promise; putSend: (id: string, request: SendRequest) => Promise; putSendRemovePassword: (id: string) => Promise; deleteSend: (id: string) => Promise; getSendFileDownloadData: (send: SendAccessView, request: SendAccessRequest) => Promise; + renewFileUploadUrl: (sendId: string, fileId: string) => Promise; getCipher: (id: string) => Promise; getCipherAdmin: (id: string) => Promise; diff --git a/src/abstractions/fileUpload.service.ts b/src/abstractions/fileUpload.service.ts new file mode 100644 index 0000000000..2bfe08b2ef --- /dev/null +++ b/src/abstractions/fileUpload.service.ts @@ -0,0 +1,7 @@ +import { CipherString } from '../models/domain'; +import { SendFileUploadDataResponse } from '../models/response/sendFileUploadDataResponse'; + +export abstract class FileUploadService { + uploadSendFile: (uploadData: SendFileUploadDataResponse, fileName: CipherString, + encryptedFileData: ArrayBuffer) => Promise; +} diff --git a/src/angular/components/send/add-edit.component.ts b/src/angular/components/send/add-edit.component.ts index 0bb78be1c1..5f2c2b263e 100644 --- a/src/angular/components/send/add-edit.component.ts +++ b/src/angular/components/send/add-edit.component.ts @@ -260,11 +260,6 @@ export class AddEditComponent implements OnInit { } file = files[0]; - if (file.size > 104857600) { // 100 MB - this.platformUtilsService.showToast('error', this.i18nService.t('errorOccurred'), - this.i18nService.t('maxFileSize')); - return; - } } if (!this.editMode) { diff --git a/src/enums/fileUploadType.ts b/src/enums/fileUploadType.ts new file mode 100644 index 0000000000..cbdd88fbe9 --- /dev/null +++ b/src/enums/fileUploadType.ts @@ -0,0 +1,4 @@ +export enum FileUploadType { + Direct = 0, + Azure = 1, +} diff --git a/src/misc/utils.ts b/src/misc/utils.ts index 91daaa8938..5e87d1e7c2 100644 --- a/src/misc/utils.ts +++ b/src/misc/utils.ts @@ -149,6 +149,14 @@ export class Utils { return Utils.fromB64ToUtf8(Utils.fromUrlB64ToB64(urlB64Str)); } + static fromUtf8ToB64(utfStr: string): string { + if (Utils.isNode || Utils.isNativeScript) { + return Buffer.from(utfStr, 'utf8').toString('base64'); + } else { + return decodeURIComponent(escape(window.btoa(utfStr))); + } + } + static fromB64ToUtf8(b64Str: string): string { if (Utils.isNode || Utils.isNativeScript) { return Buffer.from(b64Str, 'base64').toString('utf8'); @@ -281,6 +289,26 @@ export class Utils { return Object.assign(target, source); } + static getUrl(uriString: string): URL { + if (uriString == null) { + return null; + } + + uriString = uriString.trim(); + if (uriString === '') { + return null; + } + + let url = Utils.getUrlObject(uriString); + if (url == null) { + const hasHttpProtocol = uriString.indexOf('http://') === 0 || uriString.indexOf('https://') === 0; + if (!hasHttpProtocol && uriString.indexOf('.') > -1) { + url = Utils.getUrlObject('http://' + uriString); + } + } + return url; + } + private static validIpAddress(ipString: string): boolean { // tslint:disable-next-line const ipRegex = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; @@ -302,26 +330,6 @@ export class Utils { return win.navigator.userAgent.match(/iPhone/i) != null || win.navigator.userAgent.match(/iPad/i) != null; } - private static getUrl(uriString: string): URL { - if (uriString == null) { - return null; - } - - uriString = uriString.trim(); - if (uriString === '') { - return null; - } - - let url = Utils.getUrlObject(uriString); - if (url == null) { - const hasHttpProtocol = uriString.indexOf('http://') === 0 || uriString.indexOf('https://') === 0; - if (!hasHttpProtocol && uriString.indexOf('.') > -1) { - url = Utils.getUrlObject('http://' + uriString); - } - } - return url; - } - private static getUrlObject(uriString: string): URL { try { if (nodeURL != null) { diff --git a/src/models/response/sendFileUploadDataResponse.ts b/src/models/response/sendFileUploadDataResponse.ts new file mode 100644 index 0000000000..3dce1fce59 --- /dev/null +++ b/src/models/response/sendFileUploadDataResponse.ts @@ -0,0 +1,17 @@ +import { FileUploadType } from '../../enums/fileUploadType'; + +import { BaseResponse } from './baseResponse'; +import { SendResponse } from './sendResponse'; + +export class SendFileUploadDataResponse extends BaseResponse { + + fileUploadType: FileUploadType; + sendResponse: SendResponse; + url: string = null; + constructor(response: any) { + super(response); + this.fileUploadType = this.getResponseProperty('FileUploadType'); + this.sendResponse = this.getResponseProperty('SendResponse'); + this.url = this.getResponseProperty('Url'); + } +} diff --git a/src/services/api.service.ts b/src/services/api.service.ts index 9712c058d1..5e1fa0f2da 100644 --- a/src/services/api.service.ts +++ b/src/services/api.service.ts @@ -113,6 +113,7 @@ import { ProfileResponse } from '../models/response/profileResponse'; import { SelectionReadOnlyResponse } from '../models/response/selectionReadOnlyResponse'; import { SendAccessResponse } from '../models/response/sendAccessResponse'; import { SendFileDownloadDataResponse } from '../models/response/sendFileDownloadDataResponse'; +import { SendFileUploadDataResponse } from '../models/response/sendFileUploadDataResponse'; import { SendResponse } from '../models/response/sendResponse'; import { SubscriptionResponse } from '../models/response/subscriptionResponse'; import { SyncResponse } from '../models/response/syncResponse'; @@ -433,7 +434,25 @@ export class ApiService implements ApiServiceAbstraction { return new SendResponse(r); } - async postSendFile(data: FormData): Promise { + async postFileTypeSend(request: SendRequest): Promise { + const r = await this.send('POST', '/sends/file/v2', request, true, true); + return new SendFileUploadDataResponse(r); + } + + async renewFileUploadUrl(sendId: string, fileId: string): Promise { + const r = await this.send('GET', '/sends/' + sendId + '/file/' + fileId, null, true, true); + return new SendFileUploadDataResponse(r); + } + + postSendFile(sendId: string, fileId: string, data: FormData): Promise { + return this.send('POST', '/sends/' + sendId + '/file/' + fileId, data, true, false); + } + + /** + * @deprecated Mar 25 2021: This method has been deprecated in favor of direct uploads. + * This method still exists for backward compatibility with old server versions. + */ + async postSendFileLegacy(data: FormData): Promise { const r = await this.send('POST', '/sends/file', data, true, true); return new SendResponse(r); } diff --git a/src/services/azureFileUpload.service.ts b/src/services/azureFileUpload.service.ts new file mode 100644 index 0000000000..a9830b41ef --- /dev/null +++ b/src/services/azureFileUpload.service.ts @@ -0,0 +1,192 @@ +import { LogService } from '../abstractions/log.service'; + +import { Utils } from '../misc/utils'; + +const MAX_SINGLE_BLOB_UPLOAD_SIZE = 256 * 1024 * 1024; // 256 MiB +const MAX_BLOCKS_PER_BLOB = 50000; + +export class AzureFileUploadService { + constructor(private logService: LogService) { } + + async upload(url: string, data: ArrayBuffer, renewalCallback: () => Promise) { + if (data.byteLength <= MAX_SINGLE_BLOB_UPLOAD_SIZE) { + return await this.azureUploadBlob(url, data); + } else { + return await this.azureUploadBlocks(url, data, renewalCallback); + } + } + private async azureUploadBlob(url: string, data: ArrayBuffer) { + const urlObject = Utils.getUrl(url); + const headers = new Headers({ + 'x-ms-date': new Date().toUTCString(), + 'x-ms-version': urlObject.searchParams.get('sv'), + 'Content-Length': data.byteLength.toString(), + 'x-ms-blob-type': 'BlockBlob', + }); + + const request = new Request(url, { + body: data, + cache: 'no-store', + method: 'PUT', + headers: headers, + }); + + const blobResponse = await fetch(request); + } + private async azureUploadBlocks(url: string, data: ArrayBuffer, renewalCallback: () => Promise) { + const baseUrl = Utils.getUrl(url); + const blockSize = this.getMaxBlockSize(baseUrl.searchParams.get('sv')); + let blockIndex = 0; + const numBlocks = Math.ceil(data.byteLength / blockSize); + const blocksStaged: string[] = []; + + if (numBlocks > MAX_BLOCKS_PER_BLOB) { + throw new Error(`Cannot upload file, exceeds maximum size of ${blockSize * MAX_BLOCKS_PER_BLOB}`); + } + + try { + while (blockIndex < numBlocks) { + url = await this.renewUrlIfNecessary(url, renewalCallback); + const blockUrl = Utils.getUrl(url); + const blockId = this.encodedBlockId(blockIndex); + blockUrl.searchParams.append('comp', 'block'); + blockUrl.searchParams.append('blockid', blockId); + const start = blockIndex * blockSize; + const blockData = data.slice(start, start + blockSize); + const blockHeaders = new Headers({ + 'x-ms-date': new Date().toUTCString(), + 'x-ms-version': blockUrl.searchParams.get('sv'), + 'Content-Length': blockData.byteLength.toString(), + }); + + const blockRequest = new Request(blockUrl.toString(), { + body: blockData, + cache: 'no-store', + method: 'PUT', + headers: blockHeaders, + }); + + const blockResponse = await fetch(blockRequest); + + if (blockResponse.status !== 201) { + const message = `Unsuccessful block PUT. Received status ${blockResponse.status}`; + this.logService.error(message + '\n' + await blockResponse.json()); + throw new Error(message); + } + + blocksStaged.push(blockId); + blockIndex++; + } + + const blockListUrl = Utils.getUrl(url); + const blockListXml = this.blockListXml(blocksStaged); + blockListUrl.searchParams.append('comp', 'blocklist'); + const headers = new Headers({ + 'x-ms-date': new Date().toUTCString(), + 'x-ms-version': blockListUrl.searchParams.get('sv'), + 'Content-Length': blockListXml.length.toString(), + }); + + const request = new Request(blockListUrl.toString(), { + body: blockListXml, + cache: 'no-store', + method: 'PUT', + headers: headers, + }); + + const response = await fetch(request); + + if (response.status !== 201) { + const message = `Unsuccessful block list PUT. Received status ${response.status}`; + this.logService.error(message + '\n' + await response.json()); + throw new Error(message); + } + } catch (e) { + throw e; + } + } + + private async renewUrlIfNecessary(url: string, renewalCallback: () => Promise): Promise { + const urlObject = Utils.getUrl(url); + const expiry = new Date(urlObject.searchParams.get('se') ?? ''); + + if (isNaN(expiry.getTime())) { + expiry.setTime(Date.now() + 3600000); + } + + if (expiry.getTime() < Date.now() + 1000) { + return await renewalCallback(); + } + return url; + } + + private encodedBlockId(blockIndex: number) { + // Encoded blockId max size is 64, so pre-encoding max size is 48 + const utfBlockId = ('000000000000000000000000000000000000000000000000' + blockIndex.toString()).slice(-48); + return Utils.fromUtf8ToB64(utfBlockId); + } + + private blockListXml(blockIdList: string[]) { + let xml = ''; + blockIdList.forEach(blockId => { + xml += `${blockId}`; + }); + xml += ''; + return xml; + } + + private getMaxBlockSize(version: string) { + if (Version.compare(version, '2019-12-12') >= 0) { + return 4000 * 1024 * 1024; // 4000 MiB + } else if (Version.compare(version, '2016-05-31') >= 0) { + return 100 * 1024 * 1024; // 100 MiB + } else { + return 4 * 1024 * 1024; // 4 MiB + } + } +} + +class Version { + /** + * Compares two Azure Versions against each other + * @param a Version to compare + * @param b Version to compare + * @returns a number less than zero if b is newer than a, 0 if equal, + * and greater than zero if a is newer than b + */ + static compare(a: Required | string, b: Required | string) { + if (typeof (a) === 'string') { + a = new Version(a); + } + + if (typeof (b) === 'string') { + b = new Version(b); + } + + return a.year !== b.year ? a.year - b.year : + a.month !== b.month ? a.month - b.month : + a.day !== b.day ? a.day - b.day : + 0; + } + year = 0; + month = 0; + day = 0; + + constructor(version: string) { + try { + const parts = version.split('-').map(v => Number.parseInt(v, 10)); + this.year = parts[0]; + this.month = parts[1]; + this.day = parts[2]; + } catch { } + } + /** + * Compares two Azure Versions against each other + * @param compareTo Version to compare against + * @returns a number less than zero if compareTo is newer, 0 if equal, + * and greater than zero if this is greater than compareTo + */ + compare(compareTo: Required | string) { + return Version.compare(this, compareTo); + } +} diff --git a/src/services/bitwardenFileUpload.service.ts b/src/services/bitwardenFileUpload.service.ts new file mode 100644 index 0000000000..d4af2622b2 --- /dev/null +++ b/src/services/bitwardenFileUpload.service.ts @@ -0,0 +1,29 @@ +import { ApiService } from '../abstractions/api.service'; + +import { Utils } from '../misc/utils'; + +import { CipherString } from '../models/domain'; +import { SendResponse } from '../models/response/sendResponse'; + +export class BitwardenFileUploadService +{ + constructor(private apiService: ApiService) { } + + async upload(sendResponse: SendResponse, fileName: CipherString, data: ArrayBuffer) { + const fd = new FormData(); + try { + const blob = new Blob([data], { type: 'application/octet-stream' }); + fd.append('data', blob, fileName.encryptedString); + } catch (e) { + if (Utils.isNode && !Utils.isBrowser) { + fd.append('data', Buffer.from(data) as any, { + filepath: fileName.encryptedString, + contentType: 'application/octet-stream', + } as any); + } else { + throw e; + } + } + await this.apiService.postSendFile(sendResponse.id, sendResponse.file.id, fd); + } +} diff --git a/src/services/fileUpload.service.ts b/src/services/fileUpload.service.ts new file mode 100644 index 0000000000..5a56cfa112 --- /dev/null +++ b/src/services/fileUpload.service.ts @@ -0,0 +1,45 @@ +import { ApiService } from '../abstractions/api.service'; +import { FileUploadService as FileUploadServiceAbstraction } from '../abstractions/fileUpload.service'; +import { LogService } from '../abstractions/log.service'; + +import { FileUploadType } from '../enums/fileUploadType'; + +import { CipherString } from '../models/domain'; +import { SendFileUploadDataResponse } from '../models/response/sendFileUploadDataResponse'; + +import { AzureFileUploadService } from './azureFileUpload.service'; +import { BitwardenFileUploadService } from './bitwardenFileUpload.service'; + +export class FileUploadService implements FileUploadServiceAbstraction { + private azureFileUploadService: AzureFileUploadService; + private bitwardenFileUploadService: BitwardenFileUploadService; + + constructor(private logService: LogService, private apiService: ApiService) { + this.azureFileUploadService = new AzureFileUploadService(logService); + this.bitwardenFileUploadService = new BitwardenFileUploadService(apiService); + } + + async uploadSendFile(uploadData: SendFileUploadDataResponse, fileName: CipherString, encryptedFileData: ArrayBuffer) { + try { + switch (uploadData.fileUploadType) { + case FileUploadType.Direct: + await this.bitwardenFileUploadService.upload(uploadData.sendResponse, fileName, encryptedFileData); + break; + case FileUploadType.Azure: + const renewalCallback = async () => { + const renewalResponse = await this.apiService.renewFileUploadUrl(uploadData.sendResponse.id, + uploadData.sendResponse.file.id); + return renewalResponse.url; + }; + await this.azureFileUploadService.upload(uploadData.url, encryptedFileData, + renewalCallback); + break; + default: + throw new Error('Unknown file upload type'); + } + } catch (e) { + this.apiService.deleteSend(uploadData.sendResponse.id); + throw e; + } + } +} diff --git a/src/services/send.service.ts b/src/services/send.service.ts index c589e96981..9230d70373 100644 --- a/src/services/send.service.ts +++ b/src/services/send.service.ts @@ -9,6 +9,7 @@ import { SendFile } from '../models/domain/sendFile'; import { SendText } from '../models/domain/sendText'; import { SymmetricCryptoKey } from '../models/domain/symmetricCryptoKey'; +import { FileUploadType } from '../enums/fileUploadType'; import { SendType } from '../enums/sendType'; import { SendView } from '../models/view/sendView'; @@ -16,6 +17,7 @@ import { SendView } from '../models/view/sendView'; import { ApiService } from '../abstractions/api.service'; import { CryptoService } from '../abstractions/crypto.service'; import { CryptoFunctionService } from '../abstractions/cryptoFunction.service'; +import { FileUploadService } from '../abstractions/fileUpload.service'; import { I18nService } from '../abstractions/i18n.service'; import { SendService as SendServiceAbstraction } from '../abstractions/send.service'; import { StorageService } from '../abstractions/storage.service'; @@ -23,6 +25,7 @@ import { UserService } from '../abstractions/user.service'; import { Utils } from '../misc/utils'; import { CipherString } from '../models/domain'; +import { ErrorResponse } from '../models/response'; const Keys = { sendsPrefix: 'sends_', @@ -32,8 +35,9 @@ export class SendService implements SendServiceAbstraction { decryptedSendCache: SendView[]; constructor(private cryptoService: CryptoService, private userService: UserService, - private apiService: ApiService, private storageService: StorageService, - private i18nService: I18nService, private cryptoFunctionService: CryptoFunctionService) { } + private apiService: ApiService, private fileUploadService: FileUploadService, + private storageService: StorageService, private i18nService: I18nService, + private cryptoFunctionService: CryptoFunctionService) { } clearCache(): void { this.decryptedSendCache = null; @@ -133,23 +137,18 @@ export class SendService implements SendServiceAbstraction { if (sendData[0].type === SendType.Text) { response = await this.apiService.postSend(request); } else { - const fd = new FormData(); try { - const blob = new Blob([sendData[1]], { type: 'application/octet-stream' }); - fd.append('model', JSON.stringify(request)); - fd.append('data', blob, sendData[0].file.fileName.encryptedString); + const uploadDataResponse = await this.apiService.postFileTypeSend(request); + response = uploadDataResponse.sendResponse; + + this.fileUploadService.uploadSendFile(uploadDataResponse, sendData[0].file.fileName, sendData[1]); } catch (e) { - if (Utils.isNode && !Utils.isBrowser) { - fd.append('model', JSON.stringify(request)); - fd.append('data', Buffer.from(sendData[1]) as any, { - filepath: sendData[0].file.fileName.encryptedString, - contentType: 'application/octet-stream', - } as any); + if (e instanceof ErrorResponse && (e as ErrorResponse).statusCode === 404) { + response = await this.legacyServerSendFileUpload(sendData, request); } else { throw e; } } - response = await this.apiService.postSendFile(fd); } sendData[0].id = response.id; sendData[0].accessId = response.accessId; @@ -162,6 +161,31 @@ export class SendService implements SendServiceAbstraction { await this.upsert(data); } + /** + * @deprecated Mar 25 2021: This method has been deprecated in favor of direct uploads. + * This method still exists for backward compatibility with old server versions. + */ + async legacyServerSendFileUpload(sendData: [Send, ArrayBuffer], request: SendRequest): Promise + { + const fd = new FormData(); + try { + const blob = new Blob([sendData[1]], { type: 'application/octet-stream' }); + fd.append('model', JSON.stringify(request)); + fd.append('data', blob, sendData[0].file.fileName.encryptedString); + } catch (e) { + if (Utils.isNode && !Utils.isBrowser) { + fd.append('model', JSON.stringify(request)); + fd.append('data', Buffer.from(sendData[1]) as any, { + filepath: sendData[0].file.fileName.encryptedString, + contentType: 'application/octet-stream', + } as any); + } else { + throw e; + } + } + return await this.apiService.postSendFileLegacy(fd); + } + async upsert(send: SendData | SendData[]): Promise { const userId = await this.userService.getUserId(); let sends = await this.storageService.get<{ [id: string]: SendData; }>(