Upload to Azure strorage blobs (#296)

* Implemen AzureStorageService

handes uploading files to azure blob

* Correct one-shot size

* Add azureStorage.service abstraction

* Rename azure upload method

* Prefer abstractions in DI

* Abstract file upload to a single service handling uploads

* Fallback to legacy upload method

* Linter fix

* Limit legacy upload to 404 error

src/abstractions/api.service.ts

@@ -107,6 +107,7 @@ import { ProfileResponse } from '../models/response/profileResponse';
 import { SelectionReadOnlyResponse } from '../models/response/selectionReadOnlyResponse';
 import { SendAccessResponse } from '../models/response/sendAccessResponse';
 import { SendFileDownloadDataResponse } from '../models/response/sendFileDownloadDataResponse';
+import { SendFileUploadDataResponse } from '../models/response/sendFileUploadDataResponse';
 import { SendResponse } from '../models/response/sendResponse';
 import { SubscriptionResponse } from '../models/response/subscriptionResponse';
 import { SyncResponse } from '../models/response/syncResponse';
@@ -177,11 +178,18 @@ export abstract class ApiService {
     postSendAccess: (id: string, request: SendAccessRequest, apiUrl?: string) => Promise<SendAccessResponse>;
     getSends: () => Promise<ListResponse<SendResponse>>;
     postSend: (request: SendRequest) => Promise<SendResponse>;
-    postSendFile: (data: FormData) => Promise<SendResponse>;
+    postFileTypeSend: (request: SendRequest) => Promise<SendFileUploadDataResponse>;
+    postSendFile: (sendId: string, fileId: string, data: FormData) => Promise<any>;
+    /**
+     * @deprecated Mar 25 2021: This method has been deprecated in favor of direct uploads.
+     * This method still exists for backward compatibility with old server versions.
+     */
+    postSendFileLegacy: (data: FormData) => Promise<SendResponse>;
     putSend: (id: string, request: SendRequest) => Promise<SendResponse>;
     putSendRemovePassword: (id: string) => Promise<SendResponse>;
     deleteSend: (id: string) => Promise<any>;
     getSendFileDownloadData: (send: SendAccessView, request: SendAccessRequest) => Promise<SendFileDownloadDataResponse>;
+    renewFileUploadUrl: (sendId: string, fileId: string) => Promise<SendFileUploadDataResponse>;
 
     getCipher: (id: string) => Promise<CipherResponse>;
     getCipherAdmin: (id: string) => Promise<CipherResponse>;

src/abstractions/fileUpload.service.ts

@@ -0,0 +1,7 @@
+import { CipherString } from '../models/domain';
+import { SendFileUploadDataResponse } from '../models/response/sendFileUploadDataResponse';
+
+export abstract class FileUploadService {
+    uploadSendFile: (uploadData: SendFileUploadDataResponse, fileName: CipherString,
+        encryptedFileData: ArrayBuffer) => Promise<any>;
+}

src/angular/components/send/add-edit.component.ts

@@ -260,11 +260,6 @@ export class AddEditComponent implements OnInit {
             }
 
             file = files[0];
-            if (file.size > 104857600) { // 100 MB
-                this.platformUtilsService.showToast('error', this.i18nService.t('errorOccurred'),
-                    this.i18nService.t('maxFileSize'));
-                return;
-            }
         }
 
         if (!this.editMode) {

src/enums/fileUploadType.ts

@@ -0,0 +1,4 @@
+export enum FileUploadType {
+    Direct = 0,
+    Azure = 1,
+}

src/misc/utils.ts

@@ -149,6 +149,14 @@ export class Utils {
         return Utils.fromB64ToUtf8(Utils.fromUrlB64ToB64(urlB64Str));
     }
 
+    static fromUtf8ToB64(utfStr: string): string {
+        if (Utils.isNode || Utils.isNativeScript) {
+            return Buffer.from(utfStr, 'utf8').toString('base64');
+        } else {
+            return decodeURIComponent(escape(window.btoa(utfStr)));
+        }
+    }
+
     static fromB64ToUtf8(b64Str: string): string {
         if (Utils.isNode || Utils.isNativeScript) {
             return Buffer.from(b64Str, 'base64').toString('utf8');
@@ -281,6 +289,26 @@ export class Utils {
         return Object.assign(target, source);
     }
 
+    static getUrl(uriString: string): URL {
+        if (uriString == null) {
+            return null;
+        }
+
+        uriString = uriString.trim();
+        if (uriString === '') {
+            return null;
+        }
+
+        let url = Utils.getUrlObject(uriString);
+        if (url == null) {
+            const hasHttpProtocol = uriString.indexOf('http://') === 0 || uriString.indexOf('https://') === 0;
+            if (!hasHttpProtocol && uriString.indexOf('.') > -1) {
+                url = Utils.getUrlObject('http://' + uriString);
+            }
+        }
+        return url;
+    }
+
     private static validIpAddress(ipString: string): boolean {
         // tslint:disable-next-line
         const ipRegex = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
@@ -302,26 +330,6 @@ export class Utils {
         return win.navigator.userAgent.match(/iPhone/i) != null || win.navigator.userAgent.match(/iPad/i) != null;
     }
 
-    private static getUrl(uriString: string): URL {
-        if (uriString == null) {
-            return null;
-        }
-
-        uriString = uriString.trim();
-        if (uriString === '') {
-            return null;
-        }
-
-        let url = Utils.getUrlObject(uriString);
-        if (url == null) {
-            const hasHttpProtocol = uriString.indexOf('http://') === 0 || uriString.indexOf('https://') === 0;
-            if (!hasHttpProtocol && uriString.indexOf('.') > -1) {
-                url = Utils.getUrlObject('http://' + uriString);
-            }
-        }
-        return url;
-    }
-
     private static getUrlObject(uriString: string): URL {
         try {
             if (nodeURL != null) {

src/models/response/sendFileUploadDataResponse.ts

@@ -0,0 +1,17 @@
+import { FileUploadType } from '../../enums/fileUploadType';
+
+import { BaseResponse } from './baseResponse';
+import { SendResponse } from './sendResponse';
+
+export class SendFileUploadDataResponse extends BaseResponse {
+
+    fileUploadType: FileUploadType;
+    sendResponse: SendResponse;
+    url: string = null;
+    constructor(response: any) {
+        super(response);
+        this.fileUploadType = this.getResponseProperty('FileUploadType');
+        this.sendResponse = this.getResponseProperty('SendResponse');
+        this.url = this.getResponseProperty('Url');
+    }
+}

src/services/api.service.ts

@@ -113,6 +113,7 @@ import { ProfileResponse } from '../models/response/profileResponse';
 import { SelectionReadOnlyResponse } from '../models/response/selectionReadOnlyResponse';
 import { SendAccessResponse } from '../models/response/sendAccessResponse';
 import { SendFileDownloadDataResponse } from '../models/response/sendFileDownloadDataResponse';
+import { SendFileUploadDataResponse } from '../models/response/sendFileUploadDataResponse';
 import { SendResponse } from '../models/response/sendResponse';
 import { SubscriptionResponse } from '../models/response/subscriptionResponse';
 import { SyncResponse } from '../models/response/syncResponse';
@@ -433,7 +434,25 @@ export class ApiService implements ApiServiceAbstraction {
         return new SendResponse(r);
     }
 
-    async postSendFile(data: FormData): Promise<SendResponse> {
+    async postFileTypeSend(request: SendRequest): Promise<SendFileUploadDataResponse> {
+        const r = await this.send('POST', '/sends/file/v2', request, true, true);
+        return new SendFileUploadDataResponse(r);
+    }
+
+    async renewFileUploadUrl(sendId: string, fileId: string): Promise<SendFileUploadDataResponse> {
+        const r = await this.send('GET', '/sends/' + sendId + '/file/' + fileId, null, true, true);
+        return new SendFileUploadDataResponse(r);
+    }
+
+    postSendFile(sendId: string, fileId: string, data: FormData): Promise<any> {
+        return this.send('POST', '/sends/' + sendId + '/file/' + fileId, data, true, false);
+    }
+
+    /**
+     * @deprecated Mar 25 2021: This method has been deprecated in favor of direct uploads.
+     * This method still exists for backward compatibility with old server versions.
+     */
+    async postSendFileLegacy(data: FormData): Promise<SendResponse> {
         const r = await this.send('POST', '/sends/file', data, true, true);
         return new SendResponse(r);
     }

src/services/azureFileUpload.service.ts

@@ -0,0 +1,192 @@
+import { LogService } from '../abstractions/log.service';
+
+import { Utils } from '../misc/utils';
+
+const MAX_SINGLE_BLOB_UPLOAD_SIZE = 256 * 1024 * 1024; // 256 MiB
+const MAX_BLOCKS_PER_BLOB = 50000;
+
+export class AzureFileUploadService {
+    constructor(private logService: LogService) { }
+
+    async upload(url: string, data: ArrayBuffer, renewalCallback: () => Promise<string>) {
+        if (data.byteLength <= MAX_SINGLE_BLOB_UPLOAD_SIZE) {
+            return await this.azureUploadBlob(url, data);
+        } else {
+            return await this.azureUploadBlocks(url, data, renewalCallback);
+        }
+    }
+    private async azureUploadBlob(url: string, data: ArrayBuffer) {
+        const urlObject = Utils.getUrl(url);
+        const headers = new Headers({
+            'x-ms-date': new Date().toUTCString(),
+            'x-ms-version': urlObject.searchParams.get('sv'),
+            'Content-Length': data.byteLength.toString(),
+            'x-ms-blob-type': 'BlockBlob',
+        });
+
+        const request = new Request(url, {
+            body: data,
+            cache: 'no-store',
+            method: 'PUT',
+            headers: headers,
+        });
+
+        const blobResponse = await fetch(request);
+    }
+    private async azureUploadBlocks(url: string, data: ArrayBuffer, renewalCallback: () => Promise<string>) {
+        const baseUrl = Utils.getUrl(url);
+        const blockSize = this.getMaxBlockSize(baseUrl.searchParams.get('sv'));
+        let blockIndex = 0;
+        const numBlocks = Math.ceil(data.byteLength / blockSize);
+        const blocksStaged: string[] = [];
+
+        if (numBlocks > MAX_BLOCKS_PER_BLOB) {
+            throw new Error(`Cannot upload file, exceeds maximum size of ${blockSize * MAX_BLOCKS_PER_BLOB}`);
+        }
+
+        try {
+            while (blockIndex < numBlocks) {
+                url = await this.renewUrlIfNecessary(url, renewalCallback);
+                const blockUrl = Utils.getUrl(url);
+                const blockId = this.encodedBlockId(blockIndex);
+                blockUrl.searchParams.append('comp', 'block');
+                blockUrl.searchParams.append('blockid', blockId);
+                const start = blockIndex * blockSize;
+                const blockData = data.slice(start, start + blockSize);
+                const blockHeaders = new Headers({
+                    'x-ms-date': new Date().toUTCString(),
+                    'x-ms-version': blockUrl.searchParams.get('sv'),
+                    'Content-Length': blockData.byteLength.toString(),
+                });
+
+                const blockRequest = new Request(blockUrl.toString(), {
+                    body: blockData,
+                    cache: 'no-store',
+                    method: 'PUT',
+                    headers: blockHeaders,
+                });
+
+                const blockResponse = await fetch(blockRequest);
+
+                if (blockResponse.status !== 201) {
+                    const message = `Unsuccessful block PUT. Received status ${blockResponse.status}`;
+                    this.logService.error(message + '\n' + await blockResponse.json());
+                    throw new Error(message);
+                }
+
+                blocksStaged.push(blockId);
+                blockIndex++;
+            }
+
+            const blockListUrl = Utils.getUrl(url);
+            const blockListXml = this.blockListXml(blocksStaged);
+            blockListUrl.searchParams.append('comp', 'blocklist');
+            const headers = new Headers({
+                'x-ms-date': new Date().toUTCString(),
+                'x-ms-version': blockListUrl.searchParams.get('sv'),
+                'Content-Length': blockListXml.length.toString(),
+            });
+
+            const request = new Request(blockListUrl.toString(), {
+                body: blockListXml,
+                cache: 'no-store',
+                method: 'PUT',
+                headers: headers,
+            });
+
+            const response = await fetch(request);
+
+            if (response.status !== 201) {
+                const message = `Unsuccessful block list PUT. Received status ${response.status}`;
+                this.logService.error(message + '\n' + await response.json());
+                throw new Error(message);
+            }
+        } catch (e) {
+            throw e;
+        }
+    }
+
+    private async renewUrlIfNecessary(url: string, renewalCallback: () => Promise<string>): Promise<string> {
+        const urlObject = Utils.getUrl(url);
+        const expiry = new Date(urlObject.searchParams.get('se') ?? '');
+
+        if (isNaN(expiry.getTime())) {
+            expiry.setTime(Date.now() + 3600000);
+        }
+
+        if (expiry.getTime() < Date.now() + 1000) {
+            return await renewalCallback();
+        }
+        return url;
+    }
+
+    private encodedBlockId(blockIndex: number) {
+        // Encoded blockId max size is 64, so pre-encoding max size is 48
+        const utfBlockId = ('000000000000000000000000000000000000000000000000' + blockIndex.toString()).slice(-48);
+        return Utils.fromUtf8ToB64(utfBlockId);
+    }
+
+    private blockListXml(blockIdList: string[]) {
+        let xml = '<?xml version="1.0" encoding="utf-8"?><BlockList>';
+        blockIdList.forEach(blockId => {
+            xml += `<Latest>${blockId}</Latest>`;
+        });
+        xml += '</BlockList>';
+        return xml;
+    }
+
+    private getMaxBlockSize(version: string) {
+        if (Version.compare(version, '2019-12-12') >= 0) {
+            return 4000 * 1024 * 1024; // 4000 MiB
+        } else if (Version.compare(version, '2016-05-31') >= 0) {
+            return 100 * 1024 * 1024; // 100 MiB
+        } else {
+            return 4 * 1024 * 1024; // 4 MiB
+        }
+    }
+}
+
+class Version {
+    /**
+     * Compares two Azure Versions against each other
+     * @param a Version to compare
+     * @param b Version to compare
+     * @returns a number less than zero if b is newer than a, 0 if equal,
+     * and greater than zero if a is newer than b
+     */
+    static compare(a: Required<Version> | string, b: Required<Version> | string) {
+        if (typeof (a) === 'string') {
+            a = new Version(a);
+        }
+
+        if (typeof (b) === 'string') {
+            b = new Version(b);
+        }
+
+        return a.year !== b.year ? a.year - b.year :
+            a.month !== b.month ? a.month - b.month :
+                a.day !== b.day ? a.day - b.day :
+                    0;
+    }
+    year = 0;
+    month = 0;
+    day = 0;
+
+    constructor(version: string) {
+        try {
+            const parts = version.split('-').map(v => Number.parseInt(v, 10));
+            this.year = parts[0];
+            this.month = parts[1];
+            this.day = parts[2];
+        } catch { }
+    }
+    /**
+     * Compares two Azure Versions against each other
+     * @param compareTo Version to compare against
+     * @returns a number less than zero if compareTo is newer, 0 if equal,
+     * and greater than zero if this is greater than compareTo
+     */
+    compare(compareTo: Required<Version> | string) {
+        return Version.compare(this, compareTo);
+    }
+}

src/services/bitwardenFileUpload.service.ts

@@ -0,0 +1,29 @@
+import { ApiService } from '../abstractions/api.service';
+
+import { Utils } from '../misc/utils';
+
+import { CipherString } from '../models/domain';
+import { SendResponse } from '../models/response/sendResponse';
+
+export class BitwardenFileUploadService
+{
+    constructor(private apiService: ApiService) { }
+
+    async upload(sendResponse: SendResponse, fileName: CipherString, data: ArrayBuffer) {
+        const fd = new FormData();
+        try {
+            const blob = new Blob([data], { type: 'application/octet-stream' });
+            fd.append('data', blob, fileName.encryptedString);
+        } catch (e) {
+            if (Utils.isNode && !Utils.isBrowser) {
+                fd.append('data', Buffer.from(data) as any, {
+                    filepath: fileName.encryptedString,
+                    contentType: 'application/octet-stream',
+                } as any);
+            } else {
+                throw e;
+            }
+        }
+        await this.apiService.postSendFile(sendResponse.id, sendResponse.file.id, fd);
+    }
+}

src/services/fileUpload.service.ts

@@ -0,0 +1,45 @@
+import { ApiService } from '../abstractions/api.service';
+import { FileUploadService as FileUploadServiceAbstraction } from '../abstractions/fileUpload.service';
+import { LogService } from '../abstractions/log.service';
+
+import { FileUploadType } from '../enums/fileUploadType';
+
+import { CipherString } from '../models/domain';
+import { SendFileUploadDataResponse } from '../models/response/sendFileUploadDataResponse';
+
+import { AzureFileUploadService } from './azureFileUpload.service';
+import { BitwardenFileUploadService } from './bitwardenFileUpload.service';
+
+export class FileUploadService implements FileUploadServiceAbstraction {
+    private azureFileUploadService: AzureFileUploadService;
+    private bitwardenFileUploadService: BitwardenFileUploadService;
+
+    constructor(private logService: LogService, private apiService: ApiService) {
+        this.azureFileUploadService = new AzureFileUploadService(logService);
+        this.bitwardenFileUploadService = new BitwardenFileUploadService(apiService);
+    }
+
+    async uploadSendFile(uploadData: SendFileUploadDataResponse, fileName: CipherString, encryptedFileData: ArrayBuffer) {
+        try {
+            switch (uploadData.fileUploadType) {
+                case FileUploadType.Direct:
+                    await this.bitwardenFileUploadService.upload(uploadData.sendResponse, fileName, encryptedFileData);
+                    break;
+                case FileUploadType.Azure:
+                    const renewalCallback = async () => {
+                        const renewalResponse = await this.apiService.renewFileUploadUrl(uploadData.sendResponse.id,
+                            uploadData.sendResponse.file.id);
+                        return renewalResponse.url;
+                    };
+                    await this.azureFileUploadService.upload(uploadData.url, encryptedFileData,
+                        renewalCallback);
+                    break;
+                default:
+                    throw new Error('Unknown file upload type');
+            }
+        } catch (e) {
+            this.apiService.deleteSend(uploadData.sendResponse.id);
+            throw e;
+        }
+    }
+}

src/services/send.service.ts

@@ -9,6 +9,7 @@ import { SendFile } from '../models/domain/sendFile';
 import { SendText } from '../models/domain/sendText';
 import { SymmetricCryptoKey } from '../models/domain/symmetricCryptoKey';
 
+import { FileUploadType } from '../enums/fileUploadType';
 import { SendType } from '../enums/sendType';
 
 import { SendView } from '../models/view/sendView';
@@ -16,6 +17,7 @@ import { SendView } from '../models/view/sendView';
 import { ApiService } from '../abstractions/api.service';
 import { CryptoService } from '../abstractions/crypto.service';
 import { CryptoFunctionService } from '../abstractions/cryptoFunction.service';
+import { FileUploadService } from '../abstractions/fileUpload.service';
 import { I18nService } from '../abstractions/i18n.service';
 import { SendService as SendServiceAbstraction } from '../abstractions/send.service';
 import { StorageService } from '../abstractions/storage.service';
@@ -23,6 +25,7 @@ import { UserService } from '../abstractions/user.service';
 
 import { Utils } from '../misc/utils';
 import { CipherString } from '../models/domain';
+import { ErrorResponse } from '../models/response';
 
 const Keys = {
     sendsPrefix: 'sends_',
@@ -32,8 +35,9 @@ export class SendService implements SendServiceAbstraction {
     decryptedSendCache: SendView[];
 
     constructor(private cryptoService: CryptoService, private userService: UserService,
-        private apiService: ApiService, private storageService: StorageService,
+        private apiService: ApiService, private fileUploadService: FileUploadService,
-        private i18nService: I18nService, private cryptoFunctionService: CryptoFunctionService) { }
+        private storageService: StorageService, private i18nService: I18nService,
+        private cryptoFunctionService: CryptoFunctionService) { }
 
     clearCache(): void {
         this.decryptedSendCache = null;
@@ -133,23 +137,18 @@ export class SendService implements SendServiceAbstraction {
             if (sendData[0].type === SendType.Text) {
                 response = await this.apiService.postSend(request);
             } else {
-                const fd = new FormData();
                 try {
-                    const blob = new Blob([sendData[1]], { type: 'application/octet-stream' });
+                    const uploadDataResponse = await this.apiService.postFileTypeSend(request);
-                    fd.append('model', JSON.stringify(request));
+                    response = uploadDataResponse.sendResponse;
-                    fd.append('data', blob, sendData[0].file.fileName.encryptedString);
+
+                    this.fileUploadService.uploadSendFile(uploadDataResponse, sendData[0].file.fileName, sendData[1]);
                 } catch (e) {
-                    if (Utils.isNode && !Utils.isBrowser) {
+                    if (e instanceof ErrorResponse && (e as ErrorResponse).statusCode === 404) {
-                        fd.append('model', JSON.stringify(request));
+                        response = await this.legacyServerSendFileUpload(sendData, request);
-                        fd.append('data', Buffer.from(sendData[1]) as any, {
-                            filepath: sendData[0].file.fileName.encryptedString,
-                            contentType: 'application/octet-stream',
-                        } as any);
                     } else {
                         throw e;
                     }
                 }
-                response = await this.apiService.postSendFile(fd);
             }
             sendData[0].id = response.id;
             sendData[0].accessId = response.accessId;
@@ -162,6 +161,31 @@ export class SendService implements SendServiceAbstraction {
         await this.upsert(data);
     }
 
+    /**
+     * @deprecated Mar 25 2021: This method has been deprecated in favor of direct uploads.
+     * This method still exists for backward compatibility with old server versions.
+     */
+    async legacyServerSendFileUpload(sendData: [Send, ArrayBuffer], request: SendRequest): Promise<SendResponse>
+    {
+        const fd = new FormData();
+        try {
+            const blob = new Blob([sendData[1]], { type: 'application/octet-stream' });
+            fd.append('model', JSON.stringify(request));
+            fd.append('data', blob, sendData[0].file.fileName.encryptedString);
+        } catch (e) {
+            if (Utils.isNode && !Utils.isBrowser) {
+                fd.append('model', JSON.stringify(request));
+                fd.append('data', Buffer.from(sendData[1]) as any, {
+                    filepath: sendData[0].file.fileName.encryptedString,
+                    contentType: 'application/octet-stream',
+                } as any);
+            } else {
+                throw e;
+            }
+        }
+        return await this.apiService.postSendFileLegacy(fd);
+    }
+
     async upsert(send: SendData | SendData[]): Promise<any> {
         const userId = await this.userService.getUserId();
         let sends = await this.storageService.get<{ [id: string]: SendData; }>(