Add tests and fix docker files

This commit is contained in:
Yusuf Yilmaz 2022-05-27 15:04:54 +02:00
parent 9f45927593
commit f527b13535
30 changed files with 2126 additions and 4468 deletions

View File

@ -1,2 +1,3 @@
**/*.min.js
config
config.js

View File

@ -1,20 +1,14 @@
FROM node:14.8.0-stretch
FROM node:16-slim as base
RUN mkdir -p /usr/src/app && \
chown node:node /usr/src/app
ARG user
RUN mkdir /app && chown -R $user:$user /app
USER $user
WORKDIR /app
USER node:node
COPY --chown=$user:$user package.json yarn.lock /app/
RUN yarn install
WORKDIR /usr/src/app
COPY --chown=node:node . .
RUN npm install && \
npm install redis@0.8.1 && \
npm install pg@4.1.1 && \
npm install memcached@2.2.2 && \
npm install aws-sdk@2.738.0 && \
npm install rethinkdbdash@2.3.31
COPY --chown=$user:$user . /app
ENV STORAGE_TYPE=memcached \
STORAGE_HOST=127.0.0.1 \
@ -58,11 +52,15 @@ EXPOSE ${PORT}
STOPSIGNAL SIGINT
ENTRYPOINT [ "bash", "docker-entrypoint.sh" ]
RUN yarn build
COPY static /app/dist/static
HEALTHCHECK --interval=30s --timeout=30s --start-period=5s \
--retries=3 CMD [ "sh", "-c", "echo -n 'curl localhost:7777... '; \
(\
curl -sf localhost:7777 > /dev/null\
curl -sf localhost:7777 > /dev/null\
) && echo OK || (\
echo Fail && exit 2\
echo Fail && exit 2\
)"]
CMD ["npm", "start"]
CMD ["yarn", "start"]

8
config/jest.config.js Normal file
View File

@ -0,0 +1,8 @@
/** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
rootDir: './',
testRegex: '\\.test\\.ts$',
reporters: ['default']
}

View File

@ -1,10 +1,16 @@
{
"host": "0.0.0.0",
"port": 7777,
"keyLength": 10,
"maxLength": 400000,
"staticMaxAge": 86400,
"recompressStaticAssets": true,
"logging": [
{
"level": "verbose",
@ -12,9 +18,11 @@
"colorize": true
}
],
"keyGenerator": {
"type": "phonetic"
},
"rateLimits": {
"categories": {
"normal": {
@ -23,10 +31,13 @@
}
}
},
"storage": {
"type": "file"
},
"documents": {
"about": "./about.md"
}
}

View File

@ -4,6 +4,6 @@
set -e
node ./docker-entrypoint.js > ./config.js
node ./docker-entrypoint.js > ./config/project-config.js
exec "$@"

View File

@ -14,6 +14,7 @@
},
"dependencies": {
"@google-cloud/datastore": "^6.6.2",
"@types/redis": "^4.0.11",
"aws-sdk": "^2.1142.0",
"busboy": "0.2.4",
"connect": "^3.7.0",
@ -24,8 +25,7 @@
"memcached": "^2.2.2",
"mongodb": "^4.6.0",
"pg": "^8.7.3",
"redis": "0.8.1",
"redis-url": "0.1.0",
"redis": "^4.1.0",
"rethinkdbdash": "^2.3.31",
"st": "^3.0.0",
"uglify-js": "3.1.6",
@ -34,6 +34,7 @@
"devDependencies": {
"@types/busboy": "^1.5.0",
"@types/express": "^4.17.13",
"@types/jest": "^27.5.1",
"@types/memcached": "^2.2.7",
"@types/node": "^17.0.35",
"@types/pg": "^8.6.5",
@ -41,6 +42,7 @@
"@typescript-eslint/eslint-plugin": "^5.26.0",
"@typescript-eslint/parser": "^5.26.0",
"concurrently": "^7.2.1",
"copyfiles": "^2.4.1",
"eslint": "^8.10.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-config-airbnb-typescript": "^17.0.0",
@ -48,12 +50,14 @@
"eslint-import-resolver-typescript": "^2.7.1",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-jest": "^26.2.2",
"jest": "^28.1.0",
"mocha": "^8.1.3",
"module-resolver": "^1.0.0",
"nodemon": "^2.0.16",
"npm-run-all": "^4.1.5",
"prettier": "^2.5.1",
"rimraf": "^3.0.2",
"ts-jest": "^28.0.3",
"ts-node": "^9.1.1",
"typescript": "^4.6.4"
},
@ -67,10 +71,12 @@
"static"
],
"scripts": {
"test": "mocha --recursive",
"build": "rimraf dist && tsc --project ./",
"copy:files": "copyFiles -u 1 static/**/* dist/static",
"clean:files": "rimraf dist",
"test": "jest --config config/jest.config.js",
"build": "yarn clean:files && tsc --project ./",
"start:dev": "nodemon src/server.ts",
"start:prod": "node dist/src/server.js",
"start": "node dist/src/server.js",
"lint": "eslint src --fix",
"types:check": "tsc --noEmit --pretty"
}

View File

@ -125,7 +125,7 @@ class App {
winston.info('loading static document', { name, path: documentPath })
if (data) {
this.documentHandler?.store.set(
this.documentHandler?.store?.set(
name,
data,
cb => {

5
src/constants/index.ts Normal file
View File

@ -0,0 +1,5 @@
const DEFAULT_KEY_LENGTH = 10
export default {
DEFAULT_KEY_LENGTH,
}

13
src/global.d.ts vendored
View File

@ -23,24 +23,11 @@ declare module 'rethinkdbdash' {
table(s: string): RethinkFunctions
}
// function rethink<T>(obj: T[]): RethinkArray<T>
function rethink<T>(obj: T): RethinkClient<T>
export = rethink
}
// export {}
// declare module 'connect-ratelimit' {
// export = connectRateLimit
// }
// declare namespace Express {
// export interface Request {
// sturl?: string
// }
// }
declare module 'connect-ratelimit' {
function connectRateLimit(
as: RateLimits,

View File

@ -5,22 +5,21 @@ import type { Config } from '../../types/config'
import type { Store } from '../../types/store'
import type { KeyGenerator } from '../../types/key-generator'
import type { Document } from '../../types/document'
const defaultKeyLength = 10
import constants from '../../constants'
class DocumentHandler {
keyLength: number
maxLength: number
maxLength?: number
public store: Store
public store?: Store
keyGenerator: KeyGenerator
config: Config
config?: Config
constructor(options: Document) {
this.keyLength = options.keyLength || defaultKeyLength
this.keyLength = options.keyLength || constants.DEFAULT_KEY_LENGTH
this.maxLength = options.maxLength // none by default
this.store = options.store
this.config = options.config
@ -29,9 +28,9 @@ class DocumentHandler {
public handleGet(request: Request, response: Response) {
const key = request.params.id.split('.')[0]
const skipExpire = !!this.config.documents[key]
const skipExpire = !!this.config?.documents[key]
this.store.get(
this.store?.get(
key,
ret => {
if (ret) {
@ -75,7 +74,7 @@ class DocumentHandler {
}
// And then save if we should
this.chooseKey(key => {
this.store.set(key, buffer, res => {
this.store?.set(key, buffer, res => {
if (res) {
winston.verbose('added document', { key })
response.writeHead(200, { 'content-type': 'application/json' })
@ -124,9 +123,9 @@ class DocumentHandler {
public handleRawGet(request: Request, response: Response) {
const key = request.params.id.split('.')[0]
const skipExpire = !!this.config.documents[key]
const skipExpire = !!this.config?.documents[key]
this.store.get(
this.store?.get(
key,
ret => {
if (ret) {
@ -158,7 +157,7 @@ class DocumentHandler {
if (!key) return
this.store.get(
this.store?.get(
key,
(ret: string | boolean) => {
if (ret) {

View File

@ -2,21 +2,11 @@ import type { Config } from '../../types/config'
import type { Store } from '../../types/store'
const build = async (config: Config): Promise<Store> => {
const DocumentStore = (
await import(`../document-stores/${config.storage.type}`)
).default
if (process.env.REDISTOGO_URL && config.storage.type === 'redis') {
// const redisClient = require("redis-url").connect(process.env.REDISTOGO_URL);
// Store = require("./lib/document-stores/redis");
// preferredStore = new Store(config.storage, redisClient);
const DocumentStore = (await import(`../document-stores/${config.storage.type}`)).default
return new DocumentStore(config.storage)
}
const DocumentStore = (await import(`../document-stores/${config.storage.type}`)).default
return new DocumentStore(config.storage)
return new DocumentStore(config.storage)
}
export default build

View File

@ -0,0 +1,98 @@
import * as winston from 'winston'
import { createClient } from 'redis'
import { bool } from 'aws-sdk/clients/redshiftdata'
import { Callback, Store } from '../../types/store'
import { RedisStoreConfig } from '../../types/config'
export type RedisClientType = ReturnType<typeof createClient>
// For storing in redis
// options[type] = redis
// options[url] - the url to connect to redis
// options[host] - The host to connect to (default localhost)
// options[port] - The port to connect to (default 5379)
// options[db] - The db to use (default 0)
// options[expire] - The time to live for each key set (default never)
class RedisDocumentStore implements Store {
type: string
expire?: number | undefined
client?: RedisClientType
constructor(options: RedisStoreConfig) {
this.expire = options.expire
this.type = options.type
this.connect(options)
}
connect = (options: RedisStoreConfig) => {
winston.info('configuring redis')
const url = process.env.REDISTOGO_URL || options.url
const host = options.host || '127.0.0.1'
const port = options.port || 6379
const index = options.db || 0
if (url) {
this.client = createClient({ url })
this.client.connect()
} else {
this.client = createClient({
url: `http://${host}:${port}`,
database: index as number,
username: options.username,
password: options.password,
})
}
this.client.on('error', err => {
winston.error('redis disconnected', err)
})
this.client
.select(index as number)
.then(() => {
winston.info(
`connected to redis on ${url || `${host}:${port}`}/${index}`,
)
})
.catch(err => {
winston.error(`error connecting to redis index ${index}`, {
error: err,
})
process.exit(1)
})
}
getExpire = (skipExpire?: bool) => (!skipExpire ? { EX: this.expire } : {})
get = (key: string, callback: Callback): void => {
this.client
?.get(key)
.then(reply => {
callback(reply || false)
})
.catch(() => {
callback(false)
})
}
set = (
key: string,
data: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
this.client?.set(key, data, this.getExpire(skipExpire))
.then(() => {
callback(true)
})
.catch(() => {
callback(false)
})
}
}
export default RedisDocumentStore

View File

@ -1,9 +1,14 @@
import * as fs from 'fs'
import * as path from 'path'
import { Config } from '../../types/config'
const getConfig = (): Config => {
const configPath = process.argv.length <= 2 ? 'config.json' : process.argv[2]
const config = JSON.parse(fs.readFileSync(configPath, 'utf8'))
const configPath =
process.argv.length <= 2 ? 'project-config.js' : process.argv[2]
const config = JSON.parse(
fs.readFileSync(path.join('config', configPath), 'utf8'),
)
config.port = (process.env.PORT || config.port || 7777) as number
config.host = process.env.HOST || config.host || 'localhost'

View File

@ -7,7 +7,7 @@ class DictionaryGenerator implements KeyGenerator {
dictionary: string[]
constructor(options: KeyGeneratorConfig, readyCallback: () => void) {
constructor(options: KeyGeneratorConfig, readyCallback?: () => void) {
// Check options format
if (!options) throw Error('No options passed to generator')
if (!options.path) throw Error('No dictionary path specified in options')
@ -21,7 +21,7 @@ class DictionaryGenerator implements KeyGenerator {
this.dictionary = data.split(/[\n\r]+/)
if (readyCallback) readyCallback()
readyCallback?.()
})
}

View File

@ -50,6 +50,16 @@ export interface RethinkDbStoreConfig extends BaseStoreConfig {
password: string
}
export interface RedisStoreConfig extends BaseStoreConfig {
url?: string
host?: string
port?: string
db?: string
user?: string
username?: string | undefined
password?: string
}
export type GoogleStoreConfig = BaseStoreConfig
export type StoreConfig =
@ -59,6 +69,9 @@ export type StoreConfig =
| AmazonStoreConfig
| FileStoreConfig
| MongoStoreConfig
| RedisStoreConfig
| RethinkDbStoreConfig
| PostgresStoreConfig
export interface KeyGeneratorConfig {
type: string

View File

@ -3,10 +3,10 @@ import type { KeyGenerator } from './key-generator'
import type { Store } from './store'
export type Document = {
store: Store
config: Config
maxLength: number
keyLength: number
store?: Store
config?: Config
maxLength?: number
keyLength?: number
keyGenerator: KeyGenerator
}

View File

@ -1,5 +0,0 @@
import DocumentHandler from '../lib/document-handler'
export type RequestParams = {
documentHandler: DocumentHandler
}

View File

@ -0,0 +1,20 @@
import DocumentHandler from '../../src/lib/document-handler/index'
import Generator from '../../src/lib/key-generators/random'
import constants from '../../src/constants'
describe('document-handler', () => {
describe('with randomKey', () => {
it('should choose a key of the proper length', () => {
const gen = new Generator({ type: 'random' })
const dh = new DocumentHandler({ keyLength: 6, keyGenerator: gen})
expect(dh.acceptableKey()?.length).toEqual(6);
})
it('should choose a default key length', () => {
const gen = new Generator({ type: 'random' })
const dh = new DocumentHandler({ keyGenerator: gen, maxLength: 1 })
expect(dh.keyLength).toEqual(constants.DEFAULT_KEY_LENGTH);
})
})
})

View File

@ -0,0 +1,55 @@
import RedisDocumentStore from '../../src/lib/document-stores/redis'
describe('Redis document store', () => {
let store: RedisDocumentStore
/* reconnect to redis on each test */
afterEach(() => {
if (store) {
store.client?.quit()
}
})
describe('set', () => {
it('should be able to set a key and have an expiration set', async () => {
store = new RedisDocumentStore({
expire: 10,
type: 'redis',
url: 'http://localhost:6666',
})
return store.set('hello1', 'world', async () => {
const res = await store.client?.ttl('hello1')
expect(res).toBeGreaterThan(1)
})
})
it('should not set an expiration when told not to', async () => {
store = new RedisDocumentStore({
expire: 10,
type: 'redis',
url: 'http://localhost:6666',
})
store.set(
'hello2',
'world',
async () => {
const res = await store.client?.ttl('hello2')
expect(res).toEqual(-1)
},
true,
)
})
it('should not set an expiration when expiration is off', async () => {
store = new RedisDocumentStore({
type: 'redis',
url: 'http://localhost:6666',
})
store.set('hello3', 'world', async () => {
const res = await store.client?.ttl('hello3')
expect(res).toEqual(-1)
})
})
})
})

View File

@ -1,26 +0,0 @@
/* global describe, it */
var assert = require('assert');
var DocumentHandler = require('../lib/document_handler');
var Generator = require('../lib/key_generators/random');
describe('document_handler', function() {
describe('randomKey', function() {
it('should choose a key of the proper length', function() {
var gen = new Generator();
var dh = new DocumentHandler({ keyLength: 6, keyGenerator: gen });
assert.equal(6, dh.acceptableKey().length);
});
it('should choose a default key length', function() {
var gen = new Generator();
var dh = new DocumentHandler({ keyGenerator: gen });
assert.equal(dh.keyLength, DocumentHandler.defaultKeyLength);
});
});
});

View File

@ -0,0 +1,24 @@
import Generator from '../../src/lib/key-generators/dictionary'
jest.mock('fs', () => ({
readFile: jest.fn().mockImplementation((_, a, callback) =>
callback(null, 'cat'),
)
}))
describe('DictionaryGenerator', () => {
describe('options', () => {
it('should throw an error if given no options or path', () => {
expect(() => new Generator({ type: '' })).toThrow()
})
})
describe('generation', () => {
it('should return a key of the proper number of words from the given dictionary', () => {
const path = '/tmp/haste-server-test-dictionary'
const gen = new Generator({ path, type: '' })
expect(gen.createKey(3)).toEqual('catcatcat')
})
})
})

View File

@ -0,0 +1,30 @@
/* eslint-disable jest/no-conditional-expect */
import Generator from '../../src/lib/key-generators/phonetic'
const vowels = 'aeiou';
const consonants = 'bcdfghjklmnpqrstvwxyz';
describe('PhoneticKeyGenerator', () => {
describe('generation', () => {
it('should return a key of the proper length', () => {
const gen = new Generator({ type: 'phonetic'});
expect(gen.createKey(6).length).toEqual(6);
});
it('should alternate consonants and vowels', () => {
const gen = new Generator({ type: 'phonetic'});
const key = gen.createKey(3);
// if it starts with a consonant, we expect cvc
// if it starts with a vowel, we expect vcv
if(consonants.includes(key[0])) {
expect(consonants.includes(key[0])).toBeTruthy()
expect(consonants.includes(key[2])).toBeTruthy()
expect(vowels.includes(key[1])).toBeTruthy()
} else {
expect(vowels.includes(key[0])).toBeTruthy()
expect(vowels.includes(key[2])).toBeTruthy()
expect(consonants.includes(key[1])).toBeTruthy()
}
});
});
});

View File

@ -0,0 +1,20 @@
import Generator from '../../src/lib/key-generators/random'
describe('RandomKeyGenerator', () => {
describe('generation', () => {
it('should return a key of the proper length', () => {
const gen = new Generator({ type: 'random' })
expect(gen.createKey(6).length).toEqual(6)
})
it('should use a key from the given keyset if given', () => {
const gen = new Generator({ type: 'random', keyspace: 'A' })
expect(gen.createKey(6)).toEqual('AAAAAA')
})
it('should not use a key from the given keyset if not given', () => {
const gen = new Generator({ type: 'random', keyspace: 'A' })
expect(gen.createKey(6).includes('B')).toBeFalsy()
})
})
})

View File

@ -1,34 +0,0 @@
/* global describe, it */
const assert = require('assert');
const fs = require('fs');
const Generator = require('../../lib/key_generators/dictionary');
describe('DictionaryGenerator', function() {
describe('options', function() {
it('should throw an error if given no options', () => {
assert.throws(() => {
new Generator();
}, Error);
});
it('should throw an error if given no path', () => {
assert.throws(() => {
new Generator({});
}, Error);
});
});
describe('generation', function() {
it('should return a key of the proper number of words from the given dictionary', () => {
const path = '/tmp/haste-server-test-dictionary';
const words = ['cat'];
fs.writeFileSync(path, words.join('\n'));
const gen = new Generator({path}, () => {
assert.equal('catcatcat', gen.createKey(3));
});
});
});
});

View File

@ -1,35 +0,0 @@
/* global describe, it */
const assert = require('assert');
const Generator = require('../../lib/key_generators/phonetic');
const vowels = 'aeiou';
const consonants = 'bcdfghjklmnpqrstvwxyz';
describe('PhoneticKeyGenerator', () => {
describe('generation', () => {
it('should return a key of the proper length', () => {
const gen = new Generator();
assert.equal(6, gen.createKey(6).length);
});
it('should alternate consonants and vowels', () => {
const gen = new Generator();
const key = gen.createKey(3);
// if it starts with a consonant, we expect cvc
// if it starts with a vowel, we expect vcv
if(consonants.includes(key[0])) {
assert.ok(consonants.includes(key[0]));
assert.ok(consonants.includes(key[2]));
assert.ok(vowels.includes(key[1]));
} else {
assert.ok(vowels.includes(key[0]));
assert.ok(vowels.includes(key[2]));
assert.ok(consonants.includes(key[1]));
}
});
});
});

View File

@ -1,24 +0,0 @@
/* global describe, it */
const assert = require('assert');
const Generator = require('../../lib/key_generators/random');
describe('RandomKeyGenerator', () => {
describe('generation', () => {
it('should return a key of the proper length', () => {
const gen = new Generator();
assert.equal(gen.createKey(6).length, 6);
});
it('should use a key from the given keyset if given', () => {
const gen = new Generator({keyspace: 'A'});
assert.equal(gen.createKey(6), 'AAAAAA');
});
it('should not use a key from the given keyset if not given', () => {
const gen = new Generator({keyspace: 'A'});
assert.ok(!gen.createKey(6).includes('B'));
});
});
});

View File

@ -1,54 +0,0 @@
/* global it, describe, afterEach */
var assert = require('assert');
var winston = require('winston');
winston.remove(winston.transports.Console);
var RedisDocumentStore = require('../lib/document_stores/redis');
describe('redis_document_store', function() {
/* reconnect to redis on each test */
afterEach(function() {
if (RedisDocumentStore.client) {
RedisDocumentStore.client.quit();
RedisDocumentStore.client = false;
}
});
describe('set', function() {
it('should be able to set a key and have an expiration set', function(done) {
var store = new RedisDocumentStore({ expire: 10 });
store.set('hello1', 'world', function() {
RedisDocumentStore.client.ttl('hello1', function(err, res) {
assert.ok(res > 1);
done();
});
});
});
it('should not set an expiration when told not to', function(done) {
var store = new RedisDocumentStore({ expire: 10 });
store.set('hello2', 'world', function() {
RedisDocumentStore.client.ttl('hello2', function(err, res) {
assert.equal(-1, res);
done();
});
}, true);
});
it('should not set an expiration when expiration is off', function(done) {
var store = new RedisDocumentStore({ expire: false });
store.set('hello3', 'world', function() {
RedisDocumentStore.client.ttl('hello3', function(err, res) {
assert.equal(-1, res);
done();
});
});
});
});
});

View File

@ -28,10 +28,8 @@
"sourceMap": true,
"rootDir": ".",
"outDir": "dist",
// "baseUrl": "./src",
"paths": {
// "~/*": ["/src/*"],
// "~/lib/*": ["./src/lib/*"]
"~/*": ["/src/*"]
}
},
"include": ["src", "global.d.ts", "**/*.ts"],

File diff suppressed because it is too large Load Diff

1825
yarn.lock

File diff suppressed because it is too large Load Diff