fix pr comments

This commit is contained in:
Yusuf Yilmaz 2022-06-06 21:36:48 +02:00
parent a5b0a98b3f
commit b920c1f7ad
32 changed files with 222 additions and 172 deletions

View File

@ -14,7 +14,6 @@
},
"dependencies": {
"@google-cloud/datastore": "^6.6.2",
"@types/redis": "^4.0.11",
"aws-sdk": "^2.1142.0",
"busboy": "0.2.4",
"connect": "^3.7.0",
@ -38,6 +37,7 @@
"@types/memcached": "^2.2.7",
"@types/node": "^17.0.35",
"@types/pg": "^8.6.5",
"@types/redis": "^4.0.11",
"@types/uglify-js": "^3.13.2",
"@typescript-eslint/eslint-plugin": "^5.26.0",
"@typescript-eslint/parser": "^5.26.0",
@ -65,19 +65,32 @@
"bundledDependencies": [],
"main": "haste",
"bin": {
"haste-server": ".dist/src/server.js"
"haste-server": "./dist/src/server.js"
},
"files": [
"src",
"static"
],
"nodemonConfig": {
"ignore":
[
"test/**/*.test.ts",
".git",
"node_modules"
],
"watch": [
"src"
],
"exec": "node -r tsconfig-paths/register -r ts-node/register ./src/server.ts",
"ext": "ts, js"
},
"scripts": {
"copy:files": "copyFiles -u 1 static/**/* dist/static",
"clean:files": "rimraf dist",
"test": "jest --config config/jest.config.js",
"build:nostatic": "yarn clean:files && tsc --project ./",
"build": "yarn clean:files && yarn copy:files && tsc --project ./",
"dev": "nodemon src/server.ts",
"remove:files": "rimraf dist",
"test:unit": "jest --config config/jest.config.js",
"build:nostatic": "yarn remove:files && tsc --project ./",
"build": "yarn remove:files && yarn copy:files && tsc --project ./",
"dev": "nodemon",
"start": "node dist/src/server.js",
"lint": "eslint src --fix",
"types:check": "tsc --noEmit --pretty",

8
src/global.d.ts vendored
View File

@ -15,12 +15,12 @@ declare module 'rethinkdbdash' {
}
interface RethinkFunctions {
insert(object: RethinkInsertObject): RethinkRun
get(x: string): RethinkRun
insert(data: RethinkInsertObject): RethinkRun
get(id: string): RethinkRun
}
export interface RethinkClient {
table(s: string): RethinkFunctions
table(tableName: string): RethinkFunctions
}
function rethink<T>(obj: T): RethinkClient<T>
@ -54,7 +54,7 @@ declare module 'st' {
index: boolean | string
}
function connectSt(st: ConnectSt): Middleware
function connectSt(st: ConnectSt): express.NextFunction
export = connectSt
}

View File

@ -2,10 +2,10 @@ import { Request, Response } from 'express'
import * as winston from 'winston'
import Busboy from 'busboy'
import type { Config } from 'src/types/config'
import type { Store } from 'src/types/store'
import type { KeyGenerator } from 'src/types/key-generator'
import type { Store } from 'src/types/callback'
import type { Document } from 'src/types/document'
import constants from 'src/constants'
import KeyGenerator from 'src/lib/key-generators'
class DocumentHandler {
keyLength: number
@ -26,7 +26,7 @@ class DocumentHandler {
this.keyGenerator = options.keyGenerator
}
public handleGet(request: Request, response: Response) {
handleGet(request: Request, response: Response) {
const key = request.params.id.split('.')[0]
const skipExpire = !!this.config.documents[key]

View File

@ -1,21 +1,17 @@
import * as winston from 'winston'
import AWS from 'aws-sdk'
import type { Callback, Store } from 'src/types/store'
import type { AmazonStoreConfig } from 'src/types/config'
import { Callback } from 'src/types/callback'
import { Store } from '.'
class AmazonS3DocumentStore implements Store {
class AmazonS3DocumentStore extends Store {
bucket: string | undefined
client: AWS.S3
type: string
expire?: number | undefined
constructor(options: AmazonStoreConfig) {
this.expire = options.expire
super(options)
this.bucket = options.bucket
this.type = options.type
this.client = new AWS.S3({ region: options.region })
}

View File

@ -1,12 +1,51 @@
import type { Config } from 'src/types/config'
import type { Store } from 'src/types/store'
import type {
AmazonStoreConfig,
BaseStoreConfig,
Config,
GoogleStoreConfig,
MemcachedStoreConfig,
MongoStoreConfig,
PostgresStoreConfig,
RedisStoreConfig,
RethinkDbStoreConfig
} from 'src/types/config'
import type { Store } from 'src/types/callback'
import { StoreNames } from 'src/types/store-names'
const build = async (config: Config): Promise<Store> => {
const DocumentStore = (
await import(`../document-stores/${config.storage.type}`)
).default
const DocumentStore = (await import(`../document-stores/${config.storeName}`))
.default
return new DocumentStore(config.storage)
let store: BaseStoreConfig
switch (config.storeName) {
case StoreNames.amazons3:
store = config.storage as AmazonStoreConfig
break
case StoreNames.googledatastore:
store = config.storage as GoogleStoreConfig
break
case StoreNames.memcached:
store = config.storage as MemcachedStoreConfig
break
case StoreNames.mongo:
store = config.storage as MongoStoreConfig
break
case StoreNames.postgres:
store = config.storage as PostgresStoreConfig
break
case StoreNames.redis:
store = config.storage as RedisStoreConfig
break
case StoreNames.rethinkdb:
store = config.storage as RethinkDbStoreConfig
break
case StoreNames.file:
default:
store = config.storage as BaseStoreConfig
break
}
return new DocumentStore(store)
}
export default build

View File

@ -2,8 +2,9 @@ import * as winston from 'winston'
import * as fs from 'fs'
import * as crypto from 'crypto'
import type { Callback, Store } from 'src/types/store'
import type { Callback } from 'src/types/callback'
import type { FileStoreConfig } from 'src/types/config'
import { Store } from '.'
// Generate md5 of a string
const md5 = (str: string) => {
@ -16,17 +17,12 @@ const md5 = (str: string) => {
// options[type] = file
// options[path] - Where to store
class FileDocumentStore implements Store {
type: string
expire?: number | undefined
class FileDocumentStore extends Store {
basePath: string
constructor(options: FileStoreConfig) {
super(options)
this.basePath = options.path || './data'
this.expire = options.expire
this.type = options.type
}
// Get data from a file from key

View File

@ -1,23 +1,19 @@
import { Datastore, PathType } from '@google-cloud/datastore'
import * as winston from 'winston'
import type { Callback, Store } from 'src/types/store'
import type { Callback } from 'src/types/callback'
import type { GoogleStoreConfig } from 'src/types/config'
import { Store } from '.'
class GoogleDatastoreDocumentStore implements Store {
class GoogleDatastoreDocumentStore extends Store {
kind: string
expire?: number
datastore: Datastore
type: string
// Create a new store with options
constructor(options: GoogleStoreConfig) {
super(options)
this.kind = 'Haste'
this.expire = options.expire
this.type = options.type
this.datastore = new Datastore()
}

View File

@ -0,0 +1,25 @@
import { BaseStoreConfig } from 'src/types/config'
export type Callback = (data: boolean | string) => void
export abstract class Store {
type: string
expire?: number
constructor(config: BaseStoreConfig) {
this.type = config.type
if (this.expire) {
this.expire = config.expire
}
}
abstract get: (key: string, callback: Callback, skipExpire?: boolean) => void
abstract set: (
key: string,
data: string,
callback: Callback,
skipExpire?: boolean
) => void
}

View File

@ -1,32 +1,23 @@
import * as winston from 'winston'
import Memcached = require('memcached')
import type { Callback, Store } from 'src/types/store'
import type { Callback } from 'src/types/callback'
import type { MemcachedStoreConfig } from 'src/types/config'
import { Store } from '.'
class MemcachedDocumentStore implements Store {
expire: number | undefined
client?: Memcached
type: string
class MemcachedDocumentStore extends Store {
client: Memcached
// Create a new store with options
constructor(options: MemcachedStoreConfig) {
this.expire = options.expire
this.type = options.type
super(options)
const host = options.host || '127.0.0.1'
const port = options.port || 11211
const url = `${host}:${port}`
this.connect(url)
}
// Create a connection
connect = (url: string) => {
this.client = new Memcached(url)
winston.info(`connecting to memcached on ${url}`)
this.client.on('failure', (error: Memcached.IssueData) => {
winston.info('error connecting to memcached', { error })
})

View File

@ -1,21 +1,17 @@
import * as winston from 'winston'
import { MongoClient } from 'mongodb'
import type { Callback, Store } from 'src/types/store'
import type { Callback } from 'src/types/callback'
import type { MongoStoreConfig } from 'src/types/config'
import { Store } from '.'
type ConnectCallback = (error?: Error, db?: MongoClient) => void
class MongoDocumentStore implements Store {
type: string
expire?: number | undefined
class MongoDocumentStore extends Store {
connectionUrl: string
constructor(options: MongoStoreConfig) {
this.expire = options.expire
this.type = options.type
super(options)
this.connectionUrl = process.env.DATABASE_URl || options.connectionUrl
}

View File

@ -1,8 +1,9 @@
import * as winston from 'winston'
import { Pool, PoolClient } from 'pg'
import type { Callback, Store } from 'src/types/store'
import type { Callback } from 'src/types/callback'
import type { PostgresStoreConfig } from 'src/types/config'
import { Store } from '.'
type ConnectCallback = (
error?: Error,
@ -11,17 +12,11 @@ type ConnectCallback = (
) => void
// A postgres document store
class PostgresDocumentStore implements Store {
type: string
expireJS?: number
class PostgresDocumentStore extends Store {
pool: Pool
constructor(options: PostgresStoreConfig) {
this.expireJS = options.expire
this.type = options.type
super(options)
const connectionString = process.env.DATABASE_URL || options.connectionUrl
this.pool = new Pool({ connectionString })
}
@ -61,10 +56,10 @@ class PostgresDocumentStore implements Store {
return callback(false)
}
callback(result.rows.length ? result.rows[0].value : false)
if (result.rows.length && this.expireJS && !skipExpire) {
if (result.rows.length && this.expire && !skipExpire) {
return client.query(
'UPDATE entries SET expiration = $1 WHERE ID = $2',
[this.expireJS + now, result.rows[0].id],
[this.expire + now, result.rows[0].id],
(currentErr: Error) => {
if (!currentErr) {
return done?.()
@ -95,7 +90,7 @@ class PostgresDocumentStore implements Store {
}
return client?.query(
'INSERT INTO entries (key, value, expiration) VALUES ($1, $2, $3)',
[key, data, this.expireJS && !skipExpire ? this.expireJS + now : null],
[key, data, this.expire && !skipExpire ? this.expire + now : null],
(error: Error) => {
if (error) {
winston.error('error persisting value to postgres', { error })

View File

@ -1,8 +1,9 @@
import * as winston from 'winston'
import { createClient } from 'redis'
import { bool } from 'aws-sdk/clients/redshiftdata'
import { Callback, Store } from 'src/types/store'
import type { Callback } from 'src/types/callback'
import { RedisStoreConfig } from 'src/types/config'
import { Store } from '.'
export type RedisClientType = ReturnType<typeof createClient>
@ -14,27 +15,19 @@ export type RedisClientType = ReturnType<typeof createClient>
// options[db] - The db to use (default 0)
// options[expire] - The time to live for each key set (default never)
class RedisDocumentStore implements Store {
type: string
expire?: number | undefined
client?: RedisClientType
class RedisDocumentStore extends Store {
client: RedisClientType
constructor(options: RedisStoreConfig) {
this.expire = options.expire
this.type = options.type
this.connect(options)
}
connect = (options: RedisStoreConfig) => {
winston.info('configuring redis')
super(options)
const url = process.env.REDISTOGO_URL || options.url
const host = options.host || '127.0.0.1'
const port = options.port || 6379
const port = options.port || '6379'
const index = options.db || 0
winston.info('configuring redis')
const connectionParameters = url
? {
url
@ -46,12 +39,16 @@ class RedisDocumentStore implements Store {
const config = {
...connectionParameters,
database: index as number,
database: index,
...(options.username ? { username: options.username } : {}),
...(options.password ? { username: options.username } : {})
}
this.client = createClient(config)
this.connect(index)
}
connect = (index: number) => {
this.client.connect()
this.client.on('error', err => {
@ -59,9 +56,9 @@ class RedisDocumentStore implements Store {
})
this.client
.select(index as number)
.select(index)
.then(() => {
winston.info(`connected to redis on ${url}/${index}`)
winston.info(`connected to redis on ${index}`)
})
.catch(err => {
winston.error(`error connecting to redis index ${index}`, {
@ -75,7 +72,7 @@ class RedisDocumentStore implements Store {
get = (key: string, callback: Callback): void => {
this.client
?.get(key)
.get(key)
.then(reply => {
callback(reply || false)
})
@ -91,7 +88,7 @@ class RedisDocumentStore implements Store {
skipExpire?: boolean | undefined
): void => {
this.client
?.set(key, data, this.getExpire(skipExpire))
.set(key, data, this.getExpire(skipExpire))
.then(() => {
callback(true)
})

View File

@ -4,7 +4,8 @@ import * as crypto from 'crypto'
import rethink, { RethinkClient } from 'rethinkdbdash'
import type { RethinkDbStoreConfig } from 'src/types/config'
import type { Callback } from 'src/types/store'
import type { Callback } from 'src/types/callback'
import { Store } from '.'
const md5 = (str: string) => {
const md5sum = crypto.createHash('md5')
@ -12,10 +13,11 @@ const md5 = (str: string) => {
return md5sum.digest('hex')
}
class RethinkDBStore {
class RethinkDBStore extends Store {
client: RethinkClient
constructor(options: RethinkDbStoreConfig) {
super(options)
this.client = rethink({
silent: true,
host: options.host || '127.0.0.1',

View File

@ -10,14 +10,15 @@ const getConfig = (): Config => {
fs.readFileSync(path.join('config', configPath), 'utf8')
)
config.port = (process.env.PORT || config.port || 7777) as number
config.port = Number(process.env.PORT) || config.port || 7777
config.host = process.env.HOST || config.host || 'localhost'
if (!config.storage) {
config.storage = { type: 'file' }
config.storage = {}
}
if (!config.storage.type) {
config.storage.type = 'file'
if (!config.storeName) {
config.storeName = 'file'
}
return config

View File

@ -4,4 +4,4 @@ export const getStaticDirectory = (baseDirectory: string) =>
path.join(baseDirectory, '..', 'static')
export const getStaticItemDirectory = (baseDirectory: string, item: string) =>
path.join(baseDirectory, '..', 'static', item)
path.join(getStaticDirectory(baseDirectory), item)

View File

@ -1,5 +1,6 @@
import * as winston from 'winston'
import type { Config } from 'src/types/config'
import { Logging, LoggingType } from 'src/types/log'
const addLogging = (config: Config) => {
try {
@ -8,8 +9,8 @@ const addLogging = (config: Config) => {
/* was not present */
}
let detail
let type
let detail: Logging
let type: LoggingType
for (let i = 0; i < config.logging.length; i += 1) {
detail = config.logging[i]

View File

@ -1,5 +1,5 @@
import type { KeyGenerator } from 'src/types/key-generator'
import type { Config } from 'src/types/config'
import KeyGenerator from '.'
const build = async (config: Config): Promise<KeyGenerator> => {
const pwOptions = config.keyGenerator

View File

@ -1,14 +1,14 @@
import * as fs from 'fs'
import type { KeyGeneratorConfig } from 'src/types/config'
import type { KeyGenerator } from 'src/types/key-generator'
import KeyGenerator from '.'
class DictionaryGenerator implements KeyGenerator {
class DictionaryGenerator extends KeyGenerator {
type: string
dictionary: string[]
constructor(options: KeyGeneratorConfig, readyCallback?: () => void) {
// Check options format
super(options)
if (!options) throw Error('No options passed to generator')
if (!options.path) throw Error('No dictionary path specified in options')

View File

@ -0,0 +1,13 @@
import type { KeyGeneratorConfig } from 'src/types/config'
abstract class KeyGenerator {
type: string
constructor(options: KeyGeneratorConfig) {
this.type = options.type
}
abstract createKey(keyLength: number): string
}
export default KeyGenerator

View File

@ -1,7 +1,5 @@
// Draws inspiration from pwgen and http://tools.arantius.com/password
import type { KeyGeneratorConfig } from 'src/types/config'
import type { KeyGenerator } from 'src/types/key-generator'
import KeyGenerator from '.'
const randOf = (collection: string) => () =>
collection[Math.floor(Math.random() * collection.length)]
@ -10,13 +8,7 @@ const randOf = (collection: string) => () =>
const randVowel = randOf('aeiou')
const randConsonant = randOf('bcdfghjklmnpqrstvwxyz')
class PhoneticKeyGenerator implements KeyGenerator {
type: string
constructor(options: KeyGeneratorConfig) {
this.type = options.type
}
class PhoneticKeyGenerator extends KeyGenerator {
// Generate a phonetic key of alternating consonant & vowel
// eslint-disable-next-line class-methods-use-this
createKey(keyLength: number) {

View File

@ -1,17 +1,15 @@
import type { KeyGeneratorConfig } from 'src/types/config'
import type { KeyGenerator } from 'src/types/key-generator'
class RandomKeyGenerator implements KeyGenerator {
type: string
import KeyGenerator from '.'
class RandomKeyGenerator extends KeyGenerator {
keyspace: string
// Initialize a new generator with the given keySpace
constructor(options: KeyGeneratorConfig) {
super(options)
this.keyspace =
options.keyspace ||
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
this.type = options.type
}
// Generate a key of the given length

View File

@ -51,8 +51,8 @@ buildDocumenthandler(config)
}
// Send the static documents into the preferred store, skipping expirations
let documentPath
let data
let documentPath: string
let data: string
Object.keys(config.documents).forEach(name => {
documentPath = config.documents[name]

1
src/types/callback.ts Normal file
View File

@ -0,0 +1 @@
export type Callback = (data: boolean | string) => void

View File

@ -1,5 +1,6 @@
import { Logging } from './log'
import { RateLimits } from './rate-limits'
import { StoreNames } from './store-names'
export interface Config {
host: string
@ -11,8 +12,9 @@ export interface Config {
logging: Logging[]
keyGenerator: KeyGeneratorConfig
rateLimits: RateLimits
storage: StoreConfig
storage: unknown
documents: Record<string, string>
storeName: StoreNames
}
export type BaseStoreConfig = {
@ -52,7 +54,7 @@ export interface RethinkDbStoreConfig extends BaseStoreConfig {
export interface RedisStoreConfig extends BaseStoreConfig {
url?: string
db?: string
db?: number
user?: string
username?: string | undefined
password?: string
@ -62,17 +64,6 @@ export interface RedisStoreConfig extends BaseStoreConfig {
export type GoogleStoreConfig = BaseStoreConfig
export type StoreConfig =
| MongoStoreConfig
| MemcachedStoreConfig
| GoogleStoreConfig
| AmazonStoreConfig
| FileStoreConfig
| MongoStoreConfig
| RedisStoreConfig
| RethinkDbStoreConfig
| PostgresStoreConfig
export interface KeyGeneratorConfig {
type: string
keyspace?: string

View File

@ -1,6 +1,6 @@
import KeyGenerator from 'src/lib/key-generators'
import type { Config } from './config'
import type { KeyGenerator } from './key-generator'
import type { Store } from './store'
import type { Store } from './callback'
export type Document = {
store: Store

View File

@ -1,4 +0,0 @@
export interface KeyGenerator {
type: string
createKey?: (a: number) => string
}

View File

@ -1,6 +1,4 @@
export interface Logging {
level: string
type:
export type LoggingType =
| 'File'
| 'Console'
| 'Loggly'
@ -8,4 +6,8 @@ export interface Logging {
| 'Http'
| 'Memory'
| 'Webhook'
export interface Logging {
level: string
type: LoggingType
}

11
src/types/store-names.ts Normal file
View File

@ -0,0 +1,11 @@
// eslint-disable-next-line import/prefer-default-export
export enum StoreNames {
amazons3 = 'amazon-s3',
file = 'file',
googledatastore = 'google-datastore',
memcached = 'memcached',
mongo = 'mongo',
postgres = 'postgres',
redis = 'redis',
rethinkdb = 'rethinkdb'
}

View File

@ -1,13 +0,0 @@
export type Callback = (arg0: boolean | string) => void
export interface Store {
type: string
expire?: number
get: (key: string, callback: Callback, skipExpire?: boolean) => void
set: (
key: string,
data: string,
callback: Callback,
skipExpire?: boolean
) => void
}

View File

@ -2,14 +2,14 @@ import { createMock } from 'ts-auto-mock';
import DocumentHandler from 'src/lib/document-handler/index'
import Generator from 'src/lib/key-generators/random'
import constants from 'src/constants'
import { Store } from 'src/types/store'
import { Config } from 'src/types/config'
import { Store } from 'src/lib/document-stores';
const store : Store = createMock<Store>();
const config : Config = createMock<Config>();
describe('document-handler', () => {
describe('with randomKey', () => {
describe('with random key', () => {
it('should choose a key of the proper length', () => {
const gen = new Generator({ type: 'random' })
const dh = new DocumentHandler({ keyLength: 6, keyGenerator: gen, store, config})
@ -20,7 +20,6 @@ describe('document-handler', () => {
const gen = new Generator({ type: 'random' })
const dh = new DocumentHandler({ keyGenerator: gen, maxLength: 1, store, config })
expect(dh.keyLength).toEqual(constants.DEFAULT_KEY_LENGTH);
})
})
})

View File

@ -29,6 +29,9 @@
"rootDir": ".",
"outDir": "dist",
"baseUrl": ".",
"paths": {
"src/*": ["./src/*"]
}
},
"include": ["src", "**/*.ts"],
"exclude": ["node_modules"]

View File

@ -5527,6 +5527,15 @@ tsconfig-paths@^3.14.1:
minimist "^1.2.6"
strip-bom "^3.0.0"
tsconfig-paths@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.0.0.tgz#1082f5d99fd127b72397eef4809e4dd06d229b64"
integrity sha512-SLBg2GBKlR6bVtMgJJlud/o3waplKtL7skmLkExomIiaAtLGtVsoXIqP3SYdjbcH9lq/KVv7pMZeCBpLYOit6Q==
dependencies:
json5 "^2.2.1"
minimist "^1.2.6"
strip-bom "^3.0.0"
tslib@^1.8.1, tslib@^1.9.0:
version "1.14.1"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"