From 0b7d18f1cd7c52f3b77998a0fce21e797ba510bc Mon Sep 17 00:00:00 2001 From: Lezek123 Date: Tue, 22 Feb 2022 14:39:26 +0100 Subject: [PATCH] fix(Generate deterministic entity ids, remove shortId): generate deterministic entity ids, remove sh (#473) affects: @joystream/hydra-db-utils, @joystream/hydra-indexer, @joystream/hydra-processor ISSUES CLOSED: #399 --- packages/hydra-db-utils/package.json | 2 - packages/hydra-indexer/package.json | 2 - packages/hydra-processor/package.json | 1 - .../src/executor/TransactionalExecutor.ts | 100 ++++++++++++++++-- yarn.lock | 2 +- 5 files changed, 95 insertions(+), 12 deletions(-) diff --git a/packages/hydra-db-utils/package.json b/packages/hydra-db-utils/package.json index a02a9483c..980d0e654 100644 --- a/packages/hydra-db-utils/package.json +++ b/packages/hydra-db-utils/package.json @@ -24,12 +24,10 @@ "bn.js": "^5.1.3", "ioredis": "^4.17.3", "lodash": "^4.17.20", - "shortid": "^2.2.16", "typeorm": "^0.2.25" }, "devDependencies": { "@types/bn.js": "^4.11.6", - "@types/shortid": "^0.0.29", "mocha": "^8.1.3", "nyc": "^15.1.0", "ts-node": "^10.2.1", diff --git a/packages/hydra-indexer/package.json b/packages/hydra-indexer/package.json index c67122699..c83ad2a0d 100644 --- a/packages/hydra-indexer/package.json +++ b/packages/hydra-indexer/package.json @@ -43,7 +43,6 @@ "@types/express": "^4.17.8", "@types/ioredis": "^4.17.4", "@types/lodash": "^4.14.161", - "@types/shortid": "^0.0.29", "bn.js": "^5.1.2", "commander": "^6.2.0", "debug": "^4.1.1", @@ -68,7 +67,6 @@ "prom-client": "^12.0.0", "reflect-metadata": "^0.1.13", "set-interval-async": "~2.0.1", - "shortid": "^2.2.15", "typeorm": "^0.2.25", "util": "^0.12.3" }, diff --git a/packages/hydra-processor/package.json b/packages/hydra-processor/package.json index 09d1d20d2..d57d5b573 100644 --- a/packages/hydra-processor/package.json +++ b/packages/hydra-processor/package.json @@ -56,7 +56,6 @@ "p-whilst": "~2.1.0", "prom-client": "^12.0.0", "semver": "^7.3.4", - "shortid": "^2.2.16", "typedi": "^0.8.0", "yaml": "^1.10.0", "yaml-validator": "^3.0.0" diff --git a/packages/hydra-processor/src/executor/TransactionalExecutor.ts b/packages/hydra-processor/src/executor/TransactionalExecutor.ts index e0f1e3dce..2bb76f4ee 100644 --- a/packages/hydra-processor/src/executor/TransactionalExecutor.ts +++ b/packages/hydra-processor/src/executor/TransactionalExecutor.ts @@ -1,5 +1,4 @@ import { getConnection, EntityManager } from 'typeorm' -import * as shortid from 'shortid' import { getConfig as conf } from '../start/config' import Debug from 'debug' import { info } from '../util/log' @@ -12,6 +11,7 @@ import { DatabaseManager, } from '@joystream/hydra-common' import { TxAwareBlockContext } from './tx-aware' +import { ObjectType } from 'typedi' const debug = Debug('hydra-processor:mappings-executor') @@ -85,6 +85,76 @@ export class TransactionalExecutor implements IMappingExecutor { } } +class EntityIdGenerator { + private entityClass: ObjectType<{ id: string }> + private lastKnownEntityId: string | undefined + // each id is 6 chars out of 62-size alphabet, giving us 56800235584 possible ids (per entity type) + public static alphabet = + '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + + public static idSize = 6 + + constructor(entityClass: ObjectType<{ id: string }>) { + this.entityClass = entityClass + } + + private async queryLastEntityId( + em: EntityManager + ): Promise { + const lastEntity = await em.findOne(this.entityClass, { + order: { id: 'DESC' }, + }) + + return lastEntity?.id + } + + private async getLastKnownEntityId( + em: EntityManager + ): Promise { + if (this.lastKnownEntityId === undefined) { + this.lastKnownEntityId = await this.queryLastEntityId(em) + } + return this.lastKnownEntityId + } + + public async createNextEntityId(em: EntityManager): Promise { + const lastKnownId = await this.getLastKnownEntityId(em) + const { alphabet, idSize } = EntityIdGenerator + if (!lastKnownId) { + this.lastKnownEntityId = Array.from( + { length: idSize }, + () => alphabet[0] + ).join('') + return this.lastKnownEntityId + } + + let targetIdIndexToChange = idSize - 1 + while ( + targetIdIndexToChange >= 0 && + lastKnownId[targetIdIndexToChange] === alphabet[alphabet.length - 1] + ) { + --targetIdIndexToChange + } + + if (targetIdIndexToChange < 0) { + throw new Error('EntityIdGenerator: Ran out of possible ids!') + } + + const nextEntityIdChars = [...lastKnownId] + const nextAlphabetCharIndex = + alphabet.indexOf(lastKnownId[targetIdIndexToChange]) + 1 + nextEntityIdChars[targetIdIndexToChange] = alphabet[nextAlphabetCharIndex] + for (let i = idSize - 1; i > targetIdIndexToChange; --i) { + nextEntityIdChars[i] = alphabet[0] + } + + this.lastKnownEntityId = nextEntityIdChars.join('') + return this.lastKnownEntityId + } +} + +const entityIdGenerators = new Map() + /** * Create database manager. * @param entityManager EntityManager @@ -95,7 +165,7 @@ export function makeDatabaseManager( ): DatabaseManager { return { save: async (entity: DeepPartial): Promise => { - entity = fillRequiredWarthogFields(entity, blockData) + entity = await fillRequiredWarthogFields(entity, entityManager, blockData) await entityManager.save(entity) }, remove: async (entity: DeepPartial): Promise => { @@ -127,17 +197,35 @@ export function makeDatabaseManager( * * @param entity: DeepPartial */ -function fillRequiredWarthogFields( +async function fillRequiredWarthogFields( entity: DeepPartial, + entityManager: EntityManager, { block }: BlockData -): DeepPartial { +): Promise> { // eslint-disable-next-line no-prototype-builtins if (!entity.hasOwnProperty('id')) { - Object.assign(entity, { id: shortid.generate() }) + const entityClass = ((entity as unknown) as { + constructor: ObjectType<{ id: string }> + }).constructor + + if (!entityIdGenerators.has(entityClass.name)) { + entityIdGenerators.set( + entityClass.name, + new EntityIdGenerator(entityClass) + ) + } + + const idGenerator = entityIdGenerators.get( + entityClass.name + ) as EntityIdGenerator + + Object.assign(entity, { + id: await idGenerator.createNextEntityId(entityManager), + }) } // eslint-disable-next-line no-prototype-builtins if (!entity.hasOwnProperty('createdById')) { - Object.assign(entity, { createdById: shortid.generate() }) + Object.assign(entity, { createdById: '-' }) } // eslint-disable-next-line no-prototype-builtins if (!entity.hasOwnProperty('version')) { diff --git a/yarn.lock b/yarn.lock index bdf07c2e0..6cf10d2ff 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14756,7 +14756,7 @@ shimmer@^1.1.0, shimmer@^1.2.0: resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337" integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw== -shortid@^2.2.15, shortid@^2.2.16: +shortid@^2.2.15: version "2.2.16" resolved "https://registry.yarnpkg.com/shortid/-/shortid-2.2.16.tgz#b742b8f0cb96406fd391c76bfc18a67a57fe5608" integrity sha512-Ugt+GIZqvGXCIItnsL+lvFJOiN7RYqlGy7QE41O3YC1xbNSeDGIRO7xg2JJXIAj1cAGnOeC1r7/T9pgrtQbv4g==