fix(Generate deterministic entity ids, remove shortId): generate dete…

…rministic entity ids, remove sh (#473)

affects: @joystream/hydra-db-utils, @joystream/hydra-indexer, @joystream/hydra-processor

ISSUES CLOSED: #399

packages/hydra-db-utils/package.json

@@ -24,12 +24,10 @@
     "bn.js": "^5.1.3",
     "ioredis": "^4.17.3",
     "lodash": "^4.17.20",
-    "shortid": "^2.2.16",
     "typeorm": "^0.2.25"
   },
   "devDependencies": {
     "@types/bn.js": "^4.11.6",
-    "@types/shortid": "^0.0.29",
     "mocha": "^8.1.3",
     "nyc": "^15.1.0",
     "ts-node": "^10.2.1",

packages/hydra-indexer/package.json

@@ -43,7 +43,6 @@
     "@types/express": "^4.17.8",
     "@types/ioredis": "^4.17.4",
     "@types/lodash": "^4.14.161",
-    "@types/shortid": "^0.0.29",
     "bn.js": "^5.1.2",
     "commander": "^6.2.0",
     "debug": "^4.1.1",
@@ -68,7 +67,6 @@
     "prom-client": "^12.0.0",
     "reflect-metadata": "^0.1.13",
     "set-interval-async": "~2.0.1",
-    "shortid": "^2.2.15",
     "typeorm": "^0.2.25",
     "util": "^0.12.3"
   },

packages/hydra-processor/package.json

@@ -56,7 +56,6 @@
     "p-whilst": "~2.1.0",
     "prom-client": "^12.0.0",
     "semver": "^7.3.4",
-    "shortid": "^2.2.16",
     "typedi": "^0.8.0",
     "yaml": "^1.10.0",
     "yaml-validator": "^3.0.0"

packages/hydra-processor/src/executor/TransactionalExecutor.ts

@@ -1,5 +1,4 @@
 import { getConnection, EntityManager } from 'typeorm'
-import * as shortid from 'shortid'
 import { getConfig as conf } from '../start/config'
 import Debug from 'debug'
 import { info } from '../util/log'
@@ -12,6 +11,7 @@ import {
   DatabaseManager,
 } from '@joystream/hydra-common'
 import { TxAwareBlockContext } from './tx-aware'
+import { ObjectType } from 'typedi'
 
 const debug = Debug('hydra-processor:mappings-executor')
 
@@ -85,6 +85,76 @@ export class TransactionalExecutor implements IMappingExecutor {
   }
 }
 
+class EntityIdGenerator {
+  private entityClass: ObjectType<{ id: string }>
+  private lastKnownEntityId: string | undefined
+  // each id is 6 chars out of 62-size alphabet, giving us 56800235584 possible ids (per entity type)
+  public static alphabet =
+    '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
+
+  public static idSize = 6
+
+  constructor(entityClass: ObjectType<{ id: string }>) {
+    this.entityClass = entityClass
+  }
+
+  private async queryLastEntityId(
+    em: EntityManager
+  ): Promise<string | undefined> {
+    const lastEntity = await em.findOne(this.entityClass, {
+      order: { id: 'DESC' },
+    })
+
+    return lastEntity?.id
+  }
+
+  private async getLastKnownEntityId(
+    em: EntityManager
+  ): Promise<string | undefined> {
+    if (this.lastKnownEntityId === undefined) {
+      this.lastKnownEntityId = await this.queryLastEntityId(em)
+    }
+    return this.lastKnownEntityId
+  }
+
+  public async createNextEntityId(em: EntityManager): Promise<string> {
+    const lastKnownId = await this.getLastKnownEntityId(em)
+    const { alphabet, idSize } = EntityIdGenerator
+    if (!lastKnownId) {
+      this.lastKnownEntityId = Array.from(
+        { length: idSize },
+        () => alphabet[0]
+      ).join('')
+      return this.lastKnownEntityId
+    }
+
+    let targetIdIndexToChange = idSize - 1
+    while (
+      targetIdIndexToChange >= 0 &&
+      lastKnownId[targetIdIndexToChange] === alphabet[alphabet.length - 1]
+    ) {
+      --targetIdIndexToChange
+    }
+
+    if (targetIdIndexToChange < 0) {
+      throw new Error('EntityIdGenerator: Ran out of possible ids!')
+    }
+
+    const nextEntityIdChars = [...lastKnownId]
+    const nextAlphabetCharIndex =
+      alphabet.indexOf(lastKnownId[targetIdIndexToChange]) + 1
+    nextEntityIdChars[targetIdIndexToChange] = alphabet[nextAlphabetCharIndex]
+    for (let i = idSize - 1; i > targetIdIndexToChange; --i) {
+      nextEntityIdChars[i] = alphabet[0]
+    }
+
+    this.lastKnownEntityId = nextEntityIdChars.join('')
+    return this.lastKnownEntityId
+  }
+}
+
+const entityIdGenerators = new Map<string, EntityIdGenerator>()
+
 /**
  * Create database manager.
  * @param entityManager EntityManager
@@ -95,7 +165,7 @@ export function makeDatabaseManager(
 ): DatabaseManager {
   return {
     save: async <T>(entity: DeepPartial<T>): Promise<void> => {
-      entity = fillRequiredWarthogFields(entity, blockData)
+      entity = await fillRequiredWarthogFields(entity, entityManager, blockData)
       await entityManager.save(entity)
     },
     remove: async <T>(entity: DeepPartial<T>): Promise<void> => {
@@ -127,17 +197,35 @@ export function makeDatabaseManager(
  *
  * @param entity: DeepPartial<T>
  */
-function fillRequiredWarthogFields<T>(
+async function fillRequiredWarthogFields<T>(
   entity: DeepPartial<T>,
+  entityManager: EntityManager,
   { block }: BlockData
-): DeepPartial<T> {
+): Promise<DeepPartial<T>> {
   // eslint-disable-next-line no-prototype-builtins
   if (!entity.hasOwnProperty('id')) {
-    Object.assign(entity, { id: shortid.generate() })
+    const entityClass = ((entity as unknown) as {
+      constructor: ObjectType<{ id: string }>
+    }).constructor
+
+    if (!entityIdGenerators.has(entityClass.name)) {
+      entityIdGenerators.set(
+        entityClass.name,
+        new EntityIdGenerator(entityClass)
+      )
+    }
+
+    const idGenerator = entityIdGenerators.get(
+      entityClass.name
+    ) as EntityIdGenerator
+
+    Object.assign(entity, {
+      id: await idGenerator.createNextEntityId(entityManager),
+    })
   }
   // eslint-disable-next-line no-prototype-builtins
   if (!entity.hasOwnProperty('createdById')) {
-    Object.assign(entity, { createdById: shortid.generate() })
+    Object.assign(entity, { createdById: '-' })
   }
   // eslint-disable-next-line no-prototype-builtins
   if (!entity.hasOwnProperty('version')) {

yarn.lock

@@ -14756,7 +14756,7 @@ shimmer@^1.1.0, shimmer@^1.2.0:
   resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337"
   integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==
 
-shortid@^2.2.15, shortid@^2.2.16:
+shortid@^2.2.15:
   version "2.2.16"
   resolved "https://registry.yarnpkg.com/shortid/-/shortid-2.2.16.tgz#b742b8f0cb96406fd391c76bfc18a67a57fe5608"
   integrity sha512-Ugt+GIZqvGXCIItnsL+lvFJOiN7RYqlGy7QE41O3YC1xbNSeDGIRO7xg2JJXIAj1cAGnOeC1r7/T9pgrtQbv4g==