Skip to content

Commit

Permalink
feat: deterministic id generator - take deletions into account (#498)
Browse files Browse the repository at this point in the history
* feat: deterministic id generator - take deletions into account

affects: @joystream/hydra-cli, @joystream/hydra-e2e-tests, @joystream/hydra-processor, sample

* fix: tests fix - broken JSONField definition

affects: @joystream/hydra-cli, @joystream/hydra-indexer-gateway
  • Loading branch information
ondratra authored Jul 12, 2022
1 parent ca87229 commit a626fb6
Show file tree
Hide file tree
Showing 15 changed files with 179 additions and 115 deletions.
4 changes: 2 additions & 2 deletions packages/hydra-cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"@inquirer/input": "^0.0.13-alpha.0",
"@inquirer/password": "^0.0.12-alpha.0",
"@inquirer/select": "^0.0.13-alpha.0",
"@joystream/warthog": "^2.41.7",
"@joystream/warthog": "^2.41.8",
"@oclif/command": "^1.5.20",
"@oclif/config": "^1",
"@oclif/errors": "^1.3.3",
Expand Down Expand Up @@ -80,7 +80,7 @@
"lodash": "^4.17.20",
"pg": "^8.3.2",
"pg-listen": "^1.7.0",
"@joystream/warthog": "^2.41.7"
"@joystream/warthog": "^2.41.8"
},
"devDependencies": {
"@oclif/dev-cli": "^1",
Expand Down
2 changes: 1 addition & 1 deletion packages/hydra-cli/src/templates/scaffold/manifest.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: '3.0'
description: Test manifest
repository: /~https://github.com/
hydraVersion: "3"
hydraVersion: "4"
dataSource:
kind: substrate
chain: node-template
Expand Down
2 changes: 1 addition & 1 deletion packages/hydra-e2e-tests/fixtures/manifest.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: '3.0'
description: Test manifest
repository: /~https://github.com/
hydraVersion: "3"
hydraVersion: "4"
dataSource:
kind: substrate
chain: node-template
Expand Down
6 changes: 3 additions & 3 deletions packages/hydra-e2e-tests/test/e2e/transfer-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
TRANSFER_IN_QUERY,
VARIANT_FILTER_MISREABLE_ACCOUNTS,
} from './api/graphql-queries'
import { EntityIdGenerator } from '@joystream/hydra-processor/src/executor/TransactionalExecutor'
import { EntityIdGenerator } from '@joystream/hydra-processor/src/executor/EntityIdGenerator'
// You need to be connected to a development chain for this example to work.
const ALICE = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'
const BOB = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'
Expand Down Expand Up @@ -236,9 +236,9 @@ describe('end-to-end transfer tests', () => {

it('should create transfer chunks with auto-generated ids', async () => {
const chunks = await transferChunksByTransferId(
EntityIdGenerator.firstEntityId
EntityIdGenerator.entityIdAfter(EntityIdGenerator.zeroEntityId)
)
let id: string | undefined
let id: string = EntityIdGenerator.zeroEntityId
const expectedIds = Array.from(
{ length: 100 },
() => (id = EntityIdGenerator.entityIdAfter(id))
Expand Down
2 changes: 1 addition & 1 deletion packages/hydra-indexer-gateway/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
"@joystream/bn-typeorm": "^4.0.0-alpha.1",
"@joystream/hydra-common": "^4.0.0-alpha.1",
"@joystream/hydra-db-utils": "^4.0.0-alpha.1",
"@joystream/warthog": "^2.41.7",
"@joystream/warthog": "^2.41.8",
"@types/ioredis": "^4.17.4",
"bn.js": "^5.2.1",
"dotenv": "^8.2.0",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ export class SubstrateExtrinsic extends BaseModel {
@StringField()
section!: string

@JSONField()
@JSONField({ array: true })
args!: ExtrinsicArg[]

@StringField()
Expand Down
8 changes: 6 additions & 2 deletions packages/hydra-processor/src/db/ormconfig.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { ConnectionOptions } from 'typeorm'
import { SnakeNamingStrategy } from '@joystream/hydra-db-utils'
import { ProcessedEventsLogEntity } from '../entities/ProcessedEventsLogEntity'
import { ProcessedEventsLogEntity, DeterministicIdEntity } from '../entities'

const config: () => ConnectionOptions = () => {
// ugly, but we need to set the warthog envs, otherwise it fails
Expand All @@ -16,7 +16,11 @@ const config: () => ConnectionOptions = () => {
username: process.env.TYPEORM_USERNAME || process.env.DB_USER,
password: process.env.TYPEORM_PASSWORD || process.env.DB_PASS,
database: process.env.TYPEORM_DATABASE || process.env.DB_NAME,
entities: [ProcessedEventsLogEntity, process.env.TYPEORM_ENTITIES],
entities: [
ProcessedEventsLogEntity,
DeterministicIdEntity,
process.env.TYPEORM_ENTITIES,
],
migrations: [`${__dirname}/../**/migrations/*.{ts,js}`],
cli: {
migrationsDir: 'migrations',
Expand Down
18 changes: 18 additions & 0 deletions packages/hydra-processor/src/entities/DeterministicIdEntity.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { Entity, PrimaryColumn, Column } from 'typeorm'

@Entity({
name: 'deterministic_id',
})
export class DeterministicIdEntity {
@PrimaryColumn()
className!: string

@Column()
highestId!: string

// When the event is added to the database
@Column('timestamp without time zone', {
default: () => 'now()',
})
updatedAt!: Date
}
1 change: 1 addition & 0 deletions packages/hydra-processor/src/entities/index.ts
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
export * from './ProcessedEventsLogEntity'
export * from './DeterministicIdEntity'
119 changes: 119 additions & 0 deletions packages/hydra-processor/src/executor/EntityIdGenerator.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
import { EntityManager } from 'typeorm'
import { ObjectType } from 'typedi'
import AsyncLock from 'async-lock'
import { DeterministicIdEntity } from '../entities'

export class EntityIdGenerator {
private entityClass: ObjectType<{ id: string }>
private entityRecord: DeterministicIdEntity | null = null
private static lock = new AsyncLock({ maxPending: 10000 })
// each id is 8 chars out of 36-size alphabet, giving us 2821109907456 possible ids (per entity type)
public static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz'

public static idSize = 8

// this id will actually never by used in db - use .entityIdAfter(zeroEntityId) to get first id
public static zeroEntityId = Array.from(
{ length: EntityIdGenerator.idSize },
() => EntityIdGenerator.alphabet[0]
).join('')

constructor(entityClass: ObjectType<{ id: string }>) {
this.entityClass = entityClass
}

public static entityIdAfter(id: string): string {
const { alphabet, idSize } = EntityIdGenerator
let targetIdIndexToChange = idSize - 1
while (
targetIdIndexToChange >= 0 &&
id[targetIdIndexToChange] === alphabet[alphabet.length - 1]
) {
--targetIdIndexToChange
}

if (targetIdIndexToChange < 0) {
throw new Error(`EntityIdGenerator: Cannot get entity id after: ${id}!`)
}

const nextEntityIdChars = [...id]
const nextAlphabetCharIndex =
alphabet.indexOf(id[targetIdIndexToChange]) + 1
nextEntityIdChars[targetIdIndexToChange] = alphabet[nextAlphabetCharIndex]
for (let i = idSize - 1; i > targetIdIndexToChange; --i) {
nextEntityIdChars[i] = alphabet[0]
}

return nextEntityIdChars.join('')
}

private async loadEntityRecord(
em: EntityManager
): Promise<DeterministicIdEntity> {
if (this.entityRecord) {
return this.entityRecord
}

// try load record from db
this.entityRecord = await em.findOne(DeterministicIdEntity, {
where: { className: this.entityClass.name },
})

// create new record if none exists
if (!this.entityRecord) {
this.entityRecord = new DeterministicIdEntity()
this.entityRecord.className = this.entityClass.name
this.entityRecord.highestId = EntityIdGenerator.zeroEntityId
}

return this.entityRecord
}

private async generateNextEntityId(em: EntityManager): Promise<string> {
// ensure entity record
const entityRecord = await this.loadEntityRecord(em)

// generate next id
const nextEntityId = EntityIdGenerator.entityIdAfter(entityRecord.highestId)

// save new id to db
entityRecord.highestId = nextEntityId
await em.save(entityRecord)

// return next id
return nextEntityId
}

public async getNextEntityId(em: EntityManager): Promise<string> {
return EntityIdGenerator.lock.acquire(this.entityClass.name, () =>
this.generateNextEntityId(em)
)
}
}

const entityIdGenerators = new Map<string, EntityIdGenerator>()

/*
Ensures entity id generator exists for the given entity class
*/
function ensureEntityIdGenerator(
entityClass: ObjectType<{ id: string }>
): EntityIdGenerator {
if (!entityIdGenerators.has(entityClass.name)) {
entityIdGenerators.set(entityClass.name, new EntityIdGenerator(entityClass))
}

return entityIdGenerators.get(entityClass.name) as EntityIdGenerator
}

/*
Generates next sequential id for the given entity class.
*/
export async function generateNextId(
em: EntityManager,
entityClass: ObjectType<{ id: string }>
): Promise<string> {
const idGenerator = ensureEntityIdGenerator(entityClass)

return idGenerator.getNextEntityId(em)
}
100 changes: 2 additions & 98 deletions packages/hydra-processor/src/executor/TransactionalExecutor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
} from '@joystream/hydra-common'
import { TxAwareBlockContext } from './tx-aware'
import { ObjectType } from 'typedi'
import AsyncLock from 'async-lock'
import { generateNextId } from './EntityIdGenerator'

const debug = Debug('hydra-processor:mappings-executor')

Expand Down Expand Up @@ -86,91 +86,6 @@ export class TransactionalExecutor implements IMappingExecutor {
}
}

export class EntityIdGenerator {
private entityClass: ObjectType<{ id: string }>
private nextEntityIdPromise: Promise<string> | undefined
private lastKnownEntityId: string | undefined
private static lock = new AsyncLock({ maxPending: 10000 })
// each id is 8 chars out of 36-size alphabet, giving us 2821109907456 possible ids (per entity type)
public static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz'

public static idSize = 8

public static firstEntityId = Array.from(
{ length: EntityIdGenerator.idSize },
() => EntityIdGenerator.alphabet[0]
).join('')

constructor(entityClass: ObjectType<{ id: string }>) {
this.entityClass = entityClass
}

public static entityIdAfter(id?: string): string {
if (id === undefined) {
return EntityIdGenerator.firstEntityId
}

const { alphabet, idSize } = EntityIdGenerator
let targetIdIndexToChange = idSize - 1
while (
targetIdIndexToChange >= 0 &&
id[targetIdIndexToChange] === alphabet[alphabet.length - 1]
) {
--targetIdIndexToChange
}

if (targetIdIndexToChange < 0) {
throw new Error(`EntityIdGenerator: Cannot get entity id after: ${id}!`)
}

const nextEntityIdChars = [...id]
const nextAlphabetCharIndex =
alphabet.indexOf(id[targetIdIndexToChange]) + 1
nextEntityIdChars[targetIdIndexToChange] = alphabet[nextAlphabetCharIndex]
for (let i = idSize - 1; i > targetIdIndexToChange; --i) {
nextEntityIdChars[i] = alphabet[0]
}

return nextEntityIdChars.join('')
}

private async queryLastEntityId(
em: EntityManager
): Promise<string | undefined> {
const lastEntity = await em.findOne(this.entityClass, {
where: {}, // required by typeorm '0.3.5'
order: { id: 'DESC' },
})

return lastEntity?.id
}

private async getLastKnownEntityId(
em: EntityManager
): Promise<string | undefined> {
if (this.lastKnownEntityId === undefined) {
this.lastKnownEntityId = await this.queryLastEntityId(em)
}
return this.lastKnownEntityId
}

private async generateNextEntityId(em: EntityManager): Promise<string> {
const lastKnownId = await this.getLastKnownEntityId(em)
const nextEntityId = EntityIdGenerator.entityIdAfter(lastKnownId)
this.lastKnownEntityId = nextEntityId

return this.lastKnownEntityId
}

public async getNextEntityId(em: EntityManager): Promise<string> {
return EntityIdGenerator.lock.acquire(this.entityClass.name, () =>
this.generateNextEntityId(em)
)
}
}

const entityIdGenerators = new Map<string, EntityIdGenerator>()

/**
* Create database manager.
* @param entityManager EntityManager
Expand Down Expand Up @@ -245,19 +160,8 @@ async function fillRequiredWarthogFields<T extends Record<string, unknown>>(
}
).constructor

if (!entityIdGenerators.has(entityClass.name)) {
entityIdGenerators.set(
entityClass.name,
new EntityIdGenerator(entityClass)
)
}

const idGenerator = entityIdGenerators.get(
entityClass.name
) as EntityIdGenerator

Object.assign(entity, {
id: await idGenerator.getNextEntityId(entityManager),
id: await generateNextId(entityManager, entityClass),
})
}
// eslint-disable-next-line no-prototype-builtins
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { MigrationInterface, QueryRunner } from 'typeorm'

export class DeterministicIdEntity1657020899263 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "deterministic_id" (
"class_name" character varying NOT NULL,
"highest_id" character varying NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "PK_eb01f11c447ff2ae7910670ce42" PRIMARY KEY ("class_name")
)`
)
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "deterministic_id"`)
}
}
2 changes: 1 addition & 1 deletion packages/hydra-processor/test/fixtures/manifest.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: '0.2'
description: Test manifest
repository: /~https://github.com/
hydraVersion: "3"
hydraVersion: '4'
dataSource:
kind: substrate
chain: node-template
Expand Down
2 changes: 1 addition & 1 deletion packages/sample/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ version: '3.0'
description: Sample manifest
repository: /~https://github.com/
## reserved but unused fields
hydraVersion: "3"
hydraVersion: '4'
dataSource:
kind: substrate
chain: node-template
Expand Down
Loading

0 comments on commit a626fb6

Please sign in to comment.