feat(hydra-processor): use mappings contexts and support specVersion …

…filters (Joystream#375)

* refactor(hydra-common): remove util functions from hydra-common and update references

affects: @dzlzv/hydra-common, @dzlzv/hydra-indexer, @dzlzv/hydra-processor

* feat(hydra-processor): wIP: use refactored hydra-common interfaces

affects: @dzlzv/hydra-common, @dzlzv/hydra-processor

* feat(hydra-common): move BlockPayload interfaces to hydra-common

affects: @dzlzv/hydra-common, @dzlzv/hydra-indexer

* feat(hydra-processor): wip 2

* refactor(hydra-indexer): move FIFOCache to hydra-common

affects: @dzlzv/hydra-common, @dzlzv/hydra-indexer

* chore(sample): add docker-compose-dev for starting a local indexer

affects: sample

* feat(hydra-processor): wip2: block queries + handler filters

affects: @dzlzv/hydra-processor, sample, sample-mappings

* feat(hydra-common): add context interfaces and move database-manager to hydra-common

affects: @dzlzv/hydra-common

* fix(hydra-indexer): use BigInt and update DatabaseManager imports

affects: @dzlzv/hydra-indexer

* feat(hydra-processor): add specVersion filtering and cleanup

affects: @dzlzv/hydra-processor

* feat(hydra-db-utils): move DatabaseManager to hydra-common

affects: @dzlzv/hydra-db-utils

* feat(sample): use Context interfaces in the mappings

affects: sample, sample-mappings

* fix(hydra-processor): add filtering for hooks

affects: @dzlzv/hydra-processor

* fix(hydra-processor): add reviver for BigInt/Number fields that are not properly deserialized from t

affects: @dzlzv/hydra-processor

* fix(hydra-indexer): initialized from BLOCK_HEIGHT if not previous head was saved (fixes undefined he

affects: @dzlzv/hydra-indexer

* test(sample): add DateTime field to sample

affects: sample, sample-mappings

* test(hydra-e2e-tests): fix manifest and mappings in e2e-tests

affects: hydra-e2e-tests

* fix(hydra-processor): fix tests

affects: @dzlzv/hydra-processor

packages/hydra-common/package.json

@@ -14,7 +14,8 @@
     "build": "rm -rf lib && tsc --build tsconfig.json",
     "lint": "eslint . --cache --ext .ts",
     "prepack": "yarn build",
-    "test": "echo 'No tests' && exit 1"
+    "test": "nyc --extension .ts mocha --timeout 50000 --require ts-node/register --forbid-only \"src/**/*.test.ts\""
+
   },
   "dependencies": {
     "bn.js": "^5.1.3"
@@ -23,6 +24,8 @@
     "@types/bn.js": "^4.11.6",
     "@types/debug": "^4.1.5",
     "ts-node": "^9.0.0",
-    "typescript": "^3.8"
+    "typescript": "^3.8",
+    "mocha": "^8.1.3",
+    "nyc": "^15.1.0"
   }
 }

packages/hydra-common/src/interfaces/contexts.ts

@@ -0,0 +1,24 @@
+import { SubstrateBlock, SubstrateEvent, SubstrateExtrinsic } from '.'
+import { DatabaseManager } from './store'
+
+export interface StoreContext {
+  store: DatabaseManager
+}
+
+export interface BlockContext {
+  block: SubstrateBlock
+}
+
+export interface EventContext extends BlockContext {
+  event: SubstrateEvent
+  extrinsic?: SubstrateExtrinsic
+}
+
+export interface ExtrinsicContext extends BlockContext {
+  event: SubstrateEvent
+  extrinsic: SubstrateExtrinsic
+}
+
+export type MappingContext = BlockContext | EventContext | ExtrinsicContext
+
+export type ExecContext = StoreContext & MappingContext

packages/hydra-common/src/interfaces/index.ts

@@ -1,2 +1,6 @@
 export * from './json-types'
+export * from './payloads'
+export * from './contexts'
+export * from './store'
+export * from './typeorm'
 export * from './substrate-interfaces'

packages/hydra-indexer/src/model/BlockPayload.ts → packages/hydra-common/src/interfaces/payloads.ts

@@ -1,29 +1,19 @@
-import { SubstrateBlock } from '@dzlzv/hydra-common'
+import { SubstrateBlock } from './substrate-interfaces'
 import { pick } from 'lodash'
 
+/**
+ * General block information. Typically used as a payload for lightweight subscription messages.
+ */
 export interface BlockPayload {
   height: number
   hash: string
+  parentHash: string
   ts: number
   events: { id: string; name: string }[]
   extrinsics: { id: string; name: string }[]
+  runtimeVersion: { specVersion?: string }
 }
 
-// export function toPayload(qeb: {
-//   blockNumber: number
-//   blockEvents: { eventName: string }[]
-// }): BlockPayload {
-//   return (withTs({
-//     height: qeb.blockNumber,
-//     events: qeb.blockEvents.map((e, index) => {
-//       return {
-//         name: e.eventName,
-//         id: formatEventId(qeb.blockNumber, index),
-//       }
-//     }),
-//   }) as unknown) as BlockPayload
-// }
-
 export function toPayload(sb: SubstrateBlock): BlockPayload {
   return <BlockPayload>{
     ...pick(sb, [

packages/hydra-common/src/interfaces/store.ts

@@ -0,0 +1,35 @@
+import { DeepPartial, FindOneOptions } from './typeorm'
+
+export interface DatabaseManager {
+  /**
+   * Save given entity instance, if entity is exists then just update
+   * @param entity
+   */
+  save<T>(entity: DeepPartial<T>): Promise<void>
+
+  /**
+   * Removes a given entity from the database.
+   * @param entity: DeepPartial<T>
+   */
+  remove<T>(entity: DeepPartial<T>): Promise<void>
+
+  /**
+   * Finds first entity that matches given options.
+   * @param entity: T
+   * @param options: FindOneOptions<T>
+   */
+  get<T>(
+    entity: { new (...args: any[]): T },
+    options: FindOneOptions<T>
+  ): Promise<T | undefined>
+
+  /**
+   * Finds entities that match given options.
+   * @param entity: T
+   * @param options: FindOneOptions<T>
+   */
+  getMany<T>(
+    entity: { new (...args: any[]): T },
+    options: FindOneOptions<T>
+  ): Promise<T[]>
+}

packages/hydra-common/src/interfaces/substrate-interfaces.ts

@@ -1,5 +1,4 @@
 import { AnyJson, AnyJsonField } from './json-types'
-import * as BN from 'bn.js'
 
 export interface EventParam {
   type: string
@@ -25,6 +24,11 @@ export interface ExtrinsicInfo {
 }
 
 export interface SubstrateBlock {
+  /**
+   * Unique block id, following the format <block height>-<first hex digits of the hash>
+   */
+  id: string
+
   /**
    * Current block hash
    */
@@ -175,58 +179,10 @@ export interface SubstrateExtrinsic {
   /**
    * Extrinsic tip
    */
-  tip: BN
+  tip: BigInt
 
   /**
    * Ordinal index in the event array of the current block
    */
   indexInBlock: number
 }
-
-export const BLOCK_PAD_LENGTH = 10
-export const INDEX_PAD_LENGTH = 6
-export const HASH_PAD_LENGTH = 5
-
-/**
- * Formats the event id into a fixed-lentgth string. When formatted the natural string ordering
- * is the same as the ordering
- * in the blockchain (first ordered by block height, then by block ID)
- *
- * @return  id in the format 000000..00<blockNum>-000<index>
- *
- */
-export function formatEventId(blockNumber: number, index: number): string {
-  const blockPart = `${String(blockNumber).padStart(BLOCK_PAD_LENGTH, '0')}`
-  const indexPart = `${String(index).padStart(INDEX_PAD_LENGTH, '0')}`
-  return `${blockPart}-${indexPart}`
-}
-
-/**
- * Formats the event id into a fixed-lentgth string. When formatted the natural string ordering
- * is the same as the ordering
- * in the blockchain (first ordered by block height, then by block ID)
- *
- * @return  id in the format 000000..00<blockNum>-000<index>-<shorthash>
- *
- */
-export function formatId({
-  height,
-  index,
-  hash,
-}: {
-  height: number
-  index?: number
-  hash: string
-}): string {
-  const blockPart = `${String(height).padStart(BLOCK_PAD_LENGTH, '0')}`
-  const indexPart =
-    index !== undefined
-      ? `-${String(index).padStart(INDEX_PAD_LENGTH, '0')}`
-      : ''
-  const _hash = hash.startsWith('0x') ? hash.substring(2) : hash
-  const shortHash =
-    _hash.length < HASH_PAD_LENGTH
-      ? _hash.padEnd(HASH_PAD_LENGTH, '0')
-      : _hash.slice(0, HASH_PAD_LENGTH)
-  return `${blockPart}${indexPart}-${shortHash}`
-}