feat(Updated @polkadot/* dependencies, added support for metadata v14…

… in hydra-typegen): updated @po

affects: @joystream/bn-typeorm, @joystream/hydra-cli, @joystream/hydra-common,
@joystream/hydra-db-utils, @joystream/hydra-e2e-tests, @joystream/hydra-indexer-gateway,
@joystream/hydra-indexer, @joystream/hydra-processor, @joystream/hydra-typegen, sample,
sample-mappings

- Updated @polkadot/* dependencies (5.9.1 => 8.9.1)
- Updated bn.js (^5.2.1) and @types/bn.js
(^5.1.0)
- Updated prettier (^2.7.1)
- Added '@polkadot/api-augment/substrate' in
SubstrateService.ts
- Updated \`hydra-typegen\` to support v14 metadata parsing (type definitions in
metadata), dropped support for \`customTypes\` and older metadata

BREAKING CHANGE:
customTypes and metadata < v14 are no longer supported, @polkadot/api was updated from 5.9.1 =>
8.9.1

ISSUES CLOSED: Joystream#495

examples/v3/kusama-query-node/package.json

@@ -45,8 +45,8 @@
   "license": "ISC",
   "dependencies": {
     "tslib": "^2.0.0",
-    "@types/bn.js": "^4.11.6",
-    "bn.js": "^5.1.2"
+    "@types/bn.js": "^5.1.0",
+    "bn.js": "^5.2.1"
   },
   "devDependencies": {
     "@joystream/hydra-cli": "^3.1.0-alpha",

package.json

@@ -44,10 +44,10 @@
     "is-ci": "^3.0.0",
     "lerna": "^3.22.1",
     "lint-staged": "^10.5.1",
-    "prettier": "2.0.2"
+    "prettier": "^2.7.1"
   },
   "resolutions": {
-    "@polkadot/types": "5.9.1"
+    "@polkadot/types": "8.9.1"
   },
   "lint-staged": {
     "*.ts": "yarn lint --fix"

packages/bn-typeorm/package.json

@@ -19,11 +19,11 @@
     "lint": "eslint . --cache --ext .ts"
   },
   "dependencies": {
-    "bn.js": "^5.1.3",
+    "bn.js": "^5.2.1",
     "typeorm": "/~https://github.com/Joystream/typeorm/releases/download/0.3.5/typeorm-v0.3.5.tgz"
   },
   "devDependencies": {
-    "@types/bn.js": "^4.11.6",
+    "@types/bn.js": "^5.1.0",
     "eslint": "^7.12.1",
     "mocha": "^8.1.3",
     "nyc": "^15.1.0",

packages/hydra-cli/package.json

@@ -99,7 +99,7 @@
     "mocha": "^5",
     "mocha-chai-snapshot": "^1.0.0",
     "nyc": "^14",
-    "prettier": "^2.0.5",
+    "prettier": "^2.7.1",
     "spawn-command": "^0.0.2-1",
     "temp": "^0.9.1",
     "ts-node": "^10.2.1",

packages/hydra-cli/src/templates/scaffold/mappings/package.json.mst

@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "@joystream/hydra-common": "{{{hydraCommonVersion}}}",
-    "@polkadot/types": "5.9.1",
+    "@polkadot/types": "8.9.1",
     "@joystream/warthog": "{{{hydraWarthogVersion}}}"
   },
   "devDependencies": {

packages/hydra-cli/src/templates/scaffold/mappings/tsconfig.json

@@ -13,7 +13,8 @@
         "esModuleInterop": true,
         "experimentalDecorators": true,
         "emitDecoratorMetadata": true,
-        "skipLibCheck": true,    
+        "skipLibCheck": true,
+        "resolveJsonModule": true
     },
     "include": ["./**/*"]
 }

packages/hydra-common/package.json

@@ -20,10 +20,10 @@
     "test": "nyc --extension .ts mocha --timeout 50000 --require ts-node/register --forbid-only \"src/**/*.test.ts\""
   },
   "dependencies": {
-    "bn.js": "^5.1.3"
+    "bn.js": "^5.2.1"
   },
   "devDependencies": {
-    "@types/bn.js": "^4.11.6",
+    "@types/bn.js": "^5.1.0",
     "@types/debug": "^4.1.5",
     "mocha": "^8.1.3",
     "nyc": "^15.1.0",

packages/hydra-db-utils/package.json

@@ -21,13 +21,13 @@
   "dependencies": {
     "@joystream/hydra-common": "^3.1.0-alpha.27",
     "@types/ioredis": "^4.17.4",
-    "bn.js": "^5.1.3",
+    "bn.js": "^5.2.1",
     "ioredis": "^4.17.3",
     "lodash": "^4.17.20",
     "typeorm": "/~https://github.com/Joystream/typeorm/releases/download/0.3.5/typeorm-v0.3.5.tgz"
   },
   "devDependencies": {
-    "@types/bn.js": "^4.11.6",
+    "@types/bn.js": "^5.1.0",
     "mocha": "^8.1.3",
     "nyc": "^15.1.0",
     "ts-node": "^10.2.1",

packages/hydra-e2e-tests/package.json

@@ -22,8 +22,8 @@
     "e2e-test": "bash run-tests.sh"
   },
   "dependencies": {
-    "@polkadot/api": "5.9.1",
-    "@polkadot/keyring": "7.9.2",
+    "@polkadot/api": "8.9.1",
+    "@polkadot/keyring": "9.5.1",
     "fetch": "^1.1.0",
     "graphql-request": "^3.3.0",
     "graphql-subscriptions-client": "^0.16.0",
@@ -39,7 +39,7 @@
     "eslint": "^7.11.0",
     "mocha": "^8.1.3",
     "nyc": "^15.1.0",
-    "prettier": "^2.1.2",
+    "prettier": "^2.7.1",
     "ts-node": "^10.2.1",
     "typescript": "4.4.2"
   }

packages/hydra-indexer-gateway/package.json

@@ -46,7 +46,7 @@
     "@joystream/hydra-db-utils": "^3.1.0-alpha.27",
     "@joystream/warthog": "^2.41.5",
     "@types/ioredis": "^4.17.4",
-    "bn.js": "^5.1.3",
+    "bn.js": "^5.2.1",
     "dotenv": "^8.2.0",
     "graphql-parse-resolve-info": "^4.9.0",
     "graphql-redis-subscriptions": "^2.3.1",
@@ -56,7 +56,7 @@
     "reflect-metadata": "^0.1.13"
   },
   "devDependencies": {
-    "@types/bn.js": "^4.11.6",
+    "@types/bn.js": "^5.1.0",
     "@types/graphql": "^14.5.0",
     "@types/ioredis": "^4.17.4",
     "@types/jest": "^24.0.23",

packages/hydra-indexer/package.json

@@ -39,11 +39,11 @@
     "@joystream/bn-typeorm": "^3.1.0-alpha.27",
     "@joystream/hydra-common": "^3.1.0-alpha.27",
     "@joystream/hydra-db-utils": "^3.1.0-alpha.27",
-    "@polkadot/api": "5.9.1",
+    "@polkadot/api": "8.9.1",
     "@types/express": "^4.17.8",
     "@types/ioredis": "^4.17.4",
     "@types/lodash": "^4.14.161",
-    "bn.js": "^5.1.2",
+    "bn.js": "^5.2.1",
     "commander": "^6.2.0",
     "debug": "^4.1.1",
     "delay": "~5.0.0",
@@ -72,8 +72,7 @@
     "util": "^0.12.3"
   },
   "devDependencies": {
-    "@polkadot/ts": "^0.3.14",
-    "@types/bn.js": "^4.11.6",
+    "@types/bn.js": "^5.1.0",
     "@types/chai": "^4.2.12",
     "@types/debug": "^4.1.5",
     "@types/figlet": "^1.2.1",
@@ -91,7 +90,7 @@
     "eslint": "^7.1.0",
     "mocha": "^8.1.3",
     "nyc": "^15.1.0",
-    "prettier": "^2.1.2",
+    "prettier": "^2.7.1",
     "ts-mockito": "^2.6.1",
     "ts-node": "^10.2.1",
     "typescript": "4.4.2"

packages/hydra-indexer/src/substrate/SubstrateService.ts

@@ -1,3 +1,5 @@
+import '@polkadot/api-augment/substrate'
+
 import {
   Hash,
   Header,

packages/hydra-processor/package.json

@@ -43,7 +43,7 @@
     "@oclif/config": "^1",
     "@oclif/errors": "^1.3.3",
     "async-lock": "^1.3.1",
-    "bn.js": "^5.2.0",
+    "bn.js": "^5.2.1",
     "chalk": "^4.1.0",
     "delay": "^5.0.0",
     "dotenv": "^8.2.0",
@@ -64,7 +64,7 @@
   },
   "devDependencies": {
     "@joystream/hydra-cli": "^3.1.0-alpha.27",
-    "@polkadot/api": "5.9.1",
+    "@polkadot/api": "8.9.1",
     "@types/async-lock": "^1.1.3",
     "@types/bn.js": "^5.1.0",
     "@types/chai": "^4.2.19",

packages/hydra-typegen/package.json

@@ -41,7 +41,7 @@
     "@oclif/command": "^1.8.0",
     "@oclif/config": "^1",
     "@oclif/errors": "^1.3.3",
-    "@polkadot/api": "5.9.1",
+    "@polkadot/api": "8.9.1",
     "debug": "^4.3.1",
     "figlet": "^1.5.2",
     "handlebars": "^4.7.6",

packages/hydra-typegen/src/commands/typegen/index.ts

@@ -3,7 +3,7 @@ import fs from 'fs'
 import path from 'path'
 import Debug from 'debug'
 
-import { MetadataSource, registerCustomTypes } from '../../metadata/metadata'
+import { getMetadata, MetadataSource } from '../../metadata/metadata'
 import { extractMeta } from '../../metadata'
 import {
   generateModuleTypes,
@@ -14,16 +14,10 @@ import {
 import { parseConfigFile } from '../../config/parse-yaml'
 import { validate } from '../../config/validate'
 
-export type CustomTypes = {
-  lib: string // package with types. All custom types will be imported from there
-  typedefsLoc: string // path to type definitions
-}
-
 export interface IConfig {
   metadata: MetadataSource
   events: string[]
   calls: string[]
-  customTypes?: CustomTypes
   outDir: string
   strict?: boolean
 }
@@ -33,8 +27,6 @@ export type Flags = {
   calls: string | undefined
   metadata: string
   blockHash: string | undefined
-  typedefs: string | undefined
-  typelib: string | undefined
   outDir: string
   strict: boolean
   debug: boolean
@@ -74,15 +66,6 @@ Otherwise a relative path to a json file matching the RPC call response is expec
       description:
         'Hash of the block from which the metadata will be fetched. Only applied if metadata is pulled via an RPC call',
     }),
-    typedefs: flags.string({
-      char: 't',
-      description:
-        'A relative path to a file with JSON definitions for custom types used by the chain',
-    }),
-    typelib: flags.string({
-      char: 'i',
-      description: `A JavaScript module from which the custom types should be imported, e.g. '@joystream/types/augment'`,
-    }),
     outDir: flags.string({
       char: 'o',
       description:
@@ -124,19 +107,6 @@ types don't much the metadata definiton`,
   }
 
   parseFlags(flags: Flags): IConfig {
-    let customTypes: CustomTypes | undefined
-    if (flags.typedefs) {
-      if (flags.typelib === undefined) {
-        throw new Error(
-          `Please specify the library with type definitions with --typelib`
-        )
-      }
-      customTypes = {
-        lib: flags.typelib,
-        typedefsLoc: path.resolve(flags.typedefs),
-      }
-    }
-
     const events: string[] = flags.events
       ? flags.events.split(',').map((e) => e.trim())
       : []
@@ -153,25 +123,21 @@ types don't much the metadata definiton`,
         blockHash: flags.blockHash,
       },
       strict: flags.strict,
-      customTypes,
     } as IConfig
   }
 
   async buildGeneratorConfig(config: IConfig): Promise<GeneratorConfig> {
-    const { outDir, customTypes } = config
-
-    if (customTypes) {
-      registerCustomTypes(customTypes.typedefsLoc)
-    }
+    const { outDir } = config
 
-    const modules = await extractMeta(config)
+    const originalMetadata = await getMetadata(config.metadata)
+    const modules = await extractMeta(config, originalMetadata)
 
     return {
-      customTypes,
-      importsRegistry: buildImportsRegistry(customTypes),
+      importsRegistry: buildImportsRegistry(),
       modules,
       validateArgs: config.strict || false, // do not enforce validation by default
       dest: path.resolve(outDir),
+      originalMetadata,
     }
   }
 

packages/hydra-typegen/src/config/parse-yaml.ts

@@ -12,10 +12,6 @@ const options = {
         source: 'string',
         'blockHash?': 'string',
       },
-      'customTypes?': {
-        lib: 'string',
-        typedefsLoc: 'string',
-      },
       'events?': ['string'],
       'calls?': ['string'],
       outDir: 'string',

packages/hydra-typegen/src/config/validate.ts

@@ -1,23 +1,9 @@
-import path from 'path'
-import fs from 'fs'
 import { IConfig } from '../commands/typegen'
 
-export function validate({ events, calls, customTypes }: IConfig): void {
+export function validate({ events, calls }: IConfig): void {
   if (events.length === 0 && calls.length === 0) {
     throw new Error(
       `Nothing to generate: at least one event or call should be provided.`
     )
   }
-
-  if (customTypes) {
-    if (customTypes.typedefsLoc === undefined) {
-      throw new Error(
-        `Missing the type defintion location for the custom for types. Did you forget to add typedefsLoc?`
-      )
-    }
-    const typedefsPath = path.resolve(customTypes.typedefsLoc)
-    if (!fs.existsSync(typedefsPath)) {
-      throw new Error(`Cannot find type definition file at ${typedefsPath}`)
-    }
-  }
 }

packages/hydra-typegen/src/generators/gen-index.ts

@@ -10,20 +10,18 @@ const generateIndexTemplate = handlebars.compile(readTemplate('index'))
 
 export function generateIndex({
   modules,
-  customTypes,
+  originalMetadata,
   dest,
 }: GeneratorConfig): void {
-  if (customTypes) {
-    const typedefsPath = path.resolve(customTypes.typedefsLoc)
-    fs.copyFileSync(typedefsPath, path.join(dest, `typedefs.json`))
-  }
+  fs.writeFileSync(
+    path.join(dest, `metadata.json`),
+    JSON.stringify(originalMetadata.toHex())
+  )
 
   writeFile(path.join(dest, `index.ts`), () =>
     formatWithPrettier(
       generateIndexTemplate({
         modules: modules.map((m) => m.module),
-        hasTypeDefs:
-          customTypes !== undefined && customTypes.typedefsLoc !== undefined,
       })
     )
   )

packages/hydra-typegen/src/generators/gen-modules.ts

@@ -1,15 +1,21 @@
-import { ModuleMeta } from '../metadata'
+import { ExtractedModuleMeta } from '../metadata'
 import { formatWithPrettier, readTemplate, writeFile } from '../util'
 import path from 'path'
 import { buildModuleImports } from './imports'
 import { GeneratorConfig } from '.'
 import { handlebars } from './helpers'
 import { kebabCase } from 'lodash'
+import { ImportsDef } from './types'
 
 const debug = require('debug')('hydra-typegen:gen-modules')
 
 const generateModuleTemplate = handlebars.compile(readTemplate('module'))
 
+type ModuleTemplateProps = {
+  validateArgs: boolean
+  imports: ImportsDef
+} & ExtractedModuleMeta
+
 export function generateModuleTypes(config: GeneratorConfig): void {
   const { modules, dest } = config
   modules.forEach((meta) => {
@@ -23,9 +29,9 @@ export function generateModuleTypes(config: GeneratorConfig): void {
 }
 
 export function buildModuleProps(
-  meta: ModuleMeta,
+  meta: ExtractedModuleMeta,
   config: GeneratorConfig
-): unknown {
+): ModuleTemplateProps {
   const { validateArgs } = config
   const imports = buildModuleImports(meta, config)