support well known keys

packages/chopsticks/src/plugins/run-block/index.ts

@@ -1,6 +1,7 @@
 import { GenericExtrinsic } from '@polkadot/types'
 import { Header } from '@polkadot/types/interfaces'
 import { HexString } from '@polkadot/util/types'
+import { u8aToHex } from '@polkadot/util'
 import { writeFileSync } from 'node:fs'
 import { z } from 'zod'
 import _ from 'lodash'
@@ -199,6 +200,7 @@ export const name = 'runBlock'
  * Run a set of extrinsics on top of a block and get the storage diff
  * and optionally the parsed storage diff and block details.
  * NOTE: The extrinsics should include inherents or tranasctions may have unexpected results.
+ * NOTE: system.events are system.extrinsicData are excluded from storage diff to reduce size.
  *
  * This function is a dev rpc handler. Use `dev_runBlock` as the method name when calling it.
  */
@@ -235,6 +237,8 @@ export const rpc = async ({ chain }: Context, [params]: [RunBlockParams]): Promi
 
   // exclude system events because it can be stupidly large and redudant
   const systemEventsKey = compactHex(meta.query.system.events())
+  // large and not really useful
+  const systemExtrinsicDataKey = u8aToHex(meta.query.system.extrinsicData.keyPrefix())
 
   const run = async (fn: string, args: HexString[]) => {
     const result = await runTask(
@@ -261,6 +265,9 @@ export const rpc = async ({ chain }: Context, [params]: [RunBlockParams]): Promi
       if (key === systemEventsKey) {
         continue
       }
+      if (key.startsWith(systemExtrinsicDataKey)) {
+        continue
+      }
 
       const obj = {} as (typeof resp)['storageDiff'][number]
       if (includeRawStorage) {
@@ -272,8 +279,8 @@ export const rpc = async ({ chain }: Context, [params]: [RunBlockParams]): Promi
           obj.parsed = {
             section: decoded.section,
             method: decoded.method,
-            key: decoded.key?.map((x) => x.toString()),
-            value: decoded.value?.toString(),
+            key: decoded.key,
+            value: decoded.value,
           }
         }
       }

packages/core/src/utils/decoder.ts

@@ -4,10 +4,11 @@ import { DecoratedMeta } from '@polkadot/types/metadata/decorate/types'
 import { HexString } from '@polkadot/util/types'
 import { StorageEntry } from '@polkadot/types/primitive/types'
 import { StorageKey } from '@polkadot/types'
-import { blake2AsHex } from '@polkadot/util-crypto'
 import { hexToU8a, u8aToHex } from '@polkadot/util'
 import _ from 'lodash'
 
+import { decodeWellKnownKey } from './well-known-keys'
+
 const _CACHE: Record<string, Map<HexString, StorageEntry>> = {}
 
 const getCache = (uid: string): Map<HexString, StorageEntry> => {
@@ -55,6 +56,16 @@ export const decodeKeyValue = (
   value?: HexString | null,
   toHuman = true,
 ) => {
+  const res = decodeWellKnownKey(meta.registry, key, value)
+  if (res) {
+    return {
+      section: 'substrate',
+      method: res.name,
+      key: res.key,
+      value: res.value,
+    }
+  }
+
   const { storage, decodedKey } = decodeKey(meta, block, key)
 
   if (!storage || !decodedKey) {
@@ -63,9 +74,6 @@ export const decodeKeyValue = (
 
   const decodeValue = () => {
     if (!value) return null
-    if (storage.section === 'substrate' && storage.method === 'code') {
-      return `:code blake2_256 ${blake2AsHex(value, 256)} (${hexToU8a(value).length} bytes)`
-    }
     return meta.registry.createType(decodedKey.outputType, hexToU8a(value))[toHuman ? 'toHuman' : 'toJSON']()
   }
 

packages/core/src/utils/well-known-keys.ts

@@ -0,0 +1,75 @@
+import { HexString } from '@polkadot/util/types'
+import { Registry } from '@polkadot/types-codec/types'
+import { blake2AsHex } from '@polkadot/util-crypto'
+import { hexToU8a, stringToHex } from '@polkadot/util'
+
+const decodeValue = (type: string) => (registry: Registry, value: HexString) => {
+  return registry.createType(type, hexToU8a(value)).toJSON()
+}
+
+// /~https://github.com/paritytech/polkadot-sdk/issues/2126
+const wellKnownKeys = [
+  {
+    name: 'code',
+    key: ':code',
+    decodeValue: (_registry: Registry, value: HexString) => {
+      return `<:code blake2_256 ${blake2AsHex(value, 256)} (${value.length / 2 - 1} bytes)>`
+    },
+  },
+  {
+    name: 'heapPages',
+    key: ':heappages',
+    type: 'u64',
+  },
+  {
+    name: 'extrinsicIndex',
+    key: ':extrinsic_index',
+    type: 'u32',
+  },
+  {
+    name: 'intrablockEntropy',
+    key: ':intrablock_entropy',
+    type: '[u8; 32]',
+  },
+  {
+    name: 'transactionLevel',
+    key: ':transaction_level:',
+    type: 'u32',
+  },
+  {
+    name: 'grandpaAuthorities',
+    key: ':grandpa_authorities',
+    type: '(u8, AuthorityList)',
+  },
+  {
+    name: 'relayDispatchQueueRemainingCapacity',
+    prefix: ':relay_dispatch_queue_remaining_capacity',
+    decodeKey: (registry: Registry, key: HexString) => {
+      return [registry.createType('u32', hexToU8a(key)).toJSON()]
+    },
+    type: '(u32, u32)',
+  },
+].map((def) => {
+  const prefix = stringToHex(def.prefix || def.key)
+  return {
+    name: def.name,
+    prefix,
+    decodeKey: def.decodeKey || ((_registry: Registry, key: HexString) => [key]),
+    decodeValue: def.decodeValue || decodeValue(def.type),
+  }
+})
+
+export const decodeWellKnownKey = (registry: Registry, key: HexString, value?: HexString | null) => {
+  for (const defs of wellKnownKeys) {
+    if (key.startsWith(defs.prefix)) {
+      const remaining = key.slice(defs.prefix.length)
+      const decodedKey = remaining ? defs.decodeKey(registry, `0x${remaining}`) : undefined
+      const decodedValue = value ? defs.decodeValue(registry, value) : undefined
+      return {
+        name: defs.name,
+        key: decodedKey ?? [],
+        value: decodedValue,
+      }
+    }
+  }
+}

packages/e2e/src/__snapshots__/decoder.test.ts.snap

@@ -1,6 +1,6 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`decoder > decode key-value 1`] = `
+exports[`decoder > with acala > decode key-value 1`] = `
 {
   "key": [
     "25fqepuLngYL2DK9ApTejNzqPadUUZ9ALYyKWX2jyvEiuZLa",
@@ -22,7 +22,7 @@ exports[`decoder > decode key-value 1`] = `
 }
 `;
 
-exports[`decoder > decode key-value 2`] = `
+exports[`decoder > with acala > decode key-value 2`] = `
 {
   "system": {
     "account": {
@@ -43,7 +43,7 @@ exports[`decoder > decode key-value 2`] = `
 }
 `;
 
-exports[`decoder > decode key-value 3`] = `
+exports[`decoder > with acala > decode key-value 3`] = `
 {
   "key": [
     "25fqepuLngYL2DK9ApTejNzqPadUUZ9ALYyKWX2jyvEiuZLa",
@@ -61,7 +61,7 @@ exports[`decoder > decode key-value 3`] = `
 }
 `;
 
-exports[`decoder > decode key-value 4`] = `
+exports[`decoder > with acala > decode key-value 4`] = `
 {
   "tokens": {
     "accounts": {
@@ -77,6 +77,71 @@ exports[`decoder > decode key-value 4`] = `
 }
 `;
 
+exports[`decoder > with acala > decode key-value 5`] = `
+{
+  "key": [],
+  "method": "now",
+  "section": "timestamp",
+  "value": "64,188,750,128,742,400",
+}
+`;
+
+exports[`decoder > with acala > decode key-value 6`] = `
+{
+  "timestamp": {
+    "now": "64,188,750,128,742,400",
+  },
+}
+`;
+
+exports[`decoder > with acala > decode keys 1`] = `
+{
+  "decodedKey": "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d",
+  "storage": [Function],
+}
+`;
+
+exports[`decoder > with acala > works with well known keys 1`] = `
+{
+  "key": [],
+  "method": "code",
+  "section": "substrate",
+  "value": "<:code blake2_256 0x97e86e044a53385b642a902fd8ed05534d7590412a608f43dbb70e1f0e3664c7 (4 bytes)>",
+}
+`;
+
+exports[`decoder > with acala > works with well known keys 2`] = `
+{
+  "key": [
+    3340,
+  ],
+  "method": "relayDispatchQueueRemainingCapacity",
+  "section": "substrate",
+  "value": [
+    174762,
+    1048576,
+  ],
+}
+`;
+
+exports[`decoder > with acala > works with well known keys 3`] = `
+{
+  "key": [],
+  "method": "transactionLevel",
+  "section": "substrate",
+  "value": undefined,
+}
+`;
+
+exports[`decoder > with acala > works with well known keys 4`] = `
+{
+  "key": [],
+  "method": "extrinsicIndex",
+  "section": "substrate",
+  "value": 2,
+}
+`;
+
 exports[`decoder > works with multiple chains 1`] = `
 {
   "key": [

packages/e2e/src/decoder.test.ts

@@ -7,33 +7,53 @@ const SYSTEM_ACCOUNT =
   '0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'
 const TOKENS_ACCOUNTS =
   '0x99971b5749ac43e0235e41b0d37869188ee7418a6531173d60d1f6a82d8f4d51de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01a12dfa1fa4ab9a0000'
+const TIMESTAMPE_NOW = '0xf0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb'
 
 describe('decoder', async () => {
-  const { chain, teardown } = await networks.acala()
+  describe('with acala', async () => {
+    const { chain, teardown } = await networks.acala()
 
-  afterAll(async () => {
-    await teardown()
-  })
+    afterAll(async () => {
+      await teardown()
+    })
 
-  it('decode keys', async () => {
-    const { storage, decodedKey } = decodeKey(await chain.head.meta, chain.head, SYSTEM_ACCOUNT)
-    expect(storage?.section).eq('system')
-    expect(storage?.method).eq('account')
-    expect(decodedKey?.args.map((x) => x.toHuman())).contains('25fqepuLngYL2DK9ApTejNzqPadUUZ9ALYyKWX2jyvEiuZLa')
-  })
+    it('decode keys', async () => {
+      expect(decodeKey(await chain.head.meta, chain.head, SYSTEM_ACCOUNT)).toMatchSnapshot()
+    })
 
-  it('decode key-value', async () => {
-    const meta = await chain.head.meta
-    const data = { data: { free: 10000000000 } }
-    const value = meta.registry.createType('AccountInfo', data)
-    const decoded = decodeKeyValue(meta, chain.head, SYSTEM_ACCOUNT, value.toHex())
-    expect(decoded).toMatchSnapshot()
-    expect(toStorageObject(decoded)).toMatchSnapshot()
+    it('decode key-value', async () => {
+      const meta = await chain.head.meta
+      const data = { data: { free: 10000000000 } }
+      const value = meta.registry.createType('AccountInfo', data)
+      const decoded = decodeKeyValue(meta, chain.head, SYSTEM_ACCOUNT, value.toHex())
+      expect(decoded).toMatchSnapshot()
+      expect(toStorageObject(decoded)).toMatchSnapshot()
+
+      const ormlAccountData = meta.registry.createType('AccountData', data.data)
+      const decoded2 = decodeKeyValue(meta, chain.head, TOKENS_ACCOUNTS, ormlAccountData.toHex())
+      expect(decoded2).toMatchSnapshot()
+      expect(toStorageObject(decoded2)).toMatchSnapshot()
+
+      const timestampNow = meta.registry.createType('Moment', data.data)
+      const decoded3 = decodeKeyValue(meta, chain.head, TIMESTAMPE_NOW, timestampNow.toHex())
+      expect(decoded3).toMatchSnapshot()
+      expect(toStorageObject(decoded3)).toMatchSnapshot()
+    })
 
-    const ormlAccountData = meta.registry.createType('AccountData', data.data)
-    const decoded2 = decodeKeyValue(meta, chain.head, TOKENS_ACCOUNTS, ormlAccountData.toHex())
-    expect(decoded2).toMatchSnapshot()
-    expect(toStorageObject(decoded2)).toMatchSnapshot()
+    it('works with well known keys', async () => {
+      const meta = await chain.head.meta
+      expect(decodeKeyValue(meta, chain.head, '0x3a636f6465', '0x12345678')).toMatchSnapshot()
+      expect(
+        decodeKeyValue(
+          meta,
+          chain.head,
+          '0x3a72656c61795f64697370617463685f71756575655f72656d61696e696e675f63617061636974790c0d0000',
+          '0xaaaa020000001000',
+        ),
+      ).toMatchSnapshot()
+      expect(decodeKeyValue(meta, chain.head, '0x3a7472616e73616374696f6e5f6c6576656c3a')).toMatchSnapshot()
+      expect(decodeKeyValue(meta, chain.head, '0x3a65787472696e7369635f696e646578', '0x02000000')).toMatchSnapshot()
+    })
   })
 
   it('works with multiple chains', async () => {