Orion v3.1.0 (Joystream#238)

* Add granular permissions support for Gateway operator users (Joystream#231)

* Add granular permissions support for Gateway operator users

* fix lint issues

* revert docker-compose port change

* mark 'grantPermissions' & 'revokePermissions' input fields are non-nullable & return new permissions instead of boolean

* Set Channel Weight (`setChannelsWeights`) mutation (Joystream#232)

* Add granular permissions support for Gateway operator users

* fix lint issues

* add mutation to set channel weight/bias for homepage video relevance

* revert docker-compose port change

* mark 'grantPermissions' & 'revokePermissions' input fields are non-nullable & return new permissions instead of boolean

* bump package version

* update global migration counter map

* bumped package version & updated CHANGELOG

---------

Co-authored-by: Ignazio Bovo <ignazio@jsgenesis.com>

* Postgres performance improvements (Joystream#235)

* add index in video.createdAt field

* add pg_stat_extenstion extenstion for queries stats

* docs: ✨ changelog and fix data-js (Joystream#237)

---------

Co-authored-by: Zeeshan Akram <37098720+zeeshanakram3@users.noreply.github.com>

.env

@@ -38,9 +38,10 @@ VIDEO_RELEVANCE_VIEWS_TICK=50
 #   views weight,
 #   comments weight,
 #   rections weights,
-#   [joystream creation weight, YT creation weight]
+#   [joystream creation weight, YT creation weight],
+#   Default channel weight/bias
 # ]
-RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3]]"
+RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3], 1]"
 MAX_CACHED_ENTITIES=1000
 APP_PRIVATE_KEY=this-is-not-so-secret-change-it
 SESSION_EXPIRY_AFTER_INACTIVITY_MINUTES=60
@@ -59,4 +60,4 @@ SENDGRID_FROM_EMAIL=gateway@example.com
 
 # Debug settings
 SQD_DEBUG=api:*
-OPENAPI_PLAYGROUND=true
+OPENAPI_PLAYGROUND=true

CHANGELOG.md

@@ -1,3 +1,14 @@
+# 3.1.0
+
+### Entities
+- Adds `User.permission` to the `User` entity, this however doesn't require migration logic.
+- Adds `Channel.channelWeights` in order to boost channel relevance. This value can be set via the `setChannelWeights` mutation
+### Resolvers
+- Adds supports for new permissions model for gateway operator users. Now the root user can assign/revoke operator permission/s to users using `grantPermissions` & `revokePermissions` mutations
+- Adds new `setChannelWeights` operator mutation to set weight/bias for any channel/s which will be used to calculate the Atlas homepage video relevance scores
+### Performance
+- Adds `Video.createdAt` as index in order to speed up Atlas home page queries 
+
 # 3.0.4
 
 ### Misc

db/migrations/2200000000000-Operator.js

@@ -19,9 +19,13 @@ module.exports = class Operator2300000000000 {
       INSERT INTO "admin"."user" ("id", "is_root")
       VALUES ('${process.env.OPERATOR_SECRET || randomAsHex(32)}', true);
     `)
+
+    // Create pg_stat_statements extension for analyzing query stats
+    await db.query(`CREATE EXTENSION pg_stat_statements`)
   }
 
   async down(db) {
     await db.query(`DELETE FROM "admin"."user" WHERE "is_root" = true;`)
+    await db.query(`DROP EXTENSION pg_stat_statements`)
   }
 }

db/postgres.conf

@@ -3,4 +3,6 @@ log_statement = all
 autovacuum_analyze_scale_factor = 0.01
 shared_buffers=2GB
 jit=off
-random_page_cost=1.0
+random_page_cost=1.0
+shared_preload_libraries = 'pg_stat_statements'
+pg_stat_statements.track = all

docker-compose.yml

@@ -10,8 +10,7 @@ services:
       POSTGRES_DB: squid
       POSTGRES_PASSWORD: squid
     ports:
-      - '127.0.0.1:${DB_PORT}:${DB_PORT}'
-      - '[::1]:${DB_PORT}:${DB_PORT}'
+      - '${DB_PORT}:${DB_PORT}'
     command: ['postgres', '-c', 'config_file=/etc/postgresql/postgresql.conf', '-p', '${DB_PORT}']
     shm_size: 1g
     volumes:

docs/developer-guide/tutorials/updating-schema.md

@@ -25,7 +25,7 @@ In order to do that, you'll need to:
 4. Re-generate the `*-Data.js` migration inside `db/migrations`:
     ```bash
     # First, remove the old migration file(s)
-    rm db/migrations/*-Data.js
+    `rm db/migrations/*-Data.js`
     # Start PostgreSQL database service
     # Make sure it's an empty database! If the service is already running you should first run:
     # docker-compose down -v

package.json

@@ -1,6 +1,6 @@
 {
   "name": "orion",
-  "version": "3.0.4",
+  "version": "3.1.0",
   "engines": {
     "node": ">=16"
   },

schema/auth.graphql

@@ -1,10 +1,28 @@
+enum OperatorPermission {
+  GRANT_OPERATOR_PERMISSIONS
+  REVOKE_OPERATOR_PERMISSIONS
+  SET_VIDEO_WEIGHTS
+  SET_CHANNEL_WEIGHTS
+  SET_KILL_SWITCH
+  SET_VIDEO_VIEW_PER_USER_TIME_LIMIT
+  SET_VIDEO_HERO
+  SET_CATEGORY_FEATURED_VIDEOS
+  SET_SUPPORTED_CATEGORIES
+  SET_FEATURED_NFTS
+  EXCLUDE_CONTENT
+  RESTORE_CONTENT
+}
+
 type User @entity {
   "Unique identifier (32-byte string, securely random)"
   id: ID!
 
   "Whether the user has root (gateway operator) privileges"
   isRoot: Boolean!
 
+  "List of all the gateway operator permissions that this user has"
+  permissions: [OperatorPermission!]
+
   "The account associated with the user (if any)"
   account: Account @derivedFrom(field: "user")
 

schema/channels.graphql

@@ -3,7 +3,7 @@ type Channel @entity {
   id: ID!
 
   "Timestamp of the block the channel was created at"
-  createdAt: DateTime!
+  createdAt: DateTime! @index
 
   "Current member-owner of the channel (if owned by a member)"
   ownerMember: Membership
@@ -64,6 +64,9 @@ type Channel @entity {
 
   "Cumulative rewards paid to this channel"
   cumulativeReward: BigInt!
+
+  "Weight/Bias of the channel affecting video relevance in the Homepage"
+  channelWeight: Float
 }
 
 type BannedMember @entity @index(fields: ["member", "channel"], unique: true) {

src/server-extension/check.ts

@@ -1,8 +1,8 @@
 import { RequestCheckFunction } from '@subsquid/graphql-server/lib/check'
-import { Context as OpenreaderContext } from '@subsquid/openreader/lib/context'
 import { TypeormOpenreaderContext } from '@subsquid/graphql-server/lib/typeorm'
-import { AuthContext, authenticate } from '../utils/auth'
+import { Context as OpenreaderContext } from '@subsquid/openreader/lib/context'
 import { UnauthorizedError } from 'type-graphql'
+import { AuthContext, authenticate } from '../utils/auth'
 
 export type Context = OpenreaderContext & AuthContext
 

src/server-extension/resolvers/AdminResolver/index.ts

@@ -1,17 +1,47 @@
+import { generateAppActionCommitment } from '@joystream/js/utils'
+import { AppAction } from '@joystream/metadata-protobuf'
+import { hexToU8a, isHex, u8aToHex } from '@polkadot/util'
+import { ed25519PairFromString, ed25519Sign } from '@polkadot/util-crypto'
+import { Context } from '@subsquid/openreader/lib/context'
+import { getObjectSize } from '@subsquid/openreader/lib/limit.size'
+import { parseObjectTree } from '@subsquid/openreader/lib/opencrud/tree'
+import { EntityByIdQuery } from '@subsquid/openreader/lib/sql/query'
+import { getResolveTree } from '@subsquid/openreader/lib/util/resolve-tree'
+import { GraphQLResolveInfo } from 'graphql'
 import 'reflect-metadata'
-import { Args, Query, Mutation, Resolver, UseMiddleware, Info, Ctx } from 'type-graphql'
-import { EntityManager, In, Not } from 'typeorm'
+import { Args, Ctx, Info, Mutation, Query, Resolver, UseMiddleware } from 'type-graphql'
+import { EntityManager, In, Not, UpdateResult } from 'typeorm'
+import { videoRelevanceManager } from '../../../mappings/utils'
+import {
+  Channel,
+  OperatorPermission,
+  User,
+  Video,
+  VideoCategory,
+  VideoFeaturedInCategory,
+  VideoHero as VideoHeroEntity,
+} from '../../../model'
+import { ConfigVariable, config } from '../../../utils/config'
+import { withHiddenEntities } from '../../../utils/sql'
+import { VideoHero } from '../baseTypes'
+import { OperatorOnly } from '../middleware'
+import { model } from '../model'
 import {
   AppActionSignatureInput,
+  ChannelWeight,
   ExcludableContentType,
   ExcludeContentArgs,
   ExcludeContentResult,
   GeneratedSignature,
+  GrantOperatorPermissionsInput,
+  GrantOrRevokeOperatorPermissionsResult,
   KillSwitch,
   RestoreContentArgs,
   RestoreContentResult,
+  RevokeOperatorPermissionsInput,
   SetCategoryFeaturedVideosArgs,
   SetCategoryFeaturedVideosResult,
+  SetChannelsWeightsArgs,
   SetFeaturedNftsInput,
   SetFeaturedNftsResult,
   SetKillSwitchInput,
@@ -24,36 +54,49 @@ import {
   VideoViewPerUserTimeLimit,
   VideoWeights,
 } from './types'
-import { config, ConfigVariable } from '../../../utils/config'
-import { OperatorOnly } from '../middleware'
-import {
-  Video,
-  VideoCategory,
-  VideoFeaturedInCategory,
-  VideoHero as VideoHeroEntity,
-} from '../../../model'
-import { GraphQLResolveInfo } from 'graphql'
-import { Context } from '@subsquid/openreader/lib/context'
-import { parseObjectTree } from '@subsquid/openreader/lib/opencrud/tree'
-import { getResolveTree } from '@subsquid/openreader/lib/util/resolve-tree'
-import { EntityByIdQuery } from '@subsquid/openreader/lib/sql/query'
-import { getObjectSize } from '@subsquid/openreader/lib/limit.size'
-import { VideoHero } from '../baseTypes'
-import { model } from '../model'
-import { ed25519PairFromString, ed25519Sign } from '@polkadot/util-crypto'
-import { u8aToHex, hexToU8a, isHex } from '@polkadot/util'
-import { generateAppActionCommitment } from '@joystream/js/utils'
-import { AppAction } from '@joystream/metadata-protobuf'
-import { withHiddenEntities } from '../../../utils/sql'
 import { processCommentsCensorshipStatusUpdate } from './utils'
-import { videoRelevanceManager } from '../../../mappings/utils'
 
 @Resolver()
 export class AdminResolver {
-  // Set by depenency injection
+  // Set by dependency injection
   constructor(private em: () => Promise<EntityManager>) {}
 
-  @UseMiddleware(OperatorOnly)
+  @UseMiddleware(OperatorOnly(OperatorPermission.GRANT_OPERATOR_PERMISSIONS))
+  @Mutation(() => GrantOrRevokeOperatorPermissionsResult)
+  async grantPermissions(
+    @Args() args: GrantOperatorPermissionsInput
+  ): Promise<GrantOrRevokeOperatorPermissionsResult> {
+    const em = await this.em()
+    const user = await em.findOne(User, { where: { id: args.userId } })
+    if (!user) {
+      throw new Error('User not found')
+    }
+
+    // Add only new permissions that the user doesn't have yet
+    user.permissions = Array.from(new Set([...(user.permissions || []), ...args.permissions]))
+
+    await em.save(user)
+    return { newPermissions: user.permissions }
+  }
+
+  @UseMiddleware(OperatorOnly(OperatorPermission.REVOKE_OPERATOR_PERMISSIONS))
+  @Mutation(() => GrantOrRevokeOperatorPermissionsResult)
+  async revokePermission(
+    @Args() args: RevokeOperatorPermissionsInput
+  ): Promise<GrantOrRevokeOperatorPermissionsResult> {
+    const em = await this.em()
+    const user = await em.findOne(User, { where: { id: args.userId } })
+    if (!user) {
+      throw new Error('User not found')
+    }
+
+    user.permissions = (user.permissions || []).filter((perm) => !args.permissions.includes(perm))
+
+    await em.save(user)
+    return { newPermissions: user.permissions }
+  }
+
+  @UseMiddleware(OperatorOnly(OperatorPermission.SET_VIDEO_WEIGHTS))
   @Mutation(() => VideoWeights)
   async setVideoWeights(@Args() args: SetVideoWeightsInput): Promise<VideoWeights> {
     const em = await this.em()
@@ -65,14 +108,48 @@ export class AdminResolver {
         args.commentsWeight,
         args.reactionsWeight,
         [args.joysteamTimestampSubWeight, args.ytTimestampSubWeight],
+        args.defaultChannelWeight,
       ],
       em
     )
     await videoRelevanceManager.updateVideoRelevanceValue(em, true)
     return { isApplied: true }
   }
 
-  @UseMiddleware(OperatorOnly)
+  @UseMiddleware(OperatorOnly(OperatorPermission.SET_CHANNEL_WEIGHTS))
+  @Mutation(() => [ChannelWeight])
+  async setChannelsWeights(@Args() { inputs }: SetChannelsWeightsArgs): Promise<ChannelWeight[]> {
+    const em = await this.em()
+
+    const results: ChannelWeight[] = []
+
+    // Process each SetChannelWeightInput
+    for (const weightInput of inputs) {
+      const { channelId, weight } = weightInput
+
+      // Update the channel weight in the database
+      const updateResult: UpdateResult = await em.transaction(
+        async (transactionalEntityManager) => {
+          return transactionalEntityManager
+            .createQueryBuilder()
+            .update(Channel)
+            .set({ channelWeight: weight })
+            .where('id = :id', { id: channelId })
+            .execute()
+        }
+      )
+
+      // Push the result into the results array
+      results.push({
+        channelId,
+        isApplied: !!updateResult.affected,
+      })
+    }
+
+    return results
+  }
+
+  @UseMiddleware(OperatorOnly(OperatorPermission.SET_KILL_SWITCH))
   @Mutation(() => KillSwitch)
   async setKillSwitch(@Args() args: SetKillSwitchInput): Promise<KillSwitch> {
     const em = await this.em()
@@ -86,7 +163,7 @@ export class AdminResolver {
     return { isKilled: await config.get(ConfigVariable.KillSwitch, em) }
   }
 
-  @UseMiddleware(OperatorOnly)
+  @UseMiddleware(OperatorOnly(OperatorPermission.SET_VIDEO_VIEW_PER_USER_TIME_LIMIT))
   @Mutation(() => VideoViewPerUserTimeLimit)
   async setVideoViewPerUserTimeLimit(
     @Args() args: SetVideoViewPerUserTimeLimitInput
@@ -133,7 +210,7 @@ export class AdminResolver {
     return ctx.openreader.executeQuery(entityByIdQuery)
   }
 
-  @UseMiddleware(OperatorOnly)
+  @UseMiddleware(OperatorOnly(OperatorPermission.SET_VIDEO_HERO))
   @Mutation(() => SetVideoHeroResult)
   async setVideoHero(@Args() args: SetVideoHeroInput): Promise<SetVideoHeroResult> {
     const em = await this.em()
@@ -156,7 +233,7 @@ export class AdminResolver {
     return { id }
   }
 
-  @UseMiddleware(OperatorOnly)
+  @UseMiddleware(OperatorOnly(OperatorPermission.SET_CATEGORY_FEATURED_VIDEOS))
   @Mutation(() => SetCategoryFeaturedVideosResult)
   async setCategoryFeaturedVideos(
     @Args() args: SetCategoryFeaturedVideosArgs
@@ -192,7 +269,7 @@ export class AdminResolver {
     }
   }
 
-  @UseMiddleware(OperatorOnly)
+  @UseMiddleware(OperatorOnly(OperatorPermission.SET_SUPPORTED_CATEGORIES))
   @Mutation(() => SetSupportedCategoriesResult)
   async setSupportedCategories(
     @Args()
@@ -233,7 +310,7 @@ export class AdminResolver {
     }
   }
 
-  @UseMiddleware(OperatorOnly)
+  @UseMiddleware(OperatorOnly(OperatorPermission.SET_FEATURED_NFTS))
   @Mutation(() => SetFeaturedNftsResult)
   async setFeaturedNfts(
     @Args() { featuredNftsIds }: SetFeaturedNftsInput
@@ -263,7 +340,7 @@ export class AdminResolver {
     }
   }
 
-  @UseMiddleware(OperatorOnly)
+  @UseMiddleware(OperatorOnly(OperatorPermission.EXCLUDE_CONTENT))
   @Mutation(() => ExcludeContentResult)
   async excludeContent(
     @Args()
@@ -289,7 +366,7 @@ export class AdminResolver {
     })
   }
 
-  @UseMiddleware(OperatorOnly)
+  @UseMiddleware(OperatorOnly(OperatorPermission.RESTORE_CONTENT))
   @Mutation(() => RestoreContentResult)
   async restoreContent(
     @Args()