4.1.1.: Video relevance fixes (#350)

CHANGELOG.md

@@ -1,3 +1,19 @@
+# 4.1.1
+
+## Affected components:
+- Processor (non-breaking changes)
+- GraphQL server (non-breaking changes)
+
+## Bug Fixes:
+- Fixed video relevance calculation algorithm by:
+    - Ignoring `publishedBeforeJoystream` date if it's in the future,
+    - Fixing a bug with Joystream/YT creation weight formula,
+    - Introduced an age limit of 365 days to prevent negative relevance score 
+- Fixed ineffective `videoRelevanceManager` calls in GraphQL server due to `videoRelevanceManager` not being turned on.
+- Fixed a bug causing both GraphQL server and Processor services to run video relevance and language update loops (which only processor should be running)
+- Fixed imports in GraphQl server and Processor by moving shared utilities to root `utils` directory to prevent mix-ups.
+
+
 # 4.1.0
 
 ## Affected components:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "orion",
-  "version": "4.1.0",
+  "version": "4.1.1",
   "engines": {
     "node": ">=16"
   },

src/mappings/content/commentsAndReactions.ts

@@ -53,13 +53,9 @@ import {
   commentCountersManager,
   videoRelevanceManager,
 } from '../utils'
-import {
-  getAccountForMember,
-  getChannelOwnerMemberByChannelId,
-  memberHandleById,
-  parseVideoTitle,
-} from './utils'
+import { getAccountForMember, getChannelOwnerMemberByChannelId, memberHandleById } from './utils'
 import { addNotification } from '../../utils/notification'
+import { parseVideoTitle } from '../../utils/notification/helpers'
 
 function parseVideoReaction(reaction: ReactVideo.Reaction): VideoReactionOptions {
   const protobufReactionToGraphqlReaction = {

src/mappings/content/nft.ts

@@ -53,9 +53,9 @@ import {
   notifyBiddersOnAuctionCompletion,
   notifyChannelFollowers,
   parseContentActor,
-  parseVideoTitle,
   processNft,
 } from './utils'
+import { parseVideoTitle } from '../../utils/notification/helpers'
 
 export async function processOpenAuctionStartedEvent({
   overlay,

src/mappings/content/utils.ts

@@ -75,6 +75,7 @@ import { criticalError } from '../../utils/misc'
 import { addNotification } from '../../utils/notification'
 import { EntityManagerOverlay, Flat } from '../../utils/overlay'
 import { addNftActivity, addNftHistoryEntry, genericEventFields, invalidMetadata } from '../utils'
+import { parseChannelTitle, parseVideoTitle } from '../../utils/notification/helpers'
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export type AsDecoded<MetaClass> = MetaClass extends { create: (props?: infer I) => any }
@@ -887,14 +888,6 @@ export function increaseChannelCumulativeRevenue(channel: Flat<Channel>, amount:
   channel.cumulativeRevenue = (channel.cumulativeRevenue || 0n) + amount
 }
 
-export function parseChannelTitle(channel: Flat<Channel>): string {
-  return channel.title || FALLBACK_CHANNEL_TITLE
-}
-
-export function parseVideoTitle(video: Flat<Video>): string {
-  return video.title || FALLBACK_VIDEO_TITLE
-}
-
 export async function memberHandleById(
   overlay: EntityManagerOverlay,
   memberId: string
@@ -908,6 +901,3 @@ export async function getChannelTitleById(overlay: EntityManagerOverlay, channel
   const channel = await overlay.getRepository(Channel).getByIdOrFail(channelId)
   return parseChannelTitle(channel)
 }
-
-export const FALLBACK_CHANNEL_TITLE = '??'
-export const FALLBACK_VIDEO_TITLE = '??'

src/mappings/content/video.ts

@@ -29,12 +29,11 @@ import {
   deleteVideo,
   encodeAssets,
   notifyChannelFollowers,
-  parseChannelTitle,
   parseContentActor,
-  parseVideoTitle,
   processAppActionMetadata,
   processNft,
 } from './utils'
+import { parseChannelTitle, parseVideoTitle } from '../../utils/notification/helpers'
 
 export async function processVideoCreatedEvent({
   overlay,

src/mappings/token/index.ts

@@ -45,7 +45,7 @@ import { getCurrentBlockHeight } from '../../utils/blockHeight'
 import { EventHandlerContext } from '../../utils/events'
 import { criticalError } from '../../utils/misc'
 import { addNotification } from '../../utils/notification'
-import { getChannelOwnerAccount, notifyChannelFollowers, parseChannelTitle } from '../content/utils'
+import { getChannelOwnerAccount, notifyChannelFollowers } from '../content/utils'
 import { deserializeMetadata, genericEventFields } from '../utils'
 import {
   VestingScheduleData,
@@ -60,6 +60,7 @@ import {
   processTokenMetadata,
   processValidatedTransfers,
 } from './utils'
+import { parseChannelTitle } from '../../utils/notification/helpers'
 
 export async function processTokenIssuedEvent({
   overlay,

src/mappings/utils.ts

@@ -15,15 +15,6 @@ import { OrionVideoLanguageManager } from '../utils/OrionVideoLanguageManager'
 export const orionVideoLanguageManager = new OrionVideoLanguageManager()
 export const commentCountersManager = new CommentCountersManager()
 export const videoRelevanceManager = new VideoRelevanceManager()
-// eslint-disable-next-line no-void
-void orionVideoLanguageManager.init(
-  1000 * 60 * 5 // 5 mins
-)
-// eslint-disable-next-line no-void
-void videoRelevanceManager.init({
-  fullUpdateLoopTime: 1000 * 60 * 60 * 12, // 12 hrs
-  scheduledUpdateLoopTime: 1000 * 60 * 10, // 10 mins
-})
 export const JOYSTREAM_SS58_PREFIX = 126
 
 export function bytesToString(b: Uint8Array): string {

src/processor.ts

@@ -113,7 +113,11 @@ import {
   processUpcomingTokenSaleUpdatedEvent,
   processUserParticipatedInSplitEvent,
 } from './mappings/token'
-import { commentCountersManager, videoRelevanceManager } from './mappings/utils'
+import {
+  commentCountersManager,
+  videoRelevanceManager,
+  orionVideoLanguageManager,
+} from './mappings/utils'
 import { Event } from './types/support'
 import { EventHandler, EventInstance, EventNames, eventConstructors } from './utils/events'
 import { assertAssignable } from './utils/misc'
@@ -363,6 +367,25 @@ async function afterDbUpdate(em: EntityManager) {
 }
 
 const offchainState = new OffchainState()
+
+// Initialize update intervals
+orionVideoLanguageManager
+  .init(
+    1000 * 60 * 5 // 5 mins
+  )
+  .catch((e) => {
+    throw new Error(`Failed to initialize Orion video language manager: ${e.toString()}`)
+  })
+
+videoRelevanceManager
+  .init({
+    fullUpdateLoopTime: 1000 * 60 * 60 * 12, // 12 hrs
+    scheduledUpdateLoopTime: 1000 * 60 * 10, // 10 mins
+  })
+  .catch((e) => {
+    throw new Error(`Failed to initialize Orion video relevance manager: ${e.toString()}`)
+  })
+
 let exportBlockNumber: number
 
 processor.run(new TypeormDatabase({ isolationLevel: 'READ COMMITTED' }), async (ctx) => {

src/server-extension/resolvers/AdminResolver/index.ts

@@ -11,8 +11,6 @@ import { GraphQLResolveInfo } from 'graphql'
 import 'reflect-metadata'
 import { Args, Ctx, Info, Int, Mutation, Query, Resolver, UseMiddleware } from 'type-graphql'
 import { EntityManager, In, Not, UpdateResult } from 'typeorm'
-import { parseVideoTitle } from '../../../mappings/content/utils'
-import { videoRelevanceManager } from '../../../mappings/utils'
 import {
   Account,
   Channel,
@@ -76,6 +74,8 @@ import {
   VideoWeights,
 } from './types'
 import { processCommentsCensorshipStatusUpdate } from './utils'
+import { recalculateAllVideosRelevance } from '../../utils'
+import { parseVideoTitle } from '../../../utils/notification/helpers'
 
 @Resolver()
 export class AdminResolver {
@@ -161,7 +161,7 @@ export class AdminResolver {
       ],
       em
     )
-    await videoRelevanceManager.updateVideoRelevanceValue(em, true)
+    await recalculateAllVideosRelevance(em)
     return { isApplied: true }
   }
 
@@ -222,7 +222,7 @@ export class AdminResolver {
         }
       )
 
-      await videoRelevanceManager.updateVideoRelevanceValue(em, true)
+      await recalculateAllVideosRelevance(em)
 
       // Push the result into the results array
       results.push({

src/server-extension/resolvers/AdminResolver/utils.ts

@@ -1,14 +1,13 @@
 import { EntityManager, In } from 'typeorm'
-import { CommentCountersManager } from '../../../utils/CommentsCountersManager'
 import { Comment } from '../../../model'
+import { commentCountersManager } from '../../utils'
 
 export async function processCommentsCensorshipStatusUpdate(em: EntityManager, ids: string[]) {
-  const manager = new CommentCountersManager()
   const comments = await em.getRepository(Comment).find({ where: { id: In(ids) } })
   comments.forEach((c) => {
-    manager.scheduleRecalcForComment(c.parentCommentId)
-    manager.scheduleRecalcForVideo(c.videoId)
+    commentCountersManager.scheduleRecalcForComment(c.parentCommentId)
+    commentCountersManager.scheduleRecalcForVideo(c.videoId)
   })
-  await manager.updateVideoCommentsCounters(em)
-  await manager.updateParentRepliesCounters(em)
+  await commentCountersManager.updateVideoCommentsCounters(em)
+  await commentCountersManager.updateParentRepliesCounters(em)
 }

src/server-extension/resolvers/ChannelsResolver/index.ts

@@ -53,8 +53,8 @@ import { uniqueId } from '../../../utils/crypto'
 import { AccountOnly, OperatorOnly, UserOnly } from '../middleware'
 import { addNotification } from '../../../utils/notification'
 import { assertNotNull } from '@subsquid/substrate-processor'
-import { FALLBACK_CHANNEL_TITLE } from '../../../mappings/content/utils'
 import pLimit from 'p-limit'
+import { parseChannelTitle } from '../../../utils/notification/helpers'
 
 @Resolver()
 export class ChannelsResolver {
@@ -463,7 +463,7 @@ export const excludeChannelService = async (
         em,
         account,
         new MemberRecipient({ membership: channelOwnerMemberId }),
-        new ChannelExcluded({ channelTitle: channel.title ?? FALLBACK_CHANNEL_TITLE })
+        new ChannelExcluded({ channelTitle: parseChannelTitle(channel) })
       )
     }
 

src/server-extension/resolvers/VideosResolver/index.ts

@@ -21,8 +21,6 @@ import { isObject } from 'lodash'
 import 'reflect-metadata'
 import { Arg, Args, Ctx, Info, Mutation, Query, Resolver, UseMiddleware } from 'type-graphql'
 import { EntityManager, In, MoreThan } from 'typeorm'
-import { parseVideoTitle } from '../../../mappings/content/utils'
-import { videoRelevanceManager } from '../../../mappings/utils'
 import {
   Account,
   ChannelRecipient,
@@ -53,6 +51,8 @@ import {
   SetOrUnsetPublicFeedResult,
   VideoReportInfo,
 } from './types'
+import { videoRelevanceManager } from '../../utils'
+import { parseVideoTitle } from '../../../utils/notification/helpers'
 
 @Resolver()
 export class VideosResolver {

src/server-extension/utils.ts

@@ -0,0 +1,12 @@
+import { EntityManager } from 'typeorm'
+import { CommentCountersManager } from '../utils/CommentsCountersManager'
+import { VideoRelevanceManager } from '../utils/VideoRelevanceManager'
+
+export const commentCountersManager = new CommentCountersManager()
+export const videoRelevanceManager = new VideoRelevanceManager()
+
+videoRelevanceManager.turnOnVideoRelevanceManager()
+
+export async function recalculateAllVideosRelevance(em: EntityManager) {
+  return videoRelevanceManager.updateVideoRelevanceValue(em, true)
+}

src/utils/VideoRelevanceManager.ts

@@ -2,8 +2,7 @@ import { EntityManager } from 'typeorm'
 import { config, ConfigVariable } from './config'
 import { globalEm } from './globalEm'
 
-// constant used to parse seconds from creation
-export const NEWNESS_SECONDS_DIVIDER = 60 * 60 * 24
+export const SECONDS_PER_DAY = 60 * 60 * 24
 
 type VideoRelevanceManagerLoops = {
   fullUpdateLoopTime: number
@@ -67,23 +66,40 @@ export class VideoRelevanceManager {
       defaultChannelWeight,
     ] = await config.get(ConfigVariable.RelevanceWeights, em)
     const channelWeight = defaultChannelWeight ?? 1
-    const wtEpoch = `((
-          extract(epoch from video.created_at)*${joystreamTimestampWeight} +
-          COALESCE(extract(epoch from video.published_before_joystream), extract(epoch from video.created_at))*${ytTimestampWeight}
-        ) / ${joystreamTimestampWeight} + ${ytTimestampWeight})`
+    const weightedEpoch = `
+    (
+      (
+        extract(epoch from video.created_at) * ${joystreamTimestampWeight} +
+        CASE
+          WHEN (
+            video.published_before_joystream IS NOT NULL
+            AND video.published_before_joystream < now()
+          ) THEN extract(epoch from video.published_before_joystream)
+          ELSE extract(epoch from video.created_at)
+        END * ${ytTimestampWeight}
+      ) / ${joystreamTimestampWeight + ytTimestampWeight}
+    )`
+    const weightedMeanVideoAgeDays = `
+    (
+      (extract(epoch FROM now()) - ${weightedEpoch})
+      / ${SECONDS_PER_DAY}
+    )`
 
     await em.query(`
         WITH videos_with_weight AS (
           SELECT 
             video.id as videoId,
             channel.id as channelId,
-            (ROUND((
-            (extract(epoch FROM date_trunc('day', now() at time zone 'UTC')) - ${wtEpoch})
-            / ${NEWNESS_SECONDS_DIVIDER} * ${newnessWeight * -1} 
-            + (views_num * ${viewsWeight}) 
-            + (comments_count * ${commentsWeight}) 
-            + (reactions_count * ${reactionsWeight})) 
-            * COALESCE(channel.channel_weight, ${channelWeight}), 2)) as videoRelevance
+            ROUND(
+              (
+                365 * ${newnessWeight}
+                - LEAST(${weightedMeanVideoAgeDays}, 365) * ${newnessWeight}
+                + (views_num * ${viewsWeight}) 
+                + (comments_count * ${commentsWeight}) 
+                + (reactions_count * ${reactionsWeight})
+              ) * COALESCE(channel.channel_weight, ${channelWeight}),
+              2
+            ) as videoRelevance
           FROM 
             video
             INNER JOIN channel  ON video.channel_id = channel.id
@@ -129,7 +145,7 @@ export class VideoRelevanceManager {
         SET
           video_relevance = CASE
             WHEN ranked_videos.rank = 1 THEN ranked_videos.maxChannelRelevance
-            ELSE 1
+            ELSE 0
           END
         FROM
           ranked_videos

src/utils/notification/helpers.ts

@@ -1,8 +1,8 @@
-import { Flat } from 'lodash'
 import { EntityManager } from 'typeorm'
 import {
   Account,
   AccountNotificationPreferences,
+  Channel,
   Event,
   NextEntityId,
   Notification,
@@ -11,11 +11,12 @@ import {
   NotificationType,
   RecipientType,
   Unread,
+  Video,
 } from '../../model'
 import { getCurrentBlockHeight } from '../blockHeight'
 import { uniqueId } from '../crypto'
 import { getNextIdForEntity } from '../nextEntityId'
-import { EntityManagerOverlay } from '../overlay'
+import { EntityManagerOverlay, Flat } from '../overlay'
 
 export const RUNTIME_NOTIFICATION_ID_TAG = 'RuntimeNotification'
 export const OFFCHAIN_NOTIFICATION_ID_TAG = 'OffchainNotification'
@@ -339,3 +340,14 @@ const chunkFromEnd = (str: string, interval: number): string[] =>
   )
 const roundDecPart = (decPart: string, fractionDigits: number): string =>
   Number(`.${decPart}`).toFixed(fractionDigits).slice(1)
+
+export function parseChannelTitle(channel: Flat<Channel>): string {
+  return channel.title || FALLBACK_CHANNEL_TITLE
+}
+
+export function parseVideoTitle(video: Flat<Video>): string {
+  return video.title || FALLBACK_VIDEO_TITLE
+}
+
+export const FALLBACK_CHANNEL_TITLE = '??'
+export const FALLBACK_VIDEO_TITLE = '??'