query node - predictable ids

query-node/mappings/src/common.ts

@@ -1,6 +1,7 @@
 import { SubstrateEvent } from '@dzlzv/hydra-common'
 import { DatabaseManager } from '@dzlzv/hydra-db-utils'
 import { u64 } from '@polkadot/types/primitive';
+import * as crypto from 'crypto'
 
 // Asset
 import {
@@ -40,12 +41,29 @@ export function invalidMetadata(extraInfo: string, data?: unknown): void {
   logger.info(errorMessage, data)
 }
 
+/*
+  Creates a predictable and unique ID for the given content.
+*/
+export function createPredictableId(blockNumber: number, eventIndex: number, content: string | Object): string {
+  const contentType = typeof content == 'string'
+    ? content
+    : JSON.stringify(content)
+
+  const id = `${blockNumber}_${eventIndex}_${contentType}`
+
+  return crypto
+    .createHash('sha256')
+    .update(id, 'utf-8')
+    .digest('base64')
+}
+
 /*
   Prepares data object from content parameters.
 */
 export async function prepareDataObject(
   contentParameters: ContentParameters,
   blockNumber: number,
+  eventIndex: number,
   owner: typeof DataObjectOwner,
 ): Promise<DataObject> {
   // convert generic content parameters coming from processor to custom Joystream data type
@@ -64,6 +82,8 @@ export async function prepareDataObject(
     updatedById: '1',
   })
 
+  dataObject.id = createPredictableId(blockNumber, eventIndex, dataObject)
+
   return dataObject
 }
 

query-node/mappings/src/content/channel.ts

@@ -40,6 +40,7 @@ export async function content_ChannelCreated(db: DatabaseManager, event: Substra
       metadata: channelCreationParameters.meta,
       db,
       blockNumber: event.blockNumber,
+      eventIndex: event.index,
       assets: channelCreationParameters.assets,
       contentOwner: convertContentActorToDataObjectOwner(contentActor, channelId.toNumber()),
     }
@@ -108,6 +109,7 @@ export async function content_ChannelUpdated(
         metadata: newMetadata,
         db,
         blockNumber: event.blockNumber,
+        eventIndex: event.index,
         assets: channelUpdateParameters.assets.unwrapOr([]),
         contentOwner: convertContentActorToDataObjectOwner(contentActor, channelId.toNumber()),
       }
@@ -206,6 +208,7 @@ export async function content_ChannelCategoryCreated(
       metadata: channelCategoryCreationParameters.meta,
       db,
       blockNumber: event.blockNumber,
+      eventIndex: event.index,
     }
   )
 
@@ -260,6 +263,7 @@ export async function content_ChannelCategoryUpdated(
       metadata: channelCategoryUpdateParameters.new_meta,
       db,
       blockNumber: event.blockNumber,
+      eventIndex: event.index,
     }
   )
 

query-node/mappings/src/content/utils.ts

@@ -34,6 +34,7 @@ import {
   inconsistentState,
   logger,
   prepareDataObject,
+  createPredictableId,
 } from '../common'
 
 
@@ -93,6 +94,7 @@ export interface IReadProtobufArguments {
   metadata: Bytes
   db: DatabaseManager
   blockNumber: number
+  eventIndex: number
 }
 
 export interface IReadProtobufArgumentsWithAssets extends IReadProtobufArguments {
@@ -249,6 +251,7 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
         assetIndex: metaAsObject.coverPhoto,
         assets: parameters.assets,
         db: parameters.db,
+        eventIndex: parameters.eventIndex,
         blockNumber: parameters.blockNumber,
         contentOwner: parameters.contentOwner,
       })
@@ -263,6 +266,7 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
         assets: parameters.assets,
         db: parameters.db,
         blockNumber: parameters.blockNumber,
+        eventIndex: parameters.eventIndex,
         contentOwner: parameters.contentOwner,
       })
       integrateAsset('avatarPhoto', result, asset) // changes `result` inline!
@@ -271,7 +275,7 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
 
     // prepare language if needed
     if ('language' in metaAsObject) {
-      const language = await prepareLanguage(metaAsObject.language, parameters.db, parameters.blockNumber)
+      const language = await prepareLanguage(metaAsObject.language, parameters.db, parameters.blockNumber, parameters.eventIndex)
       delete metaAsObject.language // make sure temporary value will not interfere
       language.integrateInto(result, 'language')
     }
@@ -309,7 +313,7 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
 
     // prepare license if needed
     if ('license' in metaAsObject) {
-      result.license = await prepareLicense(metaAsObject.license)
+      result.license = await prepareLicense(metaAsObject.license, parameters.blockNumber, parameters.eventIndex)
     }
 
     // prepare thumbnail photo asset if needed
@@ -319,6 +323,7 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
         assets: parameters.assets,
         db: parameters.db,
         blockNumber: parameters.blockNumber,
+        eventIndex: parameters.eventIndex,
         contentOwner: parameters.contentOwner,
       })
       integrateAsset('thumbnailPhoto', result, asset) // changes `result` inline!
@@ -332,6 +337,7 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
         assets: parameters.assets,
         db: parameters.db,
         blockNumber: parameters.blockNumber,
+        eventIndex: parameters.eventIndex,
         contentOwner: parameters.contentOwner,
       })
       integrateAsset('media', result, asset) // changes `result` inline!
@@ -340,7 +346,12 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
 
     // prepare language if needed
     if ('language' in metaAsObject) {
-      const language = await prepareLanguage(metaAsObject.language, parameters.db, parameters.blockNumber)
+      const language = await prepareLanguage(
+        metaAsObject.language,
+        parameters.db,
+        parameters.blockNumber,
+        parameters.eventIndex
+      )
       delete metaAsObject.language // make sure temporary value will not interfere
       language.integrateInto(result, 'language')
     }
@@ -452,6 +463,7 @@ interface IConvertAssetParameters {
   rawAsset: NewAsset
   db: DatabaseManager
   blockNumber: number
+  eventIndex: number
   contentOwner: typeof DataObjectOwner
 }
 
@@ -470,7 +482,12 @@ async function convertAsset(parameters: IConvertAssetParameters): Promise<AssetS
 
   // prepare data object
   const contentParameters: ContentParameters = parameters.rawAsset.asUpload
-  const dataObject = await prepareDataObject(contentParameters, parameters.blockNumber, parameters.contentOwner)
+  const dataObject = await prepareDataObject(
+    contentParameters,
+    parameters.blockNumber,
+    parameters.eventIndex,
+    parameters.contentOwner,
+  )
 
   return dataObject
 }
@@ -480,6 +497,7 @@ interface IExtractAssetParameters {
   assets: NewAsset[]
   db: DatabaseManager
   blockNumber: number
+  eventIndex: number
   contentOwner: typeof DataObjectOwner
 }
 
@@ -506,6 +524,7 @@ async function extractAsset(parameters: IExtractAssetParameters): Promise<Proper
     rawAsset: parameters.assets[parameters.assetIndex],
     db: parameters.db,
     blockNumber: parameters.blockNumber,
+    eventIndex: parameters.eventIndex,
     contentOwner: parameters.contentOwner,
   })
 
@@ -591,7 +610,12 @@ function extractVideoSize(assets: NewAsset[], assetIndex: number | undefined): n
   return videoSize
 }
 
-async function prepareLanguage(languageIso: string | undefined, db: DatabaseManager, blockNumber: number): Promise<PropertyChange<Language>> {
+async function prepareLanguage(
+  languageIso: string | undefined,
+  db: DatabaseManager,
+  blockNumber: number,
+  blockIndex: number,
+): Promise<PropertyChange<Language>> {
   // is language being unset?
   if (languageIso === undefined) {
     return PropertyChange.newUnset()
@@ -625,12 +649,18 @@ async function prepareLanguage(languageIso: string | undefined, db: DatabaseMana
     updatedById: '1',
   })
 
+  newLanguage.id = createPredictableId(blockNumber, blockIndex, newLanguage)
+
   await db.save<Language>(newLanguage)
 
   return PropertyChange.newChange(newLanguage)
 }
 
-async function prepareLicense(licenseProtobuf: LicenseMetadata.AsObject | undefined): Promise<License | undefined> {
+async function prepareLicense(
+  licenseProtobuf: LicenseMetadata.AsObject | undefined,
+  blockNumber: number,
+  blockIndex: number,
+): Promise<License | undefined> {
   // NOTE: Deletion of any previous license should take place in appropriate event handling function
   //       and not here even it might appear so.
 
@@ -647,6 +677,8 @@ async function prepareLicense(licenseProtobuf: LicenseMetadata.AsObject | undefi
     updatedById: '1',
   })
 
+  license.id = createPredictableId(blockNumber, blockIndex, license)
+
   return license
 }
 

query-node/mappings/src/content/video.ts

@@ -11,6 +11,7 @@ import {
 import {
   inconsistentState,
   logger,
+  createPredictableId,
 } from '../common'
 
 import {
@@ -57,6 +58,7 @@ export async function content_VideoCategoryCreated(
       metadata: videoCategoryCreationParameters.meta,
       db,
       blockNumber: event.blockNumber,
+      eventIndex: event.index,
     }
   )
 
@@ -109,6 +111,7 @@ export async function content_VideoCategoryUpdated(
       metadata: videoCategoryUpdateParameters.new_meta,
       db,
       blockNumber: event.blockNumber,
+      eventIndex: event.index,
     }
   )
 
@@ -172,6 +175,7 @@ export async function content_VideoCreated(
       metadata: videoCreationParameters.meta,
       db,
       blockNumber: event.blockNumber,
+      eventIndex: event.index,
       assets: videoCreationParameters.assets,
       contentOwner: convertContentActorToDataObjectOwner(contentActor, channelId.toNumber()),
     }
@@ -186,7 +190,7 @@ export async function content_VideoCreated(
   }
 
   // prepare video media metadata (if any)
-  const fixedProtobuf = integrateVideoMediaMetadata(null, protobufContent, event.blockNumber)
+  const fixedProtobuf = integrateVideoMediaMetadata(null, protobufContent, event.index, event.blockNumber)
 
   // create new video
   const video = new Video({
@@ -253,13 +257,14 @@ export async function content_VideoUpdated(
         metadata: newMetadata,
         db,
         blockNumber: event.blockNumber,
+        eventIndex: event.index,
         assets: videoUpdateParameters.assets.unwrapOr([]),
         contentOwner: convertContentActorToDataObjectOwner(contentActor, (new BN(video.channel.id)).toNumber()),
       }
     )
 
     // prepare video media metadata (if any)
-    const fixedProtobuf = integrateVideoMediaMetadata(video, protobufContent, event.blockNumber)
+    const fixedProtobuf = integrateVideoMediaMetadata(video, protobufContent, event.index, event.blockNumber)
 
     // remember original license
     const originalLicense = video.license
@@ -430,6 +435,7 @@ export async function content_FeaturedVideosSet(
 function integrateVideoMediaMetadata(
   existingRecord: Video | null,
   metadata: Partial<Video>,
+  eventIndex: number,
   blockNumber: number,
 ): Partial<Video> {
   if (!metadata.mediaMetadata) {
@@ -467,6 +473,14 @@ function integrateVideoMediaMetadata(
   // connect encoding to media metadata object
   mediaMetadata.encoding = encoding
 
+  // ensure predictable ids
+  if (!mediaMetadata.encoding.id) {
+    mediaMetadata.encoding.id = createPredictableId(blockNumber, eventIndex, mediaMetadata.encoding)
+  }
+  if (!mediaMetadata.id) {
+    mediaMetadata.id = createPredictableId(blockNumber, eventIndex, mediaMetadata)
+  }
+
   return {
     ...metadata,
     mediaMetadata

query-node/mappings/src/storage.ts

@@ -45,7 +45,7 @@ export async function dataDirectory_ContentAdded(db: DatabaseManager, event: Sub
   // save all content objects
   for (let parameters of contentParameters) {
     const owner = convertStorageObjectOwner(storageObjectOwner)
-    const dataObject = await prepareDataObject(parameters, event.blockNumber, owner)
+    const dataObject = await prepareDataObject(parameters, event.blockNumber, event.index, owner)
 
     // fill in auto-generated fields
     dataObject.createdAt = new Date(fixBlockTimestamp(event.blockTimestamp).toNumber())

query-node/mappings/src/workingGroup.ts

@@ -6,6 +6,7 @@ import { Bytes } from '@polkadot/types'
 import {
   inconsistentState,
   logger,
+  createPredictableId,
 } from './common'
 
 import {
@@ -30,7 +31,7 @@ export async function storageWorkingGroup_OpeningFilled(db: DatabaseManager, eve
   const {applicationIdToWorkerIdMap} = new StorageWorkingGroup.OpeningFilledEvent(event).data
 
   // call generic processing
-  await workingGroup_OpeningFilled(db, WorkerType.STORAGE, applicationIdToWorkerIdMap)
+  await workingGroup_OpeningFilled(db, WorkerType.STORAGE, applicationIdToWorkerIdMap, event.blockNumber, event.index)
 }
 
 export async function storageWorkingGroup_WorkerStorageUpdated(db: DatabaseManager, event: SubstrateEvent): Promise<void> {
@@ -72,7 +73,7 @@ export async function gatewayWorkingGroup_OpeningFilled(db: DatabaseManager, eve
   const {applicationIdToWorkerIdMap} = new GatewayWorkingGroup.OpeningFilledEvent(event).data
 
   // call generic processing
-  await workingGroup_OpeningFilled(db, WorkerType.GATEWAY, applicationIdToWorkerIdMap)
+  await workingGroup_OpeningFilled(db, WorkerType.GATEWAY, applicationIdToWorkerIdMap, event.blockNumber, event.index)
 }
 
 export async function gatewayWorkingGroup_WorkerStorageUpdated(db: DatabaseManager, event: SubstrateEvent): Promise<void> {
@@ -112,12 +113,14 @@ export async function gatewayWorkingGroup_TerminatedLeader(db: DatabaseManager,
 export async function workingGroup_OpeningFilled(
   db: DatabaseManager,
   workerType: WorkerType,
-  applicationIdToWorkerIdMap: ApplicationIdToWorkerIdMap
+  applicationIdToWorkerIdMap: ApplicationIdToWorkerIdMap,
+  blockNumber: number,
+  eventIndex: number,
 ): Promise<void> {
   const workerIds = [...applicationIdToWorkerIdMap.values()]
 
   for (const workerId of workerIds) {
-    await createWorker(db, workerId, workerType)
+    await createWorker(db, workerId, workerType, blockNumber, eventIndex)
   }
 
   // emit log event
@@ -172,9 +175,15 @@ export async function workingGroup_TerminatedLeader(db: DatabaseManager, workerT
 
 /////////////////// Helpers ////////////////////////////////////////////////////
 
-async function createWorker(db: DatabaseManager, workerId: WorkerId, workerType: WorkerType): Promise<void> {
-  // create new worker
+async function createWorker(
+  db: DatabaseManager,
+  workerId: WorkerId,
+  workerType: WorkerType,
+  blockNumber: number,
+  eventIndex: number,
+): Promise<void> {
   const newWorker = new Worker({
+    id: createPredictableId(blockNumber, eventIndex, workerType),
     workerId: workerId.toString(),
     type: workerType,
     isActive: true,