Merge pull request #3067 from Lezek123/giza-undeterministic-ids-fix

Query node: Undeterministic ids fix

distributor-node/config.yml

@@ -34,6 +34,7 @@ operatorApi:
   hmacSecret: this-is-not-so-secret
 keys:
   - suri: //Alice
+  - suri: //testing//worker//Distribution//0
   # - mnemonic: "escape naive annual throw tragic achieve grunt verify cram note harvest problem"
   #   type: ed25519
   # - keyfile: "/path/to/keyfile.json"

query-node/mappings/common.ts

@@ -6,6 +6,7 @@ import { metaToObject } from '@joystream/metadata-protobuf/utils'
 import { AnyMetadataClass, DecodedMetadataObject } from '@joystream/metadata-protobuf/types'
 
 export const CURRENT_NETWORK = Network.GIZA
+
 /*
   Simple logger enabling error and informational reporting.
 
@@ -223,3 +224,10 @@ export async function getById<T extends BaseModel>(
 
   return result
 }
+
+export function deterministicEntityId(createdInEvent: SubstrateEvent, additionalIdentifier?: string | number): string {
+  return (
+    `${createdInEvent.blockNumber}-${createdInEvent.indexInBlock}` +
+    (additionalIdentifier ? `-${additionalIdentifier}` : '')
+  )
+}

query-node/mappings/content/utils.ts

@@ -1,5 +1,5 @@
 import { DatabaseManager, EventContext, StoreContext } from '@joystream/hydra-common'
-import { FindConditions, Raw } from 'typeorm'
+import { FindConditions } from 'typeorm'
 import {
   IVideoMetadata,
   IPublishedBeforeJoystream,
@@ -8,7 +8,7 @@ import {
   IChannelMetadata,
 } from '@joystream/metadata-protobuf'
 import { integrateMeta, isSet, isValidLanguageCode } from '@joystream/metadata-protobuf/utils'
-import { invalidMetadata, inconsistentState, logger } from '../common'
+import { invalidMetadata, inconsistentState, logger, deterministicEntityId } from '../common'
 import {
   // primary entities
   CuratorGroup,
@@ -209,9 +209,8 @@ async function processVideoMediaEncoding(
   const encoding =
     existingVideoMediaEncoding ||
     new VideoMediaEncoding({
+      id: deterministicEntityId(event),
       createdAt: new Date(event.blockTimestamp),
-      createdById: '1',
-      updatedById: '1',
     })
   // integrate media encoding-related data
   integrateMeta(encoding, metadata, ['codecName', 'container', 'mimeMediaType'])
@@ -231,10 +230,9 @@ async function processVideoMediaMetadata(
   const videoMedia =
     existingVideoMedia ||
     new VideoMediaMetadata({
+      id: deterministicEntityId(event),
       createdInBlock: event.blockNumber,
       createdAt: new Date(event.blockTimestamp),
-      createdById: '1',
-      updatedById: '1',
     })
 
   // integrate media-related data
@@ -364,13 +362,11 @@ async function processLanguage(
 
   // create new language
   const newLanguage = new Language({
+    id: deterministicEntityId(event),
     iso: languageIso,
     createdInBlock: event.blockNumber,
     createdAt: new Date(event.blockTimestamp),
     updatedAt: new Date(event.blockTimestamp),
-    // TODO: remove these lines after Hydra auto-fills the values when cascading save (remove them on all places)
-    createdById: '1',
-    updatedById: '1',
   })
 
   await store.save<Language>(newLanguage)
@@ -397,9 +393,8 @@ async function updateVideoLicense(
     license =
       previousLicense ||
       new License({
+        id: deterministicEntityId(event),
         createdAt: new Date(event.blockTimestamp),
-        createdById: '1',
-        updatedById: '1',
       })
     license.updatedAt = new Date(event.blockTimestamp)
     integrateMeta(license, licenseMetadata, ['attribution', 'code', 'customText'])

query-node/mappings/storage/index.ts

@@ -76,6 +76,7 @@ export async function storage_StorageOperatorMetadataSet({ event, store }: Event
   const [bucketId, , metadataBytes] = new Storage.StorageOperatorMetadataSetEvent(event).params
   const storageBucket = await getStorageBucketWithOperatorMetadata(store, bucketId.toString())
   storageBucket.operatorMetadata = await processStorageOperatorMetadata(
+    event,
     store,
     storageBucket.operatorMetadata,
     metadataBytes
@@ -276,7 +277,7 @@ export async function storage_DistributionBucketFamilyMetadataSet({
   const [familyId, metadataBytes] = new Storage.DistributionBucketFamilyMetadataSetEvent(event).params
 
   const family = await getDistributionBucketFamilyWithMetadata(store, familyId.toString())
-  family.metadata = await processDistributionBucketFamilyMetadata(store, family.metadata, metadataBytes)
+  family.metadata = await processDistributionBucketFamilyMetadata(event, store, family.metadata, metadataBytes)
 
   await store.save<DistributionBucketFamily>(family)
 }
@@ -426,7 +427,7 @@ export async function storage_DistributionBucketMetadataSet({
   const [workerId, bucketId, metadataBytes] = new Storage.DistributionBucketMetadataSetEvent(event).params
 
   const operator = await getDistributionBucketOperatorWithMetadata(store, distributionOperatorId(bucketId, workerId))
-  operator.metadata = await processDistributionOperatorMetadata(store, operator.metadata, metadataBytes)
+  operator.metadata = await processDistributionOperatorMetadata(event, store, operator.metadata, metadataBytes)
 
   await store.save<DistributionBucketOperator>(operator)
 }

query-node/mappings/storage/metadata.ts

@@ -1,4 +1,4 @@
-import { DatabaseManager } from '@joystream/hydra-common'
+import { DatabaseManager, SubstrateEvent } from '@joystream/hydra-common'
 import {
   DistributionBucketFamilyMetadata,
   DistributionBucketOperatorMetadata,
@@ -11,7 +11,7 @@ import {
   GeographicalAreaSubdivistion,
   DistributionBucketFamilyGeographicArea,
 } from 'query-node/dist/model'
-import { deserializeMetadata, invalidMetadata } from '../common'
+import { deserializeMetadata, deterministicEntityId, invalidMetadata } from '../common'
 import { Bytes } from '@polkadot/types'
 import {
   DistributionBucketOperatorMetadata as DistributionBucketOperatorMetadataProto,
@@ -21,6 +21,7 @@ import {
   GeographicalArea as GeographicalAreaProto,
 } from '@joystream/metadata-protobuf'
 import { isSet, isEmptyObject, isValidCountryCode, isValidSubdivisionCode } from '@joystream/metadata-protobuf/utils'
+import _ from 'lodash'
 
 const protobufContinentToGraphlContinent: { [key in GeographicalAreaProto.Continent]: Continent } = {
   [GeographicalAreaProto.Continent.AF]: Continent.AF,
@@ -33,11 +34,12 @@ const protobufContinentToGraphlContinent: { [key in GeographicalAreaProto.Contin
 }
 
 async function processNodeLocationMetadata(
+  event: SubstrateEvent,
   store: DatabaseManager,
   current: NodeLocationMetadata | undefined,
   meta: INodeLocationMetadata
 ): Promise<NodeLocationMetadata> {
-  const nodeLocation = current || new NodeLocationMetadata()
+  const nodeLocation = current || new NodeLocationMetadata({ id: deterministicEntityId(event) })
   if (isSet(meta.city)) {
     nodeLocation.city = meta.city
   }
@@ -45,7 +47,7 @@ async function processNodeLocationMetadata(
     if (isEmptyObject(meta.coordinates)) {
       nodeLocation.coordinates = null as any
     } else {
-      const coordinates = current?.coordinates || new GeoCoordinates()
+      const coordinates = current?.coordinates || new GeoCoordinates({ id: deterministicEntityId(event) })
       coordinates.latitude = meta.coordinates.latitude || coordinates.latitude || 0
       coordinates.longitude = meta.coordinates.longitude || coordinates.longitude || 0
       await store.save<GeoCoordinates>(coordinates)
@@ -65,6 +67,7 @@ async function processNodeLocationMetadata(
 }
 
 export async function processDistributionOperatorMetadata(
+  event: SubstrateEvent,
   store: DatabaseManager,
   current: DistributionBucketOperatorMetadata | undefined,
   metadataBytes: Bytes
@@ -73,14 +76,14 @@ export async function processDistributionOperatorMetadata(
   if (!meta) {
     return current
   }
-  const metadataEntity = current || new DistributionBucketOperatorMetadata()
+  const metadataEntity = current || new DistributionBucketOperatorMetadata({ id: deterministicEntityId(event) })
   if (isSet(meta.endpoint)) {
     metadataEntity.nodeEndpoint = meta.endpoint
   }
   if (isSet(meta.location)) {
     metadataEntity.nodeLocation = isEmptyObject(meta.location)
       ? (null as any)
-      : await processNodeLocationMetadata(store, metadataEntity.nodeLocation, meta.location)
+      : await processNodeLocationMetadata(event, store, metadataEntity.nodeLocation, meta.location)
   }
   if (isSet(meta.extra)) {
     metadataEntity.extra = meta.extra
@@ -92,6 +95,7 @@ export async function processDistributionOperatorMetadata(
 }
 
 export async function processStorageOperatorMetadata(
+  event: SubstrateEvent,
   store: DatabaseManager,
   current: StorageBucketOperatorMetadata | undefined,
   metadataBytes: Bytes
@@ -100,14 +104,14 @@ export async function processStorageOperatorMetadata(
   if (!meta) {
     return current
   }
-  const metadataEntity = current || new StorageBucketOperatorMetadata()
+  const metadataEntity = current || new StorageBucketOperatorMetadata({ id: deterministicEntityId(event) })
   if (isSet(meta.endpoint)) {
     metadataEntity.nodeEndpoint = meta.endpoint || (null as any)
   }
   if (isSet(meta.location)) {
     metadataEntity.nodeLocation = isEmptyObject(meta.location)
       ? (null as any)
-      : await processNodeLocationMetadata(store, metadataEntity.nodeLocation, meta.location)
+      : await processNodeLocationMetadata(event, store, metadataEntity.nodeLocation, meta.location)
   }
   if (isSet(meta.extra)) {
     metadataEntity.extra = meta.extra || (null as any)
@@ -119,6 +123,7 @@ export async function processStorageOperatorMetadata(
 }
 
 export async function processDistributionBucketFamilyMetadata(
+  event: SubstrateEvent,
   store: DatabaseManager,
   current: DistributionBucketFamilyMetadata | undefined,
   metadataBytes: Bytes
@@ -127,7 +132,7 @@ export async function processDistributionBucketFamilyMetadata(
   if (!meta) {
     return current
   }
-  const metadataEntity = current || new DistributionBucketFamilyMetadata()
+  const metadataEntity = current || new DistributionBucketFamilyMetadata({ id: deterministicEntityId(event) })
   if (isSet(meta.region)) {
     metadataEntity.region = meta.region || (null as any)
   }
@@ -138,7 +143,7 @@ export async function processDistributionBucketFamilyMetadata(
     metadataEntity.latencyTestTargets = meta.latencyTestTargets.filter((t) => t)
   }
 
-  await store.save<DistributionBucketOperatorMetadata>(metadataEntity)
+  await store.save<DistributionBucketFamilyMetadata>(metadataEntity)
 
   // Update areas after metadata is saved (since we need an id to reference)
   if (isSet(meta.areas)) {
@@ -146,45 +151,42 @@ export async function processDistributionBucketFamilyMetadata(
     await Promise.all(metadataEntity.areas?.map((a) => store.remove<DistributionBucketFamilyGeographicArea>(a)) || [])
     // Save new areas
     await Promise.all(
-      meta.areas
-        .filter((a) => !isEmptyObject(a))
-        .map(async (a) => {
-          const area = new DistributionBucketFamilyGeographicArea({
-            distributionBucketFamilyMetadata: metadataEntity,
-          })
+      _.uniqWith(
+        meta.areas.filter((a) => !isEmptyObject(a)),
+        _.isEqual
+      ).map(async (a, i) => {
+        const area = new DistributionBucketFamilyGeographicArea({
+          id: `${metadataEntity.id}-${i}`,
+          distributionBucketFamilyMetadata: metadataEntity,
+        })
 
-          if (a.continent) {
-            const continent = new GeographicalAreaContinent()
-            continent.code = protobufContinentToGraphlContinent[a.continent]
-            if (!continent.code) {
-              return invalidMetadata(`Unrecognized continent enum variant: ${a.continent}`)
-            }
-            area.id = `${metadataEntity.id}-C-${continent.code}`
-            area.area = continent
+        if (a.continent) {
+          const continent = new GeographicalAreaContinent()
+          continent.code = protobufContinentToGraphlContinent[a.continent]
+          if (!continent.code) {
+            return invalidMetadata(`Unrecognized continent enum variant: ${a.continent}`)
           }
-
-          if (a.countryCode) {
-            if (!isValidCountryCode(a.countryCode)) {
-              return invalidMetadata(`Invalid country code: ${a.countryCode}`)
-            }
-            const country = new GeographicalAreaCountry()
-            country.code = a.countryCode
-            area.id = `${metadataEntity.id}-c-${country.code}`
-            area.area = country
+          area.area = continent
+        } else if (a.countryCode) {
+          if (!isValidCountryCode(a.countryCode)) {
+            return invalidMetadata(`Invalid country code: ${a.countryCode}`)
           }
-
-          if (a.subdivisionCode) {
-            if (!isValidSubdivisionCode(a.subdivisionCode)) {
-              return invalidMetadata(`Invalid subdivision code: ${a.subdivisionCode}`)
-            }
-            const subdivision = new GeographicalAreaSubdivistion()
-            subdivision.code = a.subdivisionCode
-            area.id = `${metadataEntity.id}-s-${subdivision.code}`
-            area.area = subdivision
+          const country = new GeographicalAreaCountry()
+          country.code = a.countryCode
+          area.area = country
+        } else if (a.subdivisionCode) {
+          if (!isValidSubdivisionCode(a.subdivisionCode)) {
+            return invalidMetadata(`Invalid subdivision code: ${a.subdivisionCode}`)
           }
-
-          await store.save<DistributionBucketFamilyGeographicArea>(area)
-        })
+          const subdivision = new GeographicalAreaSubdivistion()
+          subdivision.code = a.subdivisionCode
+          area.area = subdivision
+        } else {
+          return
+        }
+
+        await store.save<DistributionBucketFamilyGeographicArea>(area)
+      })
     )
   }