Metadata & mappings updates (DistributionBucketOperatorRemoved, DistributionBucketFamilyMetadataSet)

metadata-protobuf/compiled/index.d.ts

@@ -4,10 +4,10 @@ import * as $protobuf from "protobufjs";
 export interface IGeoCoordiantes {
 
     /** GeoCoordiantes latitude */
-    latitude: number;
+    latitude?: (number|null);
 
     /** GeoCoordiantes longitude */
-    longitude: number;
+    longitude?: (number|null);
 }
 
 /** Represents a GeoCoordiantes. */

metadata-protobuf/compiled/index.js

@@ -15,8 +15,8 @@ $root.GeoCoordiantes = (function() {
      * Properties of a GeoCoordiantes.
      * @exports IGeoCoordiantes
      * @interface IGeoCoordiantes
-     * @property {number} latitude GeoCoordiantes latitude
-     * @property {number} longitude GeoCoordiantes longitude
+     * @property {number|null} [latitude] GeoCoordiantes latitude
+     * @property {number|null} [longitude] GeoCoordiantes longitude
      */
 
     /**
@@ -74,8 +74,10 @@ $root.GeoCoordiantes = (function() {
     GeoCoordiantes.encode = function encode(message, writer) {
         if (!writer)
             writer = $Writer.create();
-        writer.uint32(/* id 3, wireType 5 =*/29).float(message.latitude);
-        writer.uint32(/* id 4, wireType 5 =*/37).float(message.longitude);
+        if (message.latitude != null && Object.hasOwnProperty.call(message, "latitude"))
+            writer.uint32(/* id 3, wireType 5 =*/29).float(message.latitude);
+        if (message.longitude != null && Object.hasOwnProperty.call(message, "longitude"))
+            writer.uint32(/* id 4, wireType 5 =*/37).float(message.longitude);
         return writer;
     };
 
@@ -121,10 +123,6 @@ $root.GeoCoordiantes = (function() {
                 break;
             }
         }
-        if (!message.hasOwnProperty("latitude"))
-            throw $util.ProtocolError("missing required 'latitude'", { instance: message });
-        if (!message.hasOwnProperty("longitude"))
-            throw $util.ProtocolError("missing required 'longitude'", { instance: message });
         return message;
     };
 
@@ -155,10 +153,12 @@ $root.GeoCoordiantes = (function() {
     GeoCoordiantes.verify = function verify(message) {
         if (typeof message !== "object" || message === null)
             return "object expected";
-        if (typeof message.latitude !== "number")
-            return "latitude: number expected";
-        if (typeof message.longitude !== "number")
-            return "longitude: number expected";
+        if (message.latitude != null && message.hasOwnProperty("latitude"))
+            if (typeof message.latitude !== "number")
+                return "latitude: number expected";
+        if (message.longitude != null && message.hasOwnProperty("longitude"))
+            if (typeof message.longitude !== "number")
+                return "longitude: number expected";
         return null;
     };
 

metadata-protobuf/package.json

@@ -38,7 +38,8 @@
   "dependencies": {
     "google-protobuf": "^3.14.0",
     "long": "^4.0.0",
-    "@types/long": "^4.0.1"
+    "@types/long": "^4.0.1",
+    "i18n-iso-countries": "^6.8.0"
   },
   "devDependencies": {
     "@types/chai": "^4.2.11",

metadata-protobuf/proto/Storage.proto

@@ -1,30 +1,30 @@
 syntax = "proto2";
 
 message GeoCoordiantes {
-  required float latitude = 3;
-  required float longitude = 4;
+  optional float latitude = 3;
+  optional float longitude = 4;
 }
 
 message NodeLocationMetadata {
   optional string country_code = 1; // ISO 3166-1 alpha-2 country code (2 letters)
   optional string city = 2; // City name
-  optional GeoCoordiantes coordinates = 3; // Geographic coordinates
+  optional GeoCoordiantes coordinates = 3; // Geographic coordinates (providing {} will unset current value)
 }
 
 message StorageBucketOperatorMetadata {
   optional string endpoint = 1; // Root storage node endpoint (ie. https://example.com/storage)
-  optional NodeLocationMetadata location = 2; // Information about node's phisical location
+  optional NodeLocationMetadata location = 2; // Information about node's phisical location (providing {} will unset current value)
   optional string extra = 3; // Additional information about the node / node operator
 }
 
 message DistributionBucketOperatorMetadata {
   optional string endpoint = 1; // Root distribution node endpoint (ie. https://example.com/distribution)
-  optional NodeLocationMetadata location = 2; // Information about node's phisical location
+  optional NodeLocationMetadata location = 2; // Information about node's phisical location (providing {} will unset current value)
   optional string extra = 3; // Additional information about the node / node operator
 }
 
 message DistributionBucketFamilyMetadata {
   optional string region = 1; // ID / name of the region covered by the distribution family (ie. us-east-1). Should be unique.
   optional string description = 2; // Additional, more specific description of the region
-  repeated GeoCoordiantes boundary = 3; // Geographical boundary of the region, defined as polygon through array of coordinates
+  repeated GeoCoordiantes boundary = 3; // Geographical boundary of the region, defined as polygon through array of coordinates (providing [{}] will unset the current value)
 }

metadata-protobuf/src/utils.ts

@@ -1,9 +1,14 @@
 import { AnyMessage, AnyMetadataClass, DecodedMetadataObject } from './types'
+import countries from 'i18n-iso-countries'
 
 export function isSet<T>(v: T | null | undefined): v is T {
   return v !== null && v !== undefined
 }
 
+export function isEmptyObject(object: Record<string, any>): boolean {
+  return Object.keys(object).length === 0
+}
+
 export function integrateMeta<
   T,
   Props extends readonly (keyof T & keyof M & string)[],
@@ -26,3 +31,8 @@ export function metaToObject<T>(metaClass: AnyMetadataClass<T>, value: AnyMessag
   // Default conversion options - use Strings for "Long" values and ignore unset "repeated" fields
   return metaClass.toObject(value, { arrays: false, longs: String }) as DecodedMetadataObject<T>
 }
+
+// According to ISO 3166-1 alpha-2 standard
+export function isValidCountryCode(code: string): boolean {
+  return countries.getAlpha2Codes()[code] !== undefined
+}

query-node/manifest.yml

@@ -5,7 +5,6 @@ hydraVersion: "3"
 dataSource:
   kind: substrate
   chain: joystream
-  indexerVersion: '0.1.6'
 entities:
   - generated/graphql-server/dist/src/modules/**/*.model.js
 typegen:
@@ -15,7 +14,7 @@ typegen:
     - storage.StorageBucketCreated
     - storage.StorageBucketInvitationAccepted
     - storage.StorageBucketsUpdatedForBag
-    - storage.DataObjectdUploaded
+    - storage.DataObjectsUploaded
     - storage.StorageOperatorMetadataSet
     - storage.StorageBucketVoucherLimitsSet
     - storage.PendingDataObjectsAccepted
@@ -32,7 +31,6 @@ typegen:
     - storage.UpdateBlacklist
     - storage.DynamicBagDeleted
     - storage.DynamicBagCreated
-    - storage.DeletionPrizeChanged
     - storage.VoucherChanged
     - storage.StorageBucketDeleted
     - storage.NumberOfStorageBucketsInDynamicBagCreationPolicyUpdated
@@ -49,6 +47,8 @@ typegen:
     - storage.DistributionBucketInvitationCancelled
     - storage.DistributionBucketInvitationAccepted
     - storage.DistributionBucketMetadataSet
+    - storage.DistributionBucketOperatorRemoved
+    - storage.DistributionBucketFamilyMetadataSet
 
     # membership
     - members.MemberRegistered
@@ -158,7 +158,7 @@ typegen:
     - gateway_working_group.terminate_role
   outDir: ./mappings/generated/types
   customTypes:
-    lib: '@joystream/types/augment'
+    lib: '@joystream/types/augment/all/types'
     typedefsLoc: '../types/augment/all/defs.json'
 mappings:
   # js module that exports the handler functions
@@ -276,8 +276,8 @@ mappings:
       handler: storage_StorageBucketInvitationAccepted
     - event: storage.StorageBucketsUpdatedForBag
       handler: storage_StorageBucketsUpdatedForBag
-    - event: storage.DataObjectdUploaded
-      handler: storage_DataObjectdUploaded
+    - event: storage.DataObjectsUploaded
+      handler: storage_DataObjectsUploaded
     - event: storage.StorageOperatorMetadataSet
       handler: storage_StorageOperatorMetadataSet
     - event: storage.StorageBucketVoucherLimitsSet
@@ -310,8 +310,6 @@ mappings:
       handler: storage_DynamicBagDeleted
     - event: storage.DynamicBagCreated
       handler: storage_DynamicBagCreated
-    - event: storage.DeletionPrizeChanged
-      handler: storage_DeletionPrizeChanged
     - event: storage.VoucherChanged
       handler: storage_VoucherChanged
     - event: storage.StorageBucketDeleted
@@ -344,6 +342,10 @@ mappings:
       handler: storage_DistributionBucketInvitationAccepted
     - event: storage.DistributionBucketMetadataSet
       handler: storage_DistributionBucketMetadataSet
+    - event: storage.DistributionBucketOperatorRemoved
+      handler: storage_DistributionBucketOperatorRemoved
+    - event: storage.DistributionBucketFamilyMetadataSet
+      handler: storage_DistributionBucketFamilyMetadataSet
   extrinsicHandlers:
     # infer defaults here
     #- extrinsic: Balances.Transfer

query-node/mappings/common.ts

@@ -52,3 +52,12 @@ export function deserializeMetadata<T>(
     return null
   }
 }
+
+export function bytesToString(b: Bytes): string {
+  return (
+    Buffer.from(b.toU8a(true))
+      .toString()
+      // eslint-disable-next-line no-control-regex
+      .replace(/\u0000/g, '')
+  )
+}

query-node/mappings/storage/index.ts

@@ -7,7 +7,9 @@ import {
   DistributionBucket,
   DistributionBucketFamily,
   DistributionBucketOperator,
+  DistributionBucketOperatorMetadata,
   DistributionBucketOperatorStatus,
+  NodeLocationMetadata,
   StorageBag,
   StorageBagOwner,
   StorageBagOwnerChannel,
@@ -20,16 +22,21 @@ import {
   StorageBucketOperatorStatusMissing,
   StorageDataObject,
   StorageSystemParameters,
+  GeoCoordinates,
 } from 'query-node/dist/model'
 import BN from 'bn.js'
-import { getById, getWorkingGroupModuleName } from '../common'
+import { getById, getWorkingGroupModuleName, bytesToString } from '../common'
 import { BTreeSet } from '@polkadot/types'
 import { DataObjectCreationParameters } from '@joystream/types/storage'
 import { registry } from '@joystream/types'
 import { In } from 'typeorm'
 import _ from 'lodash'
 import { DataObjectId, BagId, DynamicBagId, StaticBagId } from '@joystream/types/augment/all'
-import { processDistributionOperatorMetadata, processStorageOperatorMetadata } from './metadata'
+import {
+  processDistributionBucketFamilyMetadata,
+  processDistributionOperatorMetadata,
+  processStorageOperatorMetadata,
+} from './metadata'
 
 async function getDataObjectsInBag(
   store: DatabaseManager,
@@ -140,7 +147,10 @@ async function getBag(
     : getDynamicBag(store, bagId.asDynamic, relations)
 }
 
-async function getDistributionBucketOperatorWithMetadata(store: DatabaseManager, id: string) {
+async function getDistributionBucketOperatorWithMetadata(
+  store: DatabaseManager,
+  id: string
+): Promise<DistributionBucketOperator> {
   const operator = await store.get(DistributionBucketOperator, {
     where: { id },
     relations: ['metadata', 'metadata.nodeLocation', 'metadata.nodeLocation.coordinates'],
@@ -151,7 +161,7 @@ async function getDistributionBucketOperatorWithMetadata(store: DatabaseManager,
   return operator
 }
 
-async function getStorageBucketWithOperatorMetadata(store: DatabaseManager, id: string) {
+async function getStorageBucketWithOperatorMetadata(store: DatabaseManager, id: string): Promise<StorageBucket> {
   const bucket = await store.get(StorageBucket, {
     where: { id },
     relations: ['operatorMetadata', 'operatorMetadata.nodeLocation', 'operatorMetadata.nodeLocation.coordinates'],
@@ -162,7 +172,10 @@ async function getStorageBucketWithOperatorMetadata(store: DatabaseManager, id:
   return bucket
 }
 
-async function getDistributionBucketFamilyWithMetadata(store: DatabaseManager, id: string) {
+async function getDistributionBucketFamilyWithMetadata(
+  store: DatabaseManager,
+  id: string
+): Promise<DistributionBucketFamily> {
   const family = await store.get(DistributionBucketFamily, {
     where: { id },
     relations: ['metadata', 'metadata.boundary'],
@@ -212,7 +225,7 @@ export async function storage_StorageOperatorMetadataSet({ event, store }: Event
 }
 
 export async function storage_StorageBucketStatusUpdated({ event, store }: EventContext & StoreContext): Promise<void> {
-  const [bucketId, , acceptingNewBags] = new Storage.StorageBucketStatusUpdatedEvent(event).params
+  const [bucketId, acceptingNewBags] = new Storage.StorageBucketStatusUpdatedEvent(event).params
 
   const storageBucket = await getById(store, StorageBucket, bucketId.toString())
   storageBucket.acceptingNewBags = acceptingNewBags.isTrue
@@ -310,17 +323,17 @@ export async function storage_DynamicBagDeleted({ event, store }: EventContext &
 // DATA OBJECTS
 
 // Note: "Uploaded" here actually means "created" (the real upload happens later)
-export async function storage_DataObjectdUploaded({ event, store }: EventContext & StoreContext): Promise<void> {
-  const [dataObjectIds, uploadParams] = new Storage.DataObjectdUploadedEvent(event).params
+export async function storage_DataObjectsUploaded({ event, store }: EventContext & StoreContext): Promise<void> {
+  const [dataObjectIds, uploadParams] = new Storage.DataObjectsUploadedEvent(event).params
   const { bagId, authenticationKey, objectCreationList } = uploadParams
   const storageBag = await getBag(store, bagId)
   const dataObjects = dataObjectIds.map((objectId, i) => {
     const objectParams = new DataObjectCreationParameters(registry, objectCreationList[i].toJSON() as any)
     return new StorageDataObject({
       id: objectId.toString(),
-      authenticationKey: authenticationKey.toString(),
+      authenticationKey: bytesToString(authenticationKey),
       isAccepted: false,
-      ipfsHash: objectParams.ipfsContentId.toString(),
+      ipfsHash: bytesToString(objectParams.ipfsContentId),
       size: new BN(objectParams.getField('size').toString()),
       storageBag,
     })
@@ -510,6 +523,39 @@ export async function storage_DistributionBucketMetadataSet({
   await store.save<DistributionBucketOperator>(operator)
 }
 
+export async function storage_DistributionBucketOperatorRemoved({
+  event,
+  store,
+}: EventContext & StoreContext): Promise<void> {
+  const [, bucketId, workerId] = new Storage.DistributionBucketOperatorRemovedEvent(event).params
+
+  // TODO: Cascade remove
+
+  const operator = await getDistributionBucketOperatorWithMetadata(store, `${bucketId}-${workerId}`)
+  await store.remove<DistributionBucketOperator>(operator)
+  if (operator.metadata) {
+    await store.remove<DistributionBucketOperatorMetadata>(operator.metadata)
+    if (operator.metadata.nodeLocation) {
+      await store.remove<NodeLocationMetadata>(operator.metadata.nodeLocation)
+      if (operator.metadata.nodeLocation.coordinates) {
+        await store.remove<GeoCoordinates>(operator.metadata.nodeLocation.coordinates)
+      }
+    }
+  }
+}
+
+export async function storage_DistributionBucketFamilyMetadataSet({
+  event,
+  store,
+}: EventContext & StoreContext): Promise<void> {
+  const [familyId, metadataBytes] = new Storage.DistributionBucketFamilyMetadataSetEvent(event).params
+
+  const family = await getDistributionBucketFamilyWithMetadata(store, familyId.toString())
+  family.metadata = await processDistributionBucketFamilyMetadata(store, family.metadata, metadataBytes)
+
+  await store.save<DistributionBucketFamily>(family)
+}
+
 export async function storage_DistributionBucketsPerBagLimitUpdated({
   event,
   store,
@@ -559,10 +605,6 @@ export async function storage_StorageBucketsVoucherMaxLimitsUpdated({
   // To be implemented
 }
 
-export async function storage_DeletionPrizeChanged({ event, store }: EventContext & StoreContext): Promise<void> {
-  // To be implemented
-}
-
 export async function storage_VoucherChanged({ event, store }: EventContext & StoreContext): Promise<void> {
   // To be implemented
 }

query-node/mappings/storage/metadata.ts

@@ -14,7 +14,7 @@ import {
   DistributionBucketFamilyMetadata as DistributionBucketFamilyMetadataProto,
   INodeLocationMetadata,
 } from '@joystream/metadata-protobuf'
-import { isSet } from '@joystream/metadata-protobuf/utils'
+import { isSet, isEmptyObject, isValidCountryCode } from '@joystream/metadata-protobuf/utils'
 
 async function processNodeLocationMetadata(
   store: DatabaseManager,
@@ -26,15 +26,23 @@ async function processNodeLocationMetadata(
     nodeLocation.city = meta.city
   }
   if (isSet(meta.coordinates)) {
-    const coordinates = current?.coordinates || new GeoCoordinates()
-    coordinates.latitude = meta.coordinates.latitude
-    coordinates.longitude = meta.coordinates.longitude
-    await store.save<GeoCoordinates>(coordinates)
-    nodeLocation.coordinates = coordinates
+    if (isEmptyObject(meta.coordinates)) {
+      nodeLocation.coordinates = null as any
+    } else {
+      const coordinates = current?.coordinates || new GeoCoordinates()
+      coordinates.latitude = meta.coordinates.latitude || coordinates.latitude || 0
+      coordinates.longitude = meta.coordinates.longitude || coordinates.longitude || 0
+      await store.save<GeoCoordinates>(coordinates)
+      nodeLocation.coordinates = coordinates
+    }
   }
   if (isSet(meta.countryCode)) {
-    // TODO: Validate the code
-    nodeLocation.countryCode = meta.countryCode
+    if (isValidCountryCode(meta.countryCode)) {
+      nodeLocation.countryCode = meta.countryCode
+    } else {
+      console.warn(`Invalid country code: ${meta.countryCode}`)
+      nodeLocation.countryCode = null as any
+    }
   }
   await store.save<NodeLocationMetadata>(nodeLocation)
   return nodeLocation
@@ -54,7 +62,9 @@ export async function processDistributionOperatorMetadata(
     metadataEntity.nodeEndpoint = meta.endpoint
   }
   if (isSet(meta.location)) {
-    metadataEntity.nodeLocation = await processNodeLocationMetadata(store, metadataEntity.nodeLocation, meta.location)
+    metadataEntity.nodeLocation = isEmptyObject(meta.location)
+      ? (null as any)
+      : await processNodeLocationMetadata(store, metadataEntity.nodeLocation, meta.location)
   }
   if (isSet(meta.extra)) {
     metadataEntity.extra = meta.extra
@@ -76,13 +86,15 @@ export async function processStorageOperatorMetadata(
   }
   const metadataEntity = current || new StorageBucketOperatorMetadata()
   if (isSet(meta.endpoint)) {
-    metadataEntity.nodeEndpoint = meta.endpoint
+    metadataEntity.nodeEndpoint = meta.endpoint || (null as any)
   }
   if (isSet(meta.location)) {
-    metadataEntity.nodeLocation = await processNodeLocationMetadata(store, metadataEntity.nodeLocation, meta.location)
+    metadataEntity.nodeLocation = isEmptyObject(meta.location)
+      ? (null as any)
+      : await processNodeLocationMetadata(store, metadataEntity.nodeLocation, meta.location)
   }
   if (isSet(meta.extra)) {
-    metadataEntity.extra = meta.extra
+    metadataEntity.extra = meta.extra || (null as any)
   }
 
   await store.save<StorageBucketOperatorMetadata>(metadataEntity)
@@ -101,10 +113,10 @@ export async function processDistributionBucketFamilyMetadata(
   }
   const metadataEntity = current || new DistributionBucketFamilyMetadata()
   if (isSet(meta.region)) {
-    metadataEntity.region = meta.region
+    metadataEntity.region = meta.region || (null as any)
   }
   if (isSet(meta.description)) {
-    metadataEntity.description = meta.description
+    metadataEntity.description = meta.description || (null as any)
   }
 
   await store.save<DistributionBucketOperatorMetadata>(metadataEntity)
@@ -113,11 +125,17 @@ export async function processDistributionBucketFamilyMetadata(
   if (isSet(meta.boundary)) {
     await Promise.all((metadataEntity.boundary || []).map((coords) => store.remove<GeoCoordinates>(coords)))
     await Promise.all(
-      meta.boundary.map(({ latitude, longitude }) =>
-        store.save<GeoCoordinates>(
-          new GeoCoordinates({ latitude, longitude, boundarySourceBucketFamilyMeta: metadataEntity })
+      meta.boundary
+        .filter((c) => !isEmptyObject(c))
+        .map(({ latitude, longitude }) =>
+          store.save<GeoCoordinates>(
+            new GeoCoordinates({
+              latitude: latitude || 0,
+              longitude: longitude || 0,
+              boundarySourceBucketFamilyMeta: metadataEntity,
+            })
+          )
         )
-      )
     )
   }