Fix https://github.com/Joystream/joystream/pull/2515 review comments

query-node/manifest.yml

@@ -33,7 +33,6 @@ typegen:
     - storage.DynamicBagCreated
     - storage.VoucherChanged
     - storage.StorageBucketDeleted
-    - storage.NumberOfStorageBucketsInDynamicBagCreationPolicyUpdated
     - storage.DistributionBucketFamilyCreated
     - storage.DistributionBucketFamilyDeleted
     - storage.DistributionBucketCreated
@@ -42,13 +41,16 @@ typegen:
     - storage.DistributionBucketsUpdatedForBag
     - storage.DistributionBucketsPerBagLimitUpdated
     - storage.DistributionBucketModeUpdated
-    - storage.FamiliesInDynamicBagCreationPolicyUpdated
     - storage.DistributionBucketOperatorInvited
     - storage.DistributionBucketInvitationCancelled
     - storage.DistributionBucketInvitationAccepted
     - storage.DistributionBucketMetadataSet
     - storage.DistributionBucketOperatorRemoved
     - storage.DistributionBucketFamilyMetadataSet
+    # Not required:
+    # - storage.NumberOfStorageBucketsInDynamicBagCreationPolicyUpdated
+    # - storage.FamiliesInDynamicBagCreationPolicyUpdated
+
   # TODO: Sumer mappings
   #   # membership
   #   - members.MemberRegistered
@@ -315,8 +317,6 @@ mappings:
       handler: storage_VoucherChanged
     - event: storage.StorageBucketDeleted
       handler: storage_StorageBucketDeleted
-    - event: storage.NumberOfStorageBucketsInDynamicBagCreationPolicyUpdated
-      handler: storage_NumberOfStorageBucketsInDynamicBagCreationPolicyUpdated
     - event: storage.DistributionBucketFamilyCreated
       handler: storage_DistributionBucketFamilyCreated
     - event: storage.DistributionBucketFamilyDeleted
@@ -333,8 +333,6 @@ mappings:
       handler: storage_DistributionBucketsPerBagLimitUpdated
     - event: storage.DistributionBucketModeUpdated
       handler: storage_DistributionBucketModeUpdated
-    - event: storage.FamiliesInDynamicBagCreationPolicyUpdated
-      handler: storage_FamiliesInDynamicBagCreationPolicyUpdated
     - event: storage.DistributionBucketOperatorInvited
       handler: storage_DistributionBucketOperatorInvited
     - event: storage.DistributionBucketInvitationCancelled

query-node/mappings/giza/genesis-data/storageSystem.json

@@ -1 +1,10 @@
-{ "blacklist": [] }
+{
+  "id": "1",
+  "blacklist": [],
+  "storageBucketsPerBagLimit": 0,
+  "distributionBucketsPerBagLimit": 0,
+  "uploadingBlocked": false,
+  "dataObjectFeePerMb": 0,
+  "storageBucketMaxObjectsCountLimit": 0,
+  "storageBucketMaxObjectsSizeLimit": 0
+}

query-node/mappings/giza/genesis.ts

@@ -1,4 +1,5 @@
 import { StoreContext } from '@joystream/hydra-common'
+import BN from 'bn.js'
 import { StorageSystemParameters } from 'query-node/dist/model'
 import { storageSystem } from './genesis-data'
 
@@ -7,6 +8,9 @@ export async function loadGenesisData({ store }: StoreContext): Promise<void> {
   await store.save<StorageSystemParameters>(
     new StorageSystemParameters({
       ...storageSystem,
+      storageBucketMaxObjectsCountLimit: new BN(storageSystem.storageBucketMaxObjectsCountLimit),
+      storageBucketMaxObjectsSizeLimit: new BN(storageSystem.storageBucketMaxObjectsSizeLimit),
+      dataObjectFeePerMb: new BN(storageSystem.dataObjectFeePerMb),
     })
   )
 }

query-node/mappings/giza/storage/index.ts

@@ -90,9 +90,9 @@ function getDynamicBagOwner(bagId: DynamicBagId) {
 
 function getStaticBagId(bagId: StaticBagId): string {
   if (bagId.isCouncil) {
-    return `CO`
+    return `static:council`
   } else if (bagId.isWorkingGroup) {
-    return `WG-${bagId.asWorkingGroup.type}`
+    return `static:wg:${bagId.asWorkingGroup.type.toLowerCase()}`
   } else {
     throw new Error(`Unexpected static bag type: ${bagId.type}`)
   }
@@ -100,9 +100,9 @@ function getStaticBagId(bagId: StaticBagId): string {
 
 function getDynamicBagId(bagId: DynamicBagId): string {
   if (bagId.isChannel) {
-    return `CH-${bagId.asChannel.toString()}`
+    return `dynamic:channel:${bagId.asChannel.toString()}`
   } else if (bagId.isMember) {
-    return `M-${bagId.asMember.toString()}`
+    return `dynamic:member:${bagId.asMember.toString()}`
   } else {
     throw new Error(`Unexpected dynamic bag type: ${bagId.type}`)
   }
@@ -180,7 +180,16 @@ async function getDistributionBucketFamilyWithMetadata(
   return family
 }
 
-// BUCKETS
+async function getStorageSystem(store: DatabaseManager) {
+  const storageSystem = await store.get(StorageSystemParameters, {})
+  if (!storageSystem) {
+    throw new Error('Storage system entity is missing!')
+  }
+
+  return storageSystem
+}
+
+// STORAGE BUCKETS
 
 export async function storage_StorageBucketCreated({ event, store }: EventContext & StoreContext): Promise<void> {
   const [
@@ -196,6 +205,8 @@ export async function storage_StorageBucketCreated({ event, store }: EventContex
     acceptingNewBags: acceptingNewBags.isTrue,
     dataObjectCountLimit: new BN(dataObjectCountLimit.toString()),
     dataObjectsSizeLimit: new BN(dataObjectSizeLimit.toString()),
+    dataObjectsCount: new BN(0),
+    dataObjectsSize: new BN(0),
   })
   if (invitedWorkerId.isSome) {
     const operatorStatus = new StorageBucketOperatorStatusInvited()
@@ -297,10 +308,33 @@ export async function storage_StorageBucketsUpdatedForBag({
   await Promise.all(assignmentsToAdd.map((a) => store.save<StorageBagStorageAssignment>(a)))
 }
 
+export async function storage_VoucherChanged({ event, store }: EventContext & StoreContext): Promise<void> {
+  const [bucketId, voucher] = new Storage.VoucherChangedEvent(event).params
+  const bucket = await getById(store, StorageBucket, bucketId.toString())
+
+  bucket.dataObjectCountLimit = voucher.objectsLimit
+  bucket.dataObjectsSizeLimit = voucher.sizeLimit
+  bucket.dataObjectsCount = voucher.objectsUsed
+  bucket.dataObjectsSize = voucher.sizeUsed
+
+  await store.save<StorageBucket>(bucket)
+}
+
+export async function storage_StorageBucketVoucherLimitsSet({
+  event,
+  store,
+}: EventContext & StoreContext): Promise<void> {
+  const [bucketId, sizeLimit, countLimit] = new Storage.StorageBucketVoucherLimitsSetEvent(event).params
+  const bucket = await getById(store, StorageBucket, bucketId.toString())
+  bucket.dataObjectsSizeLimit = sizeLimit
+  bucket.dataObjectCountLimit = countLimit
+
+  await store.save<StorageBucket>(bucket)
+}
+
 export async function storage_StorageBucketDeleted({ event, store }: EventContext & StoreContext): Promise<void> {
   const [bucketId] = new Storage.StorageBucketDeletedEvent(event).params
-  // TODO: Delete or just change status?
-  // TODO: Cascade remove on db level?
+  // TODO: Cascade remove on db level (would require changes in Hydra / comitting autogenerated files)
   const assignments = await store.getMany(StorageBagStorageAssignment, {
     where: { storageBucket: { id: bucketId.toString() } },
   })
@@ -320,10 +354,8 @@ export async function storage_DynamicBagCreated({ event, store }: EventContext &
 
 export async function storage_DynamicBagDeleted({ event, store }: EventContext & StoreContext): Promise<void> {
   const [, bagId] = new Storage.DynamicBagDeletedEvent(event).params
-  // TODO: Delete or just change status?
-  // TODO: Cascade remove on db level?
   const storageBag = await getDynamicBag(store, bagId, ['objects'])
-  await Promise.all((storageBag.objects || []).map((o) => store.remove<StorageDataObject>(o)))
+  // The bag should already be empty, so no cascade-remove required
   await store.remove<StorageBag>(storageBag)
 }
 
@@ -373,24 +405,10 @@ export async function storage_DataObjectsMoved({ event, store }: EventContext &
 export async function storage_DataObjectsDeleted({ event, store }: EventContext & StoreContext): Promise<void> {
   const [, bagId, dataObjectIds] = new Storage.DataObjectsDeletedEvent(event).params
   const dataObjects = await getDataObjectsInBag(store, bagId, dataObjectIds)
-  // TODO: Delete them or just change status?
-  // (may not be so optimal if we expect a large amount of data objects)
   await Promise.all(dataObjects.map((o) => store.remove<StorageDataObject>(o)))
 }
 
-// BLACKLIST
-export async function storage_UpdateBlacklist({ event, store }: EventContext & StoreContext): Promise<void> {
-  const [removedContentIds, addedContentIds] = new Storage.UpdateBlacklistEvent(event).params
-  const storageSystem = await store.get(StorageSystemParameters, {})
-  if (!storageSystem) {
-    throw new Error('StorageSystemParameters entity not found!')
-  }
-  storageSystem.blacklist = storageSystem.blacklist
-    .filter((cid) => !Array.from(removedContentIds).some((id) => id.eq(cid)))
-    .concat(Array.from(addedContentIds).map((id) => id.toString()))
-
-  await store.save<StorageSystemParameters>(storageSystem)
-}
+// DISTRIBUTION FAMILY
 
 export async function storage_DistributionBucketFamilyCreated({
   event,
@@ -405,6 +423,18 @@ export async function storage_DistributionBucketFamilyCreated({
   await store.save<DistributionBucketFamily>(family)
 }
 
+export async function storage_DistributionBucketFamilyMetadataSet({
+  event,
+  store,
+}: EventContext & StoreContext): Promise<void> {
+  const [familyId, metadataBytes] = new Storage.DistributionBucketFamilyMetadataSetEvent(event).params
+
+  const family = await getDistributionBucketFamilyWithMetadata(store, familyId.toString())
+  family.metadata = await processDistributionBucketFamilyMetadata(store, family.metadata, metadataBytes)
+
+  await store.save<DistributionBucketFamily>(family)
+}
+
 export async function storage_DistributionBucketFamilyDeleted({
   event,
   store,
@@ -416,6 +446,8 @@ export async function storage_DistributionBucketFamilyDeleted({
   await store.remove(family)
 }
 
+// DISTRIBUTION BUCKET
+
 export async function storage_DistributionBucketCreated({ event, store }: EventContext & StoreContext): Promise<void> {
   const [familyId, acceptingNewBags, bucketId] = new Storage.DistributionBucketCreatedEvent(event).params
 
@@ -444,8 +476,7 @@ export async function storage_DistributionBucketStatusUpdated({
 
 export async function storage_DistributionBucketDeleted({ event, store }: EventContext & StoreContext): Promise<void> {
   const [, bucketId] = new Storage.DistributionBucketDeletedEvent(event).params
-  // TODO: Delete or just change status?
-  // TODO: Cascade remove on db level?
+  // TODO: Cascade remove on db level (would require changes in Hydra / comitting autogenerated files)
   const assignments = await store.getMany(StorageBagDistributionAssignment, {
     where: { distributionBucket: { id: bucketId.toString() } },
   })
@@ -548,7 +579,7 @@ export async function storage_DistributionBucketOperatorRemoved({
 }: EventContext & StoreContext): Promise<void> {
   const [, bucketId, workerId] = new Storage.DistributionBucketOperatorRemovedEvent(event).params
 
-  // TODO: Cascade remove
+  // TODO: Cascade remove on db level (would require changes in Hydra / comitting autogenerated files)
 
   const operator = await getDistributionBucketOperatorWithMetadata(store, `${bucketId}-${workerId}`)
   await store.remove<DistributionBucketOperator>(operator)
@@ -563,74 +594,75 @@ export async function storage_DistributionBucketOperatorRemoved({
   }
 }
 
-export async function storage_DistributionBucketFamilyMetadataSet({
-  event,
-  store,
-}: EventContext & StoreContext): Promise<void> {
-  const [familyId, metadataBytes] = new Storage.DistributionBucketFamilyMetadataSetEvent(event).params
+// STORAGE SYSTEM GLOBAL PARAMS
 
-  const family = await getDistributionBucketFamilyWithMetadata(store, familyId.toString())
-  family.metadata = await processDistributionBucketFamilyMetadata(store, family.metadata, metadataBytes)
+export async function storage_UpdateBlacklist({ event, store }: EventContext & StoreContext): Promise<void> {
+  const [removedContentIds, addedContentIds] = new Storage.UpdateBlacklistEvent(event).params
+  const storageSystem = await getStorageSystem(store)
+  storageSystem.blacklist = storageSystem.blacklist
+    .filter((cid) => !Array.from(removedContentIds).some((id) => id.eq(cid)))
+    .concat(Array.from(addedContentIds).map((id) => id.toString()))
 
-  await store.save<DistributionBucketFamily>(family)
+  await store.save<StorageSystemParameters>(storageSystem)
 }
 
 export async function storage_DistributionBucketsPerBagLimitUpdated({
   event,
   store,
 }: EventContext & StoreContext): Promise<void> {
-  // To be implemented
+  const [newLimit] = new Storage.DistributionBucketsPerBagLimitUpdatedEvent(event).params
+  const storageSystem = await getStorageSystem(store)
+
+  storageSystem.distributionBucketsPerBagLimit = newLimit.toNumber()
+
+  await store.save<StorageSystemParameters>(storageSystem)
 }
 
-export async function storage_FamiliesInDynamicBagCreationPolicyUpdated({
+export async function storage_StorageBucketsPerBagLimitUpdated({
   event,
   store,
 }: EventContext & StoreContext): Promise<void> {
-  // To be implemented
+  const [newLimit] = new Storage.StorageBucketsPerBagLimitUpdatedEvent(event).params
+  const storageSystem = await getStorageSystem(store)
+
+  storageSystem.storageBucketsPerBagLimit = newLimit.toNumber()
+
+  await store.save<StorageSystemParameters>(storageSystem)
 }
 
-export async function storage_StorageBucketVoucherLimitsSet({
+export async function storage_StorageBucketsVoucherMaxLimitsUpdated({
   event,
   store,
 }: EventContext & StoreContext): Promise<void> {
-  // To be implemented
+  const [sizeLimit, countLimit] = new Storage.StorageBucketsVoucherMaxLimitsUpdatedEvent(event).params
+  const storageSystem = await getStorageSystem(store)
+
+  storageSystem.storageBucketMaxObjectsSizeLimit = sizeLimit
+  storageSystem.storageBucketMaxObjectsCountLimit = countLimit
+
+  await store.save<StorageSystemParameters>(storageSystem)
 }
 
 export async function storage_UploadingBlockStatusUpdated({
   event,
   store,
 }: EventContext & StoreContext): Promise<void> {
-  // To be implemented
-}
+  const [isBlocked] = new Storage.UploadingBlockStatusUpdatedEvent(event).params
+  const storageSystem = await getStorageSystem(store)
 
-export async function storage_DataObjectPerMegabyteFeeUpdated({
-  event,
-  store,
-}: EventContext & StoreContext): Promise<void> {
-  // To be implemented
-}
+  storageSystem.uploadingBlocked = isBlocked.isTrue
 
-export async function storage_StorageBucketsPerBagLimitUpdated({
-  event,
-  store,
-}: EventContext & StoreContext): Promise<void> {
-  // To be implemented
+  await store.save<StorageSystemParameters>(storageSystem)
 }
 
-export async function storage_StorageBucketsVoucherMaxLimitsUpdated({
+export async function storage_DataObjectPerMegabyteFeeUpdated({
   event,
   store,
 }: EventContext & StoreContext): Promise<void> {
-  // To be implemented
-}
+  const [newFee] = new Storage.DataObjectPerMegabyteFeeUpdatedEvent(event).params
+  const storageSystem = await getStorageSystem(store)
 
-export async function storage_VoucherChanged({ event, store }: EventContext & StoreContext): Promise<void> {
-  // To be implemented
-}
+  storageSystem.dataObjectFeePerMb = newFee
 
-export async function storage_NumberOfStorageBucketsInDynamicBagCreationPolicyUpdated({
-  event,
-  store,
-}: EventContext & StoreContext): Promise<void> {
-  // To be implemented
+  await store.save<StorageSystemParameters>(storageSystem)
 }

query-node/schemas/storage.graphql

@@ -3,12 +3,23 @@ type StorageSystemParameters @entity {
   "Blacklisted content hashes"
   blacklist: [String!]
 
-  # TODO: Consider if parameters like:
-  # dataObjectFeePerMB,
-  # storageBucketBagsLimit,
-  # storageBucketsVoucherMaxLimits,
-  # etc.
-  # are needed here (they can be easily queried from the node)
+  "How many buckets can be assigned to store a bag"
+  storageBucketsPerBagLimit: Int!
+
+  "How many buckets can be assigned to distribute a bag"
+  distributionBucketsPerBagLimit: Int!
+
+  "Whether the uploading is globally blocked"
+  uploadingBlocked: Boolean!
+
+  "Additional fee for storing 1 MB of data"
+  dataObjectFeePerMB: BigInt!
+
+  "Global max. number of objects a storage bucket can store (can also be further limitted the provider)"
+  storageBucketMaxObjectsCountLimit: BigInt!
+
+  "Global max. size of objects a storage bucket can store (can also be further limitted the provider)"
+  storageBucketMaxObjectsSizeLimit: BigInt!
 }
 
 type StorageBucketOperatorStatusMissing @variant {
@@ -77,12 +88,11 @@ type StorageBucket @entity {
   "Bucket's data object count limit"
   dataObjectCountLimit: BigInt!
 
-  # TODO: Are those useful for storage node?
-  # "Currently stored (assigned) data objects size"
-  # storedObjectsSize: BigInt!
+  "Number of assigned data objects"
+  dataObjectsCount: BigInt!
 
-  # "Currently stored (assigned) objects count"
-  # storedObjectsCount: BigInt!
+  "Total size of assigned data objects"
+  dataObjectsSize: BigInt!
 }
 
 type StorageBagOwnerCouncil @variant {
@@ -154,17 +164,12 @@ type StorageDataObject @entity {
   "Whether the data object was uploaded and accepted by the storage provider"
   isAccepted: Boolean!
 
-  # TODO: Is this useful for storage node?
-  # "A reward for the data object deletion"
-  # deletionPrize: BigInt!
-
   "Data object size in bytes"
   size: BigInt!
 
   "Storage bag the data object is part of"
   storageBag: StorageBag!
 
-  # TODO: Use "Bytes" for better optimalization?
   "IPFS content hash"
   ipfsHash: String!
 }