Merge branch 'giza' into giza-cli

package.json

@@ -48,9 +48,10 @@
     "typescript": "^4.4.3",
     "bn.js": "4.12.0",
     "rxjs": "^7.4.0",
-    "typeorm": "^0.2.31",
+    "typeorm": "0.2.34",
     "pg": "^8.4.0",
-    "chalk": "^4.0.0"
+    "chalk": "^4.0.0",
+    "@joystream/warthog": "2.39.0"
   },
   "devDependencies": {
     "eslint": "^7.25.0",

query-node/build.sh

@@ -23,6 +23,9 @@ yarn ts-node --project ./mappings/tsconfig.json ./mappings/scripts/postCodegen.t
 # and are inline with root workspace resolutions
 yarn
 
+# Add missing typeorm binary symlink
+ln -s ../../../../../node_modules/typeorm/cli.js ./generated/graphql-server/node_modules/.bin/typeorm
+
 yarn workspace query-node codegen
 yarn workspace query-node build
 

query-node/codegen/package.json

@@ -5,7 +5,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@joystream/hydra-cli": "3.1.0-alpha.1",
-    "@joystream/hydra-typegen": "3.1.0-alpha.1"
+    "@joystream/hydra-cli": "3.1.0-alpha.13",
+    "@joystream/hydra-typegen": "3.1.0-alpha.13"
   }
 }

query-node/mappings/common.ts

@@ -205,7 +205,7 @@ type EntityClass<T extends BaseModel> = {
   name: string
 }
 
-type RelationsArr<T extends BaseModel> = Exclude<
+export type RelationsArr<T extends BaseModel> = Exclude<
   keyof T & string,
   { [K in keyof T]: T[K] extends BaseModel | undefined ? '' : T[K] extends BaseModel[] | undefined ? '' : K }[keyof T]
 >[]

query-node/mappings/package.json

@@ -11,6 +11,7 @@
     "lint": "eslint . --quiet --ext .ts",
     "checks": "prettier ./ --check && yarn lint",
     "format": "prettier ./ --write ",
+    "postinstall": "yarn ts-node ./scripts/postInstall.ts",
     "bootstrap-data:fetch:members": "yarn ts-node ./bootstrap-data/scripts/fetchMembersData.ts",
     "bootstrap-data:fetch:categories": "yarn ts-node ./bootstrap-data/scripts/fetchCategories.ts",
     "bootstrap-data:fetch:workingGroups": "yarn ts-node ./bootstrap-data/scripts/fetchWorkingGroupsData.ts",
@@ -18,12 +19,12 @@
   },
   "dependencies": {
     "@polkadot/types": "5.9.1",
-    "@joystream/hydra-common": "3.1.0-alpha.1",
-    "@joystream/hydra-db-utils": "3.1.0-alpha.1",
+    "@joystream/hydra-common": "3.1.0-alpha.13",
+    "@joystream/hydra-db-utils": "3.1.0-alpha.13",
     "@joystream/metadata-protobuf": "^1.0.0",
     "@joystream/sumer-types": "npm:@joystream/types@^0.16.0",
     "@joystream/types": "^0.17.0",
-    "@joystream/warthog": "2.35.0",
+    "@joystream/warthog": "2.39.0",
     "@apollo/client": "^3.2.5"
   },
   "devDependencies": {

query-node/mappings/scripts/postInstall.ts

@@ -0,0 +1,35 @@
+// A script to be executed post query-node install, that may include workarounds in Hydra node_modules
+import fs from 'fs'
+import path from 'path'
+
+// FIXME: Temporarly remove broken sanitizeNullCharacter call
+const subscribersJsPath = path.resolve(
+  __dirname,
+  '../../../node_modules/@joystream/hydra-processor/lib/db/subscribers.js'
+)
+const subscribersJsContent = fs.readFileSync(subscribersJsPath).toString()
+fs.writeFileSync(
+  subscribersJsPath,
+  subscribersJsContent.replace(/sanitizeNullCharacter\(entity, field\);/g, '//sanitizeNullCharacter(entity, field)')
+)
+
+// FIXME: Temporarly replace broken relations resolution in @joystream/warthog
+const dataLoaderJsPath = path.resolve(
+  __dirname,
+  '../../../node_modules/@joystream/warthog/dist/middleware/DataLoaderMiddleware.js'
+)
+const dataLoaderJsContent = fs.readFileSync(dataLoaderJsPath).toString()
+const dataLoaderJsContentLines = dataLoaderJsContent.split('\n')
+dataLoaderJsContentLines.splice(
+  dataLoaderJsContentLines.findIndex((l) => l.match(/return context\.connection\.relationIdLoader/)),
+  0,
+  `return Promise.all(
+    entities.map(entity => context.connection.relationLoader.load(relation, entity))
+  ).then(function (results) {
+    return results.map(function (related) {
+      return (relation.isManyToOne || relation.isOneToOne) ? related[0] : related
+    })
+  })
+  `
+)
+fs.writeFileSync(dataLoaderJsPath, dataLoaderJsContentLines.join('\n'))

query-node/mappings/storage/index.ts

@@ -18,12 +18,9 @@ import {
   StorageDataObject,
   StorageSystemParameters,
   GeoCoordinates,
-  StorageBagDistributionAssignment,
-  StorageBagStorageAssignment,
 } from 'query-node/dist/model'
 import BN from 'bn.js'
-import { getById } from '../common'
-import { In } from 'typeorm'
+import { getById, inconsistentState } from '../common'
 import {
   processDistributionBucketFamilyMetadata,
   processDistributionOperatorMetadata,
@@ -141,25 +138,15 @@ export async function storage_StorageBucketsUpdatedForBag({
   event,
   store,
 }: EventContext & StoreContext): Promise<void> {
-  const [bagId, addedBucketsIds, removedBucketsIds] = new Storage.StorageBucketsUpdatedForBagEvent(event).params
+  const [bagId, addedBucketsSet, removedBucketsSet] = new Storage.StorageBucketsUpdatedForBagEvent(event).params
   // Get or create bag
-  const storageBag = await getBag(store, bagId)
-  const assignmentsToRemove = await store.getMany(StorageBagStorageAssignment, {
-    where: {
-      storageBag,
-      storageBucket: { id: In(Array.from(removedBucketsIds).map((bucketId) => bucketId.toString())) },
-    },
-  })
-  const assignmentsToAdd = Array.from(addedBucketsIds).map(
-    (bucketId) =>
-      new StorageBagStorageAssignment({
-        id: `${storageBag.id}-${bucketId.toString()}`,
-        storageBag,
-        storageBucket: new StorageBucket({ id: bucketId.toString() }),
-      })
-  )
-  await Promise.all(assignmentsToRemove.map((a) => store.remove<StorageBagStorageAssignment>(a)))
-  await Promise.all(assignmentsToAdd.map((a) => store.save<StorageBagStorageAssignment>(a)))
+  const storageBag = await getBag(store, bagId, ['storageBuckets'])
+  const removedBucketsIds = Array.from(removedBucketsSet).map((id) => id.toString())
+  const addedBucketsIds = Array.from(addedBucketsSet).map((id) => id.toString())
+  storageBag.storageBuckets = (storageBag.storageBuckets || [])
+    .filter((bucket) => !removedBucketsIds.includes(bucket.id))
+    .concat(addedBucketsIds.map((id) => new StorageBucket({ id })))
+  await store.save<StorageBag>(storageBag)
 }
 
 export async function storage_VoucherChanged({ event, store }: EventContext & StoreContext): Promise<void> {
@@ -189,11 +176,21 @@ export async function storage_StorageBucketVoucherLimitsSet({
 export async function storage_StorageBucketDeleted({ event, store }: EventContext & StoreContext): Promise<void> {
   const [bucketId] = new Storage.StorageBucketDeletedEvent(event).params
   // TODO: Cascade remove on db level (would require changes in Hydra / comitting autogenerated files)
-  const assignments = await store.getMany(StorageBagStorageAssignment, {
-    where: { storageBucket: { id: bucketId.toString() } },
+  const storageBucket = await store.get(StorageBucket, {
+    where: { id: bucketId.toString() },
+    relations: ['bags', 'bags.storageBuckets'],
   })
-  await Promise.all(assignments.map((a) => store.remove<StorageBagStorageAssignment>(a)))
-  await store.remove<StorageBucket>(new StorageBucket({ id: bucketId.toString() }))
+  if (!storageBucket) {
+    inconsistentState(`Storage bucket by id ${bucketId.toString()} not found!`)
+  }
+  // Remove relations
+  await Promise.all(
+    (storageBucket.bags || []).map((bag) => {
+      bag.storageBuckets = (bag.storageBuckets || []).filter((bucket) => bucket.id !== bucketId.toString())
+      return store.save<StorageBag>(bag)
+    })
+  )
+  await store.remove<StorageBucket>(storageBucket)
 }
 
 // DYNAMIC BAGS
@@ -202,36 +199,17 @@ export async function storage_DynamicBagCreated({ event, store }: EventContext &
   const storageBag = new StorageBag({
     id: getDynamicBagId(bagId),
     owner: getDynamicBagOwner(bagId),
+    storageBuckets: Array.from(storageBucketIdsSet).map((id) => new StorageBucket({ id: id.toString() })),
+    distributionBuckets: Array.from(distributionBucketIdsSet).map(
+      (id) => new DistributionBucket({ id: id.toString() })
+    ),
   })
-  const storageAssignments = Array.from(storageBucketIdsSet).map(
-    (bucketId) =>
-      new StorageBagStorageAssignment({
-        id: `${storageBag.id}-${bucketId.toString()}`,
-        storageBag,
-        storageBucket: new StorageBucket({ id: bucketId.toString() }),
-      })
-  )
-  const distributionAssignments = Array.from(distributionBucketIdsSet).map(
-    (bucketId) =>
-      new StorageBagDistributionAssignment({
-        id: `${storageBag.id}-${bucketId.toString()}`,
-        storageBag,
-        distributionBucket: new DistributionBucket({ id: bucketId.toString() }),
-      })
-  )
   await store.save<StorageBag>(storageBag)
-  await Promise.all(storageAssignments.map((a) => store.save<StorageBagStorageAssignment>(a)))
-  await Promise.all(distributionAssignments.map((a) => store.save<StorageBagDistributionAssignment>(a)))
 }
 
 export async function storage_DynamicBagDeleted({ event, store }: EventContext & StoreContext): Promise<void> {
   const [, bagId] = new Storage.DynamicBagDeletedEvent(event).params
   const storageBag = await getDynamicBag(store, bagId, ['objects'])
-  // TODO: Cascade remove on db level (would require changes in Hydra / comitting autogenerated files)
-  const storageAssignments = await store.getMany(StorageBagStorageAssignment, { where: { storageBag } })
-  const distributionAssignments = await store.getMany(StorageBagDistributionAssignment, { where: { storageBag } })
-  await Promise.all(storageAssignments.map((a) => store.remove<StorageBagStorageAssignment>(a)))
-  await Promise.all(distributionAssignments.map((a) => store.remove<StorageBagDistributionAssignment>(a)))
   await store.remove<StorageBag>(storageBag)
 }
 
@@ -341,36 +319,36 @@ export async function storage_DistributionBucketStatusUpdated({
 export async function storage_DistributionBucketDeleted({ event, store }: EventContext & StoreContext): Promise<void> {
   const [, bucketId] = new Storage.DistributionBucketDeletedEvent(event).params
   // TODO: Cascade remove on db level (would require changes in Hydra / comitting autogenerated files)
-  const assignments = await store.getMany(StorageBagDistributionAssignment, {
-    where: { distributionBucket: { id: bucketId.toString() } },
+  const distributionBucket = await store.get(DistributionBucket, {
+    where: { id: bucketId.toString() },
+    relations: ['bags', 'bags.distributionBuckets'],
   })
-  await Promise.all(assignments.map((a) => store.remove<StorageBagDistributionAssignment>(a)))
-  await store.remove<DistributionBucket>(new DistributionBucket({ id: bucketId.toString() }))
+  if (!distributionBucket) {
+    inconsistentState(`Distribution bucket by id ${bucketId.toString()} not found!`)
+  }
+  // Remove relations
+  await Promise.all(
+    (distributionBucket.bags || []).map((bag) => {
+      bag.distributionBuckets = (bag.distributionBuckets || []).filter((bucket) => bucket.id !== bucketId.toString())
+      return store.save<StorageBag>(bag)
+    })
+  )
+  await store.remove<DistributionBucket>(distributionBucket)
 }
 
 export async function storage_DistributionBucketsUpdatedForBag({
   event,
   store,
 }: EventContext & StoreContext): Promise<void> {
-  const [bagId, , addedBucketsIds, removedBucketsIds] = new Storage.DistributionBucketsUpdatedForBagEvent(event).params
+  const [bagId, , addedBucketsSet, removedBucketsSet] = new Storage.DistributionBucketsUpdatedForBagEvent(event).params
   // Get or create bag
-  const storageBag = await getBag(store, bagId)
-  const assignmentsToRemove = await store.getMany(StorageBagDistributionAssignment, {
-    where: {
-      storageBag,
-      distributionBucket: { id: In(Array.from(removedBucketsIds).map((bucketId) => bucketId.toString())) },
-    },
-  })
-  const assignmentsToAdd = Array.from(addedBucketsIds).map(
-    (bucketId) =>
-      new StorageBagDistributionAssignment({
-        id: `${storageBag.id}-${bucketId.toString()}`,
-        storageBag,
-        distributionBucket: new DistributionBucket({ id: bucketId.toString() }),
-      })
-  )
-  await Promise.all(assignmentsToRemove.map((a) => store.remove<StorageBagDistributionAssignment>(a)))
-  await Promise.all(assignmentsToAdd.map((a) => store.save<StorageBagDistributionAssignment>(a)))
+  const storageBag = await getBag(store, bagId, ['distributionBuckets'])
+  const removedBucketsIds = Array.from(removedBucketsSet).map((id) => id.toString())
+  const addedBucketsIds = Array.from(addedBucketsSet).map((id) => id.toString())
+  storageBag.distributionBuckets = (storageBag.distributionBuckets || [])
+    .filter((bucket) => !removedBucketsIds.includes(bucket.id))
+    .concat(addedBucketsIds.map((id) => new DistributionBucket({ id })))
+  await store.save<StorageBag>(storageBag)
 }
 
 export async function storage_DistributionBucketModeUpdated({

query-node/mappings/storage/utils.ts

@@ -17,7 +17,7 @@ import {
   DistributionBucketFamily,
 } from 'query-node/dist/model'
 import BN from 'bn.js'
-import { bytesToString, inconsistentState, getById } from '../common'
+import { bytesToString, inconsistentState, getById, RelationsArr } from '../common'
 import { In } from 'typeorm'
 import { unsetAssetRelations } from '../content/utils'
 
@@ -101,7 +101,7 @@ export function getBagId(bagId: BagId) {
 export async function getDynamicBag(
   store: DatabaseManager,
   bagId: DynamicBagId,
-  relations?: 'objects'[]
+  relations?: RelationsArr<StorageBag>
 ): Promise<StorageBag> {
   return getById(store, StorageBag, getDynamicBagId(bagId), relations)
 }
@@ -109,7 +109,7 @@ export async function getDynamicBag(
 export async function getStaticBag(
   store: DatabaseManager,
   bagId: StaticBagId,
-  relations?: 'objects'[]
+  relations?: RelationsArr<StorageBag>
 ): Promise<StorageBag> {
   const id = getStaticBagId(bagId)
   const bag = await store.get(StorageBag, { where: { id }, relations })
@@ -125,7 +125,11 @@ export async function getStaticBag(
   return bag
 }
 
-export async function getBag(store: DatabaseManager, bagId: BagId, relations?: 'objects'[]): Promise<StorageBag> {
+export async function getBag(
+  store: DatabaseManager,
+  bagId: BagId,
+  relations?: RelationsArr<StorageBag>
+): Promise<StorageBag> {
   return bagId.isStatic
     ? getStaticBag(store, bagId.asStatic, relations)
     : getDynamicBag(store, bagId.asDynamic, relations)

query-node/package.json

@@ -41,7 +41,7 @@
     "tslib": "^2.0.0",
     "@types/bn.js": "^4.11.6",
     "bn.js": "^5.1.2",
-    "@joystream/hydra-processor": "3.1.0-alpha.1"
+    "@joystream/hydra-processor": "3.1.0-alpha.13"
   },
   "volta": {
 		"extends": "../package.json"

query-node/schemas/storage.graphql

@@ -105,8 +105,8 @@ type StorageBucket @entity {
   "Whether the bucket is accepting any new storage bags"
   acceptingNewBags: Boolean!
 
-  "Assignments to store a bag"
-  bagAssignments: [StorageBagStorageAssignment!] @derivedFrom(field: "storageBucket")
+  "Storage bags assigned to the bucket"
+  bags: [StorageBag!] @derivedFrom(field: "storageBuckets")
 
   "Bucket's data object size limit in bytes"
   dataObjectsSizeLimit: BigInt!
@@ -151,46 +151,16 @@ type StorageBag @entity {
   "Data objects in the bag"
   objects: [StorageDataObject!] @derivedFrom(field: "storageBag")
 
-  "Assignments to a storage bucket"
-  storageAssignments: [StorageBagStorageAssignment!] @derivedFrom(field: "storageBag")
+  "Storage buckets assigned to the bag"
+  storageBuckets: [StorageBucket!]
 
-  "Assignments to a distribution bucket"
-  distirbutionAssignments: [StorageBagDistributionAssignment!] @derivedFrom(field: "storageBag")
+  "Distribution buckets assigned to the bag"
+  distributionBuckets: [DistributionBucket!]
 
   "Owner of the storage bag"
   owner: StorageBagOwner!
 }
 
-type StorageBagStorageAssignment @entity {
-  "{storageBagId-storageBucketId}"
-  id: ID!
-
-  "Storage bag to be stored"
-  storageBag: StorageBag!
-
-  "Storage bucket that should store the bag"
-  storageBucket: StorageBucket!
-
-  # Relationship filtering workaround
-  storageBagId: ID
-  storageBucketId: ID
-}
-
-type StorageBagDistributionAssignment @entity {
-  "{storageBagId-distributionBucketId}"
-  id: ID!
-
-  "Storage bag to be distributed"
-  storageBag: StorageBag!
-
-  "Distribution bucket that should distribute the bag"
-  distributionBucket: DistributionBucket!
-
-  # Relationship filtering workaround
-  storageBagId: ID
-  distributionBucketId: ID
-}
-
 type DataObjectTypeChannelAvatar @variant {
   "Related channel entity"
   channel: Channel!
@@ -318,8 +288,8 @@ type DistributionBucket @entity {
   "Whether the bucket is currently distributing content"
   distributing: Boolean!
 
-  "Assignments to distribute a bag"
-  bagAssignments: [StorageBagDistributionAssignment!] @derivedFrom(field: "distributionBucket")
+  "Storage bags assigned to the bucket"
+  bags: [StorageBag!] @derivedFrom(field: "distributionBuckets")
 }
 
 type DistributionBucketFamily @entity {