Update tests: Distribution bucket ids, storage transactor key, CLI accounts refactoring

start.sh

@@ -28,6 +28,7 @@ docker-compose up -d joystream-node
 export SKIP_MOCK_CONTENT=true
 HOST_IP=$(tests/network-tests/get-host-ip.sh)
 export COLOSSUS_1_URL="http://${HOST_IP}:3333"
+export COLOSSUS_1_TRANSACTOR_KEY=$(docker run --rm --pull=always docker.io/parity/subkey:2.0.1 inspect ${COLOSSUS_1_TRANSACTOR_URI} --output-type json | jq .ss58Address -r)
 export DISTRIBUTOR_1_URL="http://${HOST_IP}:3334"
 ./tests/network-tests/run-test-scenario.sh ${INIT_CHAIN_SCENARIO}
 

tests/network-tests/run-full-tests.sh

@@ -29,9 +29,12 @@ yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
 
 # Setup storage & distribution
 HOST_IP=$(./get-host-ip.sh)
+# Because proposals tests hire and then fire each lead,
+# we need to override COLOSSUS_1_WORKER_ID (0 => 1) and DISTRIBUTOR_1_WORKER_ID (0 => 1)
 export COLOSSUS_1_URL="http://${HOST_IP}:3333"
 export COLOSSUS_1_WORKER_ID=1
-export COLOSSUS_1_ACCOUNT_URI=//testing//worker//Storage//${COLOSSUS_1_WORKER_ID}
+export COLOSSUS_1_WORKER_URI=//testing//worker//Storage//${COLOSSUS_1_WORKER_ID}
+export COLOSSUS_1_TRANSACTOR_KEY=$(docker run --rm --pull=always docker.io/parity/subkey:2.0.1 inspect ${COLOSSUS_1_TRANSACTOR_URI} --output-type json | jq .ss58Address -r)
 export DISTRIBUTOR_1_URL="http://${HOST_IP}:3334"
 export DISTRIBUTOR_1_WORKER_ID=1
 export DISTRIBUTOR_1_ACCOUNT_URI=//testing//worker//Distribution//${DISTRIBUTOR_1_WORKER_ID}

tests/network-tests/src/cli/joystream.ts

@@ -8,7 +8,6 @@ const CLI_ROOT_PATH = path.resolve(__dirname, '../../../../cli')
 
 export class JoystreamCLI extends CLI {
   protected keys: string[] = []
-  protected chosenKey: string | undefined
   protected tmpFileManager: TmpFileManager
 
   constructor(tmpFileManager: TmpFileManager) {
@@ -26,20 +25,19 @@ export class JoystreamCLI extends CLI {
 
   async importKey(pair: KeyringPair): Promise<void> {
     const jsonFile = this.tmpFileManager.jsonFile(pair.toJson())
-    await this.run('account:import', [jsonFile])
+    await this.run('account:import', [
+      '--backupFilePath',
+      jsonFile,
+      '--name',
+      `Account${this.keys.length}`,
+      '--password',
+      '',
+    ])
     this.keys.push(pair.address)
   }
 
-  async chooseKey(address: string): Promise<void> {
-    if (!this.keys.includes(address)) {
-      throw new Error('Cannot choose a key that was not imported via JoystreamCLI.importKey!')
-    }
-    await this.run('account:choose', ['--address', address])
-    this.chosenKey = address
-  }
-
   async run(command: string, customArgs: string[] = [], keyLocks?: string[]): Promise<CommandResult> {
-    return super.run(command, customArgs, keyLocks || (this.chosenKey ? [this.chosenKey] : []))
+    return super.run(command, customArgs, keyLocks || this.keys)
   }
 
   async createChannel(inputData: ChannelInputParameters, args: string[]): Promise<CommandResult> {

tests/network-tests/src/flows/clis/createChannel.ts

@@ -30,7 +30,6 @@ export default async function createChannel({ api, env, query }: FlowProps): Pro
   // Init CLI, import & select channel owner key
   await joystreamCli.init()
   await joystreamCli.importKey(channelOwnerKeypair.key)
-  await joystreamCli.chooseKey(channelOwnerKeypair.key.address)
 
   // Create channel
   const avatarPhotoPath = tmpFileManager.randomImgFile(300, 300)

tests/network-tests/src/flows/clis/initDistributionBucket.ts

@@ -19,29 +19,14 @@ export default async function initDistributionBucket({ api }: FlowProps): Promis
 
   await cli.run('leader:set-buckets-per-bag-limit', ['--limit', '10'])
   const { out: familyId } = await cli.run('leader:create-bucket-family')
-  const { out: bucketId } = await cli.run('leader:create-bucket', ['--familyId', familyId, '--acceptingBags', 'yes'])
-  await cli.run('leader:update-bag', ['--bagId', 'static:council', '--familyId', familyId, '--add', bucketId])
+  const { out: bucketIndex } = await cli.run('leader:create-bucket', ['--familyId', familyId, '--acceptingBags', 'yes'])
+  const bucketId = `${familyId}:${bucketIndex}`
+  await cli.run('leader:update-bag', ['--bagId', 'static:council', '--familyId', familyId, '--add', bucketIndex])
   await cli.run('leader:update-dynamic-bag-policy', ['--type', 'Channel', '--policy', `${familyId}:1`])
-  await cli.run('leader:update-bucket-mode', ['--familyId', familyId, '--bucketId', bucketId, '--mode', 'on'])
-  await cli.run('leader:invite-bucket-operator', [
-    '--familyId',
-    familyId,
-    '--bucketId',
-    bucketId,
-    '--workerId',
-    operatorId,
-  ])
-  await cli.run('operator:accept-invitation', [
-    '--familyId',
-    familyId,
-    '--bucketId',
-    bucketId,
-    '--workerId',
-    operatorId,
-  ])
+  await cli.run('leader:update-bucket-mode', ['--bucketId', bucketId, '--mode', 'on'])
+  await cli.run('leader:invite-bucket-operator', ['--bucketId', bucketId, '--workerId', operatorId])
+  await cli.run('operator:accept-invitation', ['--bucketId', bucketId, '--workerId', operatorId])
   await cli.run('operator:set-metadata', [
-    '--familyId',
-    familyId,
     '--bucketId',
     bucketId,
     '--workerId',

tests/network-tests/src/flows/clis/initStorageBucket.ts

@@ -13,6 +13,7 @@ export default async function initStorageBucket({ api }: FlowProps): Promise<voi
     throw new Error('Active storage leader is required in this flow!')
   }
   const leaderSuri = api.getSuri(leader.role_account_id)
+  const transactorKey = '5DkE5YD8m5Yzno6EH2RTBnH268TDnnibZMEMjxwYemU4XevU' // //Colossus1
 
   const operatorId = leaderId.toString()
 
@@ -28,7 +29,14 @@ export default async function initStorageBucket({ api }: FlowProps): Promise<voi
     '--size',
     '10000000000',
   ])
-  await cli.run('operator:accept-invitation', ['--workerId', operatorId, '--bucketId', bucketId])
+  await cli.run('operator:accept-invitation', [
+    '--workerId',
+    operatorId,
+    '--bucketId',
+    bucketId,
+    '--transactorAccountId',
+    transactorKey,
+  ])
   await cli.run('leader:update-bag', ['--add', bucketId, '--bagId', 'static:council'])
   await cli.run('leader:update-dynamic-bag-policy', ['--bagType', 'Channel', '--number', '1'])
   await cli.run('operator:set-metadata', [

tests/network-tests/src/flows/storagev2/initDistribution.ts

@@ -8,13 +8,7 @@ import {
   IDistributionBucketOperatorMetadata,
 } from '@joystream/metadata-protobuf'
 import { CreateInterface, createType } from '@joystream/types'
-import {
-  BagId,
-  DistributionBucketFamilyId,
-  DistributionBucketId,
-  DynamicBagId,
-  StaticBagId,
-} from '@joystream/types/storage'
+import { BagId, DistributionBucketFamilyId, DynamicBagId, StaticBagId } from '@joystream/types/storage'
 import { SubmittableExtrinsic } from '@polkadot/api/types'
 import _ from 'lodash'
 import { Utils } from '../../utils'
@@ -169,54 +163,57 @@ export default function createFlow({ families }: InitDistributionConfig) {
       api.signAndSendMany(updateFamilyMetadataTxs, distributionLeaderKey),
       api.signAndSendMany(updateDynamicBagPolicyTxs, distributionLeaderKey),
     ])
-    const bucketEntries = createBucketResults
+    const bucketIds = createBucketResults
       .map((r) => {
-        const [familyId, , bucketId] = api.getEvent(r, 'storage', 'DistributionBucketCreated').data
-        return [familyId, bucketId] as [DistributionBucketFamilyId, DistributionBucketId]
-      })
-      .sort((a, b) => a[0].cmp(b[0]) || a[1].cmp(b[1]))
-    const bucketById = new Map<number, DistributionBucketConfig>()
-    const bucketEntriesByFamilyId = _.groupBy(bucketEntries, ([familyId]) => familyId.toString())
-    _.mapValues(bucketEntriesByFamilyId, (entries) => {
-      entries.forEach(([familyId, bucketId], bucketInFamilyIndex) => {
-        const family = familyById.get(familyId.toNumber())
-        if (!family) {
-          throw new Error('Family config not found')
-        }
-        bucketById.set(bucketId.toNumber(), family.buckets[bucketInFamilyIndex])
+        const [, , bucketId] = api.getEvent(r, 'storage', 'DistributionBucketCreated').data
+        return bucketId
       })
+      .sort(
+        (a, b) =>
+          a.distribution_bucket_family_id.cmp(b.distribution_bucket_family_id) ||
+          a.distribution_bucket_index.cmp(b.distribution_bucket_index)
+      )
+    const bucketById = new Map<string, DistributionBucketConfig>()
+    bucketIds.forEach((bucketId) => {
+      const familyId = bucketId.distribution_bucket_family_id.toNumber()
+      const bucketIndex = bucketId.distribution_bucket_index.toNumber()
+      const family = familyById.get(familyId)
+      if (!family) {
+        throw new Error(`familyById not found: ${familyId}`)
+      }
+      bucketById.set(bucketId.toString(), family.buckets[bucketIndex])
     })
 
     // Invite bucket operators
-    const bucketInviteTxs = bucketEntries.map(([familyId, bucketId], i) =>
-      api.tx.storage.inviteDistributionBucketOperator(familyId, bucketId, operatorIds[i])
+    const bucketInviteTxs = bucketIds.map((bucketId, i) =>
+      api.tx.storage.inviteDistributionBucketOperator(bucketId, operatorIds[i])
     )
     await api.signAndSendMany(bucketInviteTxs, distributionLeaderKey)
 
     // Accept invitations
-    const acceptInvitationTxs = bucketEntries.map(([familyId, bucketId], i) =>
-      api.tx.storage.acceptDistributionBucketInvitation(operatorIds[i], familyId, bucketId)
+    const acceptInvitationTxs = bucketIds.map((bucketId, i) =>
+      api.tx.storage.acceptDistributionBucketInvitation(operatorIds[i], bucketId)
     )
     await api.signAndSendManyByMany(acceptInvitationTxs, operatorKeys)
 
     // Bucket metadata and static bags
     const bucketSetupPromises = _.flatten(
-      bucketEntries.map(([familyId, bucketId], i) => {
+      bucketIds.map((bucketId, i) => {
         const operatorId = operatorIds[i]
         const operatorKey = operatorKeys[i]
-        const bucketConfig = bucketById.get(bucketId.toNumber())
+        const bucketConfig = bucketById.get(bucketId.toString())
         if (!bucketConfig) {
           throw new Error('Bucket config not found')
         }
         const metadataBytes = Utils.metadataToBytes(DistributionBucketOperatorMetadata, bucketConfig.metadata)
-        const setMetaTx = api.tx.storage.setDistributionOperatorMetadata(operatorId, familyId, bucketId, metadataBytes)
+        const setMetaTx = api.tx.storage.setDistributionOperatorMetadata(operatorId, bucketId, metadataBytes)
         const setMetaPromise = api.signAndSendMany([setMetaTx], operatorKey)
         const updateBagTxs = (bucketConfig.staticBags || []).map((sBagId) => {
           return api.tx.storage.updateDistributionBucketsForBag(
             createType<BagId, 'BagId'>('BagId', { Static: sBagId }),
-            familyId,
-            createType('DistributionBucketIdSet', [bucketId.toNumber()]),
-            createType('DistributionBucketIdSet', [])
+            bucketId.distribution_bucket_family_id,
+            createType('BTreeSet<DistributionBucketIndex>', [bucketId.distribution_bucket_index]),
+            createType('BTreeSet<DistributionBucketIndex>', [])
           )
         })
         const updateBagsPromise = api.signAndSendMany(updateBagTxs, distributionLeaderKey)

tests/network-tests/src/flows/storagev2/initStorage.ts

@@ -14,6 +14,7 @@ type StorageBucketConfig = {
   storageLimit: BN
   objectsLimit: number
   operatorId: number
+  transactorKey: string
 }
 
 type InitStorageConfig = {
@@ -46,6 +47,7 @@ export const singleBucketConfig: InitStorageConfig = {
       operatorId: parseInt(process.env.COLOSSUS_1_WORKER_ID || '0'),
       storageLimit: new BN(1_000_000_000_000),
       objectsLimit: 1000000000,
+      transactorKey: process.env.COLOSSUS_1_TRANSACTOR_KEY || '5DkE5YD8m5Yzno6EH2RTBnH268TDnnibZMEMjxwYemU4XevU', // //Colossus1
     },
   ],
 }
@@ -62,6 +64,7 @@ export const doubleBucketConfig: InitStorageConfig = {
       operatorId: parseInt(process.env.COLOSSUS_1_WORKER_ID || '0'),
       storageLimit: new BN(1_000_000_000_000),
       objectsLimit: 1000000000,
+      transactorKey: process.env.COLOSSUS_1_TRANSACTOR_KEY || '5DkE5YD8m5Yzno6EH2RTBnH268TDnnibZMEMjxwYemU4XevU', // //Colossus1
     },
     {
       metadata: { endpoint: process.env.STORAGE_2_URL || 'http://localhost:3335' },
@@ -69,6 +72,7 @@ export const doubleBucketConfig: InitStorageConfig = {
       operatorId: parseInt(process.env.STORAGE_2_WORKER_ID || '1'),
       storageLimit: new BN(1_000_000_000_000),
       objectsLimit: 1000000000,
+      transactorKey: process.env.COLOSSUS_2_TRANSACTOR_KEY || '5FbzYmQ3HogiEEDSXPYJe58yCcmSh3vsZLodTdBB6YuLDAj7', // //Colossus2
     },
   ],
 }
@@ -124,8 +128,8 @@ export default function createFlow({ buckets, dynamicBagPolicy }: InitStorageCon
     })
 
     // Accept invitations
-    const acceptInvitationTxs = Array.from(bucketById.keys()).map((bucketId, i) =>
-      api.tx.storage.acceptStorageBucketInvitation(operatorIds[i], bucketId)
+    const acceptInvitationTxs = Array.from(bucketById.entries()).map(([bucketId, bucketConfig], i) =>
+      api.tx.storage.acceptStorageBucketInvitation(operatorIds[i], bucketId, bucketConfig.transactorKey)
     )
     await api.signAndSendManyByMany(acceptInvitationTxs, operatorKeys)