Merge branch 'babylon' into cd-schemas-lib

.dockerignore

@@ -1,3 +1,4 @@
 **target*
 **node_modules*
-.tmp/
+.tmp/
+.vscode/

.github/workflows/run-network-tests.yml

@@ -104,7 +104,7 @@ jobs:
         run: tests/network-tests/run-tests.sh
 
   network_tests_2:
-    name: Query Node Tests (Placeholder)
+    name: Content Directory Initialization
     if: contains(github.event.pull_request.labels.*.name, 'run-network-tests')
     needs: build_images
     runs-on: ubuntu-latest
@@ -124,11 +124,11 @@ jobs:
       - name: Install packages and dependencies
         run: yarn install --frozen-lockfile
       - name: Ensure tests are runnable
-        run: yarn workspace network-tests build
+        run: yarn workspace cd-schemas checks --quiet
       - name: Start chain
         run: docker-compose up -d
-      # - name: Execute network tests
-      #   run: yarn workspace network-tests test
+      - name: Initialize the content directory
+        run: yarn workspace cd-schemas initialize:dev
 
   network_tests_3:
     name: Storage Node Tests
@@ -154,7 +154,12 @@ jobs:
           yarn workspace storage-node build
       - name: Build storage node
         run: yarn workspace storage-node build
-      - name: Start chain
-        run: docker-compose up -d
-      - name: Execute tests
-        run: DEBUG=* yarn storage-cli dev-init
+      - name: Start Services
+        run: docker-compose --file docker-compose-with-storage.yml up -d
+      - name: Add development storage node and initialize content directory
+        run: DEBUG=* yarn storage-cli dev-init
+      - name: Wait for storage-node to publish identity
+        run: sleep 90
+        # Better would be poll `http://localhost:3001/discover/v0/1` until we get 200 Response
+      - name: Upload a file
+        run: DEBUG=joystream:* yarn storage-cli upload ./pioneer/packages/apps/public/images/default-thumbnail.png 1 0

README.md

@@ -109,7 +109,7 @@ A step by step guide to setup a full node and validator on the Joystream testnet
 
 ```bash
 docker-compose up -d
-yarn workspace network-tests test
+DEBUG=* yarn workspace network-tests test-run src/scenarios/full.ts
 docker-compose down
 ```
 

apps.Dockerfile

@@ -0,0 +1,16 @@
+FROM node:12 as builder
+
+WORKDIR /joystream
+COPY . /joystream
+
+# Do not set NODE_ENV=production until after running yarn install
+# to ensure dev dependencies are installed.
+RUN yarn install --frozen-lockfile
+
+# Pioneer is failing to build only on github actions workflow runner
+# Error: packages/page-staking/src/index.tsx(24,21): error TS2307: Cannot find module './Targets' or its corresponding type declarations.
+# RUN yarn workspace pioneer build
+RUN yarn workspace @joystream/cli build
+RUN yarn workspace storage-node build
+
+ENTRYPOINT [ "yarn" ]

cli/.eslintignore

@@ -1 +1,2 @@
 /lib
+.eslintrc.js

cli/.eslintrc.js

@@ -2,6 +2,9 @@ module.exports = {
   env: {
     mocha: true,
   },
+  parserOptions: {
+    project: './tsconfig.json'
+  },
   extends: [
     // The oclif rules have some code-style/formatting rules which may conflict with
     // our prettier global settings. Disabling for now
@@ -11,7 +14,8 @@ module.exports = {
     // "oclif-typescript",
   ],
   rules: {
-    "no-unused-vars": "off", // Required by the typescript rule below
-    "@typescript-eslint/no-unused-vars": ["error"]
-  }
+    'no-unused-vars': 'off', // Required by the typescript rule below
+    '@typescript-eslint/no-unused-vars': ['error'],
+    '@typescript-eslint/no-floating-promises': 'error',
+  },
 }

cli/README.md

@@ -44,7 +44,7 @@ $ npm install -g @joystream/cli
 $ joystream-cli COMMAND
 running command...
 $ joystream-cli (-v|--version|version)
-@joystream/cli/0.1.0 linux-x64 node-v13.12.0
+@joystream/cli/0.2.0 linux-x64 node-v13.12.0
 $ joystream-cli --help [COMMAND]
 USAGE
   $ joystream-cli COMMAND
@@ -76,8 +76,29 @@ When using the CLI for the first time there are a few common steps you might wan
 * [`joystream-cli api:inspect`](#joystream-cli-apiinspect)
 * [`joystream-cli api:setUri [URI]`](#joystream-cli-apiseturi-uri)
 * [`joystream-cli autocomplete [SHELL]`](#joystream-cli-autocomplete-shell)
+* [`joystream-cli content-directory:addClassSchema`](#joystream-cli-content-directoryaddclassschema)
+* [`joystream-cli content-directory:addCuratorToGroup [GROUPID] [CURATORID]`](#joystream-cli-content-directoryaddcuratortogroup-groupid-curatorid)
+* [`joystream-cli content-directory:addMaintainerToClass [CLASSNAME] [GROUPID]`](#joystream-cli-content-directoryaddmaintainertoclass-classname-groupid)
+* [`joystream-cli content-directory:class CLASSNAME`](#joystream-cli-content-directoryclass-classname)
+* [`joystream-cli content-directory:classes`](#joystream-cli-content-directoryclasses)
+* [`joystream-cli content-directory:createClass`](#joystream-cli-content-directorycreateclass)
+* [`joystream-cli content-directory:createCuratorGroup`](#joystream-cli-content-directorycreatecuratorgroup)
+* [`joystream-cli content-directory:curatorGroup ID`](#joystream-cli-content-directorycuratorgroup-id)
+* [`joystream-cli content-directory:curatorGroups`](#joystream-cli-content-directorycuratorgroups)
+* [`joystream-cli content-directory:entities CLASSNAME [PROPERTIES]`](#joystream-cli-content-directoryentities-classname-properties)
+* [`joystream-cli content-directory:entity ID`](#joystream-cli-content-directoryentity-id)
+* [`joystream-cli content-directory:removeCuratorGroup [ID]`](#joystream-cli-content-directoryremovecuratorgroup-id)
+* [`joystream-cli content-directory:removeMaintainerFromClass [CLASSNAME] [GROUPID]`](#joystream-cli-content-directoryremovemaintainerfromclass-classname-groupid)
+* [`joystream-cli content-directory:setCuratorGroupStatus [ID] [STATUS]`](#joystream-cli-content-directorysetcuratorgroupstatus-id-status)
+* [`joystream-cli content-directory:updateClassPermissions [CLASSNAME]`](#joystream-cli-content-directoryupdateclasspermissions-classname)
 * [`joystream-cli council:info`](#joystream-cli-councilinfo)
 * [`joystream-cli help [COMMAND]`](#joystream-cli-help-command)
+* [`joystream-cli media:createChannel`](#joystream-cli-mediacreatechannel)
+* [`joystream-cli media:myChannels`](#joystream-cli-mediamychannels)
+* [`joystream-cli media:myVideos`](#joystream-cli-mediamyvideos)
+* [`joystream-cli media:updateChannel [ID]`](#joystream-cli-mediaupdatechannel-id)
+* [`joystream-cli media:updateVideo [ID]`](#joystream-cli-mediaupdatevideo-id)
+* [`joystream-cli media:uploadVideo FILEPATH`](#joystream-cli-mediauploadvideo-filepath)
 * [`joystream-cli working-groups:application WGAPPLICATIONID`](#joystream-cli-working-groupsapplication-wgapplicationid)
 * [`joystream-cli working-groups:createOpening`](#joystream-cli-working-groupscreateopening)
 * [`joystream-cli working-groups:decreaseWorkerStake WORKERID`](#joystream-cli-working-groupsdecreaseworkerstake-workerid)
@@ -288,6 +309,224 @@ EXAMPLES
 
 _See code: [@oclif/plugin-autocomplete](https://github.com/oclif/plugin-autocomplete/blob/v0.2.0/src/commands/autocomplete/index.ts)_
 
+## `joystream-cli content-directory:addClassSchema`
+
+Add a new schema to a class inside content directory. Requires lead access.
+
+```
+USAGE
+  $ joystream-cli content-directory:addClassSchema
+
+OPTIONS
+  -i, --input=input    Path to JSON file to use as input (if not specified - the input can be provided interactively)
+  -o, --output=output  Path where the output JSON file should be placed (can be then reused as input)
+```
+
+_See code: [src/commands/content-directory/addClassSchema.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/addClassSchema.ts)_
+
+## `joystream-cli content-directory:addCuratorToGroup [GROUPID] [CURATORID]`
+
+Add Curator to existing Curator Group.
+
+```
+USAGE
+  $ joystream-cli content-directory:addCuratorToGroup [GROUPID] [CURATORID]
+
+ARGUMENTS
+  GROUPID    ID of the Curator Group
+  CURATORID  ID of the curator
+```
+
+_See code: [src/commands/content-directory/addCuratorToGroup.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/addCuratorToGroup.ts)_
+
+## `joystream-cli content-directory:addMaintainerToClass [CLASSNAME] [GROUPID]`
+
+Add maintainer (Curator Group) to a class.
+
+```
+USAGE
+  $ joystream-cli content-directory:addMaintainerToClass [CLASSNAME] [GROUPID]
+
+ARGUMENTS
+  CLASSNAME  Name or ID of the class (ie. Video)
+  GROUPID    ID of the Curator Group to add as class maintainer
+```
+
+_See code: [src/commands/content-directory/addMaintainerToClass.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/addMaintainerToClass.ts)_
+
+## `joystream-cli content-directory:class CLASSNAME`
+
+Show Class details by id or name.
+
+```
+USAGE
+  $ joystream-cli content-directory:class CLASSNAME
+
+ARGUMENTS
+  CLASSNAME  Name or ID of the Class
+```
+
+_See code: [src/commands/content-directory/class.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/class.ts)_
+
+## `joystream-cli content-directory:classes`
+
+List existing content directory classes.
+
+```
+USAGE
+  $ joystream-cli content-directory:classes
+```
+
+_See code: [src/commands/content-directory/classes.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/classes.ts)_
+
+## `joystream-cli content-directory:createClass`
+
+Create class inside content directory. Requires lead access.
+
+```
+USAGE
+  $ joystream-cli content-directory:createClass
+
+OPTIONS
+  -i, --input=input    Path to JSON file to use as input (if not specified - the input can be provided interactively)
+  -o, --output=output  Path where the output JSON file should be placed (can be then reused as input)
+```
+
+_See code: [src/commands/content-directory/createClass.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/createClass.ts)_
+
+## `joystream-cli content-directory:createCuratorGroup`
+
+Create new Curator Group.
+
+```
+USAGE
+  $ joystream-cli content-directory:createCuratorGroup
+
+ALIASES
+  $ joystream-cli addCuratorGroup
+```
+
+_See code: [src/commands/content-directory/createCuratorGroup.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/createCuratorGroup.ts)_
+
+## `joystream-cli content-directory:curatorGroup ID`
+
+Show Curator Group details by ID.
+
+```
+USAGE
+  $ joystream-cli content-directory:curatorGroup ID
+
+ARGUMENTS
+  ID  ID of the Curator Group
+```
+
+_See code: [src/commands/content-directory/curatorGroup.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/curatorGroup.ts)_
+
+## `joystream-cli content-directory:curatorGroups`
+
+List existing Curator Groups.
+
+```
+USAGE
+  $ joystream-cli content-directory:curatorGroups
+```
+
+_See code: [src/commands/content-directory/curatorGroups.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/curatorGroups.ts)_
+
+## `joystream-cli content-directory:entities CLASSNAME [PROPERTIES]`
+
+Show entities list by class id or name.
+
+```
+USAGE
+  $ joystream-cli content-directory:entities CLASSNAME [PROPERTIES]
+
+ARGUMENTS
+  CLASSNAME   Name or ID of the Class
+
+  PROPERTIES  Comma-separated properties to include in the results table (ie. code,name). By default all property values
+              will be included.
+
+OPTIONS
+  --filters=filters  Comma-separated filters, ie. title="Some video",channelId=3.Currently only the = operator is
+                     supported.When multiple filters are provided, only the entities that match all of them together
+                     will be displayed.
+```
+
+_See code: [src/commands/content-directory/entities.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/entities.ts)_
+
+## `joystream-cli content-directory:entity ID`
+
+Show Entity details by id.
+
+```
+USAGE
+  $ joystream-cli content-directory:entity ID
+
+ARGUMENTS
+  ID  ID of the Entity
+```
+
+_See code: [src/commands/content-directory/entity.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/entity.ts)_
+
+## `joystream-cli content-directory:removeCuratorGroup [ID]`
+
+Remove existing Curator Group.
+
+```
+USAGE
+  $ joystream-cli content-directory:removeCuratorGroup [ID]
+
+ARGUMENTS
+  ID  ID of the Curator Group to remove
+```
+
+_See code: [src/commands/content-directory/removeCuratorGroup.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/removeCuratorGroup.ts)_
+
+## `joystream-cli content-directory:removeMaintainerFromClass [CLASSNAME] [GROUPID]`
+
+Remove maintainer (Curator Group) from class.
+
+```
+USAGE
+  $ joystream-cli content-directory:removeMaintainerFromClass [CLASSNAME] [GROUPID]
+
+ARGUMENTS
+  CLASSNAME  Name or ID of the class (ie. Video)
+  GROUPID    ID of the Curator Group to remove from maintainers
+```
+
+_See code: [src/commands/content-directory/removeMaintainerFromClass.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/removeMaintainerFromClass.ts)_
+
+## `joystream-cli content-directory:setCuratorGroupStatus [ID] [STATUS]`
+
+Set Curator Group status (Active/Inactive).
+
+```
+USAGE
+  $ joystream-cli content-directory:setCuratorGroupStatus [ID] [STATUS]
+
+ARGUMENTS
+  ID      ID of the Curator Group
+  STATUS  New status of the group (1 - active, 0 - inactive)
+```
+
+_See code: [src/commands/content-directory/setCuratorGroupStatus.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/setCuratorGroupStatus.ts)_
+
+## `joystream-cli content-directory:updateClassPermissions [CLASSNAME]`
+
+Update permissions in given class.
+
+```
+USAGE
+  $ joystream-cli content-directory:updateClassPermissions [CLASSNAME]
+
+ARGUMENTS
+  CLASSNAME  Name or ID of the class (ie. Video)
+```
+
+_See code: [src/commands/content-directory/updateClassPermissions.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/updateClassPermissions.ts)_
+
 ## `joystream-cli council:info`
 
 Get current council and council elections information
@@ -316,6 +555,96 @@ OPTIONS
 
 _See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v2.2.3/src/commands/help.ts)_
 
+## `joystream-cli media:createChannel`
+
+Create a new channel on Joystream (requires a membership).
+
+```
+USAGE
+  $ joystream-cli media:createChannel
+
+OPTIONS
+  -i, --input=input    Path to JSON file to use as input (if not specified - the input can be provided interactively)
+  -o, --output=output  Path where the output JSON file should be placed (can be then reused as input)
+```
+
+_See code: [src/commands/media/createChannel.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/createChannel.ts)_
+
+## `joystream-cli media:myChannels`
+
+Show the list of channels associated with current account's membership.
+
+```
+USAGE
+  $ joystream-cli media:myChannels
+```
+
+_See code: [src/commands/media/myChannels.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/myChannels.ts)_
+
+## `joystream-cli media:myVideos`
+
+Show the list of videos associated with current account's membership.
+
+```
+USAGE
+  $ joystream-cli media:myVideos
+
+OPTIONS
+  -c, --channel=channel  Channel id to filter the videos by
+```
+
+_See code: [src/commands/media/myVideos.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/myVideos.ts)_
+
+## `joystream-cli media:updateChannel [ID]`
+
+Update one of the owned channels on Joystream (requires a membership).
+
+```
+USAGE
+  $ joystream-cli media:updateChannel [ID]
+
+ARGUMENTS
+  ID  ID of the channel to update
+
+OPTIONS
+  -i, --input=input    Path to JSON file to use as input (if not specified - the input can be provided interactively)
+  -o, --output=output  Path where the output JSON file should be placed (can be then reused as input)
+```
+
+_See code: [src/commands/media/updateChannel.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/updateChannel.ts)_
+
+## `joystream-cli media:updateVideo [ID]`
+
+Update existing video information (requires a membership).
+
+```
+USAGE
+  $ joystream-cli media:updateVideo [ID]
+
+ARGUMENTS
+  ID  ID of the Video to update
+```
+
+_See code: [src/commands/media/updateVideo.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/updateVideo.ts)_
+
+## `joystream-cli media:uploadVideo FILEPATH`
+
+Upload a new Video to a channel (requires a membership).
+
+```
+USAGE
+  $ joystream-cli media:uploadVideo FILEPATH
+
+ARGUMENTS
+  FILEPATH  Path to the media file to upload
+
+OPTIONS
+  -c, --channel=channel  ID of the channel to assign the video to (if omitted - one of the owned channels can be
+                         selected from the list)
+```
+
+_See code: [src/commands/media/uploadVideo.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/uploadVideo.ts)_
+
 ## `joystream-cli working-groups:application WGAPPLICATIONID`
 
 Shows an overview of given application by Working Group Application ID
@@ -330,7 +659,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/application.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/application.ts)_
@@ -352,7 +681,7 @@ OPTIONS
 
   -g, --group=group          (required) [default: storageProviders] The working group context in which the command
                              should be executed
-                             Available values are: storageProviders.
+                             Available values are: storageProviders, curators.
 
   -n, --draftName=draftName  Name of the draft to create the opening from.
 
@@ -375,7 +704,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/decreaseWorkerStake.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/decreaseWorkerStake.ts)_
@@ -394,7 +723,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/evictWorker.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/evictWorker.ts)_
@@ -413,7 +742,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/fillOpening.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/fillOpening.ts)_
@@ -429,7 +758,7 @@ USAGE
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/increaseStake.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/increaseStake.ts)_
@@ -445,7 +774,7 @@ USAGE
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/leaveRole.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/leaveRole.ts)_
@@ -464,7 +793,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/opening.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/opening.ts)_
@@ -480,7 +809,7 @@ USAGE
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/openings.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/openings.ts)_
@@ -496,7 +825,7 @@ USAGE
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/overview.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/overview.ts)_
@@ -515,7 +844,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/slashWorker.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/slashWorker.ts)_
@@ -534,7 +863,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/startAcceptingApplications.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/startAcceptingApplications.ts)_
@@ -553,7 +882,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/startReviewPeriod.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/startReviewPeriod.ts)_
@@ -572,7 +901,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/terminateApplication.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/terminateApplication.ts)_
@@ -591,7 +920,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/updateRewardAccount.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateRewardAccount.ts)_
@@ -610,7 +939,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/updateRoleAccount.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateRoleAccount.ts)_
@@ -629,7 +958,7 @@ ARGUMENTS
 OPTIONS
   -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
                      executed
-                     Available values are: storageProviders.
+                     Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/updateWorkerReward.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateWorkerReward.ts)_

cli/package.json

@@ -8,6 +8,8 @@
   },
   "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {
+    "@apidevtools/json-schema-ref-parser": "^9.0.6",
+    "@ffmpeg-installer/ffmpeg": "^1.0.20",
     "@joystream/types": "^0.14.0",
     "@oclif/command": "^1.5.19",
     "@oclif/config": "^1.14.0",
@@ -16,12 +18,21 @@
     "@oclif/plugin-not-found": "^1.2.4",
     "@oclif/plugin-warn-if-update-available": "^1.7.0",
     "@polkadot/api": "1.26.1",
+    "@types/fluent-ffmpeg": "^2.1.16",
     "@types/inquirer": "^6.5.0",
     "@types/proper-lockfile": "^4.1.1",
     "@types/slug": "^0.9.1",
     "ajv": "^6.11.0",
     "cli-ux": "^5.4.5",
+    "fluent-ffmpeg": "^2.1.2",
     "inquirer": "^7.1.0",
+    "ipfs-http-client": "^47.0.1",
+    "ipfs-only-hash": "^1.0.2",
+    "it-all": "^1.0.4",
+    "it-drain": "^1.0.3",
+    "it-first": "^1.0.4",
+    "it-last": "^1.0.4",
+    "it-to-buffer": "^1.0.4",
     "moment": "^2.24.0",
     "proper-lockfile": "^4.1.1",
     "slug": "^2.1.1",
@@ -86,6 +97,12 @@
       },
       "working-groups": {
         "description": "Working group lead and worker actions"
+      },
+      "content-directory": {
+        "description": "Interactions with content directory module - managing classes, schemas, entities and permissions"
+      },
+      "media": {
+        "description": "Higher-level content directory interactions, ie. publishing and curating content"
       }
     }
   },
@@ -101,7 +118,7 @@
     "test": "nyc --extension .ts mocha --forbid-only \"test/**/*.test.ts\"",
     "build": "tsc --build tsconfig.json",
     "version": "oclif-dev readme && git add README.md",
-    "lint": "eslint ./ --ext .ts",
+    "lint": "eslint ./src --ext .ts",
     "checks": "tsc --noEmit --pretty && prettier ./ --check && yarn lint",
     "format": "prettier ./ --write"
   },

cli/src/@types/@ffmpeg-installer/ffmpeg/index.d.ts

@@ -0,0 +1,3 @@
+declare module '@ffmpeg-installer/ffmpeg' {
+  export const path: string
+}

cli/src/@types/ipfs-http-client/index.d.ts

@@ -0,0 +1 @@
+declare module 'ipfs-http-client'

cli/src/@types/ipfs-only-hash/index.d.ts

@@ -0,0 +1 @@
+declare module 'ipfs-only-hash'

cli/src/Api.ts

@@ -32,6 +32,7 @@ import {
   RoleStakeProfile,
   Opening as WGOpening,
   Application as WGApplication,
+  StorageProviderId,
 } from '@joystream/types/working-group'
 import {
   Opening,
@@ -47,6 +48,10 @@ import { RewardRelationship, RewardRelationshipId } from '@joystream/types/recur
 import { Stake, StakeId } from '@joystream/types/stake'
 
 import { InputValidationLengthConstraint } from '@joystream/types/common'
+import { Class, ClassId, CuratorGroup, CuratorGroupId, Entity, EntityId } from '@joystream/types/content-directory'
+import { ContentId, DataObject } from '@joystream/types/media'
+import { ServiceProviderRecord, Url } from '@joystream/types/discovery'
+import _ from 'lodash'
 
 export const DEFAULT_API_URI = 'ws://localhost:9944/'
 const DEFAULT_DECIMALS = new BN(12)
@@ -54,11 +59,13 @@ const DEFAULT_DECIMALS = new BN(12)
 // Mapping of working group to api module
 export const apiModuleByGroup: { [key in WorkingGroups]: string } = {
   [WorkingGroups.StorageProviders]: 'storageWorkingGroup',
+  [WorkingGroups.Curators]: 'contentDirectoryWorkingGroup',
 }
 
 // Api wrapper for handling most common api calls and allowing easy API implementation switch in the future
 export default class Api {
   private _api: ApiPromise
+  private _cdClassesCache: [ClassId, Class][] | null = null
 
   private constructor(originalApi: ApiPromise) {
     this._api = originalApi
@@ -68,9 +75,12 @@ export default class Api {
     return this._api
   }
 
-  private static async initApi(apiUri: string = DEFAULT_API_URI): Promise<ApiPromise> {
+  private static async initApi(
+    apiUri: string = DEFAULT_API_URI,
+    metadataCache: Record<string, any>
+  ): Promise<ApiPromise> {
     const wsProvider: WsProvider = new WsProvider(apiUri)
-    const api = await ApiPromise.create({ provider: wsProvider, types })
+    const api = await ApiPromise.create({ provider: wsProvider, types, metadata: metadataCache })
 
     // Initializing some api params based on pioneer/packages/react-api/Api.tsx
     const [properties] = await Promise.all([api.rpc.system.properties()])
@@ -87,8 +97,8 @@ export default class Api {
     return api
   }
 
-  static async create(apiUri: string = DEFAULT_API_URI): Promise<Api> {
-    const originalApi: ApiPromise = await Api.initApi(apiUri)
+  static async create(apiUri: string = DEFAULT_API_URI, metadataCache: Record<string, any>): Promise<Api> {
+    const originalApi: ApiPromise = await Api.initApi(apiUri, metadataCache)
     return new Api(originalApi)
   }
 
@@ -109,6 +119,10 @@ export default class Api {
     })
   }
 
+  async bestNumber(): Promise<number> {
+    return (await this._api.derive.chain.bestNumber()).toNumber()
+  }
+
   async getAccountsBalancesInfo(accountAddresses: string[]): Promise<DeriveBalancesAll[]> {
     const accountsBalances: DeriveBalancesAll[] = await Promise.all(
       accountAddresses.map((addr) => this._api.derive.balances.all(addr))
@@ -284,7 +298,7 @@ export default class Api {
   }
 
   async groupMembers(group: WorkingGroups): Promise<GroupMember[]> {
-    const workerEntries = await this.entriesByIds<WorkerId, Worker>(this.workingGroupApiQuery(group).workerById)
+    const workerEntries = await this.groupWorkers(group)
 
     const groupMembers: GroupMember[] = await Promise.all(
       workerEntries.map(([id, worker]) => this.parseGroupMember(id, worker))
@@ -293,6 +307,10 @@ export default class Api {
     return groupMembers.reverse() // Sort by newest
   }
 
+  groupWorkers(group: WorkingGroups): Promise<[WorkerId, Worker][]> {
+    return this.entriesByIds<WorkerId, Worker>(this.workingGroupApiQuery(group).workerById)
+  }
+
   async openingsByGroup(group: WorkingGroups): Promise<GroupOpening[]> {
     let openings: GroupOpening[] = []
     const nextId = await this.workingGroupApiQuery(group).nextOpeningId<OpeningId>()
@@ -473,4 +491,68 @@ export default class Api {
   async workerExitRationaleConstraint(group: WorkingGroups): Promise<InputValidationLengthConstraint> {
     return await this.workingGroupApiQuery(group).workerExitRationaleText<InputValidationLengthConstraint>()
   }
+
+  // Content directory
+  async availableClasses(useCache = true): Promise<[ClassId, Class][]> {
+    return useCache && this._cdClassesCache
+      ? this._cdClassesCache
+      : (this._cdClassesCache = await this.entriesByIds<ClassId, Class>(this._api.query.contentDirectory.classById))
+  }
+
+  availableCuratorGroups(): Promise<[CuratorGroupId, CuratorGroup][]> {
+    return this.entriesByIds<CuratorGroupId, CuratorGroup>(this._api.query.contentDirectory.curatorGroupById)
+  }
+
+  async curatorGroupById(id: number): Promise<CuratorGroup | null> {
+    const exists = !!(await this._api.query.contentDirectory.curatorGroupById.size(id)).toNumber()
+    return exists ? await this._api.query.contentDirectory.curatorGroupById<CuratorGroup>(id) : null
+  }
+
+  async nextCuratorGroupId(): Promise<number> {
+    return (await this._api.query.contentDirectory.nextCuratorGroupId<CuratorGroupId>()).toNumber()
+  }
+
+  async classById(id: number): Promise<Class | null> {
+    const c = await this._api.query.contentDirectory.classById<Class>(id)
+    return c.isEmpty ? null : c
+  }
+
+  async entitiesByClassId(classId: number): Promise<[EntityId, Entity][]> {
+    const entityEntries = await this.entriesByIds<EntityId, Entity>(this._api.query.contentDirectory.entityById)
+    return entityEntries.filter(([, entity]) => entity.class_id.toNumber() === classId)
+  }
+
+  async entityById(id: number): Promise<Entity | null> {
+    const exists = !!(await this._api.query.contentDirectory.curatorGroupById.size(id))
+    return exists ? await this._api.query.contentDirectory.entityById<Entity>(id) : null
+  }
+
+  async dataObjectByContentId(contentId: ContentId): Promise<DataObject | null> {
+    const dataObject = await this._api.query.dataDirectory.dataObjectByContentId<Option<DataObject>>(contentId)
+    return dataObject.unwrapOr(null)
+  }
+
+  async ipnsIdentity(storageProviderId: number): Promise<string | null> {
+    const accountInfo = await this._api.query.discovery.accountInfoByStorageProviderId<ServiceProviderRecord>(
+      storageProviderId
+    )
+    return accountInfo.isEmpty || accountInfo.expires_at.toNumber() <= (await this.bestNumber())
+      ? null
+      : accountInfo.identity.toString()
+  }
+
+  async getRandomBootstrapEndpoint(): Promise<string | null> {
+    const endpoints = await this._api.query.discovery.bootstrapEndpoints<Vec<Url>>()
+    const randomEndpoint = _.sample(endpoints.toArray())
+    return randomEndpoint ? randomEndpoint.toString() : null
+  }
+
+  async isAnyProviderAvailable(): Promise<boolean> {
+    const accounInfoEntries = await this.entriesByIds<StorageProviderId, ServiceProviderRecord>(
+      this._api.query.discovery.accountInfoByStorageProviderId
+    )
+
+    const bestNumber = await this.bestNumber()
+    return !!accounInfoEntries.filter(([, info]) => info.expires_at.toNumber() > bestNumber).length
+  }
 }

cli/src/ExitCodes.ts

@@ -11,5 +11,7 @@ enum ExitCodes {
   UnexpectedException = 500,
   FsOperationFailed = 501,
   ApiError = 502,
+  ExternalInfrastructureError = 503,
+  ActionCurrentlyUnavailable = 504,
 }
 export = ExitCodes

cli/src/Types.ts

@@ -87,10 +87,14 @@ export type NameValueObj = { name: string; value: string }
 // Working groups related types
 export enum WorkingGroups {
   StorageProviders = 'storageProviders',
+  Curators = 'curators',
 }
 
 // In contrast to Pioneer, currently only StorageProviders group is available in CLI
-export const AvailableGroups: readonly WorkingGroups[] = [WorkingGroups.StorageProviders] as const
+export const AvailableGroups: readonly WorkingGroups[] = [
+  WorkingGroups.StorageProviders,
+  WorkingGroups.Curators,
+] as const
 
 export type Reward = {
   totalRecieved: Balance

cli/src/base/AccountsCommandBase.ts

@@ -216,14 +216,41 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
   }
 
   async requestAccountDecoding(account: NamedKeyringPair): Promise<void> {
-    const password: string = await this.promptForPassword()
+    // Skip if account already unlocked
+    if (!account.isLocked) {
+      return
+    }
+
+    // First - try decoding using empty string
     try {
-      account.decodePkcs8(password)
+      account.decodePkcs8('')
+      return
     } catch (e) {
-      this.error('Invalid password!', { exit: ExitCodes.InvalidInput })
+      // Continue...
+    }
+
+    let isPassValid = false
+    while (!isPassValid) {
+      try {
+        const password = await this.promptForPassword()
+        account.decodePkcs8(password)
+        isPassValid = true
+      } catch (e) {
+        this.warn('Invalid password... Try again.')
+      }
     }
   }
 
+  async getRequiredMemberId(): Promise<number> {
+    const account = await this.getRequiredSelectedAccount()
+    const memberIds = await this.getApi().getMemberIdsByControllerAccount(account.address)
+    if (!memberIds.length) {
+      this.error('Membership required to access this command!', { exit: ExitCodes.AccessDenied })
+    }
+
+    return memberIds[0].toNumber() // FIXME: Temporary solution (just using the first one)
+  }
+
   async init() {
     await super.init()
     try {

cli/src/base/ApiCommandBase.ts

@@ -2,7 +2,7 @@ import ExitCodes from '../ExitCodes'
 import { CLIError } from '@oclif/errors'
 import StateAwareCommandBase from './StateAwareCommandBase'
 import Api from '../Api'
-import { getTypeDef, Option, Tuple, Bytes } from '@polkadot/types'
+import { getTypeDef, Option, Tuple, Bytes, TypeRegistry } from '@polkadot/types'
 import { Registry, Codec, CodecArg, TypeDef, TypeDefInfo, Constructor } from '@polkadot/types/types'
 
 import { Vec, Struct, Enum } from '@polkadot/types/codec'
@@ -13,6 +13,10 @@ import { InterfaceTypes } from '@polkadot/types/types/registry'
 import ajv from 'ajv'
 import { ApiMethodArg, ApiMethodNamedArgs, ApiParamsOptions, ApiParamOptions } from '../Types'
 import { createParamOptions } from '../helpers/promptOptions'
+import { SubmittableExtrinsic } from '@polkadot/api/types'
+import { DistinctQuestion } from 'inquirer'
+import { BOOL_PROMPT_OPTIONS } from '../helpers/prompting'
+import { DispatchError } from '@polkadot/types/interfaces/system'
 
 class ExtrinsicFailedError extends Error {}
 
@@ -48,7 +52,17 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
       this.warn("You haven't provided a node/endpoint for the CLI to connect to yet!")
       apiUri = await this.promptForApiUri()
     }
-    this.api = await Api.create(apiUri)
+
+    const { metadataCache } = this.getPreservedState()
+    this.api = await Api.create(apiUri, metadataCache)
+
+    const { genesisHash, runtimeVersion } = this.getOriginalApi()
+    const metadataKey = `${genesisHash}-${runtimeVersion.specVersion}`
+    if (!metadataCache[metadataKey]) {
+      // Add new entry to metadata cache
+      metadataCache[metadataKey] = await this.getOriginalApi().runtimeMetadata.toJSON()
+      await this.setPreservedState({ metadataCache })
+    }
   }
 
   async promptForApiUri(): Promise<string> {
@@ -131,9 +145,15 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     // If no default provided - get default value resulting from providing empty string
     const defaultValueString =
       paramOptions?.value?.default?.toString() || this.createType(typeDef.type as any, '').toString()
+
+    let typeSpecificOptions: DistinctQuestion = { type: 'input' }
+    if (typeDef.type === 'bool') {
+      typeSpecificOptions = BOOL_PROMPT_OPTIONS
+    }
+
     const providedValue = await this.simplePrompt({
       message: `Provide value for ${this.paramName(typeDef)}`,
-      type: 'input',
+      ...typeSpecificOptions,
       // We want to avoid showing default value like '0x', because it falsely suggests
       // that user needs to provide the value as hex
       default: (defaultValueString === '0x' ? '' : defaultValueString) || undefined,
@@ -313,6 +333,11 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     }
   }
 
+  // More typesafe version
+  async promptForType(type: keyof InterfaceTypes, options?: ApiParamOptions) {
+    return await this.promptForParam(type, options)
+  }
+
   async promptForJsonBytes(
     jsonStruct: Constructor<Struct>,
     argName?: string,
@@ -379,32 +404,45 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     return values
   }
 
-  sendExtrinsic(account: KeyringPair, module: string, method: string, params: CodecArg[]) {
+  sendExtrinsic(account: KeyringPair, tx: SubmittableExtrinsic<'promise'>) {
     return new Promise((resolve, reject) => {
-      const extrinsicMethod = this.getOriginalApi().tx[module][method]
       let unsubscribe: () => void
-      extrinsicMethod(...params)
-        .signAndSend(account, {}, (result) => {
-          // Implementation loosely based on /pioneer/packages/react-signer/src/Modal.tsx
-          if (!result || !result.status) {
-            return
-          }
-
-          if (result.status.isInBlock) {
-            unsubscribe()
-            result.events
-              .filter(({ event: { section } }): boolean => section === 'system')
-              .forEach(({ event: { method } }): void => {
-                if (method === 'ExtrinsicFailed') {
-                  reject(new ExtrinsicFailedError('Extrinsic execution error!'))
-                } else if (method === 'ExtrinsicSuccess') {
-                  resolve()
+      tx.signAndSend(account, {}, (result) => {
+        // Implementation loosely based on /pioneer/packages/react-signer/src/Modal.tsx
+        if (!result || !result.status) {
+          return
+        }
+
+        if (result.status.isInBlock) {
+          unsubscribe()
+          result.events
+            .filter(({ event }) => event.section === 'system')
+            .forEach(({ event }) => {
+              if (event.method === 'ExtrinsicFailed') {
+                const dispatchError = event.data[0] as DispatchError
+                let errorMsg = dispatchError.toString()
+                if (dispatchError.isModule) {
+                  try {
+                    // Need to assert that registry is of TypeRegistry type, since Registry intefrace
+                    // seems outdated and doesn't include DispatchErrorModule as possible argument for "findMetaError"
+                    const { name, documentation } = (this.getOriginalApi().registry as TypeRegistry).findMetaError(
+                      dispatchError.asModule
+                    )
+                    errorMsg = `${name} (${documentation})`
+                  } catch (e) {
+                    // This probably means we don't have this error in the metadata
+                    // In this case - continue (we'll just display dispatchError.toString())
+                  }
                 }
-              })
-          } else if (result.isError) {
-            reject(new ExtrinsicFailedError('Extrinsic execution error!'))
-          }
-        })
+                reject(new ExtrinsicFailedError(`Extrinsic execution error: ${errorMsg}`))
+              } else if (event.method === 'ExtrinsicSuccess') {
+                resolve()
+              }
+            })
+        } else if (result.isError) {
+          reject(new ExtrinsicFailedError('Extrinsic execution error!'))
+        }
+      })
         .then((unsubFunc) => (unsubscribe = unsubFunc))
         .catch((e) =>
           reject(new ExtrinsicFailedError(`Cannot send the extrinsic: ${e.message ? e.message : JSON.stringify(e)}`))
@@ -412,37 +450,46 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     })
   }
 
-  async sendAndFollowExtrinsic(
+  async sendAndFollowTx(
     account: KeyringPair,
-    module: string,
-    method: string,
-    params: CodecArg[],
-    warnOnly = false // If specified - only warning will be displayed (instead of error beeing thrown)
-  ) {
+    tx: SubmittableExtrinsic<'promise'>,
+    warnOnly = false // If specified - only warning will be displayed in case of failure (instead of error beeing thrown)
+  ): Promise<void> {
     try {
-      this.log(chalk.white(`\nSending ${module}.${method} extrinsic...`))
-      await this.sendExtrinsic(account, module, method, params)
+      await this.sendExtrinsic(account, tx)
       this.log(chalk.green(`Extrinsic successful!`))
     } catch (e) {
       if (e instanceof ExtrinsicFailedError && warnOnly) {
-        this.warn(`${module}.${method} extrinsic failed! ${e.message}`)
+        this.warn(`Extrinsic failed! ${e.message}`)
       } else if (e instanceof ExtrinsicFailedError) {
-        throw new CLIError(`${module}.${method} extrinsic failed! ${e.message}`, { exit: ExitCodes.ApiError })
+        throw new CLIError(`Extrinsic failed! ${e.message}`, { exit: ExitCodes.ApiError })
       } else {
         throw e
       }
     }
   }
 
+  async sendAndFollowNamedTx(
+    account: KeyringPair,
+    module: string,
+    method: string,
+    params: CodecArg[],
+    warnOnly = false
+  ): Promise<void> {
+    this.log(chalk.white(`\nSending ${module}.${method} extrinsic...`))
+    const tx = await this.getOriginalApi().tx[module][method](...params)
+    await this.sendAndFollowTx(account, tx, warnOnly)
+  }
+
   async buildAndSendExtrinsic(
     account: KeyringPair,
     module: string,
     method: string,
-    paramsOptions: ApiParamsOptions,
+    paramsOptions?: ApiParamsOptions,
     warnOnly = false // If specified - only warning will be displayed (instead of error beeing thrown)
   ): Promise<ApiMethodArg[]> {
     const params = await this.promptForExtrinsicParams(module, method, paramsOptions)
-    await this.sendAndFollowExtrinsic(account, module, method, params, warnOnly)
+    await this.sendAndFollowNamedTx(account, module, method, params, warnOnly)
 
     return params
   }

cli/src/base/ContentDirectoryCommandBase.ts

@@ -0,0 +1,283 @@
+import ExitCodes from '../ExitCodes'
+import AccountsCommandBase from './AccountsCommandBase'
+import { WorkingGroups, NamedKeyringPair } from '../Types'
+import { ReferenceProperty } from 'cd-schemas/types/extrinsics/AddClassSchema'
+import { FlattenRelations } from 'cd-schemas/types/utility'
+import { BOOL_PROMPT_OPTIONS } from '../helpers/prompting'
+import { Class, ClassId, CuratorGroup, CuratorGroupId, Entity, EntityId } from '@joystream/types/content-directory'
+import { Worker } from '@joystream/types/working-group'
+import { CLIError } from '@oclif/errors'
+import { Codec } from '@polkadot/types/types'
+import _ from 'lodash'
+import chalk from 'chalk'
+
+/**
+ * Abstract base class for commands related to content directory
+ */
+export default abstract class ContentDirectoryCommandBase extends AccountsCommandBase {
+  // Use when lead access is required in given command
+  async requireLead(): Promise<void> {
+    const selectedAccount: NamedKeyringPair = await this.getRequiredSelectedAccount()
+    const lead = await this.getApi().groupLead(WorkingGroups.Curators)
+
+    if (!lead || lead.roleAccount.toString() !== selectedAccount.address) {
+      this.error('Content Working Group Lead access required for this command!', { exit: ExitCodes.AccessDenied })
+    }
+  }
+
+  async promptForClass(message = 'Select a class'): Promise<Class> {
+    const classes = await this.getApi().availableClasses()
+    const choices = classes.map(([, c]) => ({ name: c.name.toString(), value: c }))
+    if (!choices.length) {
+      this.warn('No classes exist to choose from!')
+      this.exit(ExitCodes.InvalidInput)
+    }
+
+    const selectedClass = await this.simplePrompt({ message, type: 'list', choices })
+
+    return selectedClass
+  }
+
+  async classEntryByNameOrId(classNameOrId: string): Promise<[ClassId, Class]> {
+    const classes = await this.getApi().availableClasses()
+    const foundClass = classes.find(([id, c]) => id.toString() === classNameOrId || c.name.toString() === classNameOrId)
+    if (!foundClass) {
+      this.error(`Class id not found by class name or id: "${classNameOrId}"!`)
+    }
+
+    return foundClass
+  }
+
+  private async curatorGroupChoices(ids?: CuratorGroupId[]) {
+    const groups = await this.getApi().availableCuratorGroups()
+    return groups
+      .filter(([id]) => (ids ? ids.some((allowedId) => allowedId.eq(id)) : true))
+      .map(([id, group]) => ({
+        name:
+          `Group ${id.toString()} (` +
+          `${group.active.valueOf() ? 'Active' : 'Inactive'}, ` +
+          `${group.curators.toArray().length} member(s), ` +
+          `${group.number_of_classes_maintained.toNumber()} classes maintained)`,
+        value: id.toNumber(),
+      }))
+  }
+
+  async promptForCuratorGroup(message = 'Select a Curator Group', ids?: CuratorGroupId[]): Promise<number> {
+    const choices = await this.curatorGroupChoices(ids)
+    if (!choices.length) {
+      this.warn('No Curator Groups to choose from!')
+      this.exit(ExitCodes.InvalidInput)
+    }
+    const selectedId = await this.simplePrompt({ message, type: 'list', choices })
+
+    return selectedId
+  }
+
+  async promptForCuratorGroups(message = 'Select Curator Groups'): Promise<number[]> {
+    const choices = await this.curatorGroupChoices()
+    if (!choices.length) {
+      return []
+    }
+    const selectedIds = await this.simplePrompt({ message, type: 'checkbox', choices })
+
+    return selectedIds
+  }
+
+  async promptForClassReference(): Promise<ReferenceProperty['Reference']> {
+    const selectedClass = await this.promptForClass()
+    const sameOwner = await this.simplePrompt({ message: 'Same owner required?', ...BOOL_PROMPT_OPTIONS })
+    return { className: selectedClass.name.toString(), sameOwner }
+  }
+
+  async promptForCurator(message = 'Choose a Curator', ids?: number[]): Promise<number> {
+    const curators = await this.getApi().groupMembers(WorkingGroups.Curators)
+    const choices = curators
+      .filter((c) => (ids ? ids.includes(c.workerId.toNumber()) : true))
+      .map((c) => ({
+        name: `${c.profile.handle.toString()} (Worker ID: ${c.workerId})`,
+        value: c.workerId.toNumber(),
+      }))
+
+    if (!choices.length) {
+      this.warn('No Curators to choose from!')
+      this.exit(ExitCodes.InvalidInput)
+    }
+
+    const selectedCuratorId = await this.simplePrompt({
+      message,
+      type: 'list',
+      choices,
+    })
+
+    return selectedCuratorId
+  }
+
+  async getCurator(id: string | number): Promise<Worker> {
+    if (typeof id === 'string') {
+      id = parseInt(id)
+    }
+
+    let curator
+    try {
+      curator = await this.getApi().workerByWorkerId(WorkingGroups.Curators, id)
+    } catch (e) {
+      if (e instanceof CLIError) {
+        throw new CLIError('Invalid Curator id!')
+      }
+      throw e
+    }
+
+    return curator
+  }
+
+  async getCuratorGroup(id: string | number): Promise<CuratorGroup> {
+    if (typeof id === 'string') {
+      id = parseInt(id)
+    }
+
+    const group = await this.getApi().curatorGroupById(id)
+
+    if (!group) {
+      this.error('Invalid Curator Group id!', { exit: ExitCodes.InvalidInput })
+    }
+
+    return group
+  }
+
+  async getEntity(id: string | number, requiredClass?: string, ownerMemberId?: number): Promise<Entity> {
+    if (typeof id === 'string') {
+      id = parseInt(id)
+    }
+
+    const entity = await this.getApi().entityById(id)
+
+    if (!entity) {
+      this.error(`Entity not found by id: ${id}`, { exit: ExitCodes.InvalidInput })
+    }
+
+    if (requiredClass) {
+      const [classId] = await this.classEntryByNameOrId(requiredClass)
+      if (entity.class_id.toNumber() !== classId.toNumber()) {
+        this.error(`Entity of id ${id} is not of class ${requiredClass}!`, { exit: ExitCodes.InvalidInput })
+      }
+    }
+
+    const { controller } = entity.entity_permissions
+    if (
+      ownerMemberId !== undefined &&
+      (!controller.isOfType('Member') || controller.asType('Member').toNumber() !== ownerMemberId)
+    ) {
+      this.error('Cannot execute this action for specified entity - invalid ownership.', {
+        exit: ExitCodes.AccessDenied,
+      })
+    }
+
+    return entity
+  }
+
+  async getAndParseKnownEntity<T>(id: string | number): Promise<FlattenRelations<T>> {
+    const entity = await this.getEntity(id)
+    return this.parseToKnownEntityJson<T>(entity)
+  }
+
+  async entitiesByClassAndOwner(classNameOrId: number | string, ownerMemberId?: number): Promise<[EntityId, Entity][]> {
+    const classId =
+      typeof classNameOrId === 'number' ? classNameOrId : (await this.classEntryByNameOrId(classNameOrId))[0].toNumber()
+
+    return (await this.getApi().entitiesByClassId(classId)).filter(([, entity]) => {
+      const controller = entity.entity_permissions.controller
+      return ownerMemberId !== undefined
+        ? controller.isOfType('Member') && controller.asType('Member').toNumber() === ownerMemberId
+        : true
+    })
+  }
+
+  async promptForEntityEntry(
+    message: string,
+    className: string,
+    propName?: string,
+    ownerMemberId?: number,
+    defaultId?: number
+  ): Promise<[EntityId, Entity]> {
+    const [classId, entityClass] = await this.classEntryByNameOrId(className)
+    const entityEntries = await this.entitiesByClassAndOwner(classId.toNumber(), ownerMemberId)
+
+    if (!entityEntries.length) {
+      this.log(`${message}:`)
+      this.error(`No choices available! Exiting...`, { exit: ExitCodes.UnexpectedException })
+    }
+
+    const choosenEntityId = await this.simplePrompt({
+      message,
+      type: 'list',
+      choices: entityEntries.map(([id, entity]) => {
+        const parsedEntityPropertyValues = this.parseEntityPropertyValues(entity, entityClass)
+        return {
+          name: (propName && parsedEntityPropertyValues[propName]?.value.toString()) || `ID:${id.toString()}`,
+          value: id.toString(), // With numbers there are issues with "default"
+        }
+      }),
+      default: defaultId?.toString(),
+    })
+
+    return entityEntries.find(([id]) => choosenEntityId === id.toString())!
+  }
+
+  async promptForEntityId(
+    message: string,
+    className: string,
+    propName?: string,
+    ownerMemberId?: number,
+    defaultId?: number
+  ): Promise<number> {
+    return (await this.promptForEntityEntry(message, className, propName, ownerMemberId, defaultId))[0].toNumber()
+  }
+
+  parseEntityPropertyValues(
+    entity: Entity,
+    entityClass: Class,
+    includedProperties?: string[]
+  ): Record<string, { value: Codec; type: string }> {
+    const { properties } = entityClass
+    return Array.from(entity.getField('values').entries()).reduce((columns, [propId, propValue]) => {
+      const prop = properties[propId.toNumber()]
+      const propName = prop.name.toString()
+      const included = !includedProperties || includedProperties.some((p) => p.toLowerCase() === propName.toLowerCase())
+
+      if (included) {
+        columns[propName] = {
+          value: propValue.getValue(),
+          type: `${prop.property_type.type}<${prop.property_type.subtype}>`,
+        }
+      }
+      return columns
+    }, {} as Record<string, { value: Codec; type: string }>)
+  }
+
+  async parseToKnownEntityJson<T>(entity: Entity): Promise<FlattenRelations<T>> {
+    const entityClass = (await this.classEntryByNameOrId(entity.class_id.toString()))[1]
+    return (_.mapValues(this.parseEntityPropertyValues(entity, entityClass), (v) =>
+      v.value.toJSON()
+    ) as unknown) as FlattenRelations<T>
+  }
+
+  async createEntityList(
+    className: string,
+    includedProps?: string[],
+    filters: [string, string][] = [],
+    ownerMemberId?: number
+  ): Promise<Record<string, string>[]> {
+    const [classId, entityClass] = await this.classEntryByNameOrId(className)
+    const entityEntries = await this.entitiesByClassAndOwner(classId.toNumber(), ownerMemberId)
+    const parsedEntities = (await Promise.all(
+      entityEntries.map(([id, entity]) => ({
+        'ID': id.toString(),
+        ..._.mapValues(this.parseEntityPropertyValues(entity, entityClass, includedProps), (v) =>
+          v.value.toJSON() === false && v.type !== 'Single<Bool>' ? chalk.grey('[not set]') : v.value.toString()
+        ),
+      }))
+    )) as Record<string, string>[]
+
+    return parsedEntities.filter((entity) => filters.every(([pName, pValue]) => entity[pName] === pValue))
+  }
+}

cli/src/base/StateAwareCommandBase.ts

@@ -11,12 +11,14 @@ import _ from 'lodash'
 type StateObject = {
   selectedAccountFilename: string
   apiUri: string
+  metadataCache: Record<string, any>
 }
 
 // State object default values
 const DEFAULT_STATE: StateObject = {
   selectedAccountFilename: '',
   apiUri: '',
+  metadataCache: {},
 }
 
 // State file path (relative to getAppDataPath())

cli/src/commands/api/setUri.ts

@@ -16,7 +16,7 @@ export default class ApiSetUri extends ApiCommandBase {
 
   async init() {
     this.forceSkipApiUriPrompt = true
-    super.init()
+    await super.init()
   }
 
   async run() {

cli/src/commands/content-directory/addClassSchema.ts

@@ -0,0 +1,79 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import AddClassSchemaSchema from 'cd-schemas/schemas/extrinsics/AddClassSchema.schema.json'
+import { AddClassSchema } from 'cd-schemas/types/extrinsics/AddClassSchema'
+import { InputParser } from 'cd-schemas'
+import { JsonSchemaPrompter, JsonSchemaCustomPrompts } from '../../helpers/JsonSchemaPrompt'
+import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
+import { IOFlags, getInputJson, saveOutputJson } from '../../helpers/InputOutput'
+import { Class } from '@joystream/types/content-directory'
+
+export default class AddClassSchemaCommand extends ContentDirectoryCommandBase {
+  static description = 'Add a new schema to a class inside content directory. Requires lead access.'
+
+  static flags = {
+    ...IOFlags,
+  }
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+    await this.requestAccountDecoding(account)
+
+    const { input, output } = this.parse(AddClassSchemaCommand).flags
+
+    let inputJson = await getInputJson<AddClassSchema>(input)
+    if (!inputJson) {
+      let selectedClass: Class | undefined
+      const customPrompts: JsonSchemaCustomPrompts = [
+        [
+          'className',
+          async () => {
+            selectedClass = await this.promptForClass('Select a class to add schema to')
+            return selectedClass.name.toString()
+          },
+        ],
+        [
+          'existingProperties',
+          async () => {
+            const choices = selectedClass!.properties.map((p, i) => ({ name: `${i}: ${p.name.toString()}`, value: i }))
+            if (!choices.length) {
+              return []
+            }
+            return await this.simplePrompt({
+              type: 'checkbox',
+              message: 'Choose existing properties to keep',
+              choices,
+            })
+          },
+        ],
+        [
+          /^newProperties\[\d+\]\.property_type\.(Single|Vector\.vec_type)\.Reference/,
+          async () => this.promptForClassReference(),
+        ],
+        [/^newProperties\[\d+\]\.property_type\.(Single|Vector\.vec_type)\.Text/, { message: 'Provide TextMaxLength' }],
+        [
+          /^newProperties\[\d+\]\.property_type\.(Single|Vector\.vec_type)\.Hash/,
+          { message: 'Provide HashedTextMaxLength' },
+        ],
+      ]
+
+      const prompter = new JsonSchemaPrompter<AddClassSchema>(
+        AddClassSchemaSchema as JSONSchema,
+        undefined,
+        customPrompts
+      )
+
+      inputJson = await prompter.promptAll()
+    }
+
+    this.jsonPrettyPrint(JSON.stringify(inputJson))
+    const confirmed = await this.simplePrompt({ type: 'confirm', message: 'Do you confirm the provided input?' })
+
+    if (confirmed) {
+      saveOutputJson(output, `${inputJson.className}Schema.json`, inputJson)
+      const inputParser = new InputParser(this.getOriginalApi())
+      this.log('Sending the extrinsic...')
+      await this.sendAndFollowTx(account, await inputParser.parseAddClassSchemaExtrinsic(inputJson))
+    }
+  }
+}

cli/src/commands/content-directory/addCuratorToGroup.ts

@@ -0,0 +1,42 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+
+export default class AddCuratorToGroupCommand extends ContentDirectoryCommandBase {
+  static description = 'Add Curator to existing Curator Group.'
+  static args = [
+    {
+      name: 'groupId',
+      required: false,
+      description: 'ID of the Curator Group',
+    },
+    {
+      name: 'curatorId',
+      required: false,
+      description: 'ID of the curator',
+    },
+  ]
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+
+    let { groupId, curatorId } = this.parse(AddCuratorToGroupCommand).args
+
+    if (groupId === undefined) {
+      groupId = await this.promptForCuratorGroup()
+    } else {
+      await this.getCuratorGroup(groupId)
+    }
+
+    if (curatorId === undefined) {
+      curatorId = await this.promptForCurator()
+    } else {
+      await this.getCurator(curatorId)
+    }
+
+    await this.requestAccountDecoding(account)
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'addCuratorToGroup', [groupId, curatorId])
+
+    console.log(chalk.green(`Curator ${chalk.white(curatorId)} succesfully added to group ${chalk.white(groupId)}!`))
+  }
+}

cli/src/commands/content-directory/addMaintainerToClass.ts

@@ -0,0 +1,44 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+
+export default class AddMaintainerToClassCommand extends ContentDirectoryCommandBase {
+  static description = 'Add maintainer (Curator Group) to a class.'
+  static args = [
+    {
+      name: 'className',
+      required: false,
+      description: 'Name or ID of the class (ie. Video)',
+    },
+    {
+      name: 'groupId',
+      required: false,
+      description: 'ID of the Curator Group to add as class maintainer',
+    },
+  ]
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+
+    let { groupId, className } = this.parse(AddMaintainerToClassCommand).args
+
+    if (className === undefined) {
+      className = (await this.promptForClass()).name.toString()
+    }
+
+    const classId = (await this.classEntryByNameOrId(className))[0].toNumber()
+
+    if (groupId === undefined) {
+      groupId = await this.promptForCuratorGroup()
+    } else {
+      await this.getCuratorGroup(groupId)
+    }
+
+    await this.requestAccountDecoding(account)
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'addMaintainerToClass', [classId, groupId])
+
+    console.log(
+      chalk.green(`Curator Group ${chalk.white(groupId)} added as maintainer to ${chalk.white(className)} class!`)
+    )
+  }
+}

cli/src/commands/content-directory/class.ts

@@ -0,0 +1,55 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+import { displayCollapsedRow, displayHeader, displayTable } from '../../helpers/display'
+
+export default class ClassCommand extends ContentDirectoryCommandBase {
+  static description = 'Show Class details by id or name.'
+  static args = [
+    {
+      name: 'className',
+      required: true,
+      description: 'Name or ID of the Class',
+    },
+  ]
+
+  async run() {
+    const { className } = this.parse(ClassCommand).args
+    const [id, aClass] = await this.classEntryByNameOrId(className)
+    const permissions = aClass.class_permissions
+    const maintainers = permissions.maintainers.toArray()
+
+    displayCollapsedRow({
+      'Name': aClass.name.toString(),
+      'ID': id.toString(),
+      'Any member': permissions.any_member.toString(),
+      'Entity creation blocked': permissions.entity_creation_blocked.toString(),
+      'All property values locked': permissions.all_entity_property_values_locked.toString(),
+      'Number of entities': aClass.current_number_of_entities.toNumber(),
+      'Max. number of entities': aClass.maximum_entities_count.toNumber(),
+      'Default entity creation voucher max.': aClass.default_entity_creation_voucher_upper_bound.toNumber(),
+    })
+
+    displayHeader(`Maintainers`)
+    this.log(
+      maintainers.length ? maintainers.map((groupId) => chalk.white(`Group ${groupId.toString()}`)).join(', ') : 'NONE'
+    )
+
+    displayHeader(`Properties`)
+    if (aClass.properties.length) {
+      displayTable(
+        aClass.properties.map((p, i) => ({
+          'Index': i,
+          'Name': p.name.toString(),
+          'Type': JSON.stringify(p.property_type.toJSON()),
+          'Required': p.required.toString(),
+          'Unique': p.unique.toString(),
+          'Controller lock': p.locking_policy.is_locked_from_controller.toString(),
+          'Maintainer lock': p.locking_policy.is_locked_from_maintainer.toString(),
+        })),
+        3
+      )
+    } else {
+      this.log('NONE')
+    }
+  }
+}

cli/src/commands/content-directory/classes.ts

@@ -0,0 +1,24 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+// import chalk from 'chalk'
+import { displayTable } from '../../helpers/display'
+
+export default class ClassesCommand extends ContentDirectoryCommandBase {
+  static description = 'List existing content directory classes.'
+
+  async run() {
+    const classes = await this.getApi().availableClasses()
+
+    displayTable(
+      classes.map(([id, c]) => ({
+        'ID': id.toString(),
+        'Name': c.name.toString(),
+        'Any member': c.class_permissions.any_member.toString(),
+        'Entities': c.current_number_of_entities.toNumber(),
+        'Schemas': c.schemas.length,
+        'Maintainers': c.class_permissions.maintainers.toArray().length,
+        'Properties': c.properties.length,
+      })),
+      3
+    )
+  }
+}

cli/src/commands/content-directory/createClass.ts

@@ -0,0 +1,50 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import CreateClassSchema from 'cd-schemas/schemas/extrinsics/CreateClass.schema.json'
+import { CreateClass } from 'cd-schemas/types/extrinsics/CreateClass'
+import { InputParser } from 'cd-schemas'
+import { JsonSchemaPrompter, JsonSchemaCustomPrompts } from '../../helpers/JsonSchemaPrompt'
+import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
+import { IOFlags, getInputJson, saveOutputJson } from '../../helpers/InputOutput'
+
+export default class CreateClassCommand extends ContentDirectoryCommandBase {
+  static description = 'Create class inside content directory. Requires lead access.'
+  static flags = {
+    ...IOFlags,
+  }
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+    await this.requestAccountDecoding(account)
+
+    const { input, output } = this.parse(CreateClassCommand).flags
+    const existingClassnames = (await this.getApi().availableClasses()).map(([, aClass]) => aClass.name.toString())
+
+    let inputJson = await getInputJson<CreateClass>(input, CreateClassSchema as JSONSchema)
+    if (!inputJson) {
+      const customPrompts: JsonSchemaCustomPrompts<CreateClass> = [
+        [
+          'name',
+          {
+            validate: (className) => existingClassnames.includes(className) && 'A class with this name already exists!',
+          },
+        ],
+        ['class_permissions.maintainers', () => this.promptForCuratorGroups('Select class maintainers')],
+      ]
+
+      const prompter = new JsonSchemaPrompter<CreateClass>(CreateClassSchema as JSONSchema, undefined, customPrompts)
+
+      inputJson = await prompter.promptAll()
+    }
+
+    this.jsonPrettyPrint(JSON.stringify(inputJson))
+    const confirmed = await this.simplePrompt({ type: 'confirm', message: 'Do you confirm the provided input?' })
+
+    if (confirmed) {
+      saveOutputJson(output, `${inputJson.name}Class.json`, inputJson)
+      this.log('Sending the extrinsic...')
+      const inputParser = new InputParser(this.getOriginalApi())
+      await this.sendAndFollowTx(account, inputParser.parseCreateClassExtrinsic(inputJson))
+    }
+  }
+}

cli/src/commands/content-directory/createCuratorGroup.ts

@@ -0,0 +1,18 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+
+export default class AddCuratorGroupCommand extends ContentDirectoryCommandBase {
+  static description = 'Create new Curator Group.'
+  static aliases = ['addCuratorGroup']
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+
+    await this.requestAccountDecoding(account)
+    await this.buildAndSendExtrinsic(account, 'contentDirectory', 'addCuratorGroup')
+
+    const newGroupId = (await this.getApi().nextCuratorGroupId()) - 1
+    console.log(chalk.green(`New group succesfully created! (ID: ${chalk.white(newGroupId)})`))
+  }
+}

cli/src/commands/content-directory/curatorGroup.ts

@@ -0,0 +1,39 @@
+import { WorkingGroups } from '../../Types'
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+import { displayCollapsedRow, displayHeader } from '../../helpers/display'
+
+export default class CuratorGroupCommand extends ContentDirectoryCommandBase {
+  static description = 'Show Curator Group details by ID.'
+  static args = [
+    {
+      name: 'id',
+      required: true,
+      description: 'ID of the Curator Group',
+    },
+  ]
+
+  async run() {
+    const { id } = this.parse(CuratorGroupCommand).args
+    const group = await this.getCuratorGroup(id)
+    const classesMaintained = (await this.getApi().availableClasses()).filter(([, c]) =>
+      c.class_permissions.maintainers.toArray().some((gId) => gId.toNumber() === parseInt(id))
+    )
+    const members = (await this.getApi().groupMembers(WorkingGroups.Curators)).filter((curator) =>
+      group.curators.toArray().some((groupCurator) => groupCurator.eq(curator.workerId))
+    )
+
+    displayCollapsedRow({
+      'ID': id,
+      'Status': group.active.valueOf() ? 'Active' : 'Inactive',
+    })
+    displayHeader(`Classes maintained (${classesMaintained.length})`)
+    this.log(classesMaintained.map(([, c]) => chalk.white(c.name.toString())).join(', '))
+    displayHeader(`Group Members (${members.length})`)
+    this.log(
+      members
+        .map((curator) => chalk.white(`${curator.profile.handle} (WorkerID: ${curator.workerId.toString()})`))
+        .join(', ')
+    )
+  }
+}

cli/src/commands/content-directory/curatorGroups.ts

@@ -0,0 +1,25 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+// import chalk from 'chalk'
+import { displayTable } from '../../helpers/display'
+
+export default class CuratorGroupsCommand extends ContentDirectoryCommandBase {
+  static description = 'List existing Curator Groups.'
+
+  async run() {
+    const groups = await this.getApi().availableCuratorGroups()
+
+    if (groups.length) {
+      displayTable(
+        groups.map(([id, group]) => ({
+          'ID': id.toString(),
+          'Status': group.active.valueOf() ? 'Active' : 'Inactive',
+          'Classes maintained': group.number_of_classes_maintained.toNumber(),
+          'Members': group.curators.toArray().length,
+        })),
+        5
+      )
+    } else {
+      this.log('No Curator Groups available!')
+    }
+  }
+}

cli/src/commands/content-directory/entities.ts

@@ -0,0 +1,45 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import { displayTable } from '../../helpers/display'
+import { flags } from '@oclif/command'
+
+export default class EntitiesCommand extends ContentDirectoryCommandBase {
+  static description = 'Show entities list by class id or name.'
+  static args = [
+    {
+      name: 'className',
+      required: true,
+      description: 'Name or ID of the Class',
+    },
+    {
+      name: 'properties',
+      required: false,
+      description:
+        'Comma-separated properties to include in the results table (ie. code,name). ' +
+        'By default all property values will be included.',
+    },
+  ]
+
+  static flags = {
+    filters: flags.string({
+      required: false,
+      description:
+        'Comma-separated filters, ie. title="Some video",channelId=3.' +
+        'Currently only the = operator is supported.' +
+        'When multiple filters are provided, only the entities that match all of them together will be displayed.',
+    }),
+  }
+
+  async run() {
+    const { className, properties } = this.parse(EntitiesCommand).args
+    const { filters } = this.parse(EntitiesCommand).flags
+    const propsToInclude: string[] | undefined = (properties || undefined) && (properties as string).split(',')
+    const filtersArr: [string, string][] = filters
+      ? filters
+          .split(',')
+          .map((f) => f.split('='))
+          .map(([pName, pValue]) => [pName, pValue.replace(/^"(.+)"$/, '$1')])
+      : []
+
+    displayTable(await this.createEntityList(className, propsToInclude, filtersArr), 3)
+  }
+}

cli/src/commands/content-directory/entity.ts

@@ -0,0 +1,44 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+import { displayCollapsedRow, displayHeader } from '../../helpers/display'
+import _ from 'lodash'
+
+export default class EntityCommand extends ContentDirectoryCommandBase {
+  static description = 'Show Entity details by id.'
+  static args = [
+    {
+      name: 'id',
+      required: true,
+      description: 'ID of the Entity',
+    },
+  ]
+
+  async run() {
+    const { id } = this.parse(EntityCommand).args
+    const entity = await this.getEntity(id)
+    const { controller, frozen, referenceable } = entity.entity_permissions
+    const [classId, entityClass] = await this.classEntryByNameOrId(entity.class_id.toString())
+    const propertyValues = this.parseEntityPropertyValues(entity, entityClass)
+
+    displayCollapsedRow({
+      'ID': id,
+      'Class name': entityClass.name.toString(),
+      'Class ID': classId.toNumber(),
+      'Supported schemas': JSON.stringify(entity.supported_schemas.toJSON()),
+      'Controller': controller.type + (controller.isOfType('Member') ? `(${controller.asType('Member')})` : ''),
+      'Frozen': frozen.toString(),
+      'Refrecencable': referenceable.toString(),
+      'Same owner references': entity.reference_counter.same_owner.toNumber(),
+      'Total references': entity.reference_counter.total.toNumber(),
+    })
+    displayHeader('Property values')
+    displayCollapsedRow(
+      _.mapValues(
+        propertyValues,
+        (v) =>
+          (v.value.toJSON() === false && v.type !== 'Single<Bool>' ? chalk.grey('[not set]') : v.value.toString()) +
+          ` ${chalk.green(`${v.type}`)}`
+      )
+    )
+  }
+}

cli/src/commands/content-directory/removeCuratorFromGroup.ts

@@ -0,0 +1,46 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+
+export default class RemoveCuratorFromGroupCommand extends ContentDirectoryCommandBase {
+  static description = 'Remove Curator from Curator Group.'
+  static args = [
+    {
+      name: 'groupId',
+      required: false,
+      description: 'ID of the Curator Group',
+    },
+    {
+      name: 'curatorId',
+      required: false,
+      description: 'ID of the curator',
+    },
+  ]
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+
+    let { groupId, curatorId } = this.parse(RemoveCuratorFromGroupCommand).args
+
+    if (groupId === undefined) {
+      groupId = await this.promptForCuratorGroup()
+    }
+
+    const group = await this.getCuratorGroup(groupId)
+    const groupCuratorIds = group.curators.toArray().map((id) => id.toNumber())
+
+    if (curatorId === undefined) {
+      curatorId = await this.promptForCurator('Choose a Curator to remove', groupCuratorIds)
+    } else {
+      if (!groupCuratorIds.includes(parseInt(curatorId))) {
+        this.error(`Curator ${chalk.white(curatorId)} is not part of group ${chalk.white(groupId)}`)
+      }
+      await this.getCurator(curatorId)
+    }
+
+    await this.requestAccountDecoding(account)
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'removeCuratorFromGroup', [groupId, curatorId])
+
+    this.log(chalk.green(`Curator ${chalk.white(curatorId)} successfully removed from group ${chalk.white(groupId)}!`))
+  }
+}

cli/src/commands/content-directory/removeCuratorGroup.ts

@@ -0,0 +1,35 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+import ExitCodes from '../../ExitCodes'
+
+export default class AddCuratorGroupCommand extends ContentDirectoryCommandBase {
+  static description = 'Remove existing Curator Group.'
+  static args = [
+    {
+      name: 'id',
+      required: false,
+      description: 'ID of the Curator Group to remove',
+    },
+  ]
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+
+    let { id } = this.parse(AddCuratorGroupCommand).args
+    if (id === undefined) {
+      id = await this.promptForCuratorGroup('Select Curator Group to remove')
+    }
+
+    const group = await this.getCuratorGroup(id)
+
+    if (group.number_of_classes_maintained.toNumber() > 0) {
+      this.error('Cannot remove a group which has some maintained classes!', { exit: ExitCodes.InvalidInput })
+    }
+
+    await this.requestAccountDecoding(account)
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'removeCuratorGroup', [id])
+
+    console.log(chalk.green(`Curator Group ${chalk.white(id)} succesfully removed!`))
+  }
+}

cli/src/commands/content-directory/removeMaintainerFromClass.ts

@@ -0,0 +1,44 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+
+export default class AddMaintainerToClassCommand extends ContentDirectoryCommandBase {
+  static description = 'Remove maintainer (Curator Group) from class.'
+  static args = [
+    {
+      name: 'className',
+      required: false,
+      description: 'Name or ID of the class (ie. Video)',
+    },
+    {
+      name: 'groupId',
+      required: false,
+      description: 'ID of the Curator Group to remove from maintainers',
+    },
+  ]
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+
+    let { groupId, className } = this.parse(AddMaintainerToClassCommand).args
+
+    if (className === undefined) {
+      className = (await this.promptForClass()).name.toString()
+    }
+
+    const [classId, aClass] = await this.classEntryByNameOrId(className)
+
+    if (groupId === undefined) {
+      groupId = await this.promptForCuratorGroup('Select a maintainer', aClass.class_permissions.maintainers.toArray())
+    } else {
+      await this.getCuratorGroup(groupId)
+    }
+
+    await this.requestAccountDecoding(account)
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'removeMaintainerFromClass', [classId, groupId])
+
+    console.log(
+      chalk.green(`Curator Group ${chalk.white(groupId)} removed as maintainer of ${chalk.white(className)} class!`)
+    )
+  }
+}

cli/src/commands/content-directory/setCuratorGroupStatus.ts

@@ -0,0 +1,61 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import chalk from 'chalk'
+import ExitCodes from '../../ExitCodes'
+
+export default class SetCuratorGroupStatusCommand extends ContentDirectoryCommandBase {
+  static description = 'Set Curator Group status (Active/Inactive).'
+  static args = [
+    {
+      name: 'id',
+      required: false,
+      description: 'ID of the Curator Group',
+    },
+    {
+      name: 'status',
+      required: false,
+      description: 'New status of the group (1 - active, 0 - inactive)',
+    },
+  ]
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+
+    let { id, status } = this.parse(SetCuratorGroupStatusCommand).args
+
+    if (id === undefined) {
+      id = await this.promptForCuratorGroup()
+    } else {
+      await this.getCuratorGroup(id)
+    }
+
+    if (status === undefined) {
+      status = await this.simplePrompt({
+        type: 'list',
+        message: 'Select new status',
+        choices: [
+          { name: 'Active', value: true },
+          { name: 'Inactive', value: false },
+        ],
+      })
+    } else {
+      if (status !== '0' && status !== '1') {
+        this.error('Invalid status provided. Use "1" for Active and "0" for Inactive.', {
+          exit: ExitCodes.InvalidInput,
+        })
+      }
+      status = !!parseInt(status)
+    }
+
+    await this.requestAccountDecoding(account)
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'setCuratorGroupStatus', [id, status])
+
+    console.log(
+      chalk.green(
+        `Curator Group ${chalk.white(id)} status succesfully changed to: ${chalk.white(
+          status ? 'Active' : 'Inactive'
+        )}!`
+      )
+    )
+  }
+}

cli/src/commands/content-directory/updateClassPermissions.ts

@@ -0,0 +1,55 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import CreateClassSchema from 'cd-schemas/schemas/extrinsics/CreateClass.schema.json'
+import chalk from 'chalk'
+import { JsonSchemaCustomPrompts, JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
+import { CreateClass } from 'cd-schemas/types/extrinsics/CreateClass'
+import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
+
+export default class UpdateClassPermissionsCommand extends ContentDirectoryCommandBase {
+  static description = 'Update permissions in given class.'
+  static args = [
+    {
+      name: 'className',
+      required: false,
+      description: 'Name or ID of the class (ie. Video)',
+    },
+  ]
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    await this.requireLead()
+
+    let { className } = this.parse(UpdateClassPermissionsCommand).args
+
+    if (className === undefined) {
+      className = (await this.promptForClass()).name.toString()
+    }
+
+    const [classId, aClass] = await this.classEntryByNameOrId(className)
+    const currentPermissions = aClass.class_permissions
+
+    const customPrompts: JsonSchemaCustomPrompts = [
+      ['class_permissions.maintainers', () => this.promptForCuratorGroups('Select class maintainers')],
+    ]
+
+    const prompter = new JsonSchemaPrompter<CreateClass>(
+      CreateClassSchema as JSONSchema,
+      { class_permissions: currentPermissions.toJSON() as CreateClass['class_permissions'] },
+      customPrompts
+    )
+
+    const newPermissions = await prompter.promptSingleProp('class_permissions')
+
+    await this.requestAccountDecoding(account)
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'updateClassPermissions', [
+      classId,
+      newPermissions.any_member,
+      newPermissions.entity_creation_blocked,
+      newPermissions.all_entity_property_values_locked,
+      newPermissions.maintainers,
+    ])
+
+    console.log(chalk.green(`${chalk.white(className)} class permissions updated to:`))
+    this.jsonPrettyPrint(JSON.stringify(newPermissions))
+  }
+}

cli/src/commands/media/createChannel.ts

@@ -0,0 +1,53 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import ChannelEntitySchema from 'cd-schemas/schemas/entities/ChannelEntity.schema.json'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { InputParser } from 'cd-schemas'
+import { IOFlags, getInputJson, saveOutputJson } from '../../helpers/InputOutput'
+import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
+import { JsonSchemaCustomPrompts, JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
+
+export default class CreateChannelCommand extends ContentDirectoryCommandBase {
+  static description = 'Create a new channel on Joystream (requires a membership).'
+  static flags = {
+    ...IOFlags,
+  }
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    const memberId = await this.getRequiredMemberId()
+    const actor = { Member: memberId }
+
+    await this.requestAccountDecoding(account)
+
+    const channelJsonSchema = (ChannelEntitySchema as unknown) as JSONSchema
+
+    const { input, output } = this.parse(CreateChannelCommand).flags
+
+    let inputJson = await getInputJson<ChannelEntity>(input, channelJsonSchema)
+    if (!inputJson) {
+      const customPrompts: JsonSchemaCustomPrompts = [
+        ['language', () => this.promptForEntityId('Choose channel language', 'Language', 'name')],
+        ['curationStatus', async () => undefined],
+      ]
+
+      const prompter = new JsonSchemaPrompter<ChannelEntity>(channelJsonSchema, undefined, customPrompts)
+
+      inputJson = await prompter.promptAll()
+    }
+
+    this.jsonPrettyPrint(JSON.stringify(inputJson))
+    const confirmed = await this.simplePrompt({ type: 'confirm', message: 'Do you confirm the provided input?' })
+
+    if (confirmed) {
+      saveOutputJson(output, `${inputJson.title}Channel.json`, inputJson)
+      const inputParser = InputParser.createWithKnownSchemas(this.getOriginalApi(), [
+        {
+          className: 'Channel',
+          entries: [inputJson],
+        },
+      ])
+      const operations = await inputParser.getEntityBatchOperations()
+      await this.sendAndFollowNamedTx(account, 'contentDirectory', 'transaction', [actor, operations])
+    }
+  }
+}

cli/src/commands/media/myChannels.ts

@@ -0,0 +1,25 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { displayTable } from '../../helpers/display'
+import chalk from 'chalk'
+
+export default class MyChannelsCommand extends ContentDirectoryCommandBase {
+  static description = "Show the list of channels associated with current account's membership."
+
+  async run() {
+    const memberId = await this.getRequiredMemberId()
+
+    const props: (keyof ChannelEntity)[] = ['title', 'isPublic']
+
+    const list = await this.createEntityList('Channel', props, [], memberId)
+
+    if (list.length) {
+      displayTable(list, 3)
+      this.log(
+        `\nTIP: Use ${chalk.bold('content-directory:entity ID')} command to see more details about given channel`
+      )
+    } else {
+      this.log(`No channels created yet! Create a channel with ${chalk.bold('media:createChannel')}`)
+    }
+  }
+}

cli/src/commands/media/myVideos.ts

@@ -0,0 +1,33 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { displayTable } from '../../helpers/display'
+import chalk from 'chalk'
+import { flags } from '@oclif/command'
+
+export default class MyVideosCommand extends ContentDirectoryCommandBase {
+  static description = "Show the list of videos associated with current account's membership."
+  static flags = {
+    channel: flags.integer({
+      char: 'c',
+      required: false,
+      description: 'Channel id to filter the videos by',
+    }),
+  }
+
+  async run() {
+    const memberId = await this.getRequiredMemberId()
+
+    const { channel } = this.parse(MyVideosCommand).flags
+    const props: (keyof VideoEntity)[] = ['title', 'isPublic', 'channel']
+    const filters: [string, string][] = channel !== undefined ? [['channel', channel.toString()]] : []
+
+    const list = await this.createEntityList('Video', props, filters, memberId)
+
+    if (list.length) {
+      displayTable(list, 3)
+      this.log(`\nTIP: Use ${chalk.bold('content-directory:entity ID')} command to see more details about given video`)
+    } else {
+      this.log(`No videos uploaded yet! Upload a video with ${chalk.bold('media:uploadVideo')}`)
+    }
+  }
+}

cli/src/commands/media/updateChannel.ts

@@ -0,0 +1,77 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import ChannelEntitySchema from 'cd-schemas/schemas/entities/ChannelEntity.schema.json'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { InputParser } from 'cd-schemas'
+import { IOFlags, getInputJson, saveOutputJson } from '../../helpers/InputOutput'
+import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
+import { JsonSchemaCustomPrompts, JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
+import { Entity } from '@joystream/types/content-directory'
+
+export default class UpdateChannelCommand extends ContentDirectoryCommandBase {
+  static description = 'Update one of the owned channels on Joystream (requires a membership).'
+  static flags = {
+    ...IOFlags,
+  }
+
+  static args = [
+    {
+      name: 'id',
+      description: 'ID of the channel to update',
+      required: false,
+    },
+  ]
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    const memberId = await this.getRequiredMemberId()
+    const actor = { Member: memberId }
+
+    await this.requestAccountDecoding(account)
+
+    const { id } = this.parse(UpdateChannelCommand).args
+
+    let channelEntity: Entity, channelId: number
+    if (id) {
+      channelId = parseInt(id)
+      channelEntity = await this.getEntity(channelId, 'Channel', memberId)
+    } else {
+      const [id, channel] = await this.promptForEntityEntry('Select a channel to update', 'Channel', 'title', memberId)
+      channelId = id.toNumber()
+      channelEntity = channel
+    }
+
+    const currentValues = await this.parseToKnownEntityJson<ChannelEntity>(channelEntity)
+    this.jsonPrettyPrint(JSON.stringify(currentValues))
+
+    const channelJsonSchema = (ChannelEntitySchema as unknown) as JSONSchema
+
+    const { input, output } = this.parse(UpdateChannelCommand).flags
+
+    let inputJson = await getInputJson<ChannelEntity>(input, channelJsonSchema)
+    if (!inputJson) {
+      const customPrompts: JsonSchemaCustomPrompts = [
+        [
+          'language',
+          () =>
+            this.promptForEntityId('Choose channel language', 'Language', 'name', undefined, currentValues.language),
+        ],
+        ['curationStatus', async () => undefined],
+      ]
+
+      const prompter = new JsonSchemaPrompter<ChannelEntity>(channelJsonSchema, currentValues, customPrompts)
+
+      inputJson = await prompter.promptAll()
+    }
+
+    this.jsonPrettyPrint(JSON.stringify(inputJson))
+    const confirmed = await this.simplePrompt({ type: 'confirm', message: 'Do you confirm the provided input?' })
+
+    if (confirmed) {
+      saveOutputJson(output, `${inputJson.title}Channel.json`, inputJson)
+      const inputParser = InputParser.createWithKnownSchemas(this.getOriginalApi())
+      const updateOperation = await inputParser.createEntityUpdateOperation(inputJson, 'Channel', channelId)
+      this.log('Sending the extrinsic...')
+      await this.sendAndFollowNamedTx(account, 'contentDirectory', 'transaction', [actor, [updateOperation]])
+    }
+  }
+}

cli/src/commands/media/updateVideo.ts

@@ -0,0 +1,96 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import VideoEntitySchema from 'cd-schemas/schemas/entities/VideoEntity.schema.json'
+import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { LicenseEntity } from 'cd-schemas/types/entities/LicenseEntity'
+import { InputParser } from 'cd-schemas'
+import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
+import { JsonSchemaCustomPrompts, JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
+import { Entity } from '@joystream/types/content-directory'
+
+export default class UpdateVideoCommand extends ContentDirectoryCommandBase {
+  static description = 'Update existing video information (requires a membership).'
+  static flags = {
+    // TODO: ...IOFlags, - providing input as json
+  }
+
+  static args = [
+    {
+      name: 'id',
+      description: 'ID of the Video to update',
+      required: false,
+    },
+  ]
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    const memberId = await this.getRequiredMemberId()
+    const actor = { Member: memberId }
+
+    await this.requestAccountDecoding(account)
+
+    const { id } = this.parse(UpdateVideoCommand).args
+
+    let videoEntity: Entity, videoId: number
+    if (id) {
+      videoId = parseInt(id)
+      videoEntity = await this.getEntity(videoId, 'Video', memberId)
+    } else {
+      const [id, video] = await this.promptForEntityEntry('Select a video to update', 'Video', 'title', memberId)
+      videoId = id.toNumber()
+      videoEntity = video
+    }
+
+    const currentValues = await this.parseToKnownEntityJson<VideoEntity>(videoEntity)
+    const videoJsonSchema = (VideoEntitySchema as unknown) as JSONSchema
+
+    const { language: currLanguageId, category: currCategoryId, license: currLicenseId } = currentValues
+
+    const customizedPrompts: JsonSchemaCustomPrompts<VideoEntity> = [
+      [
+        'language',
+        () => this.promptForEntityId('Choose Video language', 'Language', 'name', undefined, currLanguageId),
+      ],
+      [
+        'category',
+        () => this.promptForEntityId('Choose Video category', 'ContentCategory', 'name', undefined, currCategoryId),
+      ],
+    ]
+    const videoPrompter = new JsonSchemaPrompter<VideoEntity>(videoJsonSchema, currentValues, customizedPrompts)
+
+    // Updating a license is currently a bit more tricky since it's a nested relation
+    const currKnownLicenseId = (await this.getAndParseKnownEntity<LicenseEntity>(currLicenseId)).knownLicense
+    const knownLicenseId = await this.promptForEntityId(
+      'Choose a license',
+      'KnownLicense',
+      'code',
+      undefined,
+      currKnownLicenseId
+    )
+    const updatedLicense: LicenseEntity = { knownLicense: knownLicenseId }
+
+    // Prompt for other video data
+    const updatedProps: Partial<VideoEntity> = await videoPrompter.promptMultipleProps([
+      'language',
+      'category',
+      'title',
+      'description',
+      'thumbnailURL',
+      'duration',
+      'isPublic',
+      'hasMarketing',
+    ])
+
+    this.jsonPrettyPrint(JSON.stringify(updatedProps))
+
+    // Parse inputs into operations and send final extrinsic
+    const inputParser = InputParser.createWithKnownSchemas(this.getOriginalApi())
+    const videoUpdateOperation = await inputParser.createEntityUpdateOperation(updatedProps, 'Video', videoId)
+    const licenseUpdateOperation = await inputParser.createEntityUpdateOperation(
+      updatedLicense,
+      'License',
+      currentValues.license
+    )
+    const operations = [videoUpdateOperation, licenseUpdateOperation]
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'transaction', [actor, operations], true)
+  }
+}

cli/src/commands/media/uploadVideo.ts

@@ -0,0 +1,372 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import VideoEntitySchema from 'cd-schemas/schemas/entities/VideoEntity.schema.json'
+import VideoMediaEntitySchema from 'cd-schemas/schemas/entities/VideoMediaEntity.schema.json'
+import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { VideoMediaEntity } from 'cd-schemas/types/entities/VideoMediaEntity'
+import { InputParser } from 'cd-schemas'
+import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
+import { JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
+import { flags } from '@oclif/command'
+import fs from 'fs'
+import ExitCodes from '../../ExitCodes'
+import { ContentId } from '@joystream/types/media'
+import ipfsHash from 'ipfs-only-hash'
+import { cli } from 'cli-ux'
+import axios, { AxiosRequestConfig } from 'axios'
+import { URL } from 'url'
+import ipfsHttpClient from 'ipfs-http-client'
+import first from 'it-first'
+import last from 'it-last'
+import toBuffer from 'it-to-buffer'
+import ffmpegInstaller from '@ffmpeg-installer/ffmpeg'
+import ffmpeg from 'fluent-ffmpeg'
+
+ffmpeg.setFfmpegPath(ffmpegInstaller.path)
+
+const DATA_OBJECT_TYPE_ID = 1
+const MAX_FILE_SIZE = 500 * 1024 * 1024
+
+type VideoMetadata = {
+  width?: number
+  height?: number
+  codecName?: string
+  codecFullName?: string
+  duration?: number
+}
+
+export default class UploadVideoCommand extends ContentDirectoryCommandBase {
+  static description = 'Upload a new Video to a channel (requires a membership).'
+  static flags = {
+    // TODO: ...IOFlags, - providing input as json
+    channel: flags.integer({
+      char: 'c',
+      required: false,
+      description:
+        'ID of the channel to assign the video to (if omitted - one of the owned channels can be selected from the list)',
+    }),
+  }
+
+  static args = [
+    {
+      name: 'filePath',
+      required: true,
+      description: 'Path to the media file to upload',
+    },
+  ]
+
+  private createReadStreamWithProgressBar(filePath: string, barTitle: string, fileSize?: number) {
+    // Progress CLI UX:
+    // https://github.com/oclif/cli-ux#cliprogress
+    // https://www.npmjs.com/package/cli-progress
+    if (!fileSize) {
+      fileSize = fs.statSync(filePath).size
+    }
+    const progress = cli.progress({ format: `${barTitle} | {bar} | {value}/{total} KB processed` })
+    let processedKB = 0
+    const fileSizeKB = Math.ceil(fileSize / 1024)
+    progress.start(fileSizeKB, processedKB)
+    return {
+      fileStream: fs
+        .createReadStream(filePath)
+        .pause() // Explicitly pause to prevent switching to flowing mode (https://nodejs.org/api/stream.html#stream_event_data)
+        .on('error', () => {
+          progress.stop()
+          this.error(`Error while trying to read data from: ${filePath}!`, {
+            exit: ExitCodes.FsOperationFailed,
+          })
+        })
+        .on('data', (data) => {
+          processedKB += data.length / 1024
+          progress.update(processedKB)
+        })
+        .on('end', () => {
+          progress.update(fileSizeKB)
+          progress.stop()
+        }),
+      progressBar: progress,
+    }
+  }
+
+  private async calculateFileIpfsHash(filePath: string, fileSize: number): Promise<string> {
+    const { fileStream } = this.createReadStreamWithProgressBar(filePath, 'Calculating file hash', fileSize)
+    const hash: string = await ipfsHash.of(fileStream)
+
+    return hash
+  }
+
+  private async getDiscoveryDataViaLocalIpfsNode(ipnsIdentity: string): Promise<any> {
+    const ipfs = ipfsHttpClient({
+      // TODO: Allow customizing node url:
+      // host: 'localhost', port: '5001', protocol: 'http',
+      timeout: 10000,
+    })
+
+    const ipnsAddress = `/ipns/${ipnsIdentity}/`
+    const ipfsName = await last(
+      ipfs.name.resolve(ipnsAddress, {
+        recursive: false,
+        nocache: false,
+      })
+    )
+    const data: any = await first(ipfs.get(ipfsName))
+    const buffer = await toBuffer(data.content)
+
+    return JSON.parse(buffer.toString())
+  }
+
+  private async getDiscoveryDataViaBootstrapEndpoint(storageProviderId: number): Promise<any> {
+    const bootstrapEndpoint = await this.getApi().getRandomBootstrapEndpoint()
+    if (!bootstrapEndpoint) {
+      this.error('No bootstrap endpoints available', { exit: ExitCodes.ApiError })
+    }
+    this.log('Bootstrap endpoint:', bootstrapEndpoint)
+    const discoveryEndpoint = new URL(`/discover/v0/${storageProviderId}`, bootstrapEndpoint).toString()
+    try {
+      const data = (await axios.get(discoveryEndpoint)).data
+      return data
+    } catch (e) {
+      this.error(`Cannot retrieve data from bootstrap enpoint (${discoveryEndpoint})`, {
+        exit: ExitCodes.ExternalInfrastructureError,
+      })
+    }
+  }
+
+  private async getUploadUrlFromDiscoveryData(data: any, contentId: ContentId): Promise<string> {
+    if (typeof data === 'object' && data !== null && data.serialized) {
+      const unserialized = JSON.parse(data.serialized)
+      if (unserialized.asset && unserialized.asset.endpoint && typeof unserialized.asset.endpoint === 'string') {
+        return new URL(`/asset/v0/${contentId.encode()}`, unserialized.asset.endpoint).toString()
+      }
+    }
+    this.error(`Unexpected discovery data: ${JSON.stringify(data)}`)
+  }
+
+  private async getUploadUrl(ipnsIdentity: string, storageProviderId: number, contentId: ContentId): Promise<string> {
+    let data: any
+    try {
+      this.log('Trying to connect to local ipfs node...')
+      data = await this.getDiscoveryDataViaLocalIpfsNode(ipnsIdentity)
+    } catch (e) {
+      this.warn("Couldn't get data from local ipfs node, resolving to bootstrap endpoint...")
+      data = await this.getDiscoveryDataViaBootstrapEndpoint(storageProviderId)
+    }
+
+    const uploadUrl = await this.getUploadUrlFromDiscoveryData(data, contentId)
+
+    return uploadUrl
+  }
+
+  private async getVideoMetadata(filePath: string): Promise<VideoMetadata | null> {
+    let metadata: VideoMetadata | null = null
+    const metadataPromise = new Promise<VideoMetadata>((resolve, reject) => {
+      ffmpeg.ffprobe(filePath, (err, data) => {
+        if (err) {
+          reject(err)
+          return
+        }
+        const videoStream = data.streams.find((s) => s.codec_type === 'video')
+        if (videoStream) {
+          resolve({
+            width: videoStream.width,
+            height: videoStream.height,
+            codecName: videoStream.codec_name,
+            codecFullName: videoStream.codec_long_name,
+            duration: videoStream.duration !== undefined ? Math.ceil(Number(videoStream.duration)) || 0 : undefined,
+          })
+        } else {
+          reject(new Error('No video stream found in file'))
+        }
+      })
+    })
+
+    try {
+      metadata = await metadataPromise
+    } catch (e) {
+      const message = e.message || e
+      this.warn(`Failed to get video metadata via ffprobe (${message})`)
+    }
+
+    return metadata
+  }
+
+  private async uploadVideo(filePath: string, fileSize: number, uploadUrl: string) {
+    const { fileStream, progressBar } = this.createReadStreamWithProgressBar(filePath, 'Uploading', fileSize)
+    fileStream.on('end', () => {
+      cli.action.start('Waiting for the file to be processed...')
+    })
+
+    try {
+      const config: AxiosRequestConfig = {
+        headers: {
+          'Content-Type': '', // https://github.com/Joystream/storage-node-joystream/issues/16
+          'Content-Length': fileSize.toString(),
+        },
+        maxContentLength: MAX_FILE_SIZE,
+      }
+      await axios.put(uploadUrl, fileStream, config)
+      cli.action.stop()
+
+      this.log('File uploaded!')
+    } catch (e) {
+      progressBar.stop()
+      cli.action.stop()
+      const msg = (e.response && e.response.data && e.response.data.message) || e.message || e
+      this.error(`Unexpected error when trying to upload a file: ${msg}`, {
+        exit: ExitCodes.ExternalInfrastructureError,
+      })
+    }
+  }
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    const memberId = await this.getRequiredMemberId()
+    const actor = { Member: memberId }
+
+    await this.requestAccountDecoding(account)
+
+    const {
+      args: { filePath },
+      flags: { channel: inputChannelId },
+    } = this.parse(UploadVideoCommand)
+
+    // Basic file validation
+    if (!fs.existsSync(filePath)) {
+      this.error('File does not exist under provided path!', { exit: ExitCodes.FileNotFound })
+    }
+
+    const { size: fileSize } = fs.statSync(filePath)
+    if (fileSize > MAX_FILE_SIZE) {
+      this.error(`File size too large! Max. file size is: ${(MAX_FILE_SIZE / 1024 / 1024).toFixed(2)} MB`)
+    }
+
+    const videoMetadata = await this.getVideoMetadata(filePath)
+    this.log('Video media file parameters established:', { ...(videoMetadata || {}), size: fileSize })
+
+    // Check if any providers are available
+    if (!(await this.getApi().isAnyProviderAvailable())) {
+      this.error('No active storage providers available! Try again later...', {
+        exit: ExitCodes.ActionCurrentlyUnavailable,
+      })
+    }
+
+    // Start by prompting for a channel to make sure user has one available
+    let channelId: number
+    if (inputChannelId === undefined) {
+      channelId = await this.promptForEntityId(
+        'Select a channel to publish the video under',
+        'Channel',
+        'title',
+        memberId
+      )
+    } else {
+      await this.getEntity(inputChannelId, 'Channel', memberId) // Validates if exists and belongs to member
+      channelId = inputChannelId
+    }
+
+    // Calculate hash and create content id
+    const contentId = ContentId.generate(this.getTypesRegistry())
+    const ipfsCid = await this.calculateFileIpfsHash(filePath, fileSize)
+
+    this.log('Video identification established:', {
+      contentId: contentId.toString(),
+      encodedContentId: contentId.encode(),
+      ipfsHash: ipfsCid,
+    })
+
+    // Send dataDirectory.addContent extrinsic
+    await this.sendAndFollowNamedTx(account, 'dataDirectory', 'addContent', [
+      memberId,
+      contentId,
+      DATA_OBJECT_TYPE_ID,
+      fileSize,
+      ipfsCid,
+    ])
+
+    const dataObject = await this.getApi().dataObjectByContentId(contentId)
+    if (!dataObject) {
+      this.error('Data object could not be retrieved from chain', { exit: ExitCodes.ApiError })
+    }
+
+    this.log('Data object:', dataObject.toJSON())
+
+    // Get storage provider identity
+    const storageProviderId = dataObject.liaison.toNumber()
+    const ipnsIdentity = await this.getApi().ipnsIdentity(storageProviderId)
+
+    if (!ipnsIdentity) {
+      this.error('Storage provider IPNS identity could not be determined', { exit: ExitCodes.ApiError })
+    }
+
+    // Resolve upload url and upload the video
+    const uploadUrl = await this.getUploadUrl(ipnsIdentity, storageProviderId, contentId)
+    this.log('Resolved upload url:', uploadUrl)
+
+    await this.uploadVideo(filePath, fileSize, uploadUrl)
+
+    // Prompting for the data:
+
+    // Set the defaults
+    const videoMediaDefaults: Partial<VideoMediaEntity> = {
+      pixelWidth: videoMetadata?.width,
+      pixelHeight: videoMetadata?.height,
+    }
+    const videoDefaults: Partial<VideoEntity> = {
+      duration: videoMetadata?.duration,
+    }
+    // Create prompting helpers
+    const videoJsonSchema = (VideoEntitySchema as unknown) as JSONSchema
+    const videoMediaJsonSchema = (VideoMediaEntitySchema as unknown) as JSONSchema
+
+    const videoMediaPrompter = new JsonSchemaPrompter<VideoMediaEntity>(videoMediaJsonSchema, videoMediaDefaults)
+    const videoPrompter = new JsonSchemaPrompter<VideoEntity>(videoJsonSchema, videoDefaults)
+
+    // Prompt for the data
+    const encodingSuggestion =
+      videoMetadata && videoMetadata.codecFullName ? ` (suggested: ${videoMetadata.codecFullName})` : ''
+    const encoding = await this.promptForEntityId(
+      `Choose Video encoding${encodingSuggestion}`,
+      'VideoMediaEncoding',
+      'name'
+    )
+    const { pixelWidth, pixelHeight } = await videoMediaPrompter.promptMultipleProps(['pixelWidth', 'pixelHeight'])
+    const language = await this.promptForEntityId('Choose Video language', 'Language', 'name')
+    const category = await this.promptForEntityId('Choose Video category', 'ContentCategory', 'name')
+    const license = await this.promptForEntityId('Choose License', 'KnownLicense', 'code')
+    const videoProps = await videoPrompter.promptMultipleProps([
+      'title',
+      'description',
+      'thumbnailURL',
+      'duration',
+      'isPublic',
+      'isExplicit',
+      'hasMarketing',
+    ])
+
+    // Create final inputs
+    const videoMediaInput: VideoMediaEntity = {
+      encoding,
+      pixelWidth,
+      pixelHeight,
+      size: fileSize,
+      location: { new: { joystreamMediaLocation: { new: { dataObjectId: contentId.encode() } } } },
+    }
+    const videoInput: VideoEntity = {
+      ...videoProps,
+      channel: channelId,
+      language,
+      category,
+      license: { new: { knownLicense: license } },
+      media: { new: videoMediaInput },
+    }
+
+    // Parse inputs into operations and send final extrinsic
+    const inputParser = InputParser.createWithKnownSchemas(this.getOriginalApi(), [
+      {
+        className: 'Video',
+        entries: [videoInput],
+      },
+    ])
+    const operations = await inputParser.getEntityBatchOperations()
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'transaction', [actor, operations])
+  }
+}

cli/src/commands/working-groups/createOpening.ts

@@ -79,9 +79,7 @@ export default class WorkingGroupsCreateOpening extends WorkingGroupsCommandBase
       this.log(chalk.white('Sending the extrinsic...'))
       await this.sendExtrinsic(
         account,
-        apiModuleByGroup[this.group],
-        'addOpening',
-        defaultValues!.map((v) => v.value)
+        this.getOriginalApi().tx[apiModuleByGroup[this.group]].addOpening(...defaultValues!.map((v) => v.value))
       )
       this.log(chalk.green('Opening succesfully created!'))
     }

cli/src/commands/working-groups/decreaseWorkerStake.ts

@@ -42,7 +42,7 @@ export default class WorkingGroupsDecreaseWorkerStake extends WorkingGroupsComma
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'decreaseStake', [workerId, balance])
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'decreaseStake', [workerId, balance])
 
     this.log(
       chalk.green(

cli/src/commands/working-groups/evictWorker.ts

@@ -41,7 +41,7 @@ export default class WorkingGroupsEvictWorker extends WorkingGroupsCommandBase {
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'terminateRole', [
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'terminateRole', [
       workerId,
       rationale,
       shouldSlash,

cli/src/commands/working-groups/fillOpening.ts

@@ -33,7 +33,7 @@ export default class WorkingGroupsFillOpening extends WorkingGroupsCommandBase {
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'fillOpening', [
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'fillOpening', [
       openingId,
       applicationIds,
       rewardPolicyOpt,

cli/src/commands/working-groups/increaseStake.ts

@@ -30,10 +30,7 @@ export default class WorkingGroupsIncreaseStake extends WorkingGroupsCommandBase
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'increaseStake', [
-      worker.workerId,
-      balance,
-    ])
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'increaseStake', [worker.workerId, balance])
 
     this.log(
       chalk.green(

cli/src/commands/working-groups/leaveRole.ts

@@ -21,7 +21,7 @@ export default class WorkingGroupsLeaveRole extends WorkingGroupsCommandBase {
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'leaveRole', [worker.workerId, rationale])
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'leaveRole', [worker.workerId, rationale])
 
     this.log(chalk.green(`Succesfully left the role! (worker id: ${chalk.white(worker.workerId.toNumber())})`))
   }

cli/src/commands/working-groups/slashWorker.ts

@@ -39,7 +39,7 @@ export default class WorkingGroupsSlashWorker extends WorkingGroupsCommandBase {
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'slashStake', [workerId, balance])
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'slashStake', [workerId, balance])
 
     this.log(
       chalk.green(

cli/src/commands/working-groups/startAcceptingApplications.ts

@@ -29,7 +29,7 @@ export default class WorkingGroupsStartAcceptingApplications extends WorkingGrou
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'acceptApplications', [openingId])
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'acceptApplications', [openingId])
 
     this.log(
       chalk.green(`Opening ${chalk.white(openingId)} status changed to: ${chalk.white('Accepting Applications')}`)

cli/src/commands/working-groups/startReviewPeriod.ts

@@ -29,7 +29,7 @@ export default class WorkingGroupsStartReviewPeriod extends WorkingGroupsCommand
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'beginApplicantReview', [openingId])
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'beginApplicantReview', [openingId])
 
     this.log(chalk.green(`Opening ${chalk.white(openingId)} status changed to: ${chalk.white('In Review')}`))
   }

cli/src/commands/working-groups/terminateApplication.ts

@@ -30,7 +30,7 @@ export default class WorkingGroupsTerminateApplication extends WorkingGroupsComm
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'terminateApplication', [applicationId])
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'terminateApplication', [applicationId])
 
     this.log(chalk.green(`Application ${chalk.white(applicationId)} has been succesfully terminated!`))
   }

cli/src/commands/working-groups/updateRewardAccount.ts

@@ -38,7 +38,7 @@ export default class WorkingGroupsUpdateRewardAccount extends WorkingGroupsComma
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'updateRewardAccount', [
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'updateRewardAccount', [
       worker.workerId,
       newRewardAccount,
     ])

cli/src/commands/working-groups/updateRoleAccount.ts

@@ -32,7 +32,7 @@ export default class WorkingGroupsUpdateRoleAccount extends WorkingGroupsCommand
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'updateRoleAccount', [
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'updateRoleAccount', [
       worker.workerId,
       newRoleAccount,
     ])

cli/src/commands/working-groups/updateWorkerReward.ts

@@ -55,7 +55,7 @@ export default class WorkingGroupsUpdateWorkerReward extends WorkingGroupsComman
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'updateRewardAmount', [
+    await this.sendAndFollowNamedTx(account, apiModuleByGroup[this.group], 'updateRewardAmount', [
       workerId,
       newRewardValue,
     ])

cli/src/helpers/InputOutput.ts

@@ -0,0 +1,74 @@
+import { flags } from '@oclif/command'
+import { CLIError } from '@oclif/errors'
+import ExitCodes from '../ExitCodes'
+import fs from 'fs'
+import path from 'path'
+import Ajv from 'ajv'
+import $RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser'
+import { getSchemasLocation } from 'cd-schemas'
+import chalk from 'chalk'
+
+// Default schema path for resolving refs
+const DEFAULT_SCHEMA_PATH = getSchemasLocation('entities') + path.sep
+
+export const IOFlags = {
+  input: flags.string({
+    char: 'i',
+    required: false,
+    description: `Path to JSON file to use as input (if not specified - the input can be provided interactively)`,
+  }),
+  output: flags.string({
+    char: 'o',
+    required: false,
+    description:
+      'Path to the directory where the output JSON file should be placed (the output file can be then reused as input)',
+  }),
+}
+
+export async function getInputJson<T>(inputPath?: string, schema?: JSONSchema, schemaPath?: string): Promise<T | null> {
+  if (inputPath) {
+    let content, jsonObj
+    try {
+      content = fs.readFileSync(inputPath).toString()
+    } catch (e) {
+      throw new CLIError(`Cannot access the input file at: ${inputPath}`, { exit: ExitCodes.FsOperationFailed })
+    }
+    try {
+      jsonObj = JSON.parse(content)
+    } catch (e) {
+      throw new CLIError(`JSON parsing failed for file: ${inputPath}`, { exit: ExitCodes.InvalidInput })
+    }
+    if (schema) {
+      const ajv = new Ajv()
+      schema = await $RefParser.dereference(schemaPath || DEFAULT_SCHEMA_PATH, schema, {})
+      const valid = ajv.validate(schema, jsonObj) as boolean
+      if (!valid) {
+        throw new CLIError(`Input JSON file is not valid: ${ajv.errorsText()}`)
+      }
+    }
+
+    return jsonObj as T
+  }
+
+  return null
+}
+
+export function saveOutputJson(outputPath: string | undefined, fileName: string, data: any): void {
+  if (outputPath) {
+    let outputFilePath = path.join(outputPath, fileName)
+    let postfix = 0
+    while (fs.existsSync(outputFilePath)) {
+      fileName = fileName.replace(/(_[0-9]+)?\.json/, `_${++postfix}.json`)
+      outputFilePath = path.join(outputPath, fileName)
+    }
+    try {
+      fs.writeFileSync(outputFilePath, JSON.stringify(data, null, 4))
+    } catch (e) {
+      throw new CLIError(`Could not save the output to: ${outputFilePath}. Check directory permissions`, {
+        exit: ExitCodes.FsOperationFailed,
+      })
+    }
+
+    console.log(`${chalk.green('Output succesfully saved to:')} ${chalk.white(outputFilePath)}`)
+  }
+}

cli/src/helpers/JsonSchemaPrompt.ts

@@ -0,0 +1,237 @@
+import Ajv from 'ajv'
+import inquirer, { DistinctQuestion } from 'inquirer'
+import _ from 'lodash'
+import RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser'
+import chalk from 'chalk'
+import { BOOL_PROMPT_OPTIONS } from './prompting'
+import { getSchemasLocation } from 'cd-schemas'
+import path from 'path'
+
+type CustomPromptMethod = () => Promise<any>
+type CustomPrompt = DistinctQuestion | CustomPromptMethod | { $item: CustomPrompt }
+
+// For the explaination of "string & { x: never }", see: https://github.com/microsoft/TypeScript/issues/29729
+// eslint-disable-next-line @typescript-eslint/ban-types
+export type JsonSchemaCustomPrompts<T = Record<string, unknown>> = [keyof T | (string & {}) | RegExp, CustomPrompt][]
+
+// Default schema path for resolving refs
+// TODO: Would be nice to skip the filename part (but without it it doesn't work)
+const DEFAULT_SCHEMA_PATH = getSchemasLocation('entities') + path.sep
+
+export class JsonSchemaPrompter<JsonResult> {
+  schema: JSONSchema
+  schemaPath: string
+  customPropmpts?: JsonSchemaCustomPrompts
+  ajv: Ajv.Ajv
+  filledObject: Partial<JsonResult>
+
+  constructor(
+    schema: JSONSchema,
+    defaults?: Partial<JsonResult>,
+    customPrompts?: JsonSchemaCustomPrompts,
+    schemaPath: string = DEFAULT_SCHEMA_PATH
+  ) {
+    this.customPropmpts = customPrompts
+    this.schema = schema
+    this.schemaPath = schemaPath
+    // allErrors prevents .validate from setting only one error when in fact there are multiple
+    this.ajv = new Ajv({ allErrors: true })
+    this.filledObject = defaults || {}
+  }
+
+  private oneOfToChoices(oneOf: JSONSchema[]) {
+    const choices: { name: string; value: number | string }[] = []
+
+    oneOf.forEach((pSchema, index) => {
+      if (pSchema.description) {
+        choices.push({ name: pSchema.description, value: index })
+      } else if (pSchema.type === 'object' && pSchema.properties) {
+        choices.push({ name: `{ ${Object.keys(pSchema.properties).join(', ')} }`, value: index })
+      } else {
+        choices.push({ name: index.toString(), value: index })
+      }
+    })
+
+    return choices
+  }
+
+  private getCustomPrompt(propertyPath: string): CustomPrompt | undefined {
+    const found = this.customPropmpts?.find(([pathToMatch]) =>
+      pathToMatch instanceof RegExp ? pathToMatch.test(propertyPath) : propertyPath === pathToMatch
+    )
+
+    return found ? found[1] : undefined
+  }
+
+  private propertyDisplayName(propertyPath: string) {
+    return chalk.green(propertyPath)
+  }
+
+  private async prompt(schema: JSONSchema, propertyPath = '', custom?: CustomPrompt): Promise<any> {
+    const customPrompt: CustomPrompt | undefined = custom || this.getCustomPrompt(propertyPath)
+    const propDisplayName = this.propertyDisplayName(propertyPath)
+
+    // Custom prompt
+    if (typeof customPrompt === 'function') {
+      return await this.promptWithRetry(customPrompt, propertyPath, true)
+    }
+
+    // oneOf
+    if (schema.oneOf) {
+      const oneOf = schema.oneOf as JSONSchema[]
+      const choices = this.oneOfToChoices(oneOf)
+      const { choosen } = await inquirer.prompt({ name: 'choosen', message: propDisplayName, type: 'list', choices })
+      return await this.prompt(oneOf[choosen], propertyPath)
+    }
+
+    // object
+    if (schema.type === 'object' && schema.properties) {
+      const value: Record<string, any> = {}
+      for (const [pName, pSchema] of Object.entries(schema.properties)) {
+        value[pName] = await this.prompt(pSchema, propertyPath ? `${propertyPath}.${pName}` : pName)
+      }
+      return value
+    }
+
+    // array
+    if (schema.type === 'array' && schema.items) {
+      return await this.promptWithRetry(() => this.promptArray(schema, propertyPath), propertyPath, true)
+    }
+
+    // "primitive" values:
+    const currentValue = _.get(this.filledObject, propertyPath)
+    const basicPromptOptions: DistinctQuestion = {
+      message: propDisplayName,
+      default: currentValue !== undefined ? currentValue : schema.default,
+    }
+
+    let additionalPromptOptions: DistinctQuestion | undefined
+    let normalizer: (v: any) => any = (v) => v
+
+    // Prompt options
+    if (schema.enum) {
+      additionalPromptOptions = { type: 'list', choices: schema.enum as any[] }
+    } else if (schema.type === 'boolean') {
+      additionalPromptOptions = BOOL_PROMPT_OPTIONS
+    }
+
+    // Normalizers
+    if (schema.type === 'integer') {
+      normalizer = (v) => (parseInt(v).toString() === v ? parseInt(v) : v)
+    }
+
+    if (schema.type === 'number') {
+      normalizer = (v) => (Number(v).toString() === v ? Number(v) : v)
+    }
+
+    const promptOptions = { ...basicPromptOptions, ...additionalPromptOptions, ...customPrompt }
+    // Need to wrap in retry, because "validate" will not get called if "type" is "list" etc.
+    return await this.promptWithRetry(
+      async () => normalizer(await this.promptSimple(promptOptions, propertyPath, normalizer)),
+      propertyPath
+    )
+  }
+
+  private setValueAndGetError(propertyPath: string, value: any, nestedErrors = false): string | null {
+    _.set(this.filledObject as Record<string, unknown>, propertyPath, value)
+    this.ajv.validate(this.schema, this.filledObject) as boolean
+    return this.ajv.errors
+      ? this.ajv.errors
+          .filter((e) => (nestedErrors ? e.dataPath.startsWith(`.${propertyPath}`) : e.dataPath === `.${propertyPath}`))
+          .map((e) => (e.dataPath.replace(`.${propertyPath}`, '') || 'This value') + ` ${e.message}`)
+          .join(', ')
+      : null
+  }
+
+  private async promptArray(schema: JSONSchema, propertyPath: string) {
+    if (!schema.items) {
+      return []
+    }
+    const { maxItems = Number.MAX_SAFE_INTEGER } = schema
+    let currItem = 0
+    const result = []
+    while (currItem < maxItems) {
+      const { next } = await inquirer.prompt([
+        {
+          ...BOOL_PROMPT_OPTIONS,
+          name: 'next',
+          message: `Do you want to add another item to ${this.propertyDisplayName(propertyPath)} array?`,
+        },
+      ])
+      if (!next) {
+        break
+      }
+      const itemSchema = Array.isArray(schema.items) ? schema.items[schema.items.length % currItem] : schema.items
+      result.push(await this.prompt(typeof itemSchema === 'boolean' ? {} : itemSchema, `${propertyPath}[${currItem}]`))
+
+      ++currItem
+    }
+
+    return result
+  }
+
+  private async promptSimple(promptOptions: DistinctQuestion, propertyPath: string, normalize?: (v: any) => any) {
+    const { result } = await inquirer.prompt([
+      {
+        ...promptOptions,
+        name: 'result',
+        validate: (v) => {
+          v = normalize ? normalize(v) : v
+          return (
+            this.setValueAndGetError(propertyPath, v) ||
+            (promptOptions.validate ? promptOptions.validate(v) : true) ||
+            true
+          )
+        },
+      },
+    ])
+
+    return result
+  }
+
+  private async promptWithRetry(customMethod: CustomPromptMethod, propertyPath: string, nestedErrors = false) {
+    let error: string | null
+    let value: any
+    do {
+      value = await customMethod()
+      error = this.setValueAndGetError(propertyPath, value, nestedErrors)
+      if (error) {
+        console.log('\n')
+        console.log('Provided value:', value)
+        console.warn(`ERROR: ${error}`)
+        console.warn(`Try providing the input for ${propertyPath} again...`)
+      }
+    } while (error)
+
+    return value
+  }
+
+  async getMainSchema() {
+    return await RefParser.dereference(this.schemaPath, this.schema, {})
+  }
+
+  async promptAll() {
+    await this.prompt(await this.getMainSchema())
+    return this.filledObject as JsonResult
+  }
+
+  async promptMultipleProps<P extends keyof JsonResult & string, PA extends readonly P[]>(
+    props: PA
+  ): Promise<{ [K in PA[number]]: Exclude<JsonResult[K], undefined> }> {
+    const result: Partial<{ [K in PA[number]]: Exclude<JsonResult[K], undefined> }> = {}
+    for (const prop of props) {
+      result[prop] = await this.promptSingleProp(prop)
+    }
+
+    return result as { [K in PA[number]]: Exclude<JsonResult[K], undefined> }
+  }
+
+  async promptSingleProp<P extends keyof JsonResult & string>(
+    p: P,
+    customPrompt?: CustomPrompt
+  ): Promise<Exclude<JsonResult[P], undefined>> {
+    const mainSchema = await this.getMainSchema()
+    await this.prompt(mainSchema.properties![p] as JSONSchema, p, customPrompt)
+    return this.filledObject[p] as Exclude<JsonResult[P], undefined>
+  }
+}

cli/src/helpers/display.ts

@@ -48,6 +48,7 @@ export function displayTable(rows: { [k: string]: string | number }[], cellHoriz
       return Math.max(maxLength, valLength)
     }, columnName.length)
   const columnDef = (columnName: string) => ({
+    header: columnName,
     get: (row: typeof rows[number]) => chalk.white(`${row[columnName]}`),
     minWidth: maxLength(columnName) + cellHorizontalPadding,
   })

cli/src/helpers/prompting.ts

@@ -0,0 +1,9 @@
+import { DistinctQuestion } from 'inquirer'
+
+export const BOOL_PROMPT_OPTIONS: DistinctQuestion = {
+  type: 'list',
+  choices: [
+    { name: 'Yes', value: true },
+    { name: 'No', value: false },
+  ],
+}

cli/tsconfig.json

@@ -13,7 +13,9 @@
     "baseUrl": ".",
     "paths": {
       "@polkadot/types/augment": ["../types/augment-codec/augment-types.ts"],
-    }
+    },
+    "resolveJsonModule": true,
+    "skipLibCheck": true
   },
   "include": [
     "src/**/*"

content-directory-schemas/schemas/extrinsics/AddClassSchema.schema.json

@@ -50,11 +50,12 @@
     "PropertyName": {
       "type": "string",
       "minLength": 1,
-      "maxLength": 100
+      "maxLength": 49
     },
     "PropertyDescription": {
       "type": "string",
-      "minLength": 0,
+      "minLength": 1,
+      "maxLength": 500,
       "default": ""
     },
     "SinglePropertyType": {

content-directory-schemas/schemas/extrinsics/CreateClass.schema.json

@@ -9,11 +9,15 @@
   "properties": {
     "name": {
       "type": "string",
-      "description": "Name of this class. Required property."
+      "description": "Name of this class. Required property.",
+      "minLength": 1,
+      "maxLength": 49
     },
     "description": {
       "type": "string",
-      "description": "Description of this class."
+      "description": "Description of this class.",
+      "minLength": 1,
+      "maxLength": 500
     },
     "class_permissions": {
       "type": "object",

content-directory-schemas/src/helpers/InputParser.ts

@@ -214,16 +214,14 @@ export class InputParser {
       let value = customHandler && (await customHandler(schemaProperty, propertyValue))
       if (value === undefined) {
         value = createType('ParametrizedPropertyValue', {
-          InputPropertyValue: this.parsePropertyType(schemaProperty.property_type)
-            .toInputPropertyValue(propertyValue)
-            .toJSON() as any,
+          InputPropertyValue: this.parsePropertyType(schemaProperty.property_type).toInputPropertyValue(propertyValue),
         })
       }
 
       parametrizedClassPropValues.push(
         createType('ParametrizedClassPropertyValue', {
           in_class_index: schemaPropertyIndex,
-          value: value.toJSON() as any,
+          value,
         })
       )
     }

docker-compose-with-storage.yml

@@ -5,8 +5,6 @@ services:
     ports:
       - '127.0.0.1:5001:5001'
       - '127.0.0.1:8080:8080'
-    volumes:
-      - ipfs-data:/data/ipfs
     entrypoint: ''
     command: |
       /bin/sh -c "
@@ -22,11 +20,20 @@ services:
       dockerfile: joystream-node.Dockerfile
     ports:
       - '127.0.0.1:9944:9944'
-    volumes:
-      - chain-data:/data
-    command: --dev --ws-external --base-path /data
-volumes:
-  ipfs-data:
-    driver: local
-  chain-data:
-    driver: local
+    command: --dev --ws-external --base-path /data --log runtime
+
+  colossus:
+    image: joystream/apps
+    restart: on-failure
+    depends_on:
+      - "chain"
+      - "ipfs"
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - '127.0.0.1:3001:3001'
+    command: colossus --dev --ws-provider ws://chain:9944 --ipfs-host ipfs
+    environment:
+      - DEBUG=*
+

docker-compose.yml

@@ -11,7 +11,7 @@ services:
       # dockerfile is relative to the context
       dockerfile: joystream-node.Dockerfile
     container_name: joystream-node
-    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all
+    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all --log runtime
     ports:
       - "9944:9944"
   

pioneer/packages/joy-forum/src/Context.tsx

@@ -232,10 +232,10 @@ function reducer (state: ForumState, action: ForumAction): ForumState {
         moderator_id: createType('AccountId', moderator),
         rationale: createType('Text', rationale)
       });
-      const threadUpd = createType('Thread', Object.assign(
-        thread.cloneValues(),
-        { moderation: createType('Option<ModerationAction>', moderation) }
-      ));
+      const threadUpd = createType('Thread', {
+        ...thread.cloneValues(),
+        moderation: createType('Option<ModerationAction>', moderation)
+      });
 
       threadById.set(id, threadUpd);
 

pioneer/packages/joy-forum/src/calls.tsx

@@ -17,12 +17,12 @@ const storage: StorageType = 'substrate';
 type EntityMapName = 'categoryById' | 'threadById' | 'replyById';
 
 const getReactValue = (state: ForumState, endpoint: string, paramValue: any) => {
-  function getEntityById<T extends keyof InterfaceTypes>
+  function getEntityById<T extends 'Category' | 'Thread' | 'Reply'>
   (mapName: EntityMapName, type: T): InterfaceTypes[T] {
     const id = (paramValue as u64).toNumber();
     const entity = state[mapName].get(id);
 
-    return createType(type, entity);
+    return createType(type, entity as any);
   }
 
   switch (endpoint) {

pioneer/packages/joy-proposals/src/stories/data/ProposalDetails.mock.ts

@@ -20,8 +20,8 @@ const mockedProposal: ParsedProposal = {
   proposerId: 303,
   status: createType('ProposalStatus', {
     Active: {
-      stakeId: 0,
-      sourceAccountId: '5C4hrfkRjSLwQSFVtCvtbV6wctV1WFnkiexUZWLAh4Bc7jib'
+      stake_id: 0,
+      source_account_id: '5C4hrfkRjSLwQSFVtCvtbV6wctV1WFnkiexUZWLAh4Bc7jib'
     }
   }),
   proposer: {

pioneer/packages/joy-roles/src/classifiers.spec.ts

@@ -75,7 +75,7 @@ describe('hiring.Opening-> OpeningStageClassification', (): void => {
           stage: createType('OpeningStage', {
             Active: {
               stage: createType('ActiveOpeningStage', {
-                acceptingApplications: {
+                AcceptingApplications: {
                   started_accepting_applicants_at_block: 100
                 }
               })
@@ -101,7 +101,7 @@ describe('hiring.Opening-> OpeningStageClassification', (): void => {
           stage: createType('OpeningStage', {
             Active: {
               stage: createType('ActiveOpeningStage', {
-                reviewPeriod: {
+                ReviewPeriod: {
                   started_accepting_applicants_at_block: 100,
                   started_review_period_at_block: 100
                 }

pioneer/packages/joy-roles/src/tabs/Admin.controller.tsx

@@ -601,8 +601,10 @@ const NewOpening = (props: NewOpeningProps) => {
   };
 
   const onChangeExactBlock = (e: any, { value }: InputOnChangeData) => {
-    setExactBlock(typeof value === 'number' ? value : (parseInt(value) || 0));
-    setStart(createType('ActivateOpeningAt', { ExactBlock: value }));
+    const valueInt = typeof value === 'number' ? value : (parseInt(value) || 0);
+
+    setExactBlock(valueInt);
+    setStart(createType('ActivateOpeningAt', { ExactBlock: valueInt }));
   };
 
   const [policy, setPolicy] = useState(props.desc.policy);

storage-node/packages/cli/package.json

@@ -44,7 +44,7 @@
     "@joystream/storage-runtime-api": "^0.1.0",
     "@joystream/service-discovery": "^0.1.0",
     "@joystream/storage-utils": "^0.1.0",
-    "@joystream/types": "^0.13.0",
+    "@joystream/types": "^0.14.0",
     "axios": "^0.19.2",
     "chalk": "^2.4.2",
     "lodash": "^4.17.11",

storage-node/packages/cli/src/cli.ts

@@ -77,11 +77,11 @@ const commands = {
     api: any,
     filePath: string,
     dataObjectTypeId: string,
+    memberId: string,
     keyFile: string,
-    passPhrase: string,
-    memberId: string
+    passPhrase: string
   ) => {
-    const uploadCmd = new UploadCommand(api, filePath, dataObjectTypeId, keyFile, passPhrase, memberId)
+    const uploadCmd = new UploadCommand(api, filePath, dataObjectTypeId, memberId, keyFile, passPhrase)
 
     await uploadCmd.run()
   },

storage-node/packages/cli/src/commands/dev.ts

@@ -100,12 +100,6 @@ const init = async (api: RuntimeApi): Promise<any> => {
     debug('Alice is already a member.')
   }
 
-  debug('Setting Alice as content working group lead.')
-  await api.signAndSend(alice, api.api.tx.sudo.sudo(api.api.tx.contentWorkingGroup.replaceLead([aliceMemberId, alice])))
-
-  // Initialize classes and entities in the content-directory
-  // TODO: when cli tools are ready re-use here
-
   // set localhost colossus as discovery provider
   // assuming pioneer dev server is running on port 3000 we should run
   // the storage dev server on a different port than the default for colossus which is also

storage-node/packages/cli/src/commands/upload.ts

@@ -5,7 +5,7 @@ import { ContentId, DataObject } from '@joystream/types/media'
 import BN from 'bn.js'
 import { Option } from '@polkadot/types/codec'
 import { BaseCommand } from './base'
-import { discover } from '@joystream/service-discovery/discover'
+import { DiscoveryClient } from '@joystream/service-discovery'
 import Debug from 'debug'
 import chalk from 'chalk'
 import { aliceKeyPair } from './dev'
@@ -32,6 +32,7 @@ export class UploadCommand extends BaseCommand {
   private readonly keyFile: string
   private readonly passPhrase: string
   private readonly memberId: string
+  private readonly discoveryClient: DiscoveryClient
 
   constructor(
     api: any,
@@ -44,6 +45,7 @@ export class UploadCommand extends BaseCommand {
     super()
 
     this.api = api
+    this.discoveryClient = new DiscoveryClient({ api })
     this.mediaSourceFilePath = mediaSourceFilePath
     this.dataObjectTypeId = dataObjectTypeId
     this.memberId = memberId
@@ -153,7 +155,7 @@ export class UploadCommand extends BaseCommand {
   // Requests the runtime and obtains the storage node endpoint URL.
   private async discoverStorageProviderEndpoint(storageProviderId: string): Promise<string> {
     try {
-      const serviceInfo = await discover(storageProviderId, this.api)
+      const serviceInfo = await this.discoveryClient.discover(storageProviderId)
 
       if (serviceInfo === null) {
         this.fail('Storage node discovery failed.')

storage-node/packages/colossus/bin/cli.js

@@ -11,6 +11,7 @@ const meow = require('meow')
 const chalk = require('chalk')
 const figlet = require('figlet')
 const _ = require('lodash')
+const { sleep } = require('@joystream/storage-utils/sleep')
 
 const debug = require('debug')('joystream:colossus')
 
@@ -60,6 +61,10 @@ const FLAG_DEFINITIONS = {
       return !flags.dev && serverCmd
     },
   },
+  ipfsHost: {
+    type: 'string',
+    default: 'localhost',
+  },
 }
 
 const cli = meow(
@@ -82,6 +87,7 @@ const cli = meow(
     --passphrase            Optional passphrase to use to decrypt the key-file.
     --port=PORT, -p PORT    Port number to listen on, defaults to 3000.
     --ws-provider WS_URL    Joystream-node websocket provider, defaults to ws://localhost:9944
+    --ipfs-host   hostname  ipfs host to use, default to 'localhost'. Default port 5001 is always used
   `,
   { flags: FLAG_DEFINITIONS }
 )
@@ -110,19 +116,19 @@ function startExpressApp(app, port) {
 }
 
 // Start app
-function startAllServices({ store, api, port }) {
-  const app = require('../lib/app')(PROJECT_ROOT, store, api)
+function startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl }) {
+  const app = require('../lib/app')(PROJECT_ROOT, store, api, discoveryClient, ipfsHttpGatewayUrl)
   return startExpressApp(app, port)
 }
 
 // Start discovery service app only
-function startDiscoveryService({ api, port }) {
-  const app = require('../lib/discovery')(PROJECT_ROOT, api)
+function startDiscoveryService({ port, discoveryClient }) {
+  const app = require('../lib/discovery')(PROJECT_ROOT, discoveryClient)
   return startExpressApp(app, port)
 }
 
 // Get an initialized storage instance
-function getStorage(runtimeApi) {
+function getStorage(runtimeApi, { ipfsHost }) {
   // TODO at some point, we can figure out what backend-specific connection
   // options make sense. For now, just don't use any configuration.
   const { Storage } = require('@joystream/storage-node-backend')
@@ -137,6 +143,7 @@ function getStorage(runtimeApi) {
       // if obj.liaison_judgement !== Accepted .. throw ?
       return obj.unwrap().ipfs_content_id.toString()
     },
+    ipfsHost,
   }
 
   return Storage.create(options)
@@ -146,14 +153,6 @@ async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }
   // Load key information
   const { RuntimeApi } = require('@joystream/storage-runtime-api')
 
-  if (!keyFile) {
-    throw new Error('Must specify a --key-file argument for running a storage node.')
-  }
-
-  if (providerId === undefined) {
-    throw new Error('Must specify a --provider-id argument for running a storage node')
-  }
-
   const api = await RuntimeApi.create({
     account_file: keyFile,
     passphrase,
@@ -167,19 +166,19 @@ async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }
 
   await api.untilChainIsSynced()
 
-  if (!(await api.workers.isRoleAccountOfStorageProvider(api.storageProviderId, api.identities.key.address))) {
-    throw new Error('storage provider role account and storageProviderId are not associated with a worker')
+  // We allow the node to startup without correct provider id and account, but syncing and
+  // publishing of identity will be skipped.
+  if (!(await api.providerIsActiveWorker())) {
+    debug('storage provider role account and storageProviderId are not associated with a worker')
   }
 
   return api
 }
 
-async function initApiDevelopment() {
+async function initApiDevelopment({ wsProvider }) {
   // Load key information
   const { RuntimeApi } = require('@joystream/storage-runtime-api')
 
-  const wsProvider = 'ws://localhost:9944'
-
   const api = await RuntimeApi.create({
     provider_url: wsProvider,
   })
@@ -188,7 +187,17 @@ async function initApiDevelopment() {
 
   api.identities.useKeyPair(dev.roleKeyPair(api))
 
-  api.storageProviderId = await dev.check(api)
+  // Wait until dev provider is added to role
+  while (true) {
+    try {
+      api.storageProviderId = await dev.check(api)
+      break
+    } catch (err) {
+      debug(err)
+    }
+
+    await sleep(10000)
+  }
 
   return api
 }
@@ -209,10 +218,10 @@ function getServiceInformation(publicUrl) {
 
 // TODO: instead of recursion use while/async-await and use promise/setTimout based sleep
 // or cleaner code with generators?
-async function announcePublicUrl(api, publicUrl) {
+async function announcePublicUrl(api, publicUrl, publisherClient) {
   // re-announce in future
   const reannounce = function (timeoutMs) {
-    setTimeout(announcePublicUrl, timeoutMs, api, publicUrl)
+    setTimeout(announcePublicUrl, timeoutMs, api, publicUrl, publisherClient)
   }
 
   const chainIsSyncing = await api.chainIsSyncing()
@@ -221,6 +230,12 @@ async function announcePublicUrl(api, publicUrl) {
     return reannounce(10 * 60 * 1000)
   }
 
+  // postpone if provider not active
+  if (!(await api.providerIsActiveWorker())) {
+    debug('storage provider role account and storageProviderId are not associated with a worker')
+    return reannounce(10 * 60 * 1000)
+  }
+
   const sufficientBalance = await api.providerHasMinimumBalance(1)
   if (!sufficientBalance) {
     debug('Provider role account does not have sufficient balance. Postponing announcing public url.')
@@ -228,12 +243,11 @@ async function announcePublicUrl(api, publicUrl) {
   }
 
   debug('announcing public url')
-  const { publish } = require('@joystream/service-discovery')
 
   try {
     const serviceInformation = getServiceInformation(publicUrl)
 
-    const keyId = await publish.publish(serviceInformation)
+    const keyId = await publisherClient.publish(serviceInformation)
 
     await api.discovery.setAccountInfo(keyId)
 
@@ -260,23 +274,14 @@ if (!command) {
   command = 'server'
 }
 
-async function startColossus({ api, publicUrl, port }) {
-  // TODO: check valid url, and valid port number
-  const store = getStorage(api)
-  banner()
-  const { startSyncing } = require('../lib/sync')
-  startSyncing(api, { syncPeriod: SYNC_PERIOD_MS }, store)
-  announcePublicUrl(api, publicUrl)
-  return startAllServices({ store, api, port })
-}
-
 const commands = {
   server: async () => {
+    banner()
     let publicUrl, port, api
 
     if (cli.flags.dev) {
       const dev = require('../../cli/dist/commands/dev')
-      api = await initApiDevelopment()
+      api = await initApiDevelopment(cli.flags)
       port = dev.developmentPort()
       publicUrl = `http://localhost:${port}/`
     } else {
@@ -285,7 +290,22 @@ const commands = {
       port = cli.flags.port
     }
 
-    return startColossus({ api, publicUrl, port })
+    // TODO: check valid url, and valid port number
+    const store = getStorage(api, cli.flags)
+
+    const ipfsHost = cli.flags.ipfsHost
+    const ipfs = require('ipfs-http-client')(ipfsHost, '5001', { protocol: 'http' })
+    const { PublisherClient, DiscoveryClient } = require('@joystream/service-discovery')
+    const publisherClient = new PublisherClient(ipfs)
+    const discoveryClient = new DiscoveryClient({ ipfs, api })
+    const ipfsHttpGatewayUrl = `http://${ipfsHost}:8080/`
+
+    const { startSyncing } = require('../lib/sync')
+    startSyncing(api, { syncPeriod: SYNC_PERIOD_MS }, store)
+
+    announcePublicUrl(api, publicUrl, publisherClient)
+
+    return startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl })
   },
   discovery: async () => {
     banner()
@@ -294,8 +314,12 @@ const commands = {
     const wsProvider = cli.flags.wsProvider
     const api = await RuntimeApi.create({ provider_url: wsProvider })
     const port = cli.flags.port
+    const ipfsHost = cli.flags.ipfsHost
+    const ipfs = require('ipfs-http-client')(ipfsHost, '5001', { protocol: 'http' })
+    const { DiscoveryClient } = require('@joystream/service-discovery')
+    const discoveryClient = new DiscoveryClient({ ipfs, api })
     await api.untilChainIsSynced()
-    await startDiscoveryService({ api, port })
+    await startDiscoveryService({ api, port, discoveryClient })
   },
 }
 

storage-node/packages/colossus/lib/app.js

@@ -35,7 +35,7 @@ const fileUploads = require('./middleware/file_uploads')
 const pagination = require('@joystream/storage-utils/pagination')
 
 // Configure app
-function createApp(projectRoot, storage, runtime) {
+function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatewayUrl) {
   const app = express()
   app.use(cors())
   app.use(bodyParser.json())
@@ -59,6 +59,8 @@ function createApp(projectRoot, storage, runtime) {
     dependencies: {
       storage,
       runtime,
+      discoveryClient,
+      ipfsHttpGatewayUrl,
     },
   })
 

storage-node/packages/colossus/lib/discovery.js

@@ -33,7 +33,7 @@ const path = require('path')
 const validateResponses = require('./middleware/validate_responses')
 
 // Configure app
-function createApp(projectRoot, runtime) {
+function createApp(projectRoot, discoveryClient) {
   const app = express()
   app.use(cors())
   app.use(bodyParser.json())
@@ -54,7 +54,7 @@ function createApp(projectRoot, runtime) {
     },
     docsPath: '/swagger.json',
     dependencies: {
-      runtime,
+      discoveryClient,
     },
   })
 

storage-node/packages/colossus/lib/middleware/ipfs_proxy.js

@@ -41,12 +41,12 @@ const createResolver = (storage) => {
   return async (id) => await storage.resolveContentIdWithTimeout(5000, id)
 }
 
-const createProxy = (storage) => {
+const createProxy = (storage, ipfsHttpGatewayUrl) => {
   const pathRewrite = createPathRewriter(createResolver(storage))
 
   return createProxyMiddleware(pathFilter, {
     // Default path to local IPFS HTTP GATEWAY
-    target: 'http://localhost:8080/',
+    target: ipfsHttpGatewayUrl || 'http://localhost:8080/',
     pathRewrite,
     onProxyRes: function (proxRes, req, res) {
       /*

storage-node/packages/colossus/lib/sync.js

@@ -132,6 +132,14 @@ async function syncPeriodic({ api, flags, storage, contentBeingSynced, contentCo
       return retry()
     }
 
+    // Retry later if provider is not active
+    if (!(await api.providerIsActiveWorker())) {
+      debug(
+        'storage provider role account and storageProviderId are not associated with a worker. Postponing sync run.'
+      )
+      return retry()
+    }
+
     const recommendedBalance = await api.providerHasMinimumBalance(300)
     if (!recommendedBalance) {
       debug('Warning: Provider role account is running low on balance.')

storage-node/packages/colossus/paths/asset/v0/{id}.js

@@ -27,9 +27,9 @@ function errorHandler(response, err, code) {
   response.status(err.code || code || 500).send({ message: err.toString() })
 }
 
-module.exports = function (storage, runtime) {
+module.exports = function (storage, runtime, ipfsHttpGatewayUrl) {
   // Creat the IPFS HTTP Gateway proxy middleware
-  const proxy = ipfsProxy.createProxy(storage)
+  const proxy = ipfsProxy.createProxy(storage, ipfsHttpGatewayUrl)
 
   const doc = {
     // parameters for all operations in this path

storage-node/packages/colossus/paths/discover/v0/{id}.js

@@ -1,10 +1,9 @@
-const { discover } = require('@joystream/service-discovery')
 const debug = require('debug')('joystream:colossus:api:discovery')
 
 const MAX_CACHE_AGE = 30 * 60 * 1000
 const USE_CACHE = true
 
-module.exports = function (runtime) {
+module.exports = function (discoveryClient) {
   const doc = {
     // parameters for all operations in this path
     parameters: [
@@ -45,7 +44,13 @@ module.exports = function (runtime) {
 
       try {
         debug(`resolving ${id}`)
-        const info = await discover.discover(id, runtime, USE_CACHE, cacheMaxAge)
+        // Storage providers discoveryClient must use ipfs client and not rely
+        // on joystream http discovery to avoid potentially an infinite request loop
+        // back to our own api endpoint.
+        if (!discoveryClient.ipfs) {
+          return res.status(500)
+        }
+        const info = await discoveryClient.discover(id, USE_CACHE, cacheMaxAge)
         if (info === null) {
           debug('info not found')
           res.status(404).end()

storage-node/packages/discovery/discover.js

@@ -2,7 +2,6 @@ const axios = require('axios')
 const debug = require('debug')('joystream:discovery:discover')
 const stripEndingSlash = require('@joystream/storage-utils/stripEndingSlash')
 
-const ipfs = require('ipfs-http-client')('localhost', '5001', { protocol: 'http' })
 const BN = require('bn.js')
 const { newExternallyControlledPromise } = require('@joystream/storage-utils/externalPromise')
 
@@ -14,259 +13,264 @@ function inBrowser() {
   return typeof window !== 'undefined'
 }
 
-/**
- * Map storage-provider id to a Promise of a discovery result. The purpose
- * is to avoid concurrent active discoveries for the same provider.
- */
-const activeDiscoveries = {}
-
-/**
- * Map of storage provider id to string
- * Cache of past discovery lookup results
- */
-const accountInfoCache = {}
-
 /**
  * After what period of time a cached record is considered stale, and would
  * trigger a re-discovery, but only if a query is made for the same provider.
  */
 const CACHE_TTL = 60 * 60 * 1000
 
-/**
- * Queries the ipns id (service key) of the storage provider from the blockchain.
- * If the storage provider is not registered it will return null.
- * @param {number | BN | u64} storageProviderId - the provider id to lookup
- * @param { RuntimeApi } runtimeApi - api instance to query the chain
- * @returns { Promise<string | null> } - ipns multiformat address
- */
-async function getIpnsIdentity(storageProviderId, runtimeApi) {
-  storageProviderId = new BN(storageProviderId)
-  // lookup ipns identity from chain corresponding to storageProviderId
-  const info = await runtimeApi.discovery.getAccountInfo(storageProviderId)
-
-  if (info === null) {
-    // no identity found on chain for account
-    return null
+class DiscoveryClient {
+  /**
+   * Map storage-provider id to a Promise of a discovery result. The purpose
+   * is to avoid concurrent active discoveries for the same provider.
+   */
+  activeDiscoveries = {}
+
+  /**
+   * Map of storage provider id to string
+   * Cache of past discovery lookup results
+   */
+  accountInfoCache = {}
+
+  /*
+   * @param {RuntimeApi} api - api instance to query the chain
+   * @param {string} ipfsHttpGatewayUrl - optional ipfs http gateway
+   * @param {IpfsClient} ipfs - optinoal instance of an ipfs-http-client
+   */
+  constructor({ api, ipfs, ipfsHttpGatewayUrl }) {
+    this.runtimeApi = api
+    this.ipfs = ipfs
+    this.ipfsHttpGatewayUrl = ipfsHttpGatewayUrl
   }
-  return info.identity.toString()
-}
-
-/**
- * Resolves provider id to its service information.
- * Will use an IPFS HTTP gateway. If caller doesn't provide a url the default gateway on
- * the local ipfs node will be used.
- * If the storage provider is not registered it will throw an error
- * @param {number | BN | u64} storageProviderId - the provider id to lookup
- * @param {RuntimeApi} runtimeApi - api instance to query the chain
- * @param {string} gateway - optional ipfs http gateway url to perform ipfs queries
- * @returns { Promise<object> } - the published service information
- */
-async function discoverOverIpfsHttpGateway(storageProviderId, runtimeApi, gateway = 'http://localhost:8080') {
-  storageProviderId = new BN(storageProviderId)
-  const isProvider = await runtimeApi.workers.isStorageProvider(storageProviderId)
 
-  if (!isProvider) {
-    throw new Error('Cannot discover non storage providers')
+  /**
+   * Queries the ipns id (service key) of the storage provider from the blockchain.
+   * If the storage provider is not registered it will return null.
+   * @param {number | BN | u64} storageProviderId - the provider id to lookup
+   * @returns { Promise<string | null> } - ipns multiformat address
+   */
+  async getIpnsIdentity(storageProviderId) {
+    storageProviderId = new BN(storageProviderId)
+    // lookup ipns identity from chain corresponding to storageProviderId
+    const info = await this.runtimeApi.discovery.getAccountInfo(storageProviderId)
+
+    if (info === null) {
+      // no identity found on chain for account
+      return null
+    }
+    return info.identity.toString()
   }
 
-  const identity = await getIpnsIdentity(storageProviderId, runtimeApi)
-
-  if (identity === null) {
-    // dont waste time trying to resolve if no identity was found
-    throw new Error('no identity to resolve')
-  }
+  /**
+   * Resolves provider id to its service information.
+   * Will use an IPFS HTTP gateway. If caller doesn't provide a url the default gateway on
+   * the local ipfs node will be used.
+   * If the storage provider is not registered it will throw an error
+   * @param {number | BN | u64} storageProviderId - the provider id to lookup
+   * @param {string} ipfsHttpGatewayUrl - optional ipfs http gateway url to perform ipfs queries
+   * @returns { Promise<object> } - the published service information
+   */
+  async discoverOverIpfsHttpGateway(storageProviderId, ipfsHttpGatewayUrl) {
+    let gateway = ipfsHttpGatewayUrl || this.ipfsHttpGatewayUrl || 'http://localhost:8080'
+    storageProviderId = new BN(storageProviderId)
+    const isProvider = await this.runtimeApi.workers.isStorageProvider(storageProviderId)
+
+    if (!isProvider) {
+      throw new Error('Cannot discover non storage providers')
+    }
 
-  gateway = stripEndingSlash(gateway)
+    const identity = await this.getIpnsIdentity(storageProviderId)
 
-  const url = `${gateway}/ipns/${identity}`
+    if (identity === null) {
+      // dont waste time trying to resolve if no identity was found
+      throw new Error('no identity to resolve')
+    }
 
-  const response = await axios.get(url)
+    gateway = stripEndingSlash(gateway)
 
-  return response.data
-}
+    const url = `${gateway}/ipns/${identity}`
 
-/**
- * Resolves id of provider to its service information.
- * Will use the provided colossus discovery api endpoint. If no api endpoint
- * is provided it attempts to use the configured endpoints from the chain.
- * If the storage provider is not registered it will throw an error
- * @param {number | BN | u64 } storageProviderId - provider id to lookup
- * @param {RuntimeApi} runtimeApi - api instance to query the chain
- * @param {string} discoverApiEndpoint - url for a colossus discovery api endpoint
- * @returns { Promise<object> } - the published service information
- */
-async function discoverOverJoystreamDiscoveryService(storageProviderId, runtimeApi, discoverApiEndpoint) {
-  storageProviderId = new BN(storageProviderId)
-  const isProvider = await runtimeApi.workers.isStorageProvider(storageProviderId)
+    const response = await axios.get(url)
 
-  if (!isProvider) {
-    throw new Error('Cannot discover non storage providers')
+    return response.data
   }
 
-  const identity = await getIpnsIdentity(storageProviderId, runtimeApi)
-
-  // dont waste time trying to resolve if no identity was found
-  if (identity === null) {
-    throw new Error('no identity to resolve')
-  }
+  /**
+   * Resolves id of provider to its service information.
+   * Will use the provided colossus discovery api endpoint. If no api endpoint
+   * is provided it attempts to use the configured endpoints from the chain.
+   * If the storage provider is not registered it will throw an error
+   * @param {number | BN | u64 } storageProviderId - provider id to lookup
+   * @param {string} discoverApiEndpoint - url for a colossus discovery api endpoint
+   * @returns { Promise<object> } - the published service information
+   */
+  async discoverOverJoystreamDiscoveryService(storageProviderId, discoverApiEndpoint) {
+    storageProviderId = new BN(storageProviderId)
+    const isProvider = await this.runtimeApi.workers.isStorageProvider(storageProviderId)
+
+    if (!isProvider) {
+      throw new Error('Cannot discover non storage providers')
+    }
 
-  if (!discoverApiEndpoint) {
-    // Use bootstrap nodes
-    const discoveryBootstrapNodes = await runtimeApi.discovery.getBootstrapEndpoints()
+    const identity = await this.getIpnsIdentity(storageProviderId)
 
-    if (discoveryBootstrapNodes.length) {
-      discoverApiEndpoint = stripEndingSlash(discoveryBootstrapNodes[0].toString())
-    } else {
-      throw new Error('No known discovery bootstrap nodes found on network')
+    // dont waste time trying to resolve if no identity was found
+    if (identity === null) {
+      throw new Error('no identity to resolve')
     }
-  }
 
-  const url = `${discoverApiEndpoint}/discover/v0/${storageProviderId.toNumber()}`
+    if (!discoverApiEndpoint) {
+      // Use bootstrap nodes
+      const discoveryBootstrapNodes = await this.runtimeApi.discovery.getBootstrapEndpoints()
 
-  // should have parsed if data was json?
-  const response = await axios.get(url)
+      if (discoveryBootstrapNodes.length) {
+        discoverApiEndpoint = stripEndingSlash(discoveryBootstrapNodes[0].toString())
+      } else {
+        throw new Error('No known discovery bootstrap nodes found on network')
+      }
+    }
 
-  return response.data
-}
+    const url = `${discoverApiEndpoint}/discover/v0/${storageProviderId.toNumber()}`
 
-/**
- * Resolves id of provider to its service information.
- * Will use the local IPFS node over RPC interface.
- * If the storage provider is not registered it will throw an error.
- * @param {number | BN | u64 } storageProviderId - provider id to lookup
- * @param {RuntimeApi} runtimeApi - api instance to query the chain
- * @returns { Promise<object> } - the published service information
- */
-async function discoverOverLocalIpfsNode(storageProviderId, runtimeApi) {
-  storageProviderId = new BN(storageProviderId)
-  const isProvider = await runtimeApi.workers.isStorageProvider(storageProviderId)
+    // should have parsed if data was json?
+    const response = await axios.get(url)
 
-  if (!isProvider) {
-    throw new Error('Cannot discover non storage providers')
+    return response.data
   }
 
-  const identity = await getIpnsIdentity(storageProviderId, runtimeApi)
+  /**
+   * Resolves id of provider to its service information.
+   * Will use the local IPFS node over RPC interface.
+   * If the storage provider is not registered it will throw an error.
+   * @param {number | BN | u64 } storageProviderId - provider id to lookup
+   * @returns { Promise<object> } - the published service information
+   */
+  async discoverOverLocalIpfsNode(storageProviderId) {
+    storageProviderId = new BN(storageProviderId)
+    const isProvider = await this.runtimeApi.workers.isStorageProvider(storageProviderId)
+
+    if (!isProvider) {
+      throw new Error('Cannot discover non storage providers')
+    }
 
-  if (identity === null) {
-    // dont waste time trying to resolve if no identity was found
-    throw new Error('no identity to resolve')
-  }
+    const identity = await this.getIpnsIdentity(storageProviderId)
 
-  const ipnsAddress = `/ipns/${identity}/`
+    if (identity === null) {
+      // dont waste time trying to resolve if no identity was found
+      throw new Error('no identity to resolve')
+    }
 
-  debug('resolved ipns to ipfs object')
-  // Can this call hang forever!? can/should we set a timeout?
-  const ipfsName = await ipfs.name.resolve(ipnsAddress, {
-    // don't recurse, there should only be one indirection to the service info file
-    recursive: false,
-    nocache: false,
-  })
+    const ipnsAddress = `/ipns/${identity}/`
 
-  debug('getting ipfs object', ipfsName)
-  const data = await ipfs.get(ipfsName) // this can sometimes hang forever!?! can we set a timeout?
+    debug('resolved ipns to ipfs object')
+    // Can this call hang forever!? can/should we set a timeout?
+    const ipfsName = await this.ipfs.name.resolve(ipnsAddress, {
+      // don't recurse, there should only be one indirection to the service info file
+      recursive: false,
+      nocache: false,
+    })
 
-  // there should only be one file published under the resolved path
-  const content = data[0].content
+    debug('getting ipfs object', ipfsName)
+    const data = await this.ipfs.get(ipfsName) // this can sometimes hang forever!?! can we set a timeout?
 
-  return JSON.parse(content)
-}
+    // there should only be one file published under the resolved path
+    const content = data[0].content
 
-/**
- * Internal method that handles concurrent discoveries and caching of results. Will
- * select the appropriate discovery protocol based on whether we are in a browser environment or not.
- * If not in a browser it expects a local ipfs node to be running.
- * @param {number | BN | u64} storageProviderId - ID of the storage provider
- * @param {RuntimeApi} runtimeApi - api instance for querying the chain
- * @returns { Promise<object | null> } - the published service information
- */
-async function _discover(storageProviderId, runtimeApi) {
-  storageProviderId = new BN(storageProviderId)
-  const id = storageProviderId.toNumber()
-
-  const discoveryResult = activeDiscoveries[id]
-  if (discoveryResult) {
-    debug('discovery in progress waiting for result for', id)
-    return discoveryResult
+    return JSON.parse(content)
   }
 
-  debug('starting new discovery for', id)
-  const deferredDiscovery = newExternallyControlledPromise()
-  activeDiscoveries[id] = deferredDiscovery.promise
-
-  let result
-  try {
-    if (inBrowser()) {
-      result = await discoverOverJoystreamDiscoveryService(storageProviderId, runtimeApi)
-    } else {
-      result = await discoverOverLocalIpfsNode(storageProviderId, runtimeApi)
+  /**
+   * Internal method that handles concurrent discoveries and caching of results. Will
+   * select the appropriate discovery protocol based on browser environment or not,
+   * and if an ipfs client was passed in the constructor.
+   * @param {number | BN | u64} storageProviderId - ID of the storage provider
+   * @returns { Promise<object | null> } - the published service information
+   */
+  async _discover(storageProviderId) {
+    storageProviderId = new BN(storageProviderId)
+    const id = storageProviderId.toNumber()
+
+    const discoveryResult = this.activeDiscoveries[id]
+    if (discoveryResult) {
+      debug('discovery in progress waiting for result for', id)
+      return discoveryResult
     }
 
-    debug(result)
-    result = JSON.stringify(result)
-    accountInfoCache[id] = {
-      value: result,
-      updated: Date.now(),
-    }
+    debug('starting new discovery for', id)
+    const deferredDiscovery = newExternallyControlledPromise()
+    this.activeDiscoveries[id] = deferredDiscovery.promise
 
-    deferredDiscovery.resolve(result)
-    delete activeDiscoveries[id]
-    return result
-  } catch (err) {
-    // we catch the error so we can update all callers
-    // and throw again to inform the first caller.
-    debug(err.message)
-    delete activeDiscoveries[id]
-    // deferredDiscovery.reject(err)
-    deferredDiscovery.resolve(null) // resolve to null until we figure out the issue below
-    // throw err // <-- throwing but this isn't being
-    // caught correctly in express server! Is it because there is an uncaught promise somewhere
-    // in the prior .reject() call ?
-    // I've only seen this behaviour when error is from ipfs-client
-    // ... is this unique to errors thrown from ipfs-client?
-    // Problem is its crashing the node so just return null for now
-    return null
-  }
-}
+    let result
+    try {
+      if (inBrowser() || !this.ipfs) {
+        result = await this.discoverOverJoystreamDiscoveryService(storageProviderId)
+      } else {
+        result = await this.discoverOverLocalIpfsNode(storageProviderId)
+      }
 
-/**
- * Cached discovery of storage provider service information. If useCachedValue is
- * set to true, will always return the cached result if found. New discovery will be triggered
- * if record is found to be stale. If a stale record is not desired (CACHE_TTL old) pass a non zero
- * value for maxCacheAge, which will force a new discovery and return the new resolved value.
- * This method in turn calls _discovery which handles concurrent discoveries and selects the appropriate
- * protocol to perform the query.
- * If the storage provider is not registered it will resolve to null
- * @param {number | BN | u64} storageProviderId - provider to discover
- * @param {RuntimeApi} runtimeApi - api instance to query the chain
- * @param {bool} useCachedValue - optionaly use chached queries
- * @param {number} maxCacheAge - maximum age of a cached query that triggers automatic re-discovery
- * @returns { Promise<object | null> } - the published service information
- */
-async function discover(storageProviderId, runtimeApi, useCachedValue = false, maxCacheAge = 0) {
-  storageProviderId = new BN(storageProviderId)
-  const id = storageProviderId.toNumber()
-  const cached = accountInfoCache[id]
-
-  if (cached && useCachedValue) {
-    if (maxCacheAge > 0) {
-      // get latest value
-      if (Date.now() > cached.updated + maxCacheAge) {
-        return _discover(storageProviderId, runtimeApi)
+      debug(result)
+      result = JSON.stringify(result)
+      this.accountInfoCache[id] = {
+        value: result,
+        updated: Date.now(),
       }
+
+      deferredDiscovery.resolve(result)
+      delete this.activeDiscoveries[id]
+      return result
+    } catch (err) {
+      // we catch the error so we can update all callers
+      // and throw again to inform the first caller.
+      debug(err.message)
+      delete this.activeDiscoveries[id]
+      // deferredDiscovery.reject(err)
+      deferredDiscovery.resolve(null) // resolve to null until we figure out the issue below
+      // throw err // <-- throwing but this isn't being
+      // caught correctly in express server! Is it because there is an uncaught promise somewhere
+      // in the prior .reject() call ?
+      // I've only seen this behaviour when error is from ipfs-client
+      // ... is this unique to errors thrown from ipfs-client?
+      // Problem is its crashing the node so just return null for now
+      return null
     }
-    // refresh if cache if stale, new value returned on next cached query
-    if (Date.now() > cached.updated + CACHE_TTL) {
-      _discover(storageProviderId, runtimeApi)
+  }
+
+  /**
+   * Cached discovery of storage provider service information. If useCachedValue is
+   * set to true, will always return the cached result if found. New discovery will be triggered
+   * if record is found to be stale. If a stale record is not desired (CACHE_TTL old) pass a non zero
+   * value for maxCacheAge, which will force a new discovery and return the new resolved value.
+   * This method in turn calls _discovery which handles concurrent discoveries and selects the appropriate
+   * protocol to perform the query.
+   * If the storage provider is not registered it will resolve to null
+   * @param {number | BN | u64} storageProviderId - provider to discover
+   * @param {bool} useCachedValue - optionaly use chached queries
+   * @param {number} maxCacheAge - maximum age of a cached query that triggers automatic re-discovery
+   * @returns { Promise<object | null> } - the published service information
+   */
+  async discover(storageProviderId, useCachedValue = false, maxCacheAge = 0) {
+    storageProviderId = new BN(storageProviderId)
+    const id = storageProviderId.toNumber()
+    const cached = this.accountInfoCache[id]
+
+    if (cached && useCachedValue) {
+      if (maxCacheAge > 0) {
+        // get latest value
+        if (Date.now() > cached.updated + maxCacheAge) {
+          return this._discover(storageProviderId)
+        }
+      }
+      // refresh if cache if stale, new value returned on next cached query
+      if (Date.now() > cached.updated + CACHE_TTL) {
+        this._discover(storageProviderId)
+      }
+      // return best known value
+      return cached.value
     }
-    // return best known value
-    return cached.value
+    return this._discover(storageProviderId)
   }
-  return _discover(storageProviderId, runtimeApi)
 }
 
 module.exports = {
-  discover,
-  discoverOverJoystreamDiscoveryService,
-  discoverOverIpfsHttpGateway,
-  discoverOverLocalIpfsNode,
+  DiscoveryClient,
 }

storage-node/packages/discovery/example.js

@@ -1,37 +0,0 @@
-const { RuntimeApi } = require('@joystream/storage-runtime-api')
-
-const { discover, publish } = require('./')
-
-async function main() {
-  // The assigned storage-provider id
-  const providerId = 0
-
-  const runtimeApi = await RuntimeApi.create({
-    // Path to the role account key file of the provider
-    account_file: '/path/to/role_account_key_file.json',
-    storageProviderId: providerId,
-  })
-
-  const ipnsId = await publish.publish(
-    {
-      asset: {
-        version: 1,
-        endpoint: 'http://endpoint.com',
-      },
-    },
-    runtimeApi
-  )
-
-  console.log(ipnsId)
-
-  // register ipnsId on chain
-  await runtimeApi.setAccountInfo(ipnsId)
-
-  const serviceInfo = await discover.discover(providerId, runtimeApi)
-
-  console.log(serviceInfo)
-
-  runtimeApi.api.disconnect()
-}
-
-main()

storage-node/packages/discovery/index.js

@@ -1,4 +1,7 @@
+const { PublisherClient } = require('./publish')
+const { DiscoveryClient } = require('./discover')
+
 module.exports = {
-  discover: require('./discover'),
-  publish: require('./publish'),
+  PublisherClient,
+  DiscoveryClient,
 }

storage-node/packages/discovery/publish.js

@@ -1,7 +1,3 @@
-const ipfsClient = require('ipfs-http-client')
-
-const ipfs = ipfsClient('localhost', '5001', { protocol: 'http' })
-
 const debug = require('debug')('joystream:discovery:publish')
 
 /**
@@ -32,57 +28,66 @@ function encodeServiceInfo(info) {
     serialized: JSON.stringify(info),
   })
 }
-
 /**
- * Publishes the service information, encoded using the standard defined in encodeServiceInfo()
- * to ipfs, using the local ipfs node's PUBLISH_KEY, and returns the key id used to publish.
- * What we refer to as the ipns id.
- * @param {object} serviceInfo - the service information to publish
- * @returns {string} - the ipns id
+ * A PublisherClient is used to store a JSON serializable piece of "service information" in the ipfs network
+ * using the `self` key of the ipfs node. This makes looking up that information available through IPNS.
  */
-async function publish(serviceInfo) {
-  const keys = await ipfs.key.list()
-  let servicesKey = keys.find((key) => key.name === PUBLISH_KEY)
-
-  // An ipfs node will always have the self key.
-  // If the publish key is specified as anything else and it doesn't exist
-  // we create it.
-  if (PUBLISH_KEY !== 'self' && !servicesKey) {
-    debug('generating ipns services key')
-    servicesKey = await ipfs.key.gen(PUBLISH_KEY, {
-      type: 'rsa',
-      size: 2048,
-    })
+class PublisherClient {
+  /**
+   * Create an instance of a PublisherClient, taking an optional ipfs client instance. If not provided
+   * a default client using default localhost node will be used.
+   * @param {IpfsClient} ipfs - optional instance of an ipfs-http-client.
+   */
+  constructor(ipfs) {
+    this.ipfs = ipfs || require('ipfs-http-client')('localhost', '5001', { protocol: 'http' })
   }
 
-  if (!servicesKey) {
-    throw new Error('No IPFS publishing key available!')
-  }
+  /**
+   * Publishes the service information, encoded using the standard defined in encodeServiceInfo()
+   * to ipfs, using the local ipfs node's PUBLISH_KEY, and returns the key id used to publish.
+   * What we refer to as the ipns id.
+   * @param {object} serviceInfo - the service information to publish
+   * @return {string} - the ipns id
+   */
+  async publish(serviceInfo) {
+    const keys = await this.ipfs.key.list()
+    let servicesKey = keys.find((key) => key.name === PUBLISH_KEY)
 
-  debug('adding service info file to node')
-  const files = await ipfs.add(encodeServiceInfo(serviceInfo))
+    // An ipfs node will always have the self key.
+    // If the publish key is specified as anything else and it doesn't exist
+    // we create it.
+    if (PUBLISH_KEY !== 'self' && !servicesKey) {
+      debug('generating ipns services key')
+      servicesKey = await this.ipfs.key.gen(PUBLISH_KEY, {
+        type: 'rsa',
+        size: 2048,
+      })
+    }
 
-  debug('publishing...')
-  const published = await ipfs.name.publish(files[0].hash, {
-    key: PUBLISH_KEY,
-    resolve: false,
-    // lifetime: // string - Time duration of the record. Default: 24h
-    // ttl:      // string - Time duration this record should be cached
-  })
+    if (!servicesKey) {
+      throw new Error('No IPFS publishing key available!')
+    }
+
+    debug('adding service info file to node')
+    const files = await this.ipfs.add(encodeServiceInfo(serviceInfo))
 
-  // The name and ipfs hash of the published service information file, eg.
-  // {
-  //   name: 'QmUNQCkaU1TRnc1WGixqEP3Q3fazM8guSdFRsdnSJTN36A',
-  //   value: '/ipfs/QmcSjtVMfDSSNYCxNAb9PxNpEigCw7h1UZ77gip3ghfbnA'
-  // }
-  // .. The name is equivalent to the key id that was used.
-  debug(published)
+    debug('publishing...')
+    const { name, value } = await this.ipfs.name.publish(files[0].hash, {
+      key: PUBLISH_KEY,
+      resolve: false,
+      // lifetime: // string - Time duration of the record. Default: 24h
+      // ttl:      // string - Time duration this record should be cached
+    })
 
-  // Return the key id under which the content was published. Which is used
-  // to lookup the actual ipfs content id of the published service information
-  return servicesKey.id
+    debug(`published ipns name: ${name} -> ${value}`)
+
+    // Return the key id under which the content was published. Which is used
+    // to lookup the actual ipfs content id of the published service information
+    // Note: name === servicesKey.id
+    return servicesKey.id
+  }
 }
 
 module.exports = {
-  publish,
+  PublisherClient,
 }

storage-node/packages/helios/bin/cli.js

@@ -2,7 +2,7 @@
 
 const { RuntimeApi } = require('@joystream/storage-runtime-api')
 const { encodeAddress } = require('@polkadot/keyring')
-const { discover } = require('@joystream/service-discovery')
+const { DiscoveryClient } = require('@joystream/service-discovery')
 const axios = require('axios')
 const stripEndingSlash = require('@joystream/storage-utils/stripEndingSlash')
 
@@ -124,12 +124,14 @@ async function main() {
     })
   )
 
+  const discoveryClient = new DiscoveryClient({ api: runtime })
+
   // Resolve IPNS identities of providers
   console.log('\nResolving live provider API Endpoints...')
   const endpoints = await Promise.all(
     providersStatuses.map(async ({ providerId }) => {
       try {
-        const serviceInfo = await discover.discoverOverJoystreamDiscoveryService(providerId, runtime)
+        const serviceInfo = await discoveryClient.discoverOverJoystreamDiscoveryService(providerId)
 
         if (serviceInfo === null) {
           console.log(`provider ${providerId} has not published service information`)

storage-node/packages/runtime-api/index.js

@@ -57,7 +57,17 @@ class RuntimeApi {
     const provider = new WsProvider(options.provider_url || 'ws://localhost:9944')
 
     // Create the API instrance
-    this.api = await ApiPromise.create({ provider, types: types })
+    while (true) {
+      try {
+        this.api = await ApiPromise.create({ provider, types: types })
+        break
+      } catch (err) {
+        debug('connecting to node failed, will retry..')
+      }
+      await sleep(5000)
+    }
+
+    await this.api.isReady
 
     this.asyncLock = new AsyncLock()
 
@@ -104,6 +114,10 @@ class RuntimeApi {
     return this.balances.hasMinimumBalanceOf(providerAccountId, minimumBalance)
   }
 
+  async providerIsActiveWorker() {
+    return this.workers.isRoleAccountOfStorageProvider(this.storageProviderId, this.identities.key.address)
+  }
+
   executeWithAccountLock(accountId, func) {
     return this.asyncLock.acquire(`${accountId}`, func)
   }

storage-node/packages/storage/storage.js

@@ -215,7 +215,7 @@ class Storage {
     this._timeout = this.options.timeout || DEFAULT_TIMEOUT
     this._resolve_content_id = this.options.resolve_content_id || DEFAULT_RESOLVE_CONTENT_ID
 
-    this.ipfs = ipfsClient(this.options.ipfs.connect_options)
+    this.ipfs = ipfsClient(this.options.ipfsHost || 'localhost', '5001', { protocol: 'http' })
 
     this.pinned = {}
     this.pinning = {}

tests/network-tests/.env

@@ -1,8 +1,8 @@
 # Address of the Joystream node.
 NODE_URL = ws://127.0.0.1:9944
-# Path to the database for shared keys and nonce
-DB_PATH = .tmp/db.json
 # Account which is expected to provide sufficient funds to test accounts.
+TREASURY_ACCOUNT_URI = //Alice
+# Sudo Account
 SUDO_ACCOUNT_URI = //Alice
 # Amount of members able to buy membership in membership creation test.
 MEMBERSHIP_CREATION_N = 2

tests/network-tests/package.json

@@ -4,32 +4,31 @@
   "license": "GPL-3.0-only",
   "scripts": {
     "build": "tsc --noEmit",
-    "test": "yarn db-path-setup && tap --files src/tests/unknown.unknown src/tests/councilSetup.ts src/tests/proposals/*Test.ts src/tests/leaderSetup.ts src/tests/workingGroup/*Test.ts -T",
+    "run-tests": "./run-tests.sh",
+    "test-run": "node -r ts-node/register --unhandled-rejections=strict",
     "lint": "eslint . --quiet --ext .ts",
     "checks": "tsc --noEmit --pretty && prettier ./ --check && yarn lint",
-    "format": "prettier ./ --write ",
-    "db-path-setup": "mkdir .tmp/ || rm .tmp/db.json || echo ''"
+    "format": "prettier ./ --write "
   },
   "dependencies": {
     "@joystream/types": "link:../../types",
     "@polkadot/api": "1.26.1",
     "@polkadot/keyring": "3.0.1",
+    "@types/async-lock": "^1.1.2",
     "@types/bn.js": "^4.11.5",
     "@types/lowdb": "^1.0.9",
+    "async-lock": "^1.2.0",
     "bn.js": "^4.11.8",
     "dotenv": "^8.2.0",
     "fs": "^0.0.1-security",
-    "lowdb": "^1.0.0",
     "uuid": "^7.0.3"
   },
   "devDependencies": {
     "@polkadot/ts": "^0.3.14",
     "@types/chai": "^4.2.11",
-    "@types/tap": "^14.10.0",
     "@types/uuid": "^7.0.2",
     "chai": "^4.2.0",
     "prettier": "2.0.2",
-    "tap": "^14.10.7",
     "ts-node": "^8.8.1",
     "typescript": "^3.8.3"
   }

tests/network-tests/run-tests.sh

@@ -50,6 +50,7 @@ CONTAINER_ID=`docker run -d -v ${DATA_PATH}:/data -p 9944:9944 joystream/node \
   --chain /data/chain-spec-raw.json`
 
 function cleanup() {
+    docker logs ${CONTAINER_ID} --tail 15
     docker stop ${CONTAINER_ID}
     docker rm ${CONTAINER_ID}
 }
@@ -57,4 +58,4 @@ function cleanup() {
 trap cleanup EXIT
 
 # Execute the tests
-yarn workspace network-tests test
+time DEBUG=* yarn workspace network-tests test-run src/scenarios/full.ts

tests/network-tests/src/Fixture.ts

@@ -0,0 +1,30 @@
+import { Api } from './Api'
+
+export interface Fixture {
+  runner(expectFailure: boolean): Promise<void>
+}
+
+// Fixture that measures start and end blocks
+// ensures fixture only runs once
+export class BaseFixture implements Fixture {
+  protected api: Api
+  private ran = false
+
+  constructor(api: Api) {
+    this.api = api
+    // record starting block
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    if (this.ran) {
+      return
+    }
+    this.ran = true
+    return this.execute(expectFailure)
+    // record end blocks
+  }
+
+  protected async execute(expectFailure: boolean): Promise<void> {
+    return
+  }
+}

tests/network-tests/src/fixtures/councilElectionHappyCase.ts

@@ -0,0 +1,34 @@
+import { Fixture } from '../Fixture'
+import { ElectCouncilFixture } from './councilElectionModule'
+import { Api } from '../Api'
+import BN from 'bn.js'
+
+export class CouncilElectionHappyCaseFixture implements Fixture {
+  private api: Api
+  private voters: string[]
+  private applicants: string[]
+  private k: number
+  private greaterStake: BN
+  private lesserStake: BN
+
+  constructor(api: Api, voters: string[], applicants: string[], k: number, greaterStake: BN, lesserStake: BN) {
+    this.api = api
+    this.voters = voters
+    this.applicants = applicants
+    this.k = k
+    this.greaterStake = greaterStake
+    this.lesserStake = lesserStake
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const electCouncilFixture: ElectCouncilFixture = new ElectCouncilFixture(
+      this.api,
+      this.voters,
+      this.applicants,
+      this.k,
+      this.greaterStake,
+      this.lesserStake
+    )
+    await electCouncilFixture.runner(false)
+  }
+}

tests/network-tests/src/fixtures/councilElectionModule.ts

@@ -0,0 +1,112 @@
+import { Api } from '../Api'
+import BN from 'bn.js'
+import { assert } from 'chai'
+import { Seat } from '@joystream/types/council'
+import { v4 as uuid } from 'uuid'
+import { Utils } from '../utils'
+import { Fixture } from '../Fixture'
+
+export class ElectCouncilFixture implements Fixture {
+  private api: Api
+  private voters: string[]
+  private applicants: string[]
+  private k: number
+  private greaterStake: BN
+  private lesserStake: BN
+
+  public constructor(api: Api, voters: string[], applicants: string[], k: number, greaterStake: BN, lesserStake: BN) {
+    this.api = api
+    this.voters = voters
+    this.applicants = applicants
+    this.k = k
+    this.greaterStake = greaterStake
+    this.lesserStake = lesserStake
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Assert no council exists
+    assert((await this.api.getCouncil()).length === 0)
+
+    let now = await this.api.getBestBlock()
+    const applyForCouncilFee: BN = this.api.estimateApplyForCouncilFee(this.greaterStake)
+    const voteForCouncilFee: BN = this.api.estimateVoteForCouncilFee(
+      this.applicants[0],
+      this.applicants[0],
+      this.greaterStake
+    )
+    const salt: string[] = this.voters.map(() => {
+      return ''.concat(uuid().replace(/-/g, ''))
+    })
+    const revealVoteFee: BN = this.api.estimateRevealVoteFee(this.applicants[0], salt[0])
+
+    // Topping the balances
+    this.api.treasuryTransferBalanceToAccounts(this.applicants, applyForCouncilFee.add(this.greaterStake))
+    this.api.treasuryTransferBalanceToAccounts(this.voters, voteForCouncilFee.add(revealVoteFee).add(this.greaterStake))
+
+    // First K members stake more
+    await this.api.sudoStartAnnouncingPeriod(now.addn(100))
+    await this.api.batchApplyForCouncilElection(this.applicants.slice(0, this.k), this.greaterStake)
+    this.applicants.slice(0, this.k).forEach((account) =>
+      this.api.getCouncilElectionStake(account).then((stake) => {
+        assert(
+          stake.eq(this.greaterStake),
+          `${account} not applied correctly for council election with stake ${stake} versus expected ${this.greaterStake}`
+        )
+      })
+    )
+
+    // Last members stake less
+    await this.api.batchApplyForCouncilElection(this.applicants.slice(this.k), this.lesserStake)
+    this.applicants.slice(this.k).forEach((account) =>
+      this.api.getCouncilElectionStake(account).then((stake) => {
+        assert(
+          stake.eq(this.lesserStake),
+          `${account} not applied correctrly for council election with stake ${stake} versus expected ${this.lesserStake}`
+        )
+      })
+    )
+
+    // Voting
+    await this.api.sudoStartVotingPeriod(now.addn(100))
+    await this.api.batchVoteForCouncilMember(
+      this.voters.slice(0, this.k),
+      this.applicants.slice(0, this.k),
+      salt.slice(0, this.k),
+      this.lesserStake
+    )
+    await this.api.batchVoteForCouncilMember(
+      this.voters.slice(this.k),
+      this.applicants.slice(this.k),
+      salt.slice(this.k),
+      this.greaterStake
+    )
+
+    // Revealing
+    await this.api.sudoStartRevealingPeriod(now.addn(100))
+    await this.api.batchRevealVote(
+      this.voters.slice(0, this.k),
+      this.applicants.slice(0, this.k),
+      salt.slice(0, this.k)
+    )
+    await this.api.batchRevealVote(this.voters.slice(this.k), this.applicants.slice(this.k), salt.slice(this.k))
+    now = await this.api.getBestBlock()
+
+    // Resolving election
+    // 3 is to ensure the revealing block is in future
+    await this.api.sudoStartRevealingPeriod(now.addn(3))
+    await Utils.wait(this.api.getBlockDuration().muln(2.5).toNumber())
+    const seats: Seat[] = await this.api.getCouncil()
+
+    // Assert a council was created
+    assert(seats.length)
+
+    // const applicantAddresses: string[] = this.applicantKeyPairs.map((keyPair) => keyPair.address)
+    // const voterAddresses: string[] = this.voterKeyPairs.map((keyPair) => keyPair.address)
+    // const councilMembers: string[] = seats.map((seat) => seat.member.toString())
+    // const backers: string[] = seats.map((seat) => seat.backers.map((backer) => backer.member.toString())).flat()
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}

tests/network-tests/src/fixtures/membershipModule.ts

@@ -0,0 +1,94 @@
+import { Api } from '../Api'
+import BN from 'bn.js'
+import { assert } from 'chai'
+import { Fixture, BaseFixture } from '../Fixture'
+import { PaidTermId, MemberId } from '@joystream/types/members'
+import Debugger from 'debug'
+
+export class BuyMembershipHappyCaseFixture extends BaseFixture {
+  private accounts: string[]
+  private paidTerms: PaidTermId
+  private debug: Debugger.Debugger
+  private memberIds: MemberId[] = []
+
+  public constructor(api: Api, accounts: string[], paidTerms: PaidTermId) {
+    super(api)
+    this.accounts = accounts
+    this.paidTerms = paidTerms
+    this.debug = Debugger('fixture:BuyMembershipHappyCaseFixture')
+  }
+
+  public getCreatedMembers(): MemberId[] {
+    return this.memberIds.slice()
+  }
+
+  public async execute(expectFailure: boolean): Promise<void> {
+    this.debug(`Registering ${this.accounts.length} new members`)
+    // Fee estimation and transfer
+    const membershipFee: BN = await this.api.getMembershipFee(this.paidTerms)
+    const membershipTransactionFee: BN = this.api.estimateBuyMembershipFee(
+      this.accounts[0],
+      this.paidTerms,
+      'member_name_which_is_longer_than_expected'
+    )
+    this.api.treasuryTransferBalanceToAccounts(this.accounts, membershipTransactionFee.add(new BN(membershipFee)))
+
+    this.memberIds = (
+      await Promise.all(
+        this.accounts.map((account) =>
+          this.api.buyMembership(account, this.paidTerms, `member${account.substring(0, 14)}`)
+        )
+      )
+    ).map(({ events }) => this.api.expectMemberRegisteredEvent(events))
+
+    this.debug(`New member ids: ${this.memberIds}`)
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class BuyMembershipWithInsufficienFundsFixture implements Fixture {
+  private api: Api
+  private account: string
+  private paidTerms: PaidTermId
+
+  public constructor(api: Api, account: string, paidTerms: PaidTermId) {
+    this.api = api
+    this.account = account
+    this.paidTerms = paidTerms
+  }
+
+  public async runner(expectFailure: boolean) {
+    // Assertions
+    this.api.getMemberIds(this.account).then((membership) => assert(membership.length === 0, 'Account A is a member'))
+
+    // Fee estimation and transfer
+    const membershipFee: BN = await this.api.getMembershipFee(this.paidTerms)
+    const membershipTransactionFee: BN = this.api.estimateBuyMembershipFee(
+      this.account,
+      this.paidTerms,
+      'member_name_which_is_longer_than_expected'
+    )
+    this.api.treasuryTransferBalance(this.account, membershipTransactionFee)
+
+    // Balance assertion
+    await this.api
+      .getBalance(this.account)
+      .then((balance) =>
+        assert(
+          balance.toBn() < membershipFee.add(membershipTransactionFee),
+          'Account A already have sufficient balance to purchase membership'
+        )
+      )
+
+    // Buying memebership
+    await this.api.buyMembership(this.account, this.paidTerms, `late_member_${this.account.substring(0, 8)}`, true)
+
+    // Assertions
+    this.api.getMemberIds(this.account).then((membership) => assert(membership.length === 0, 'Account A is a member'))
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}

tests/network-tests/src/fixtures/proposalsModule.ts

@@ -0,0 +1,801 @@
+import { Api, WorkingGroups } from '../Api'
+import { v4 as uuid } from 'uuid'
+import BN from 'bn.js'
+import { ProposalId } from '@joystream/types/proposals'
+import { Fixture } from '../Fixture'
+import { assert } from 'chai'
+import { ApplicationId, OpeningId } from '@joystream/types/hiring'
+import { WorkerId } from '@joystream/types/working-group'
+import { Utils } from '../utils'
+import { EventRecord } from '@polkadot/types/interfaces'
+
+export class CreateWorkingGroupLeaderOpeningFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private applicationStake: BN
+  private roleStake: BN
+  private workingGroup: string
+
+  private result: ProposalId | undefined
+
+  constructor(api: Api, proposer: string, applicationStake: BN, roleStake: BN, workingGroup: string) {
+    this.api = api
+    this.proposer = proposer
+    this.applicationStake = applicationStake
+    this.roleStake = roleStake
+    this.workingGroup = workingGroup
+  }
+
+  public getCreatedProposalId(): ProposalId | undefined {
+    return this.result
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing working group lead opening proposal ' + uuid().substring(0, 8)
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(100000)
+    const proposalFee: BN = this.api.estimateProposeCreateWorkingGroupLeaderOpeningFee()
+    this.api.treasuryTransferBalance(this.proposer, proposalFee.add(proposalStake))
+
+    // Proposal creation
+    const result = await this.api.proposeCreateWorkingGroupLeaderOpening({
+      account: this.proposer,
+      title: proposalTitle,
+      description: description,
+      proposalStake: proposalStake,
+      actiavteAt: 'CurrentBlock',
+      maxActiveApplicants: new BN(10),
+      maxReviewPeriodLength: new BN(32),
+      applicationStakingPolicyAmount: this.applicationStake,
+      applicationCrowdedOutUnstakingPeriodLength: new BN(1),
+      applicationReviewPeriodExpiredUnstakingPeriodLength: new BN(1),
+      roleStakingPolicyAmount: this.roleStake,
+      roleCrowdedOutUnstakingPeriodLength: new BN(1),
+      roleReviewPeriodExpiredUnstakingPeriodLength: new BN(1),
+      slashableMaxCount: new BN(1),
+      slashableMaxPercentPtsPerTime: new BN(100),
+      fillOpeningSuccessfulApplicantApplicationStakeUnstakingPeriod: new BN(1),
+      fillOpeningFailedApplicantApplicationStakeUnstakingPeriod: new BN(1),
+      fillOpeningFailedApplicantRoleStakeUnstakingPeriod: new BN(1),
+      terminateApplicationStakeUnstakingPeriod: new BN(1),
+      terminateRoleStakeUnstakingPeriod: new BN(1),
+      exitRoleApplicationStakeUnstakingPeriod: new BN(1),
+      exitRoleStakeUnstakingPeriod: new BN(1),
+      text: uuid().substring(0, 8),
+      workingGroup: this.workingGroup,
+    })
+
+    this.result = this.api.expectProposalCreatedEvent(result.events)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class BeginWorkingGroupLeaderApplicationReviewFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private openingId: OpeningId
+  private workingGroup: string
+
+  private result: ProposalId | undefined
+
+  constructor(api: Api, proposer: string, openingId: OpeningId, workingGroup: string) {
+    this.api = api
+    this.proposer = proposer
+    this.openingId = openingId
+    this.workingGroup = workingGroup
+  }
+
+  public getCreatedProposalId(): ProposalId | undefined {
+    return this.result
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing begin working group lead application review proposal ' + uuid().substring(0, 8)
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(25000)
+    const proposalFee: BN = this.api.estimateProposeBeginWorkingGroupLeaderApplicationReviewFee()
+    this.api.treasuryTransferBalance(this.proposer, proposalFee.add(proposalStake))
+
+    // Proposal creation
+    const result = await this.api.proposeBeginWorkingGroupLeaderApplicationReview(
+      this.proposer,
+      proposalTitle,
+      description,
+      proposalStake,
+      this.openingId,
+      this.workingGroup
+    )
+    this.result = this.api.expectProposalCreatedEvent(result.events)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class FillLeaderOpeningProposalFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private applicationId: ApplicationId
+  private firstRewardInterval: BN
+  private rewardInterval: BN
+  private payoutAmount: BN
+  private openingId: OpeningId
+  private workingGroup: WorkingGroups
+
+  private result: ProposalId | undefined
+
+  constructor(
+    api: Api,
+    proposer: string,
+    applicationId: ApplicationId,
+    firstRewardInterval: BN,
+    rewardInterval: BN,
+    payoutAmount: BN,
+    openingId: OpeningId,
+    workingGroup: WorkingGroups
+  ) {
+    this.api = api
+    this.proposer = proposer
+    this.applicationId = applicationId
+    this.firstRewardInterval = firstRewardInterval
+    this.rewardInterval = rewardInterval
+    this.payoutAmount = payoutAmount
+    this.openingId = openingId
+    this.workingGroup = workingGroup
+  }
+
+  public getCreatedProposalId(): ProposalId | undefined {
+    return this.result
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing fill opening proposal ' + uuid().substring(0, 8)
+    const workingGroupString: string = this.api.getWorkingGroupString(this.workingGroup)
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(50000)
+    const proposalFee: BN = this.api.estimateProposeFillLeaderOpeningFee()
+    this.api.treasuryTransferBalance(this.proposer, proposalFee.add(proposalStake))
+
+    const now: BN = await this.api.getBestBlock()
+
+    // Proposal creation
+    const result = await this.api.proposeFillLeaderOpening({
+      account: this.proposer,
+      title: proposalTitle,
+      description: description,
+      proposalStake: proposalStake,
+      openingId: this.openingId,
+      successfulApplicationId: this.applicationId,
+      amountPerPayout: this.payoutAmount,
+      nextPaymentAtBlock: now.add(this.firstRewardInterval),
+      payoutInterval: this.rewardInterval,
+      workingGroup: workingGroupString,
+    })
+
+    this.result = this.api.expectProposalCreatedEvent(result.events)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class TerminateLeaderRoleProposalFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private slash: boolean
+  private workingGroup: WorkingGroups
+
+  private result: ProposalId | undefined
+
+  constructor(api: Api, proposer: string, slash: boolean, workingGroup: WorkingGroups) {
+    this.api = api
+    this.proposer = proposer
+    this.slash = slash
+    this.workingGroup = workingGroup
+  }
+
+  public getCreatedProposalId(): ProposalId | undefined {
+    return this.result
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing begin working group lead application review proposal ' + uuid().substring(0, 8)
+    const rationale: string = 'Testing leader termination ' + uuid().substring(0, 8)
+    const workingGroupString: string = this.api.getWorkingGroupString(this.workingGroup)
+    // assert worker exists
+    const workerId: WorkerId = (await this.api.getLeadWorkerId(this.workingGroup))!
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(100000)
+    const proposalFee: BN = this.api.estimateProposeTerminateLeaderRoleFee()
+    this.api.treasuryTransferBalance(this.proposer, proposalFee.add(proposalStake))
+
+    // Proposal creation
+    const result = await this.api.proposeTerminateLeaderRole(
+      this.proposer,
+      proposalTitle,
+      description,
+      proposalStake,
+      workerId,
+      rationale,
+      this.slash,
+      workingGroupString
+    )
+    this.result = this.api.expectProposalCreatedEvent(result.events)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class SetLeaderRewardProposalFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private payoutAmount: BN
+  private workingGroup: WorkingGroups
+
+  private result: ProposalId | undefined
+
+  constructor(api: Api, proposer: string, payoutAmount: BN, workingGroup: WorkingGroups) {
+    this.api = api
+    this.proposer = proposer
+    this.payoutAmount = payoutAmount
+    this.workingGroup = workingGroup
+  }
+
+  public getCreatedProposalId(): ProposalId | undefined {
+    return this.result
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing set leader reward proposal ' + uuid().substring(0, 8)
+    const workingGroupString: string = this.api.getWorkingGroupString(this.workingGroup)
+    // assert worker exists?
+    const workerId: WorkerId = (await this.api.getLeadWorkerId(this.workingGroup))!
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(50000)
+    const proposalFee: BN = this.api.estimateProposeLeaderRewardFee()
+    this.api.treasuryTransferBalance(this.proposer, proposalFee.add(proposalStake))
+
+    // Proposal creation
+    const result = await this.api.proposeLeaderReward(
+      this.proposer,
+      proposalTitle,
+      description,
+      proposalStake,
+      workerId,
+      this.payoutAmount,
+      workingGroupString
+    )
+
+    this.result = this.api.expectProposalCreatedEvent(result.events)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class DecreaseLeaderStakeProposalFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private stakeDecrement: BN
+  private workingGroup: WorkingGroups
+
+  private result: ProposalId | undefined
+
+  constructor(api: Api, proposer: string, stakeDecrement: BN, workingGroup: WorkingGroups) {
+    this.api = api
+    this.proposer = proposer
+    this.stakeDecrement = stakeDecrement
+    this.workingGroup = workingGroup
+  }
+
+  public getCreatedProposalId(): ProposalId | undefined {
+    return this.result
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing decrease leader stake proposal ' + uuid().substring(0, 8)
+    const workingGroupString: string = this.api.getWorkingGroupString(this.workingGroup)
+    // assert worker exists ?
+    const workerId: WorkerId = (await this.api.getLeadWorkerId(this.workingGroup))!
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(50000)
+    const proposalFee: BN = this.api.estimateProposeDecreaseLeaderStakeFee()
+    this.api.treasuryTransferBalance(this.proposer, proposalFee.add(proposalStake))
+
+    // Proposal creation
+    const result = await this.api.proposeDecreaseLeaderStake(
+      this.proposer,
+      proposalTitle,
+      description,
+      proposalStake,
+      workerId,
+      this.stakeDecrement,
+      workingGroupString
+    )
+
+    this.result = this.api.expectProposalCreatedEvent(result.events)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class SlashLeaderProposalFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private slashAmount: BN
+  private workingGroup: WorkingGroups
+
+  private result: ProposalId | undefined
+
+  constructor(api: Api, proposer: string, slashAmount: BN, workingGroup: WorkingGroups) {
+    this.api = api
+    this.proposer = proposer
+    this.slashAmount = slashAmount
+    this.workingGroup = workingGroup
+  }
+
+  public getCreatedProposalId(): ProposalId | undefined {
+    return this.result
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing slash leader stake proposal ' + uuid().substring(0, 8)
+    const workingGroupString: string = this.api.getWorkingGroupString(this.workingGroup)
+    const workerId: WorkerId = (await this.api.getLeadWorkerId(this.workingGroup))!
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(50000)
+    const proposalFee: BN = this.api.estimateProposeSlashLeaderStakeFee()
+    this.api.treasuryTransferBalance(this.proposer, proposalFee.add(proposalStake))
+
+    // Proposal creation
+    const result = await this.api.proposeSlashLeaderStake(
+      this.proposer,
+      proposalTitle,
+      description,
+      proposalStake,
+      workerId,
+      this.slashAmount,
+      workingGroupString
+    )
+    this.result = this.api.expectProposalCreatedEvent(result.events)
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class WorkingGroupMintCapacityProposalFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private mintCapacity: BN
+  private workingGroup: WorkingGroups
+
+  private result: ProposalId | undefined
+
+  constructor(api: Api, proposer: string, mintCapacity: BN, workingGroup: WorkingGroups) {
+    this.api = api
+    this.proposer = proposer
+    this.mintCapacity = mintCapacity
+    this.workingGroup = workingGroup
+  }
+
+  public getCreatedProposalId(): ProposalId | undefined {
+    return this.result
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing working group mint capacity proposal ' + uuid().substring(0, 8)
+    const workingGroupString: string = this.api.getWorkingGroupString(this.workingGroup)
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(50000)
+    const proposalFee: BN = this.api.estimateProposeWorkingGroupMintCapacityFee()
+    this.api.treasuryTransferBalance(this.proposer, proposalFee.add(proposalStake))
+
+    // Proposal creation
+    const result = await this.api.proposeWorkingGroupMintCapacity(
+      this.proposer,
+      proposalTitle,
+      description,
+      proposalStake,
+      this.mintCapacity,
+      workingGroupString
+    )
+    this.result = this.api.expectProposalCreatedEvent(result.events)
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class ElectionParametersProposalFixture implements Fixture {
+  private api: Api
+  private proposerAccount: string
+
+  constructor(api: Api, proposerAccount: string) {
+    this.api = api
+    this.proposerAccount = proposerAccount
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing validator count proposal ' + uuid().substring(0, 8)
+
+    // Council accounts enough balance to ensure they can vote
+    const councilAccounts = await this.api.getCouncilAccounts()
+    const runtimeVoteFee: BN = this.api.estimateVoteForProposalFee()
+    this.api.treasuryTransferBalanceToAccounts(councilAccounts, runtimeVoteFee)
+
+    const announcingPeriod: BN = new BN(28800)
+    const votingPeriod: BN = new BN(14400)
+    const revealingPeriod: BN = new BN(14400)
+    const councilSize: BN = await this.api.getCouncilSize()
+    const candidacyLimit: BN = await this.api.getCandidacyLimit()
+    const newTermDuration: BN = new BN(144000)
+    const minCouncilStake: BN = await this.api.getMinCouncilStake()
+    const minVotingStake: BN = await this.api.getMinVotingStake()
+
+    // Proposal stake calculation
+    // Required stake is hardcoded in runtime-module (but not available as const)
+    const proposalStake: BN = new BN(200000)
+    const proposalFee: BN = this.api.estimateProposeElectionParametersFee(
+      description,
+      description,
+      proposalStake,
+      announcingPeriod,
+      votingPeriod,
+      revealingPeriod,
+      councilSize,
+      candidacyLimit,
+      newTermDuration,
+      minCouncilStake,
+      minVotingStake
+    )
+
+    this.api.treasuryTransferBalance(this.proposerAccount, proposalFee.add(proposalStake))
+
+    // Proposal creation
+    const proposedAnnouncingPeriod: BN = announcingPeriod.subn(1)
+    const proposedVotingPeriod: BN = votingPeriod.addn(1)
+    const proposedRevealingPeriod: BN = revealingPeriod.addn(1)
+    const proposedCouncilSize: BN = councilSize.addn(1)
+    const proposedCandidacyLimit: BN = candidacyLimit.addn(1)
+    const proposedNewTermDuration: BN = newTermDuration.addn(1)
+    const proposedMinCouncilStake: BN = minCouncilStake.addn(1)
+    const proposedMinVotingStake: BN = minVotingStake.addn(1)
+
+    const proposalCreationResult = await this.api.proposeElectionParameters(
+      this.proposerAccount,
+      proposalTitle,
+      description,
+      proposalStake,
+      proposedAnnouncingPeriod,
+      proposedVotingPeriod,
+      proposedRevealingPeriod,
+      proposedCouncilSize,
+      proposedCandidacyLimit,
+      proposedNewTermDuration,
+      proposedMinCouncilStake,
+      proposedMinVotingStake
+    )
+    const proposalNumber = this.api.expectProposalCreatedEvent(proposalCreationResult.events)
+
+    // Approving the proposal
+    this.api.batchApproveProposal(proposalNumber)
+    await this.api.waitForProposalToFinalize(proposalNumber)
+
+    // Assertions
+    const newAnnouncingPeriod: BN = await this.api.getAnnouncingPeriod()
+    const newVotingPeriod: BN = await this.api.getVotingPeriod()
+    const newRevealingPeriod: BN = await this.api.getRevealingPeriod()
+    const newCouncilSize: BN = await this.api.getCouncilSize()
+    const newCandidacyLimit: BN = await this.api.getCandidacyLimit()
+    const newNewTermDuration: BN = await this.api.getNewTermDuration()
+    const newMinCouncilStake: BN = await this.api.getMinCouncilStake()
+    const newMinVotingStake: BN = await this.api.getMinVotingStake()
+    assert(
+      proposedAnnouncingPeriod.eq(newAnnouncingPeriod),
+      `Announcing period has unexpected value ${newAnnouncingPeriod}, expected ${proposedAnnouncingPeriod}`
+    )
+    assert(
+      proposedVotingPeriod.eq(newVotingPeriod),
+      `Voting period has unexpected value ${newVotingPeriod}, expected ${proposedVotingPeriod}`
+    )
+    assert(
+      proposedRevealingPeriod.eq(newRevealingPeriod),
+      `Revealing has unexpected value ${newRevealingPeriod}, expected ${proposedRevealingPeriod}`
+    )
+    assert(
+      proposedCouncilSize.eq(newCouncilSize),
+      `Council size has unexpected value ${newCouncilSize}, expected ${proposedCouncilSize}`
+    )
+    assert(
+      proposedCandidacyLimit.eq(newCandidacyLimit),
+      `Candidacy limit has unexpected value ${newCandidacyLimit}, expected ${proposedCandidacyLimit}`
+    )
+    assert(
+      proposedNewTermDuration.eq(newNewTermDuration),
+      `New term duration has unexpected value ${newNewTermDuration}, expected ${proposedNewTermDuration}`
+    )
+    assert(
+      proposedMinCouncilStake.eq(newMinCouncilStake),
+      `Min council stake has unexpected value ${newMinCouncilStake}, expected ${proposedMinCouncilStake}`
+    )
+    assert(
+      proposedMinVotingStake.eq(newMinVotingStake),
+      `Min voting stake has unexpected value ${newMinVotingStake}, expected ${proposedMinVotingStake}`
+    )
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class SpendingProposalFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private spendingBalance: BN
+  private mintCapacity: BN
+
+  constructor(api: Api, proposer: string, spendingBalance: BN, mintCapacity: BN) {
+    this.api = api
+    this.proposer = proposer
+    this.spendingBalance = spendingBalance
+    this.mintCapacity = mintCapacity
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const description = 'spending proposal which is used for API network testing with some mock data'
+    const runtimeVoteFee: BN = this.api.estimateVoteForProposalFee()
+
+    // Topping the balances
+    const proposalStake: BN = new BN(25000)
+    const runtimeProposalFee: BN = this.api.estimateProposeSpendingFee(
+      description,
+      description,
+      proposalStake,
+      this.spendingBalance,
+      this.proposer
+    )
+    this.api.treasuryTransferBalance(this.proposer, runtimeProposalFee.add(proposalStake))
+    const councilAccounts = await this.api.getCouncilAccounts()
+    this.api.treasuryTransferBalanceToAccounts(councilAccounts, runtimeVoteFee)
+    await this.api.sudoSetCouncilMintCapacity(this.mintCapacity)
+
+    const fundingRecipient = this.api.createKeyPairs(1)[0].address
+
+    // Proposal creation
+    const result = await this.api.proposeSpending(
+      this.proposer,
+      'testing spending' + uuid().substring(0, 8),
+      'spending to test proposal functionality' + uuid().substring(0, 8),
+      proposalStake,
+      this.spendingBalance,
+      fundingRecipient
+    )
+    const proposalNumber: ProposalId = this.api.expectProposalCreatedEvent(result.events)
+
+    // Approving spending proposal
+    const balanceBeforeMinting: BN = await this.api.getBalance(fundingRecipient)
+    this.api.batchApproveProposal(proposalNumber)
+    await this.api.waitForProposalToFinalize(proposalNumber)
+
+    const balanceAfterMinting: BN = await this.api.getBalance(fundingRecipient)
+    assert(
+      balanceAfterMinting.eq(balanceBeforeMinting.add(this.spendingBalance)),
+      `member ${fundingRecipient} has unexpected balance ${balanceAfterMinting}, expected ${balanceBeforeMinting.add(
+        this.spendingBalance
+      )}`
+    )
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class TextProposalFixture implements Fixture {
+  private api: Api
+  private proposer: string
+
+  constructor(api: Api, proposer: string) {
+    this.api = api
+    this.proposer = proposer
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing text proposal ' + uuid().substring(0, 8)
+    const proposalText: string = 'Text of the testing proposal ' + uuid().substring(0, 8)
+    const runtimeVoteFee: BN = this.api.estimateVoteForProposalFee()
+    const councilAccounts = await this.api.getCouncilAccounts()
+    this.api.treasuryTransferBalanceToAccounts(councilAccounts, runtimeVoteFee)
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(25000)
+    const runtimeProposalFee: BN = this.api.estimateProposeTextFee(
+      proposalStake,
+      description,
+      description,
+      proposalText
+    )
+    this.api.treasuryTransferBalance(this.proposer, runtimeProposalFee.add(proposalStake))
+
+    // Proposal creation
+
+    const result = await this.api.proposeText(this.proposer, proposalStake, proposalTitle, description, proposalText)
+    const proposalNumber: ProposalId = this.api.expectProposalCreatedEvent(result.events)
+
+    // Approving text proposal
+    this.api.batchApproveProposal(proposalNumber)
+    await this.api.waitForProposalToFinalize(proposalNumber)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class ValidatorCountProposalFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private validatorCountIncrement: BN
+
+  constructor(api: Api, proposer: string, validatorCountIncrement: BN) {
+    this.api = api
+    this.proposer = proposer
+    this.validatorCountIncrement = validatorCountIncrement
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const proposalTitle: string = 'Testing proposal ' + uuid().substring(0, 8)
+    const description: string = 'Testing validator count proposal ' + uuid().substring(0, 8)
+    const runtimeVoteFee: BN = this.api.estimateVoteForProposalFee()
+    const councilAccounts = await this.api.getCouncilAccounts()
+    this.api.treasuryTransferBalanceToAccounts(councilAccounts, runtimeVoteFee)
+
+    // Proposal stake calculation
+    const proposalStake: BN = new BN(100000)
+    const proposalFee: BN = this.api.estimateProposeValidatorCountFee(description, description, proposalStake)
+    this.api.treasuryTransferBalance(this.proposer, proposalFee.add(proposalStake))
+    const validatorCount: BN = await this.api.getValidatorCount()
+
+    // Proposal creation
+    const proposedValidatorCount: BN = validatorCount.add(this.validatorCountIncrement)
+    const result = await this.api.proposeValidatorCount(
+      this.proposer,
+      proposalTitle,
+      description,
+      proposalStake,
+      proposedValidatorCount
+    )
+    const proposalNumber: ProposalId = this.api.expectProposalCreatedEvent(result.events)
+
+    // Approving the proposal
+    this.api.batchApproveProposal(proposalNumber)
+    await this.api.waitForProposalToFinalize(proposalNumber)
+
+    const newValidatorCount: BN = await this.api.getValidatorCount()
+    assert(
+      proposedValidatorCount.eq(newValidatorCount),
+      `Validator count has unexpeccted value ${newValidatorCount}, expected ${proposedValidatorCount}`
+    )
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class UpdateRuntimeFixture implements Fixture {
+  private api: Api
+  private proposer: string
+  private runtimePath: string
+
+  constructor(api: Api, proposer: string, runtimePath: string) {
+    this.api = api
+    this.proposer = proposer
+    this.runtimePath = runtimePath
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Setup
+    const runtime: string = Utils.readRuntimeFromFile(this.runtimePath)
+    const description = 'runtime upgrade proposal which is used for API network testing'
+    const runtimeVoteFee: BN = this.api.estimateVoteForProposalFee()
+
+    // Topping the balances
+    const proposalStake: BN = new BN(1000000)
+    const runtimeProposalFee: BN = this.api.estimateProposeRuntimeUpgradeFee(
+      proposalStake,
+      description,
+      description,
+      runtime
+    )
+    this.api.treasuryTransferBalance(this.proposer, runtimeProposalFee.add(proposalStake))
+    const councilAccounts = await this.api.getCouncilAccounts()
+    this.api.treasuryTransferBalanceToAccounts(councilAccounts, runtimeVoteFee)
+
+    // Proposal creation
+    const result = await this.api.proposeRuntime(
+      this.proposer,
+      proposalStake,
+      'testing runtime' + uuid().substring(0, 8),
+      'runtime to test proposal functionality' + uuid().substring(0, 8),
+      runtime
+    )
+    const proposalNumber: ProposalId = this.api.expectProposalCreatedEvent(result.events)
+
+    // Approving runtime update proposal
+    this.api.batchApproveProposal(proposalNumber)
+    await this.api.waitForProposalToFinalize(proposalNumber)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class VoteForProposalFixture implements Fixture {
+  private api: Api
+  private proposalNumber: ProposalId
+  private events: EventRecord[] = []
+
+  constructor(api: Api, proposalNumber: ProposalId) {
+    this.api = api
+    this.proposalNumber = proposalNumber
+  }
+
+  public getEvents(): EventRecord[] {
+    return this.events
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const proposalVoteFee: BN = this.api.estimateVoteForProposalFee()
+    const councilAccounts = await this.api.getCouncilAccounts()
+    this.api.treasuryTransferBalanceToAccounts(councilAccounts, proposalVoteFee)
+
+    // Approving the proposal
+    this.api.batchApproveProposal(this.proposalNumber)
+    this.events = await this.api.waitForProposalToFinalize(this.proposalNumber)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}

tests/network-tests/src/fixtures/sudoHireLead.ts

@@ -0,0 +1,100 @@
+import { Fixture } from '../Fixture'
+import {
+  SudoAddLeaderOpeningFixture,
+  ApplyForOpeningFixture,
+  SudoBeginLeaderApplicationReviewFixture,
+  SudoFillLeaderOpeningFixture,
+} from './workingGroupModule'
+import { BuyMembershipHappyCaseFixture } from './membershipModule'
+import { Api, WorkingGroups } from '../Api'
+import { OpeningId } from '@joystream/types/hiring'
+import { PaidTermId } from '@joystream/types/members'
+import BN from 'bn.js'
+import { assert } from 'chai'
+
+export class SudoHireLeadFixture implements Fixture {
+  private api: Api
+  private leadAccount: string
+  private paidTerms: PaidTermId
+  private applicationStake: BN
+  private roleStake: BN
+  private openingActivationDelay: BN
+  private rewardInterval: BN
+  private firstRewardInterval: BN
+  private payoutAmount: BN
+  private workingGroup: WorkingGroups
+
+  constructor(
+    api: Api,
+    leadAccount: string,
+    paidTerms: PaidTermId,
+    applicationStake: BN,
+    roleStake: BN,
+    openingActivationDelay: BN,
+    rewardInterval: BN,
+    firstRewardInterval: BN,
+    payoutAmount: BN,
+    workingGroup: WorkingGroups
+  ) {
+    this.api = api
+    this.leadAccount = leadAccount
+    this.paidTerms = paidTerms
+    this.applicationStake = applicationStake
+    this.roleStake = roleStake
+    this.openingActivationDelay = openingActivationDelay
+    this.rewardInterval = rewardInterval
+    this.firstRewardInterval = firstRewardInterval
+    this.payoutAmount = payoutAmount
+    this.workingGroup = workingGroup
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const leaderHappyCaseFixture: BuyMembershipHappyCaseFixture = new BuyMembershipHappyCaseFixture(
+      this.api,
+      [this.leadAccount],
+      this.paidTerms
+    )
+    // Buying membership for leader account
+    await leaderHappyCaseFixture.runner(false)
+
+    const addLeaderOpeningFixture: SudoAddLeaderOpeningFixture = new SudoAddLeaderOpeningFixture(
+      this.api,
+      this.applicationStake,
+      this.roleStake,
+      this.openingActivationDelay,
+      this.workingGroup
+    )
+    // Add lead opening
+    await addLeaderOpeningFixture.runner(false)
+
+    const applyForLeaderOpeningFixture = new ApplyForOpeningFixture(
+      this.api,
+      [this.leadAccount],
+      this.applicationStake,
+      this.roleStake,
+      addLeaderOpeningFixture.getCreatedOpeningId() as OpeningId,
+      this.workingGroup
+    )
+    await applyForLeaderOpeningFixture.runner(false)
+
+    assert(applyForLeaderOpeningFixture.getApplicationIds().length === 1)
+
+    const beginLeaderApplicationReviewFixture = new SudoBeginLeaderApplicationReviewFixture(
+      this.api,
+      addLeaderOpeningFixture.getCreatedOpeningId() as OpeningId,
+      this.workingGroup
+    )
+    await beginLeaderApplicationReviewFixture.runner(false)
+
+    const fillLeaderOpeningFixture = new SudoFillLeaderOpeningFixture(
+      this.api,
+      applyForLeaderOpeningFixture.getApplicationIds()[0],
+      addLeaderOpeningFixture.getCreatedOpeningId() as OpeningId,
+      this.firstRewardInterval,
+      this.rewardInterval,
+      this.payoutAmount,
+      this.workingGroup
+    )
+    await fillLeaderOpeningFixture.runner(false)
+  }
+}

tests/network-tests/src/fixtures/workingGroupModule.ts

@@ -0,0 +1,770 @@
+import BN from 'bn.js'
+import { assert } from 'chai'
+import { Api, WorkingGroups } from '../Api'
+import { KeyringPair } from '@polkadot/keyring/types'
+import { v4 as uuid } from 'uuid'
+import { RewardRelationship } from '@joystream/types/recurring-rewards'
+import { Application, ApplicationIdToWorkerIdMap, Worker, WorkerId } from '@joystream/types/working-group'
+import { Utils } from '../utils'
+import { ApplicationId, Opening as HiringOpening, OpeningId } from '@joystream/types/hiring'
+import { Fixture } from '../Fixture'
+
+export class AddWorkerOpeningFixture implements Fixture {
+  private api: Api
+  private applicationStake: BN
+  private roleStake: BN
+  private activationDelay: BN
+  private unstakingPeriod: BN
+  private module: WorkingGroups
+
+  private result: OpeningId | undefined
+
+  public getCreatedOpeningId(): OpeningId | undefined {
+    return this.result
+  }
+
+  public constructor(
+    api: Api,
+    applicationStake: BN,
+    roleStake: BN,
+    activationDelay: BN,
+    unstakingPeriod: BN,
+    module: WorkingGroups
+  ) {
+    this.api = api
+    this.applicationStake = applicationStake
+    this.roleStake = roleStake
+    this.activationDelay = activationDelay
+    this.unstakingPeriod = unstakingPeriod
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const lead = await this.api.getGroupLead(this.module)
+    if (!lead) {
+      throw new Error('No Lead')
+    }
+    // Fee estimation and transfer
+    const addOpeningFee: BN = this.api.estimateAddOpeningFee(this.module)
+    this.api.treasuryTransferBalance(lead.role_account_id.toString(), addOpeningFee)
+
+    // Worker opening creation
+    const result = await this.api.addOpening(
+      lead.role_account_id.toString(),
+      {
+        activationDelay: this.activationDelay,
+        maxActiveApplicants: new BN(10),
+        maxReviewPeriodLength: new BN(32),
+        applicationStakingPolicyAmount: this.applicationStake,
+        applicationCrowdedOutUnstakingPeriodLength: new BN(1),
+        applicationReviewPeriodExpiredUnstakingPeriodLength: new BN(1),
+        roleStakingPolicyAmount: this.roleStake,
+        roleCrowdedOutUnstakingPeriodLength: new BN(1),
+        roleReviewPeriodExpiredUnstakingPeriodLength: new BN(1),
+        slashableMaxCount: new BN(1),
+        slashableMaxPercentPtsPerTime: new BN(100),
+        fillOpeningSuccessfulApplicantApplicationStakeUnstakingPeriod: this.unstakingPeriod,
+        fillOpeningFailedApplicantApplicationStakeUnstakingPeriod: this.unstakingPeriod,
+        fillOpeningFailedApplicantRoleStakeUnstakingPeriod: this.unstakingPeriod,
+        terminateApplicationStakeUnstakingPeriod: this.unstakingPeriod,
+        terminateRoleStakeUnstakingPeriod: this.unstakingPeriod,
+        exitRoleApplicationStakeUnstakingPeriod: this.unstakingPeriod,
+        exitRoleStakeUnstakingPeriod: this.unstakingPeriod,
+        text: uuid().substring(0, 8),
+        type: 'Worker',
+      },
+      this.module,
+      expectFailure
+    )
+
+    if (!expectFailure) {
+      this.result = this.api.expectOpeningAddedEvent(result.events)
+    }
+  }
+}
+
+export class SudoAddLeaderOpeningFixture implements Fixture {
+  private api: Api
+  private applicationStake: BN
+  private roleStake: BN
+  private activationDelay: BN
+  private module: WorkingGroups
+
+  private result: OpeningId | undefined
+
+  public getCreatedOpeningId(): OpeningId | undefined {
+    return this.result
+  }
+
+  public constructor(api: Api, applicationStake: BN, roleStake: BN, activationDelay: BN, module: WorkingGroups) {
+    this.api = api
+    this.applicationStake = applicationStake
+    this.roleStake = roleStake
+    this.activationDelay = activationDelay
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const result = await this.api.sudoAddOpening(
+      {
+        activationDelay: this.activationDelay,
+        maxActiveApplicants: new BN(10),
+        maxReviewPeriodLength: new BN(32),
+        applicationStakingPolicyAmount: this.applicationStake,
+        applicationCrowdedOutUnstakingPeriodLength: new BN(1),
+        applicationReviewPeriodExpiredUnstakingPeriodLength: new BN(1),
+        roleStakingPolicyAmount: this.roleStake,
+        roleCrowdedOutUnstakingPeriodLength: new BN(1),
+        roleReviewPeriodExpiredUnstakingPeriodLength: new BN(1),
+        slashableMaxCount: new BN(1),
+        slashableMaxPercentPtsPerTime: new BN(100),
+        fillOpeningSuccessfulApplicantApplicationStakeUnstakingPeriod: new BN(1),
+        fillOpeningFailedApplicantApplicationStakeUnstakingPeriod: new BN(1),
+        fillOpeningFailedApplicantRoleStakeUnstakingPeriod: new BN(1),
+        terminateApplicationStakeUnstakingPeriod: new BN(1),
+        terminateRoleStakeUnstakingPeriod: new BN(1),
+        exitRoleApplicationStakeUnstakingPeriod: new BN(1),
+        exitRoleStakeUnstakingPeriod: new BN(1),
+        text: uuid().substring(0, 8),
+        type: 'Leader',
+      },
+      this.module
+    )
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    } else {
+      this.result = this.api.expectOpeningAddedEvent(result.events)
+    }
+  }
+}
+
+export class AcceptApplicationsFixture implements Fixture {
+  private api: Api
+  private openingId: OpeningId
+  private module: WorkingGroups
+
+  public constructor(api: Api, openingId: OpeningId, module: WorkingGroups) {
+    this.api = api
+    this.openingId = openingId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const lead = await this.api.getGroupLead(this.module)
+    if (!lead) {
+      throw new Error('No Lead')
+    }
+    const leadAccount = lead.role_account_id.toString()
+    // Fee estimation and transfer
+    const acceptApplicationsFee: BN = this.api.estimateAcceptApplicationsFee(this.module)
+    this.api.treasuryTransferBalance(leadAccount, acceptApplicationsFee)
+
+    // Begin accepting applications
+    await this.api.acceptApplications(leadAccount, this.openingId, this.module)
+    const wgOpening = await this.api.getWorkingGroupOpening(this.openingId, this.module)
+    const opening: HiringOpening = await this.api.getHiringOpening(wgOpening.hiring_opening_id)
+    assert(opening.is_active, `${this.module} Opening ${this.openingId} is not active`)
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class ApplyForOpeningFixture implements Fixture {
+  private api: Api
+  private applicants: string[]
+  private applicationStake: BN
+  private roleStake: BN
+  private openingId: OpeningId
+  private module: WorkingGroups
+  private result: ApplicationId[] = []
+
+  public constructor(
+    api: Api,
+    applicants: string[],
+    applicationStake: BN,
+    roleStake: BN,
+    openingId: OpeningId,
+    module: WorkingGroups
+  ) {
+    this.api = api
+    this.applicants = applicants
+    this.applicationStake = applicationStake
+    this.roleStake = roleStake
+    this.openingId = openingId
+    this.module = module
+  }
+
+  public getApplicationIds(): ApplicationId[] {
+    return this.result
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Fee estimation and transfer
+    const applyOnOpeningFee: BN = this.api
+      .estimateApplyOnOpeningFee(this.applicants[0], this.module)
+      .add(this.applicationStake)
+      .add(this.roleStake)
+    this.api.treasuryTransferBalanceToAccounts(this.applicants, applyOnOpeningFee)
+
+    // Applying for created worker opening
+    const results = await this.api.batchApplyOnOpening(
+      this.applicants,
+      this.openingId,
+      this.roleStake,
+      this.applicationStake,
+      uuid().substring(0, 8),
+      this.module,
+      expectFailure
+    )
+
+    const applicationIds = results.map(({ events }) => {
+      const record = events.find(
+        (record) => record.event.method && record.event.method.toString() === 'AppliedOnOpening'
+      )
+      if (record) {
+        return (record.event.data[1] as unknown) as ApplicationId
+      }
+      throw new Error('Application on opening failed')
+    })
+
+    this.result = applicationIds
+  }
+}
+
+export class WithdrawApplicationFixture implements Fixture {
+  private api: Api
+  private applicationIds: ApplicationId[]
+  private module: WorkingGroups
+
+  constructor(api: Api, applicationIds: ApplicationId[], module: WorkingGroups) {
+    this.api = api
+    this.applicationIds = applicationIds
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Fee estimation and transfer
+    const withdrawApplicaitonFee: BN = this.api.estimateWithdrawApplicationFee(this.module)
+
+    // get role accounts of applicants
+    const roleAccounts = await this.api.getApplicantRoleAccounts(this.applicationIds, this.module)
+    this.api.treasuryTransferBalanceToAccounts(roleAccounts, withdrawApplicaitonFee)
+
+    // Application withdrawal
+    await this.api.batchWithdrawActiveApplications(this.applicationIds, this.module)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class BeginApplicationReviewFixture implements Fixture {
+  private api: Api
+  private openingId: OpeningId
+  private module: WorkingGroups
+
+  constructor(api: Api, openingId: OpeningId, module: WorkingGroups) {
+    this.api = api
+    this.openingId = openingId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const lead = await this.api.getGroupLead(this.module)
+    if (!lead) {
+      throw new Error('No Lead')
+    }
+    const leadAccount = lead.role_account_id.toString()
+    // Fee estimation and transfer
+    const beginReviewFee: BN = this.api.estimateBeginApplicantReviewFee(this.module)
+    this.api.treasuryTransferBalance(leadAccount, beginReviewFee)
+
+    // Begin application review
+    // const beginApplicantReviewPromise: Promise<ApplicationId> = this.api.expectApplicationReviewBegan()
+    const result = await this.api.beginApplicantReview(leadAccount, this.openingId, this.module)
+
+    this.api.expectApplicationReviewBeganEvent(result.events)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class SudoBeginLeaderApplicationReviewFixture implements Fixture {
+  private api: Api
+  private openingId: OpeningId
+  private module: WorkingGroups
+
+  constructor(api: Api, openingId: OpeningId, module: WorkingGroups) {
+    this.api = api
+    this.openingId = openingId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Begin application review
+    await this.api.sudoBeginApplicantReview(this.openingId, this.module)
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class FillOpeningFixture implements Fixture {
+  private api: Api
+  private applicationIds: ApplicationId[]
+  private openingId: OpeningId
+  private firstPayoutInterval: BN
+  private payoutInterval: BN
+  private amountPerPayout: BN
+  private module: WorkingGroups
+  private workerIds: WorkerId[] = []
+
+  constructor(
+    api: Api,
+    applicationIds: ApplicationId[],
+    openingId: OpeningId,
+    firstPayoutInterval: BN,
+    payoutInterval: BN,
+    amountPerPayout: BN,
+    module: WorkingGroups
+  ) {
+    this.api = api
+    this.applicationIds = applicationIds
+    this.openingId = openingId
+    this.firstPayoutInterval = firstPayoutInterval
+    this.payoutInterval = payoutInterval
+    this.amountPerPayout = amountPerPayout
+    this.module = module
+  }
+
+  public getWorkerIds(): WorkerId[] {
+    return this.workerIds
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const lead = await this.api.getGroupLead(this.module)
+    if (!lead) {
+      throw new Error('No Lead')
+    }
+    const leadAccount = lead.role_account_id.toString()
+    // Fee estimation and transfer
+    const beginReviewFee: BN = this.api.estimateFillOpeningFee(this.module)
+    this.api.treasuryTransferBalance(leadAccount, beginReviewFee)
+
+    // Assert max number of workers is not exceeded
+    const activeWorkersCount: BN = await this.api.getActiveWorkersCount(this.module)
+    const maxWorkersCount: BN = this.api.getMaxWorkersCount(this.module)
+    assert(
+      activeWorkersCount.addn(this.applicationIds.length).lte(maxWorkersCount),
+      `The number of workers ${activeWorkersCount.addn(
+        this.applicationIds.length
+      )} will exceed max workers count ${maxWorkersCount}`
+    )
+
+    // Fill worker opening
+    const now: BN = await this.api.getBestBlock()
+    const result = await this.api.fillOpening(
+      leadAccount,
+      this.openingId,
+      this.applicationIds,
+      this.amountPerPayout,
+      now.add(this.firstPayoutInterval),
+      this.payoutInterval,
+      this.module
+    )
+    const applicationIdToWorkerIdMap: ApplicationIdToWorkerIdMap = this.api.expectOpeningFilledEvent(result.events)
+    this.workerIds = []
+    applicationIdToWorkerIdMap.forEach((workerId) => this.workerIds.push(workerId))
+
+    // Assertions
+    applicationIdToWorkerIdMap.forEach(async (workerId, applicationId) => {
+      const worker: Worker = await this.api.getWorkerById(workerId, this.module)
+      const application: Application = await this.api.getApplicationById(applicationId, this.module)
+      assert(
+        worker.role_account_id.toString() === application.role_account_id.toString(),
+        `Role account ids does not match, worker account: ${worker.role_account_id}, application account ${application.role_account_id}`
+      )
+    })
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class SudoFillLeaderOpeningFixture implements Fixture {
+  private api: Api
+  private applicationId: ApplicationId
+  private openingId: OpeningId
+  private firstPayoutInterval: BN
+  private payoutInterval: BN
+  private amountPerPayout: BN
+  private module: WorkingGroups
+
+  constructor(
+    api: Api,
+    applicationId: ApplicationId,
+    openingId: OpeningId,
+    firstPayoutInterval: BN,
+    payoutInterval: BN,
+    amountPerPayout: BN,
+    module: WorkingGroups
+  ) {
+    this.api = api
+    this.applicationId = applicationId
+    this.openingId = openingId
+    this.firstPayoutInterval = firstPayoutInterval
+    this.payoutInterval = payoutInterval
+    this.amountPerPayout = amountPerPayout
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Fill leader opening
+    const now: BN = await this.api.getBestBlock()
+    const result = await this.api.sudoFillOpening(
+      this.openingId,
+      [this.applicationId],
+      this.amountPerPayout,
+      now.add(this.firstPayoutInterval),
+      this.payoutInterval,
+      this.module
+    )
+
+    // Assertions
+    const applicationIdToWorkerIdMap = this.api.expectOpeningFilledEvent(result.events)
+    assert(applicationIdToWorkerIdMap.size === 1)
+    applicationIdToWorkerIdMap.forEach(async (workerId, applicationId) => {
+      const worker: Worker = await this.api.getWorkerById(workerId, this.module)
+      const application: Application = await this.api.getApplicationById(applicationId, this.module)
+      const leadWorkerId: WorkerId = (await this.api.getLeadWorkerId(this.module))!
+      assert(
+        worker.role_account_id.toString() === application.role_account_id.toString(),
+        `Role account ids does not match, leader account: ${worker.role_account_id}, application account ${application.role_account_id}`
+      )
+      assert(
+        leadWorkerId.eq(workerId),
+        `Role account ids does not match, leader account: ${worker.role_account_id}, application account ${application.role_account_id}`
+      )
+    })
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class IncreaseStakeFixture implements Fixture {
+  private api: Api
+  private workerId: WorkerId
+  private module: WorkingGroups
+
+  constructor(api: Api, workerId: WorkerId, module: WorkingGroups) {
+    this.api = api
+    this.workerId = workerId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    // Fee estimation and transfer
+    const increaseStakeFee: BN = this.api.estimateIncreaseStakeFee(this.module)
+    const stakeIncrement: BN = new BN(1)
+    const worker = await this.api.getWorkerById(this.workerId, this.module)
+    const workerRoleAccount = worker.role_account_id.toString()
+    this.api.treasuryTransferBalance(workerRoleAccount, increaseStakeFee.add(stakeIncrement))
+
+    // Increase worker stake
+    const increasedWorkerStake: BN = (await this.api.getWorkerStakeAmount(this.workerId, this.module)).add(
+      stakeIncrement
+    )
+    await this.api.increaseStake(workerRoleAccount, this.workerId, stakeIncrement, this.module)
+    const newWorkerStake: BN = await this.api.getWorkerStakeAmount(this.workerId, this.module)
+    assert(
+      increasedWorkerStake.eq(newWorkerStake),
+      `Unexpected worker stake ${newWorkerStake}, expected ${increasedWorkerStake}`
+    )
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class UpdateRewardAccountFixture implements Fixture {
+  public api: Api
+  public workerId: WorkerId
+  public module: WorkingGroups
+
+  constructor(api: Api, workerId: WorkerId, module: WorkingGroups) {
+    this.api = api
+    this.workerId = workerId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const worker = await this.api.getWorkerById(this.workerId, this.module)
+    const workerRoleAccount = worker.role_account_id.toString()
+    // Fee estimation and transfer
+    const updateRewardAccountFee: BN = this.api.estimateUpdateRewardAccountFee(workerRoleAccount, this.module)
+    this.api.treasuryTransferBalance(workerRoleAccount, updateRewardAccountFee)
+
+    // Update reward account
+    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0]
+    await this.api.updateRewardAccount(workerRoleAccount, this.workerId, createdAccount.address, this.module)
+    const newRewardAccount: string = await this.api.getWorkerRewardAccount(this.workerId, this.module)
+    assert(
+      newRewardAccount === createdAccount.address,
+      `Unexpected role account ${newRewardAccount}, expected ${createdAccount.address}`
+    )
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class UpdateRoleAccountFixture implements Fixture {
+  private api: Api
+  private workerId: WorkerId
+  private module: WorkingGroups
+
+  constructor(api: Api, workerId: WorkerId, module: WorkingGroups) {
+    this.api = api
+    this.workerId = workerId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const worker = await this.api.getWorkerById(this.workerId, this.module)
+    const workerRoleAccount = worker.role_account_id.toString()
+    // Fee estimation and transfer
+    const updateRoleAccountFee: BN = this.api.estimateUpdateRoleAccountFee(workerRoleAccount, this.module)
+
+    this.api.treasuryTransferBalance(workerRoleAccount, updateRoleAccountFee)
+
+    // Update role account
+    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0]
+    await this.api.updateRoleAccount(workerRoleAccount, this.workerId, createdAccount.address, this.module)
+    const newRoleAccount: string = (await this.api.getWorkerById(this.workerId, this.module)).role_account_id.toString()
+    assert(
+      newRoleAccount === createdAccount.address,
+      `Unexpected role account ${newRoleAccount}, expected ${createdAccount.address}`
+    )
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class TerminateApplicationsFixture implements Fixture {
+  private api: Api
+  private applicationIds: ApplicationId[]
+  private module: WorkingGroups
+
+  constructor(api: Api, applicationIds: ApplicationId[], module: WorkingGroups) {
+    this.api = api
+    this.applicationIds = applicationIds
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const lead = await this.api.getGroupLead(this.module)
+    if (!lead) {
+      throw new Error('No Lead')
+    }
+    const leadAccount = lead.role_account_id.toString()
+
+    // Fee estimation and transfer
+    const terminateApplicationFee: BN = this.api.estimateTerminateApplicationFee(this.module)
+    this.api.treasuryTransferBalance(leadAccount, terminateApplicationFee.muln(this.applicationIds.length))
+
+    // Terminate worker applications
+    await this.api.batchTerminateApplication(leadAccount, this.applicationIds, this.module)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class DecreaseStakeFixture implements Fixture {
+  private api: Api
+  private workerId: WorkerId
+  private module: WorkingGroups
+
+  constructor(api: Api, workerId: WorkerId, module: WorkingGroups) {
+    this.api = api
+    this.workerId = workerId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const lead = await this.api.getGroupLead(this.module)
+    if (!lead) {
+      throw new Error('No Lead')
+    }
+    const leadAccount = lead.role_account_id.toString()
+
+    // Fee estimation and transfer
+    const decreaseStakeFee: BN = this.api.estimateDecreaseStakeFee(this.module)
+    this.api.treasuryTransferBalance(leadAccount, decreaseStakeFee)
+    const workerStakeDecrement: BN = new BN(1)
+
+    // Worker stake decrement
+    const decreasedWorkerStake: BN = (await this.api.getWorkerStakeAmount(this.workerId, this.module)).sub(
+      workerStakeDecrement
+    )
+    await this.api.decreaseStake(leadAccount, this.workerId, workerStakeDecrement, this.module, expectFailure)
+    const newWorkerStake: BN = await this.api.getWorkerStakeAmount(this.workerId, this.module)
+
+    // Assertions
+    if (!expectFailure) {
+      assert(
+        decreasedWorkerStake.eq(newWorkerStake),
+        `Unexpected worker stake ${newWorkerStake}, expected ${decreasedWorkerStake}`
+      )
+    }
+  }
+}
+
+export class SlashFixture implements Fixture {
+  private api: Api
+  private workerId: WorkerId
+  private module: WorkingGroups
+
+  constructor(api: Api, workerId: WorkerId, module: WorkingGroups) {
+    this.api = api
+    this.workerId = workerId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const lead = await this.api.getGroupLead(this.module)
+    if (!lead) {
+      throw new Error('No Lead')
+    }
+    const leadAccount = lead.role_account_id.toString()
+
+    // Fee estimation and transfer
+    const slashStakeFee: BN = this.api.estimateSlashStakeFee(this.module)
+    this.api.treasuryTransferBalance(leadAccount, slashStakeFee)
+    const slashAmount: BN = new BN(1)
+
+    // Slash worker
+    const slashedStake: BN = (await this.api.getWorkerStakeAmount(this.workerId, this.module)).sub(slashAmount)
+    await this.api.slashStake(leadAccount, this.workerId, slashAmount, this.module, expectFailure)
+    const newStake: BN = await this.api.getWorkerStakeAmount(this.workerId, this.module)
+
+    // Assertions
+    assert(slashedStake.eq(newStake), `Unexpected worker stake ${newStake}, expected ${slashedStake}`)
+  }
+}
+
+export class TerminateRoleFixture implements Fixture {
+  private api: Api
+  private workerId: WorkerId
+  private module: WorkingGroups
+
+  constructor(api: Api, workerId: WorkerId, module: WorkingGroups) {
+    this.api = api
+    this.workerId = workerId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const lead = await this.api.getGroupLead(this.module)
+    if (!lead) {
+      throw new Error('No Lead')
+    }
+    const leadAccount = lead.role_account_id.toString()
+
+    // Fee estimation and transfer
+    const terminateRoleFee: BN = this.api.estimateTerminateRoleFee(this.module)
+    this.api.treasuryTransferBalance(leadAccount, terminateRoleFee)
+
+    // Terminate worker role
+    await this.api.terminateRole(leadAccount, this.workerId, uuid().substring(0, 8), this.module, expectFailure)
+
+    // Assertions
+    const isWorker: boolean = await this.api.isWorker(this.workerId, this.module)
+    assert(!isWorker, `Worker ${this.workerId} is not terminated`)
+  }
+}
+
+export class LeaveRoleFixture implements Fixture {
+  private api: Api
+  private workerIds: WorkerId[]
+  private module: WorkingGroups
+
+  constructor(api: Api, workerIds: WorkerId[], module: WorkingGroups) {
+    this.api = api
+    this.workerIds = workerIds
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const roleAccounts = await this.api.getWorkerRoleAccounts(this.workerIds, this.module)
+    // Fee estimation and transfer
+    const leaveRoleFee: BN = this.api.estimateLeaveRoleFee(this.module)
+    this.api.treasuryTransferBalanceToAccounts(roleAccounts, leaveRoleFee)
+
+    await this.api.batchLeaveRole(this.workerIds, uuid().substring(0, 8), expectFailure, this.module)
+
+    // Assertions
+    this.workerIds.forEach(async (workerId) => {
+      const isWorker: boolean = await this.api.isWorker(workerId, this.module)
+      assert(!isWorker, `Worker${workerId} is not terminated`)
+    })
+  }
+}
+
+export class AwaitPayoutFixture implements Fixture {
+  private api: Api
+  private workerId: WorkerId
+  private module: WorkingGroups
+
+  constructor(api: Api, workerId: WorkerId, module: WorkingGroups) {
+    this.api = api
+    this.workerId = workerId
+    this.module = module
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    const worker: Worker = await this.api.getWorkerById(this.workerId, this.module)
+    const reward: RewardRelationship = await this.api.getRewardRelationship(worker.reward_relationship.unwrap())
+    const now: BN = await this.api.getBestBlock()
+    const nextPaymentBlock: BN = new BN(reward.getField('next_payment_at_block').toString())
+    const payoutInterval: BN = new BN(reward.getField('payout_interval').toString())
+    const amountPerPayout: BN = new BN(reward.getField('amount_per_payout').toString())
+
+    assert(now.lt(nextPaymentBlock), `Payout already happened in block ${nextPaymentBlock} now ${now}`)
+    const balance: BN = await this.api.getBalance(reward.account.toString())
+
+    const firstPayoutWaitingPeriod: BN = nextPaymentBlock.sub(now).addn(1)
+    await Utils.wait(this.api.getBlockDuration().mul(firstPayoutWaitingPeriod).toNumber())
+
+    const balanceAfterFirstPayout: BN = await this.api.getBalance(reward.account.toString())
+    const expectedBalanceFirst: BN = balance.add(amountPerPayout)
+    assert(
+      balanceAfterFirstPayout.eq(expectedBalanceFirst),
+      `Unexpected balance, expected ${expectedBalanceFirst} got ${balanceAfterFirstPayout}`
+    )
+
+    const secondPayoutWaitingPeriod: BN = payoutInterval.addn(1)
+    await Utils.wait(this.api.getBlockDuration().mul(secondPayoutWaitingPeriod).toNumber())
+
+    const balanceAfterSecondPayout: BN = await this.api.getBalance(reward.account.toString())
+    const expectedBalanceSecond: BN = expectedBalanceFirst.add(amountPerPayout)
+    assert(
+      balanceAfterSecondPayout.eq(expectedBalanceSecond),
+      `Unexpected balance, expected ${expectedBalanceSecond} got ${balanceAfterSecondPayout}`
+    )
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}

tests/network-tests/src/flows/membership/creatingMemberships.ts

@@ -0,0 +1,36 @@
+import { Api } from '../../Api'
+import {
+  BuyMembershipHappyCaseFixture,
+  BuyMembershipWithInsufficienFundsFixture,
+} from '../../fixtures/membershipModule'
+import { PaidTermId } from '@joystream/types/members'
+import BN from 'bn.js'
+import Debugger from 'debug'
+
+export default async function membershipCreation(api: Api, env: NodeJS.ProcessEnv) {
+  const debug = Debugger('flow:memberships')
+  debug('started')
+
+  const N: number = +env.MEMBERSHIP_CREATION_N!
+  const nAccounts = api.createKeyPairs(N).map((key) => key.address)
+  const aAccount = api.createKeyPairs(1)[0].address
+  const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
+
+  const happyCaseFixture = new BuyMembershipHappyCaseFixture(api, nAccounts, paidTerms)
+  // Buy membeship is accepted with sufficient funds
+  await happyCaseFixture.runner(false)
+
+  const insufficientFundsFixture: BuyMembershipWithInsufficienFundsFixture = new BuyMembershipWithInsufficienFundsFixture(
+    api,
+    aAccount,
+    paidTerms
+  )
+  // Account A can not buy the membership with insufficient funds
+  await insufficientFundsFixture.runner(false)
+
+  const buyMembershipAfterAccountTopUp = new BuyMembershipHappyCaseFixture(api, [aAccount], paidTerms)
+
+  // Account A was able to buy the membership with sufficient funds
+  await buyMembershipAfterAccountTopUp.runner(false)
+  debug('finished')
+}

tests/network-tests/src/flows/proposals/councilSetup.ts

@@ -0,0 +1,46 @@
+import BN from 'bn.js'
+import { PaidTermId } from '@joystream/types/members'
+import { Api } from '../../Api'
+import { CouncilElectionHappyCaseFixture } from '../../fixtures/councilElectionHappyCase'
+import { BuyMembershipHappyCaseFixture } from '../../fixtures/membershipModule'
+import Debugger from 'debug'
+import { assert } from 'chai'
+
+const debug = Debugger('flow:councilSetup')
+
+export default async function councilSetup(api: Api, env: NodeJS.ProcessEnv) {
+  // Skip creating council if already elected
+  if ((await api.getCouncil()).length) {
+    debug('Skipping Council Setup, Council already elected')
+    return
+  }
+
+  const numberOfApplicants = (await api.getCouncilSize()).toNumber() * 2
+  const applicants = api.createKeyPairs(numberOfApplicants).map((key) => key.address)
+  const voters = api.createKeyPairs(5).map((key) => key.address)
+
+  const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
+  const K: number = +env.COUNCIL_ELECTION_K!
+  const greaterStake: BN = new BN(+env.COUNCIL_STAKE_GREATER_AMOUNT!)
+  const lesserStake: BN = new BN(+env.COUNCIL_STAKE_LESSER_AMOUNT!)
+
+  const createMembersFixture = new BuyMembershipHappyCaseFixture(api, [...voters, ...applicants], paidTerms)
+  await createMembersFixture.runner(false)
+
+  // The fixture moves manually with sudo the election stages, so proper processing
+  // that normally occurs during stage transitions does not happen. This can lead to a council
+  // that is smaller than the council size if not enough members apply.
+  const councilElectionHappyCaseFixture = new CouncilElectionHappyCaseFixture(
+    api,
+    voters, // should be member ids
+    applicants, // should be member ids
+    K,
+    greaterStake,
+    lesserStake
+  )
+
+  await councilElectionHappyCaseFixture.runner(false)
+
+  // Elected council
+  assert((await api.getCouncil()).length)
+}