Merge branch 'master' into query_node_sumer_features_lint

.dockerignore

@@ -6,3 +6,4 @@ query-node/**/dist
 query-node/lib
 cli/
 tests/
+devops/

.github/workflows/create-ami.yml

@@ -0,0 +1,63 @@
+name: Build code and create AMI
+
+on:
+  push:
+    branches:
+      - master
+      - olympia
+      - create-joystream-node-ami
+
+jobs:
+  build:
+    name: Build the code and run setup
+    runs-on: ubuntu-latest
+    env:
+      STACK_NAME: joystream-github-action-${{ github.run_number }}
+      KEY_NAME: joystream-github-action-key
+    steps:
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+
+    - name: Set AMI Name environment variable
+      shell: bash
+      run: echo "ami_name=joystream-${{ steps.extract_branch.outputs.branch }}-${{ github.run_number }}" >> $GITHUB_ENV
+      id: ami_name
+
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Deploy to AWS CloudFormation
+      uses: aws-actions/aws-cloudformation-github-deploy@v1
+      id: deploy_stack
+      with:
+        name: ${{ env.STACK_NAME }}
+        template: devops/infrastructure/single-instance.yml
+        no-fail-on-empty-changeset: "1"
+        parameter-overrides: "KeyName=${{ env.KEY_NAME }}"
+
+    - name: Install Ansible dependencies
+      run: pipx inject ansible-base boto3 botocore
+
+    - name: Run playbook
+      uses: dawidd6/action-ansible-playbook@v2
+      with:
+        playbook: github-action-playbook.yml
+        directory: devops/infrastructure
+        requirements: requirements.yml
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        inventory: |
+          [all]
+          ${{ steps.deploy_stack.outputs.PublicIp }}
+        options: |
+          --extra-vars "git_repo=https://github.com/${{ github.repository }} \
+                        branch_name=${{ steps.extract_branch.outputs.branch }} instance_id=${{ steps.deploy_stack.outputs.InstanceId }}
+                        stack_name=${{ env.STACK_NAME }} ami_name=${{ env.ami_name }}"

.github/workflows/run-network-tests.yml

@@ -103,7 +103,7 @@ jobs:
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       - name: Execute network tests
-        run: RUNTIME=antioch tests/network-tests/run-tests.sh full
+        run: RUNTIME=sumer tests/network-tests/run-tests.sh full
 
   basic_runtime:
     name: Integration Tests (New Chain)
@@ -193,7 +193,7 @@ jobs:
           docker-compose up -d joystream-node
       - name: Configure and start development storage node
         run: |
-          DEBUG=* yarn storage-cli dev-init
+          DEBUG=joystream:* yarn storage-cli dev-init
           docker-compose up -d colossus
       - name: Test uploading
         run: |

Cargo.lock

@@ -2332,7 +2332,7 @@ dependencies = [
 
 [[package]]
 name = "joystream-node"
-version = "5.5.0"
+version = "5.6.0"
 dependencies = [
  "frame-benchmarking",
  "frame-benchmarking-cli",
@@ -2393,7 +2393,7 @@ dependencies = [
 
 [[package]]
 name = "joystream-node-runtime"
-version = "9.7.0"
+version = "9.8.0"
 dependencies = [
  "frame-benchmarking",
  "frame-executive",

colossus.Dockerfile

@@ -0,0 +1,16 @@
+FROM --platform=linux/x86-64 node:14 as builder
+
+WORKDIR /joystream
+COPY . /joystream
+RUN  rm -fr /joystream/pioneer
+
+EXPOSE 3001
+
+RUN yarn --frozen-lockfile
+
+RUN yarn workspace @joystream/types build
+RUN yarn workspace storage-node build
+
+RUN yarn
+
+ENTRYPOINT yarn colossus --dev --ws-provider $WS_PROVIDER_ENDPOINT_URI

devops/infrastructure/.gitignore

@@ -0,0 +1,5 @@
+# Ignore files created by deployment scripts
+bash-config.cfg
+inventory
+data-*
+chain_spec_output.txt

devops/infrastructure/README.md

@@ -0,0 +1,35 @@
+## Setup
+
+### Configuring the AWS CLI
+We’re going to use the AWS CLI to access AWS resources from the command line. 
+
+Follow [the official directions](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) for your system.
+
+Once the AWS CLI is installed, configure a profile
+
+`aws configure --profile joystream-user`
+
+### Create a key pair
+Change profile and region parameters according to your configuration
+```
+aws ec2 create-key-pair --key-name joystream-key --profile joystream-user --region us-east-1 --query 'KeyMaterial' --output text > joystream-key.pem
+```
+
+Set the permissions for the key pair 
+
+`chmod 400 joystream-key.pem`
+
+### Install Ansible
+On Mac run the command:
+* `brew install ansible`
+
+Follow [the official installation guide](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) for your system.
+
+# How to run
+Copy and edit the file `deploy-config.sample.cfg` and update parameters like AWS_KEY_PAIR_NAME, KEY_PATH
+Run the `deploy-infra.sh` script to deploy the infrastructure
+
+```
+cd devops/infrastructure
+./deploy-infra.sh your-deploy-config.cfg
+```

devops/infrastructure/ansible.cfg

@@ -0,0 +1,8 @@
+[defaults]
+host_key_checking = False
+remote_user = ubuntu
+# Use the YAML callback plugin.
+stdout_callback = yaml
+# Use the stdout_callback when running ad-hoc commands.
+bin_ansible_callbacks = True
+interpreter_python = /usr/bin/python3

devops/infrastructure/build-code.yml

@@ -0,0 +1,36 @@
+---
+
+- name: Get latest Joystream code, build it and copy binary to local
+  hosts: build
+  gather_facts: no
+  tasks:
+    - name: Get code from local or git repo
+      include_role:
+        name: common
+        tasks_from: "{{ 'get-code-local' if build_local_code|bool else 'get-code-git' }}"
+
+    - name: Run setup and build
+      include_role:
+        name: common
+        tasks_from: run-setup-build
+
+    - name: Copy joystream-node binary from build to local
+      fetch:
+        src: "{{ remote_code_path }}/target/release/joystream-node"
+        dest: "{{ data_path }}/joystream-node"
+        flat: yes
+
+- name: Copy binary to remote servers
+  hosts: all
+  gather_facts: no
+  tasks:
+    - name: Create release directory
+      file:
+        path: "{{ remote_code_path }}/target/release"
+        state: directory
+
+    - name: Copying joystream-node binary to all servers
+      copy:
+        src: "{{ data_path }}/joystream-node"
+        dest: "{{ remote_code_path }}/target/release/joystream-node"
+        mode: "0775"

devops/infrastructure/chain-spec-pioneer.yml

@@ -0,0 +1,37 @@
+---
+# Configure chain spec, start joystream-node service on the servers and build Pioneer
+
+- name: Create and copy the chain-spec file
+  hosts: all
+
+  tasks:
+    - name: Generate chain-spec file and data keys either on localhost or admin server
+      include_role:
+        name: common
+        tasks_from: chain-spec-node-keys
+      vars:
+        local_or_admin: "{{ groups['build'][0] if run_on_admin_server|bool else 'localhost' }}"
+        admin_code_dir: "{{ remote_code_path if run_on_admin_server|bool else local_dir }}"
+
+- name: Copy secret, auth and start joystream-node service for validators
+  hosts: validators
+  gather_facts: no
+
+  roles:
+    - validators
+
+- name: Configure RPC service and start it
+  hosts: rpc
+  gather_facts: no
+
+  roles:
+    - rpc
+
+- name: Build Pioneer and copy artifacts to S3
+  hosts: build
+  gather_facts: no
+
+  tasks:
+    - include_role:
+        name: admin
+        tasks_from: deploy-pioneer

devops/infrastructure/common.sh

@@ -0,0 +1,9 @@
+get_aws_export () {
+  STACK=$1
+  PROPERTY=$2
+  RESULT=$(aws cloudformation list-exports \
+    --profile $CLI_PROFILE \
+    --query "Exports[?starts_with(Name,'$STACK$PROPERTY')].Value" \
+    --output text | sed 's/\t\t*/\n/g')
+  echo -e $RESULT | tr " " "\n"
+}

devops/infrastructure/delete-stack.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+source common.sh
+
+if [ -z "$1" ]; then
+  echo "ERROR: Configuration file not passed"
+  echo "Please use ./delete-stack.sh PATH/TO/CONFIG to run this script"
+  exit 1
+else
+  echo "Using $1 file for config"
+  source $1
+fi
+
+BUCKET_NAME=$(get_aws_export $NEW_STACK_NAME "S3BucketName")
+
+# Delete the CloudFormation stack
+
+echo -e "\n\n=========== Emptying bucket $BUCKET_NAME ==========="
+
+aws s3 rm s3://$BUCKET_NAME --recursive --profile $CLI_PROFILE || echo "No bucket"
+
+echo -e "\n\n=========== Deleting stack $NEW_STACK_NAME ==========="
+
+aws cloudformation delete-stack --stack-name $NEW_STACK_NAME --profile $CLI_PROFILE
+
+echo -e "\n\n=========== Waiting for stack deletion to complete ==========="
+
+aws cloudformation wait stack-delete-complete --stack-name $NEW_STACK_NAME --profile $CLI_PROFILE
+
+echo -e "\n\n=========== Stack $NEW_STACK_NAME deleted ==========="

devops/infrastructure/deploy-config.sample.cfg

@@ -0,0 +1,43 @@
+#### PARAMETERS USED BY AWS
+
+STACK_NAME=joystream-node
+REGION=us-east-1
+CLI_PROFILE=joystream-user
+KEY_PATH="/Users/joystream/Joystream/joystream-key.pem"
+AWS_KEY_PAIR_NAME="joystream-key"
+DEFAULT_EC2_INSTANCE_TYPE=t2.micro
+VALIDATOR_EC2_INSTANCE_TYPE=t2.micro
+BUILD_EC2_INSTANCE_TYPE=t2.xlarge
+RPC_EC2_INSTANCE_TYPE=t2.micro
+
+# prebuilt AMI with joystream-node, chain-spec and subkey already built
+EC2_AMI_ID="ami-08ffec5991ca99db9"
+
+ACCOUNT_ID=$(aws sts get-caller-identity --profile $CLI_PROFILE --query Account --output text)
+
+NEW_STACK_NAME="${STACK_NAME}-${ACCOUNT_ID}"
+
+DATA_PATH="data-$NEW_STACK_NAME"
+
+INVENTORY_PATH="$DATA_PATH/inventory"
+
+NUMBER_OF_VALIDATORS=2
+
+## Used for Deploying a new node
+DATE_TIME=$(date +"%d-%b-%Y-%H-%M-%S")
+
+SINGLE_NODE_STACK_NAME="new-node-$DATE_TIME"
+
+BINARY_FILE="https://github.com/Joystream/joystream/releases/download/v9.3.0/joystream-node-5.1.0-9d9e77751-x86_64-linux-gnu.tar.gz"
+CHAIN_SPEC_FILE="https://github.com/Joystream/joystream/releases/download/v9.3.0/joy-testnet-5.json"
+
+#### PARAMETERS USED BY ANSIBLE
+
+LOCAL_CODE_PATH="~/Joystream/joystream"
+NETWORK_SUFFIX=7891
+
+GIT_REPO="https://github.com/Joystream/joystream.git"
+BRANCH_NAME=sumer
+
+# If true will build LOCAL_CODE_PATH otherwise will pull from GIT_REPO:BRANCH_NAME
+BUILD_LOCAL_CODE=false

devops/infrastructure/deploy-infra.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+set -e
+
+source common.sh
+
+if [ -z "$1" ]; then
+  echo "ERROR: Configuration file not passed"
+  echo "Please use ./deploy-infra.sh PATH/TO/CONFIG to run this script"
+  exit 1
+else
+  echo "Using $1 file for config"
+  source $1
+fi
+
+if [ $ACCOUNT_ID == None ]; then
+    echo "Couldn't find Account ID, please check if AWS Profile $CLI_PROFILE is set"
+    exit 1
+fi
+
+if [ ! -f "$KEY_PATH" ]; then
+    echo "Key file not found at $KEY_PATH"
+    exit 1
+fi
+
+# Deploy the CloudFormation template
+echo -e "\n\n=========== Deploying main.yml ==========="
+aws cloudformation deploy \
+  --region $REGION \
+  --profile $CLI_PROFILE \
+  --stack-name $NEW_STACK_NAME \
+  --template-file infrastructure.yml \
+  --no-fail-on-empty-changeset \
+  --capabilities CAPABILITY_NAMED_IAM \
+  --parameter-overrides \
+    EC2InstanceType=$DEFAULT_EC2_INSTANCE_TYPE \
+    ValidatorEC2InstanceType=$VALIDATOR_EC2_INSTANCE_TYPE \
+    RPCEC2InstanceType=$RPC_EC2_INSTANCE_TYPE \
+    BuildEC2InstanceType=$BUILD_EC2_INSTANCE_TYPE \
+    KeyName=$AWS_KEY_PAIR_NAME \
+    EC2AMI=$EC2_AMI_ID \
+    NumberOfValidators=$NUMBER_OF_VALIDATORS
+
+# If the deploy succeeded, get the IP, create inventory and configure the created instances
+if [ $? -eq 0 ]; then
+  # Install additional Ansible roles from requirements
+  ansible-galaxy install -r requirements.yml
+
+  ASG=$(get_aws_export $NEW_STACK_NAME "AutoScalingGroup")
+
+  VALIDATORS=""
+
+  INSTANCES=$(aws autoscaling describe-auto-scaling-instances --profile $CLI_PROFILE \
+    --query "AutoScalingInstances[?AutoScalingGroupName=='${ASG}'].InstanceId" --output text);
+
+  for ID in $INSTANCES
+  do
+    IP=$(aws ec2 describe-instances --instance-ids $ID --query "Reservations[].Instances[].PublicIpAddress" --profile $CLI_PROFILE --output text)
+    VALIDATORS+="$IP\n"
+  done
+
+  RPC_NODES=$(get_aws_export $NEW_STACK_NAME "RPCPublicIp")
+
+  BUILD_SERVER=$(get_aws_export $NEW_STACK_NAME "BuildPublicIp")
+
+  BUCKET_NAME=$(get_aws_export $NEW_STACK_NAME "S3BucketName")
+
+  DOMAIN_NAME=$(get_aws_export $NEW_STACK_NAME "DomainName")
+
+  mkdir -p $DATA_PATH
+
+  echo -e "[build]\n$BUILD_SERVER\n\n[validators]\n$VALIDATORS\n[rpc]\n$RPC_NODES" > $INVENTORY_PATH
+
+  if [ -z "$EC2_AMI_ID" ]
+  then
+    echo -e "\n\n=========== Compile joystream-node on build server ==========="
+    ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH build-code.yml \
+      --extra-vars "branch_name=$BRANCH_NAME git_repo=$GIT_REPO build_local_code=$BUILD_LOCAL_CODE data_path=data-$NEW_STACK_NAME"
+
+    echo -e "\n\n=========== Install additional utils on build server ==========="
+    ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH setup-admin.yml
+  fi
+
+  echo -e "\n\n=========== Configure and start new validators, rpc node and pioneer ==========="
+  ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH chain-spec-pioneer.yml \
+    --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX
+                  data_path=data-$NEW_STACK_NAME bucket_name=$BUCKET_NAME number_of_validators=$NUMBER_OF_VALIDATORS"
+
+  echo -e "\n\n Pioneer URL: https://$DOMAIN_NAME"
+fi

devops/infrastructure/deploy-single-node.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -e
+
+source common.sh
+
+if [ -z "$1" ]; then
+  echo "ERROR: Configuration file not passed"
+  echo "Please use ./deploy-single-node.sh PATH/TO/CONFIG to run this script"
+  exit 1
+else
+  echo "Using $1 file for config"
+  source $1
+fi
+
+if [ $ACCOUNT_ID == None ]; then
+    echo "Couldn't find Account ID, please check if AWS Profile $CLI_PROFILE is set"
+    exit 1
+fi
+
+if [ ! -f "$KEY_PATH" ]; then
+    echo "Key file not found at $KEY_PATH"
+    exit 1
+fi
+
+# # Deploy the CloudFormation template
+echo -e "\n\n=========== Deploying single node ==========="
+aws cloudformation deploy \
+  --region $REGION \
+  --profile $CLI_PROFILE \
+  --stack-name $SINGLE_NODE_STACK_NAME \
+  --template-file single-instance.yml \
+  --no-fail-on-empty-changeset \
+  --capabilities CAPABILITY_NAMED_IAM \
+  --parameter-overrides \
+    EC2InstanceType=$DEFAULT_EC2_INSTANCE_TYPE \
+    KeyName=$AWS_KEY_PAIR_NAME
+
+# If the deploy succeeded, get the IP and configure the created instance
+if [ $? -eq 0 ]; then
+  # Install additional Ansible roles from requirements
+  ansible-galaxy install -r requirements.yml
+
+  SERVER_IP=$(get_aws_export $SINGLE_NODE_STACK_NAME "PublicIp")
+
+  echo -e "New Node Public IP: $SERVER_IP"
+
+  echo -e "\n\n=========== Configuring node ==========="
+  ansible-playbook -i $SERVER_IP, --private-key $KEY_PATH single-node-playbook.yml \
+    --extra-vars "binary_file=$BINARY_FILE chain_spec_file=$CHAIN_SPEC_FILE"
+fi

devops/infrastructure/github-action-playbook.yml

@@ -0,0 +1,45 @@
+---
+# Setup joystream code, build and Create AMI
+
+- name: Setup instance
+  hosts: all
+
+  tasks:
+    - block:
+      - name: Get code from git repo
+        include_role:
+          name: common
+          tasks_from: get-code-git
+
+      - name: Run setup and build
+        include_role:
+          name: common
+          tasks_from: run-setup-build
+
+      - name: Install subkey
+        include_role:
+          name: admin
+          tasks_from: main
+
+      - name: Basic AMI Creation
+        amazon.aws.ec2_ami:
+          instance_id: "{{ instance_id }}"
+          wait: yes
+          name: "{{ ami_name }}"
+          launch_permissions:
+            group_names: ['all']
+          tags:
+            Name: "{{ ami_name }}"
+        register: ami_data
+        delegate_to: localhost
+
+      - name: Print AMI ID
+        debug:
+          msg: "AMI ID is: {{ ami_data.image_id }}"
+
+      always:
+      - name: Delete the stack
+        amazon.aws.cloudformation:
+          stack_name: "{{ stack_name }}"
+          state: "absent"
+        delegate_to: localhost

devops/infrastructure/group_vars/all

@@ -0,0 +1,21 @@
+---
+# Variables applicable to all hosts
+
+branch_name: sumer
+git_repo: "https://github.com/Joystream/joystream.git"
+
+local_dir: ~/Joystream/joystream
+
+# Generates random number between 1000..9999
+network_suffix: "{{ 10000 | random(1000) }}"
+
+data_path: ./data
+chain_spec_path: "{{ data_path }}/chainspec.json"
+raw_chain_spec_path: "{{ data_path }}/chainspec-raw.json"
+remote_code_path: "/home/ubuntu/joystream"
+remote_chain_spec_path: "{{ remote_code_path }}/chainspec.json"
+run_on_admin_server: true
+build_local_code: false
+number_of_validators: 2
+
+bucket_name: s3-bucket-joystream

devops/infrastructure/infrastructure.yml

@@ -0,0 +1,248 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Parameters:
+  EC2InstanceType:
+    Type: String
+    Default: t2.micro
+  ValidatorEC2InstanceType:
+    Type: String
+    Default: t2.micro
+  RPCEC2InstanceType:
+    Type: String
+    Default: t2.micro
+  BuildEC2InstanceType:
+    Type: String
+    Default: t2.micro
+  EC2AMI:
+    Type: String
+    Default: 'ami-09e67e426f25ce0d7'
+  DefaultAMI:
+    Type: String
+    Default: 'ami-09e67e426f25ce0d7'
+  KeyName:
+    Description: Name of an existing EC2 KeyPair to enable SSH access to the instance
+    Type: 'AWS::EC2::KeyPair::KeyName'
+    Default: 'joystream-key'
+    ConstraintDescription: must be the name of an existing EC2 KeyPair.
+  NumberOfValidators:
+    Description: Number of validator instances to launch
+    Type: Number
+    Default: 2
+
+Conditions:
+  HasAMIId: !Not [!Equals [!Ref EC2AMI, ""]]
+
+Resources:
+  SecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription:
+        !Sub 'Internal Security group for validator nodes ${AWS::StackName}'
+      SecurityGroupIngress:
+        - IpProtocol: tcp
+          FromPort: 30333
+          ToPort: 30333
+          CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 22
+          ToPort: 22
+          CidrIp: 0.0.0.0/0
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_validator'
+
+  RPCSecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription:
+        !Sub 'Internal Security group for RPC nodes ${AWS::StackName}'
+      SecurityGroupIngress:
+        - IpProtocol: tcp
+          FromPort: 9933
+          ToPort: 9933
+          CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 9944
+          ToPort: 9944
+          CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 30333
+          ToPort: 30333
+          CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 443
+          ToPort: 443
+          CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 22
+          ToPort: 22
+          CidrIp: 0.0.0.0/0
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_rpc'
+
+  InstanceLaunchTemplate:
+    Type: AWS::EC2::LaunchTemplate
+    Metadata:
+      AWS::CloudFormation::Init:
+        config:
+          packages:
+            apt:
+              wget: []
+              unzip: []
+    Properties:
+      LaunchTemplateName: !Sub 'LaunchTemplate_${AWS::StackName}'
+      LaunchTemplateData:
+        ImageId: !If [HasAMIId, !Ref EC2AMI, !Ref DefaultAMI]
+        InstanceType: !Ref EC2InstanceType
+        KeyName: !Ref KeyName
+        SecurityGroupIds:
+          - !GetAtt SecurityGroup.GroupId
+        BlockDeviceMappings:
+          - DeviceName: /dev/sda1
+            Ebs:
+              VolumeSize: '40'
+        UserData:
+          Fn::Base64: !Sub |
+            #!/bin/bash -xe
+
+            # send script output to /tmp so we can debug boot failures
+            exec > /tmp/userdata.log 2>&1
+
+            # Update all packages
+            apt-get update -y
+
+            # Install the updates
+            apt-get upgrade -y
+
+            # Get latest cfn scripts and install them;
+            apt-get install -y python3-setuptools
+            mkdir -p /opt/aws/bin
+            wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz
+            python3 -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-py3-latest.tar.gz
+
+            /opt/aws/bin/cfn-signal -e $? -r "Instance Created" '${WaitHandle}'
+
+  AutoScalingGroup:
+    Type: AWS::AutoScaling::AutoScalingGroup
+    Properties:
+      MinSize: '0'
+      MaxSize: '10'
+      DesiredCapacity: !Ref NumberOfValidators
+      AvailabilityZones:
+        Fn::GetAZs:
+          Ref: "AWS::Region"
+      MixedInstancesPolicy:
+        LaunchTemplate:
+          LaunchTemplateSpecification:
+            LaunchTemplateId: !Ref InstanceLaunchTemplate
+            Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
+          Overrides:
+            - InstanceType: !Ref ValidatorEC2InstanceType
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}'
+          PropagateAtLaunch: "true"
+
+  RPCInstance:
+    Type: AWS::EC2::Instance
+    Properties:
+      SecurityGroupIds:
+        - !GetAtt RPCSecurityGroup.GroupId
+      InstanceType: !Ref RPCEC2InstanceType
+      LaunchTemplate:
+        LaunchTemplateId: !Ref InstanceLaunchTemplate
+        Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_rpc'
+
+  BuildInstance:
+    Type: AWS::EC2::Instance
+    Properties:
+      InstanceType: !Ref BuildEC2InstanceType
+      LaunchTemplate:
+        LaunchTemplateId: !Ref InstanceLaunchTemplate
+        Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_build'
+
+  WaitHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+
+  WaitCondition:
+    Type: AWS::CloudFormation::WaitCondition
+    Properties:
+      Handle: !Ref 'WaitHandle'
+      Timeout: '600'
+      Count: !Ref NumberOfValidators
+
+  S3Bucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      AccessControl: PublicRead
+      WebsiteConfiguration:
+        IndexDocument: index.html
+
+  BucketPolicy:
+    Type: AWS::S3::BucketPolicy
+    Properties:
+      PolicyDocument:
+        Id: PublicPolicy
+        Version: 2012-10-17
+        Statement:
+          - Sid: PublicReadForGetBucketObjects
+            Effect: Allow
+            Principal: '*'
+            Action: 's3:GetObject'
+            Resource: !Sub "arn:aws:s3:::${S3Bucket}/*"
+      Bucket: !Ref S3Bucket
+
+  CloudFrontDistribution:
+    Type: AWS::CloudFront::Distribution
+    Properties:
+      DistributionConfig:
+        Origins:
+        - DomainName: !Select [1, !Split ["//", !GetAtt S3Bucket.WebsiteURL]]
+          Id: pioneer-origin-s3
+          CustomOriginConfig:
+            OriginProtocolPolicy: http-only
+        DefaultCacheBehavior:
+          TargetOriginId: pioneer-origin-s3
+          ViewerProtocolPolicy: redirect-to-https
+          ForwardedValues:
+            QueryString: true
+        Enabled: true
+        HttpVersion: http2
+
+Outputs:
+  AutoScalingId:
+    Description: The Auto Scaling ID
+    Value:  !Ref AutoScalingGroup
+    Export:
+      Name: !Sub "${AWS::StackName}AutoScalingGroup"
+
+  RPCPublicIp:
+    Description: The DNS name for the created instance
+    Value:  !Sub "${RPCInstance.PublicIp}"
+    Export:
+      Name: !Sub "${AWS::StackName}RPCPublicIp"
+
+  BuildPublicIp:
+    Description: The DNS name for the created instance
+    Value:  !Sub "${BuildInstance.PublicIp}"
+    Export:
+      Name: !Sub "${AWS::StackName}BuildPublicIp"
+
+  S3BucketName:
+    Value: !Ref S3Bucket
+    Description: Name of S3 bucket to hold website content
+    Export:
+      Name: !Sub "${AWS::StackName}S3BucketName"
+
+  DomainName:
+    Description: CloudFront Domain Name
+    Value:  !Sub "${CloudFrontDistribution.DomainName}"
+    Export:
+      Name: !Sub "${AWS::StackName}DomainName"

devops/infrastructure/library/json_modify.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python
+
+import json
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    fields = {
+        "chain_spec_path": {"required": True, "type": "str"},
+        "file_content": {"required": False, "type": "str" },
+        "prefix": {"required": False, "type": "str" },
+        "all_nodes": {"required": False, "type": "dict" }
+    }
+    module = AnsibleModule(argument_spec=fields)
+    prefix = module.params["prefix"]
+    chain_spec_path = module.params["chain_spec_path"]
+    all_nodes = module.params["all_nodes"]
+
+    with open(chain_spec_path) as f:
+        data = json.load(f)
+
+    response = {
+        "name": f'{data["name"]} {prefix}',
+        "id": f'{data["id"]}_{prefix}',
+        "protocolId": f'{data["protocolId"]}{prefix}'
+    }
+
+    boot_node_list = data["bootNodes"]
+    for key in all_nodes:
+        if "validators" in all_nodes[key]["group_names"]:
+            public_key = all_nodes[key]["subkey_output"]["stderr"]
+            boot_node_list.append(f"/ip4/{key}/tcp/30333/p2p/{public_key}")
+
+    telemetry_endpoints = data["telemetryEndpoints"]
+    telemetry_endpoints.append([
+        "/dns/telemetry.joystream.org/tcp/443/x-parity-wss/%2Fsubmit%2F", 0])
+
+    response["bootNodes"] = boot_node_list
+    response["telemetryEndpoints"] = telemetry_endpoints
+
+    data.update(response)
+    with open(chain_spec_path, 'w') as outfile:
+        json.dump(data, outfile, indent=4)
+    module.exit_json(changed=False, result=response)
+
+if __name__ == '__main__':
+    main()

devops/infrastructure/requirements.yml

@@ -0,0 +1,6 @@
+---
+roles:
+- caddy_ansible.caddy_ansible
+collections:
+- community.aws
+- amazon.aws

devops/infrastructure/roles/admin/tasks/deploy-pioneer.yml

@@ -0,0 +1,30 @@
+---
+# Build Pioneer, copy build artifacts and sync to S3
+
+- name: Set ws_rpc for build node
+  set_fact:
+    ws_rpc: "{{ hostvars[groups['rpc'][0]].ws_rpc }}"
+
+- name: Build Pioneer code
+  shell: "WS_URL=wss://{{ ws_rpc }} yarn && yarn workspace @joystream/types build && yarn workspace pioneer build"
+  args:
+    chdir: "{{ remote_code_path }}"
+  retries: 3
+  delay: 5
+  register: result
+  until: result is not failed
+
+- name: Copying build files to local
+  synchronize:
+    src: "{{ remote_code_path }}/pioneer/packages/apps/build"
+    dest: "{{ data_path }}"
+    mode: pull
+  run_once: true
+
+- name: Run S3 Sync to upload build files to bucket
+  community.aws.s3_sync:
+    bucket: "{{ bucket_name }}"
+    file_root: "{{ data_path }}/build"
+    profile: joystream-user
+    region: us-east-1
+  delegate_to: localhost

devops/infrastructure/roles/admin/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+# Configure admin server to be able to create chain-spec file and subkey commands
+
+- name: Copy bash_profile content
+  shell: cat ~/.bash_profile
+  register: bash_data
+
+- name: Copy bash_profile content to bashrc for non-interactive sessions
+  blockinfile:
+    block: '{{ bash_data.stdout }}'
+    path: ~/.bashrc
+    insertbefore: BOF
+
+- name: Get dependencies for subkey
+  shell: curl https://getsubstrate.io -sSf | bash -s -- --fast
+
+- name: Install subkey
+  shell: cargo install --force subkey --git https://github.com/paritytech/substrate --version 2.0.1 --locked
+  async: 3600
+  poll: 0
+  register: install_result
+
+- name: Check whether install subkey task has finished
+  async_status:
+    jid: '{{ install_result.ansible_job_id }}'
+  register: job_result
+  until: job_result.finished
+  retries: 36
+  delay: 100

devops/infrastructure/roles/common/tasks/chain-spec-node-keys.yml

@@ -0,0 +1,76 @@
+---
+# Create chain spec files and keys and copy to all the servers
+
+- name: Debug to test variable
+  debug:
+    msg: "Data path: {{ data_path }}, Chain Spec path: {{ chain_spec_path }}"
+  run_once: true
+
+- name: Run chain-spec-builder to generate chainspec.json file
+  command: "{{ admin_code_dir }}/target/release/chain-spec-builder generate -a {{ number_of_validators }} --chain-spec-path {{ chain_spec_path }} --deployment live --endowed 1 --keystore-path {{ data_path }}"
+  register: chain_spec_output
+  delegate_to: "{{ local_or_admin }}"
+  run_once: true
+
+- name: Run subkey to generate node keys
+  shell: subkey generate-node-key
+  delegate_to: "{{ local_or_admin }}"
+  register: subkey_output
+
+- name: Print to stdout
+  debug:
+    msg:
+    - "Public Key: {{ subkey_output.stderr }}"
+    - "Private Key: {{ subkey_output.stdout }}"
+
+- name: Print to stdout chain spec
+  debug: var=chain_spec_output.stdout
+  run_once: true
+
+- name: Save output of chain spec to local file
+  copy:
+    content: '{{ chain_spec_output.stdout | regex_replace("\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]", "") }}'
+    dest: "{{ data_path }}/chain_spec_output.txt"
+  delegate_to: "{{ local_or_admin }}"
+  run_once: true
+
+- name: Change chain spec name, id, protocolId
+  json_modify:
+    chain_spec_path: "{{ chain_spec_path }}"
+    prefix: "{{ network_suffix }}"
+    all_nodes: "{{ hostvars }}"
+  delegate_to: "{{ local_or_admin }}"
+  register: result
+  run_once: true
+
+- name: Print output of modified chainspec
+  debug:
+    var: result.result
+  run_once: true
+
+- name: Run build-spec to generate raw chainspec file
+  shell: "{{ admin_code_dir }}/target/release/joystream-node build-spec --chain {{ chain_spec_path }} --raw > {{ raw_chain_spec_path }}"
+  delegate_to: "{{ local_or_admin }}"
+  run_once: true
+
+- name: Copying chain spec files to localhost
+  synchronize:
+    src: "/home/ubuntu/{{ data_path }}/"
+    dest: "{{ data_path }}"
+    mode: pull
+  run_once: true
+  when: run_on_admin_server|bool
+
+- name: Copy joystream-node binary to localhost
+  fetch:
+    src: "{{ admin_code_dir }}/target/release/joystream-node"
+    dest: "{{ data_path }}/joystream-node"
+    flat: yes
+  delegate_to: "{{ local_or_admin }}"
+  run_once: true
+  when: run_on_admin_server|bool
+
+- name: Copying raw chain spec file to all servers
+  copy:
+    src: "{{ raw_chain_spec_path }}"
+    dest: "{{ remote_chain_spec_path }}"

devops/infrastructure/roles/common/tasks/get-code-git.yml

@@ -0,0 +1,13 @@
+---
+# Get the latest code
+
+- name: Delete remote code directory if exists
+  file:
+    state: absent
+    path: "{{ remote_code_path }}"
+
+- name: Git checkout
+  git:
+    repo: "{{ git_repo }}"
+    dest: "{{ remote_code_path }}"
+    version: "{{ branch_name }}"

devops/infrastructure/roles/common/tasks/get-code-local.yml

@@ -0,0 +1,27 @@
+---
+# Get the latest code
+
+- name: Archive the current Git repository
+  command: git archive --format tar HEAD
+  args:  
+    chdir: "{{ local_dir }}"
+  delegate_to: localhost
+  register: archive_output
+
+- name: Save output the git repo as an archive
+  local_action: copy content={{ archive_output.stdout }} dest="{{ local_dir }}/code-archive.tar"
+
+- name: Delete remote code directory if exists
+  file:
+    state: absent
+    path: "{{ remote_code_path }}"
+
+- name: create directory for unarchiving
+  file:
+    path: "{{ remote_code_path }}"
+    state: directory
+
+- name: Extract code into path
+  unarchive:
+    src: "{{ local_dir }}/code-archive.tar"
+    dest: "{{ remote_code_path }}"

devops/infrastructure/roles/common/tasks/run-setup-build.yml

@@ -0,0 +1,26 @@
+---
+# Run setup and build code
+
+- name: Creat bash profile file
+  command: "touch /home/ubuntu/.bash_profile"
+
+- name: Run setup script
+  command: ./setup.sh
+  args:
+    chdir: "{{ remote_code_path }}"
+
+- name: Build joystream node
+  shell: . ~/.bash_profile && yarn cargo-build
+  args:
+    chdir: "{{ remote_code_path }}"
+  async: 3600
+  poll: 0
+  register: build_result
+
+- name: Check on build async task
+  async_status:
+    jid: "{{ build_result.ansible_job_id }}"
+  register: job_result
+  until: job_result.finished
+  retries: 36
+  delay: 100

devops/infrastructure/roles/node/templates/joystream-node.service.j2

@@ -0,0 +1,18 @@
+[Unit]
+Description=Joystream Node
+After=network.target
+
+[Service]
+Type=simple
+User=ubuntu
+WorkingDirectory=/home/ubuntu/
+ExecStart={{ template_binary_path }} \
+        --chain {{ template_remote_chain_spec_path }} \
+        --pruning archive \
+        --log runtime,txpool,transaction-pool,trace=sync
+Restart=on-failure
+RestartSec=3
+LimitNOFILE=10000
+
+[Install]
+WantedBy=multi-user.target

devops/infrastructure/roles/rpc/tasks/main.yml

@@ -0,0 +1,42 @@
+---
+# Configure and start joystream-node RPC service on the servers
+
+- name: Print bootNodes
+  debug:
+    var: result.result.bootNodes
+  run_once: true
+
+- name: Create a service file
+  template:
+    src: joystream-node.service.j2
+    dest: /etc/systemd/system/joystream-node.service
+  vars:
+    template_remote_chain_spec_path: "{{ remote_chain_spec_path }}"
+    boot_nodes: "{{ result.result.bootNodes }}"
+  become: yes
+
+- name: Start service joystream-node, if not started
+  service:
+    name: joystream-node
+    state: started
+  become: yes
+
+- name: Set websocket and http endpoint variables
+  set_fact:
+    ws_rpc: "{{ inventory_hostname }}.nip.io/ws-rpc"
+    http_rpc: "{{ inventory_hostname }}.nip.io/http-rpc"
+  run_once: yes
+
+- name: Install and configure Caddy
+  include_role:
+    name: caddy_ansible.caddy_ansible
+    apply:
+      become: yes
+  vars:
+    caddy_config: "{{ lookup('template', 'templates/Caddyfile.j2') }}"
+    caddy_systemd_capabilities_enabled: true
+    caddy_update: false
+
+- name: Print RPC node DNS
+  debug:
+    msg: "RPC Endpoint: wss://{{ ws_rpc }}"

devops/infrastructure/roles/rpc/templates/Caddyfile.j2

@@ -0,0 +1,7 @@
+{{ ws_rpc }} {
+    reverse_proxy localhost:9944
+}
+
+{{ http_rpc }} {
+    reverse_proxy localhost:9933
+}

devops/infrastructure/roles/rpc/templates/joystream-node.service.j2

@@ -0,0 +1,25 @@
+[Unit]
+Description=Joystream Node
+After=network.target
+
+[Service]
+Type=simple
+User=ubuntu
+WorkingDirectory=/home/ubuntu/joystream/
+ExecStart=/home/ubuntu/joystream/target/release/joystream-node \
+        --chain {{ template_remote_chain_spec_path }} \
+        --ws-external \
+        --rpc-cors all \
+        --pruning archive \
+        --ws-max-connections 512 \
+        --telemetry-url "wss://telemetry.joystream.org/submit/ 0" \
+        --telemetry-url "wss://telemetry.polkadot.io/submit/ 0"
+        --reserved-nodes \
+                {{ boot_nodes|join(" ") }}
+
+Restart=on-failure
+RestartSec=3
+LimitNOFILE=16384
+
+[Install]
+WantedBy=multi-user.target

devops/infrastructure/roles/validators/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+# Configure chain spec and start joystream-node service on the servers
+
+- set_fact:
+    chain_path: "{{ remote_code_path }}/chains/{{ result.result.id }}"
+
+- set_fact:
+    network_path: "{{ chain_path }}/network"
+    keystore_path: "{{ chain_path }}/keystore/"
+
+- set_fact:
+    secret_path: "{{ network_path }}/secret"
+
+- name: Creating chains directory
+  file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - "{{ network_path }}"
+
+- name: Copy node key to remote host
+  copy:
+    dest: "{{ secret_path }}"
+    content: "{{ subkey_output.stdout }}"
+
+- name: Copy auth directory to remote host
+  copy:
+    src: "{{ data_path }}/auth-{{ ansible_play_batch.index(inventory_hostname) }}/"
+    dest: "{{ keystore_path }}"
+
+- name: Create a service file
+  template:
+    src: joystream-node.service.j2
+    dest: /etc/systemd/system/joystream-node.service
+  vars:
+    template_keystore_path: "{{ keystore_path }}"
+    template_secret_path: "{{ secret_path }}"
+    template_remote_chain_spec_path: "{{ remote_chain_spec_path }}"
+  become: yes
+
+- name: Start service joystream-node, if not started
+  service:
+    name: joystream-node
+    state: started
+  become: yes

devops/infrastructure/roles/validators/templates/joystream-node.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=Joystream Node
+After=network.target
+
+[Service]
+Type=simple
+User=ubuntu
+WorkingDirectory=/home/ubuntu/joystream/
+ExecStart=/home/ubuntu/joystream/target/release/joystream-node \
+        --chain {{ template_remote_chain_spec_path }} \
+        --pruning archive \
+        --node-key-file {{ template_secret_path }} \
+        --keystore-path {{ template_keystore_path }} \
+        --validator \
+        --log runtime,txpool,transaction-pool,trace=sync
+Restart=on-failure
+RestartSec=3
+LimitNOFILE=10000
+
+[Install]
+WantedBy=multi-user.target

devops/infrastructure/setup-admin.yml

@@ -0,0 +1,9 @@
+---
+# Setup Build server install subkey
+
+- name: Setup build server, install subkey
+  hosts: build
+
+  roles:
+    - role: admin
+      when: run_on_admin_server|bool

devops/infrastructure/single-instance.yml

@@ -0,0 +1,102 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Parameters:
+  EC2InstanceType:
+    Type: String
+    Default: t2.xlarge
+  EC2AMI:
+    Type: String
+    Default: 'ami-09e67e426f25ce0d7'
+  KeyName:
+    Description: Name of an existing EC2 KeyPair to enable SSH access to the instance
+    Type: 'AWS::EC2::KeyPair::KeyName'
+    Default: 'joystream-key'
+    ConstraintDescription: must be the name of an existing EC2 KeyPair.
+
+Resources:
+  SecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription:
+        !Sub 'Internal Security group for validator nodes ${AWS::StackName}'
+      SecurityGroupIngress:
+        - IpProtocol: tcp
+          FromPort: 22
+          ToPort: 22
+          CidrIp: 0.0.0.0/0
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_validator'
+
+  InstanceLaunchTemplate:
+    Type: AWS::EC2::LaunchTemplate
+    Metadata:
+      AWS::CloudFormation::Init:
+        config:
+          packages:
+            apt:
+              wget: []
+              unzip: []
+    Properties:
+      LaunchTemplateName: !Sub 'LaunchTemplate_${AWS::StackName}'
+      LaunchTemplateData:
+        ImageId: !Ref EC2AMI
+        InstanceType: !Ref EC2InstanceType
+        KeyName: !Ref KeyName
+        SecurityGroupIds:
+          - !GetAtt SecurityGroup.GroupId
+        BlockDeviceMappings:
+          - DeviceName: /dev/sda1
+            Ebs:
+              VolumeSize: '30'
+        UserData:
+          Fn::Base64: !Sub |
+            #!/bin/bash -xe
+
+            # send script output to /tmp so we can debug boot failures
+            exec > /tmp/userdata.log 2>&1
+
+            # Update all packages
+            apt-get update -y
+
+            # Install the updates
+            apt-get upgrade -y
+
+            # Get latest cfn scripts and install them;
+            apt-get install -y python3-setuptools
+            mkdir -p /opt/aws/bin
+            wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz
+            python3 -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-py3-latest.tar.gz
+
+            /opt/aws/bin/cfn-signal -e $? -r "Instance Created" '${WaitHandle}'
+
+  Instance:
+    Type: AWS::EC2::Instance
+    Properties:
+      LaunchTemplate:
+        LaunchTemplateId: !Ref InstanceLaunchTemplate
+        Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_1'
+
+  WaitHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+
+  WaitCondition:
+    Type: AWS::CloudFormation::WaitCondition
+    Properties:
+      Handle: !Ref 'WaitHandle'
+      Timeout: '600'
+      Count: 1
+
+Outputs:
+  PublicIp:
+    Description: The DNS name for the created instance
+    Value:  !Sub "${Instance.PublicIp}"
+    Export:
+      Name: !Sub "${AWS::StackName}PublicIp"
+
+  InstanceId:
+    Description: The Instance ID
+    Value:  !Ref Instance

devops/infrastructure/single-node-playbook.yml

@@ -0,0 +1,48 @@
+---
+# Configure chain spec file, copy joystream-node binary and run the service
+
+- name: Create and copy the chain-spec file
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Download chain spec file using link
+      get_url:
+        url: "{{ chain_spec_file }}"
+        dest: ~/chain-spec.json
+      when: chain_spec_file is search("http")
+
+    - name: Copy chain spec file from local
+      copy:
+        src: "{{ chain_spec_file }}"
+        dest: ~/chain-spec.json
+      when: chain_spec_file is not search("http")
+
+    - name: Download and unarchive binary using link
+      unarchive:
+        src: "{{ binary_file }}"
+        dest: ~/
+        remote_src: yes
+      when: binary_file is search("http")
+
+    - name: Copy binary from local
+      copy:
+        src: "{{ binary_file }}"
+        dest: ~/joystream-node
+        mode: "0775"
+      when: binary_file is not search("http")
+
+    - name: Create a service file
+      template:
+        src: roles/node/templates/joystream-node.service.j2
+        dest: /etc/systemd/system/joystream-node.service
+      vars:
+        template_remote_chain_spec_path: "/home/ubuntu/chain-spec.json"
+        template_binary_path: "/home/ubuntu/joystream-node"
+      become: yes
+
+    - name: Start service joystream-node, if not started
+      service:
+        name: joystream-node
+        state: started
+      become: yes

devops/infrastructure/storage-node/.gitignore

@@ -0,0 +1,5 @@
+/bin/
+/node_modules/
+kubeconfig.yml
+package-lock.json
+Pulumi.*.yaml

devops/infrastructure/storage-node/Pulumi.yaml

@@ -0,0 +1,33 @@
+name: eks-cluster
+runtime: nodejs
+description: A Pulumi program to deploy storage node to cloud environment
+template:
+  config:
+    aws:profile:
+      default: joystream-user
+    aws:region:
+      default: us-east-1
+    wsProviderEndpointURI:
+      description: Chain RPC endpoint
+      default: 'wss://rome-rpc-endpoint.joystream.org:9944/'
+    isAnonymous:
+      description: Whether you are deploying an anonymous storage node
+      default: true
+    isLoadBalancerReady:
+      description: Whether the load balancer service is ready and has been assigned an IP
+      default: false
+    colossusPort:
+      description: Port that is exposed for the colossus container
+      default: 3000
+    storage:
+      description: Amount of storage in gigabytes for ipfs volume
+      default: 40
+    providerId:
+      description: StorageProviderId assigned to you in working group
+    keyFile:
+      description: Path to JSON key export file to use as the storage provider (role account)
+    publicURL:
+      description: API Public URL to announce
+    passphrase:
+      description: Optional passphrase to use to decrypt the key-file
+      secret: true

devops/infrastructure/storage-node/README.md

@@ -0,0 +1,120 @@
+# Amazon EKS Cluster: Hello World!
+
+This example deploys an EKS Kubernetes cluster with custom ipfs image
+
+## Deploying the App
+
+To deploy your infrastructure, follow the below steps.
+
+### Prerequisites
+
+1. [Install Pulumi](https://www.pulumi.com/docs/get-started/install/)
+1. [Install Node.js](https://nodejs.org/en/download/)
+1. Install a package manager for Node.js, such as [npm](https://www.npmjs.com/get-npm) or [Yarn](https://yarnpkg.com/en/docs/install).
+1. [Configure AWS Credentials](https://www.pulumi.com/docs/intro/cloud-providers/aws/setup/)
+1. Optional (for debugging): [Install kubectl](https://kubernetes.io/docs/tasks/tools/)
+
+### Steps
+
+After cloning this repo, from this working directory, run these commands:
+
+1. Install the required Node.js packages:
+
+   This installs the dependent packages [needed](https://www.pulumi.com/docs/intro/concepts/how-pulumi-works/) for our Pulumi program.
+
+   ```bash
+   $ npm install
+   ```
+
+1. Create a new stack, which is an isolated deployment target for this example:
+
+   This will initialize the Pulumi program in TypeScript.
+
+   ```bash
+   $ pulumi stack init
+   ```
+
+1. Set the required configuration variables in `Pulumi.<stack>.yaml`
+
+   ```bash
+   $ pulumi config set-all --plaintext aws:region=us-east-1 --plaintext aws:profile=joystream-user \
+    --plaintext wsProviderEndpointURI='wss://rome-rpc-endpoint.joystream.org:9944/' \
+    --plaintext isAnonymous=true
+   ```
+
+   If running for production use the below mentioned config
+
+   ```bash
+   $ pulumi config set-all --plaintext aws:region=us-east-1 --plaintext aws:profile=joystream-user \
+    --plaintext wsProviderEndpointURI='wss://rome-rpc-endpoint.joystream.org:9944/' --plaintext isAnonymous=false \
+    --plaintext providerId=<ID> --plaintext keyFile=<PATH> --plaintext publicURL=<DOMAIN> --secret passphrase=<PASSPHRASE>
+   ```
+
+   You can also set the `storage` and the `colossusPort` config parameters if required
+
+1. Stand up the EKS cluster:
+
+   Running `pulumi up -y` will deploy the EKS cluster. Note, provisioning a
+   new EKS cluster takes between 10-15 minutes.
+
+1. Once the stack if up and running, we will modify the Caddy config to get SSL certificate for the load balancer
+
+   Modify the config variable `isLoadBalancerReady`
+
+   ```bash
+   $ pulumi config set isLoadBalancerReady true
+   ```
+
+   Run `pulumi up -y` to update the Caddy config
+
+1. Access the Kubernetes Cluster using `kubectl`
+
+   To access your new Kubernetes cluster using `kubectl`, we need to set up the
+   `kubeconfig` file and download `kubectl`. We can leverage the Pulumi
+   stack output in the CLI, as Pulumi facilitates exporting these objects for us.
+
+   ```bash
+   $ pulumi stack output kubeconfig --show-secrets > kubeconfig
+   $ export KUBECONFIG=$PWD/kubeconfig
+   $ kubectl get nodes
+   ```
+
+   We can also use the stack output to query the cluster for our newly created Deployment:
+
+   ```bash
+   $ kubectl get deployment $(pulumi stack output deploymentName) --namespace=$(pulumi stack output namespaceName)
+   $ kubectl get service $(pulumi stack output serviceName) --namespace=$(pulumi stack output namespaceName)
+   ```
+
+   To get logs
+
+   ```bash
+   $ kubectl config set-context --current --namespace=$(pulumi stack output namespaceName)
+   $ kubectl get pods
+   $ kubectl logs <PODNAME> --all-containers
+   ```
+
+   To run a command on a pod
+
+   ```bash
+   $ kubectl exec ${POD_NAME} -c ${CONTAINER_NAME} -- ${CMD} ${ARG1}
+   ```
+
+   To see complete pulumi stack output
+
+   ```bash
+   $ pulumi stack output
+   ```
+
+   To execute a command
+
+   ```bash
+   $ kubectl exec --stdin --tty <PODNAME> -c colossus -- /bin/bash
+   ```
+
+1. Once you've finished experimenting, tear down your stack's resources by destroying and removing it:
+
+   ```bash
+   $ pulumi destroy --yes
+   $ pulumi stack rm --yes
+   ```

devops/infrastructure/storage-node/index.ts

@@ -0,0 +1,283 @@
+import * as awsx from '@pulumi/awsx'
+import * as aws from '@pulumi/aws'
+import * as eks from '@pulumi/eks'
+import * as k8s from '@pulumi/kubernetes'
+import * as pulumi from '@pulumi/pulumi'
+import * as fs from 'fs'
+
+const dns = require('dns')
+
+const awsConfig = new pulumi.Config('aws')
+const config = new pulumi.Config()
+
+const wsProviderEndpointURI = config.require('wsProviderEndpointURI')
+const isAnonymous = config.require('isAnonymous') === 'true'
+const lbReady = config.get('isLoadBalancerReady') === 'true'
+const name = 'storage-node'
+const colossusPort = parseInt(config.get('colossusPort') || '3000')
+const storage = parseInt(config.get('storage') || '40')
+
+let additionalParams: string[] | pulumi.Input<string>[] = []
+let volumeMounts: pulumi.Input<pulumi.Input<k8s.types.input.core.v1.VolumeMount>[]> = []
+let caddyVolumeMounts: pulumi.Input<pulumi.Input<k8s.types.input.core.v1.VolumeMount>[]> = []
+let volumes: pulumi.Input<pulumi.Input<k8s.types.input.core.v1.Volume>[]> = []
+
+// Create a VPC for our cluster.
+const vpc = new awsx.ec2.Vpc('vpc', { numberOfAvailabilityZones: 2 })
+
+// Create an EKS cluster with the default configuration.
+const cluster = new eks.Cluster('eksctl-my-cluster', {
+  vpcId: vpc.id,
+  subnetIds: vpc.publicSubnetIds,
+  instanceType: 't2.micro',
+  providerCredentialOpts: {
+    profileName: awsConfig.get('profile'),
+  },
+})
+
+// Export the cluster's kubeconfig.
+export const kubeconfig = cluster.kubeconfig
+
+// Create a repository
+const repo = new awsx.ecr.Repository('colossus-image')
+
+// Build an image and publish it to our ECR repository.
+export const colossusImage = repo.buildAndPushImage({
+  dockerfile: '../../../colossus.Dockerfile',
+  context: '../../../',
+})
+
+// Create a Kubernetes Namespace
+const ns = new k8s.core.v1.Namespace(name, {}, { provider: cluster.provider })
+
+// Export the Namespace name
+export const namespaceName = ns.metadata.name
+
+const appLabels = { appClass: name }
+
+const pvc = new k8s.core.v1.PersistentVolumeClaim(
+  `${name}-pvc`,
+  {
+    metadata: {
+      labels: appLabels,
+      namespace: namespaceName,
+      name: `${name}-pvc`,
+    },
+    spec: {
+      accessModes: ['ReadWriteOnce'],
+      resources: {
+        requests: {
+          storage: `${storage}Gi`,
+        },
+      },
+    },
+  },
+  { provider: cluster.provider }
+)
+
+volumes.push({
+  name: 'ipfs-data',
+  persistentVolumeClaim: {
+    claimName: `${name}-pvc`,
+  },
+})
+
+// Create a LoadBalancer Service for the Deployment
+const service = new k8s.core.v1.Service(
+  name,
+  {
+    metadata: {
+      labels: appLabels,
+      namespace: namespaceName,
+    },
+    spec: {
+      type: 'LoadBalancer',
+      ports: [
+        { name: 'http', port: 80 },
+        { name: 'https', port: 443 },
+      ],
+      selector: appLabels,
+    },
+  },
+  {
+    provider: cluster.provider,
+  }
+)
+
+// Export the Service name and public LoadBalancer Endpoint
+export const serviceName = service.metadata.name
+// When "done", this will print the hostname
+export let serviceHostname: pulumi.Output<string>
+serviceHostname = service.status.loadBalancer.ingress[0].hostname
+
+export let appLink: pulumi.Output<string>
+
+if (lbReady) {
+  async function lookupPromise(url: string) {
+    return new Promise((resolve, reject) => {
+      dns.lookup(url, (err: any, address: any) => {
+        if (err) reject(err)
+        resolve(address)
+      })
+    })
+  }
+
+  const lbIp = serviceHostname.apply((dnsName) => {
+    return lookupPromise(dnsName)
+  })
+
+  const caddyConfig = pulumi.interpolate`${lbIp}.nip.io {
+  reverse_proxy localhost:${colossusPort}
+}`
+
+  const keyConfig = new k8s.core.v1.ConfigMap(name, {
+    metadata: { namespace: namespaceName, labels: appLabels },
+    data: { 'fileData': caddyConfig },
+  })
+  const keyConfigName = keyConfig.metadata.apply((m) => m.name)
+
+  caddyVolumeMounts.push({
+    mountPath: '/etc/caddy/Caddyfile',
+    name: 'caddy-volume',
+    subPath: 'fileData',
+  })
+
+  volumes.push({
+    name: 'caddy-volume',
+    configMap: {
+      name: keyConfigName,
+    },
+  })
+
+  appLink = pulumi.interpolate`https://${lbIp}.nip.io`
+
+  lbIp.apply((value) => console.log(`You can now access the app at: ${value}.nip.io`))
+
+  if (!isAnonymous) {
+    const remoteKeyFilePath = '/joystream/key-file.json'
+    const providerId = config.require('providerId')
+    const keyFile = config.require('keyFile')
+    const publicUrl = config.get('publicURL') ? config.get('publicURL')! : appLink
+
+    const keyConfig = new k8s.core.v1.ConfigMap('key-config', {
+      metadata: { namespace: namespaceName, labels: appLabels },
+      data: { 'fileData': fs.readFileSync(keyFile).toString() },
+    })
+    const keyConfigName = keyConfig.metadata.apply((m) => m.name)
+
+    additionalParams = ['--provider-id', providerId, '--key-file', remoteKeyFilePath, '--public-url', publicUrl]
+
+    volumeMounts.push({
+      mountPath: remoteKeyFilePath,
+      name: 'keyfile-volume',
+      subPath: 'fileData',
+    })
+
+    volumes.push({
+      name: 'keyfile-volume',
+      configMap: {
+        name: keyConfigName,
+      },
+    })
+
+    const passphrase = config.get('passphrase')
+    if (passphrase) {
+      additionalParams.push('--passphrase', passphrase)
+    }
+  }
+}
+
+if (isAnonymous) {
+  additionalParams.push('--anonymous')
+}
+
+// Create a Deployment
+const deployment = new k8s.apps.v1.Deployment(
+  name,
+  {
+    metadata: {
+      namespace: namespaceName,
+      labels: appLabels,
+    },
+    spec: {
+      replicas: 1,
+      selector: { matchLabels: appLabels },
+      template: {
+        metadata: {
+          labels: appLabels,
+        },
+        spec: {
+          hostname: 'ipfs',
+          containers: [
+            {
+              name: 'ipfs',
+              image: 'ipfs/go-ipfs:latest',
+              ports: [{ containerPort: 5001 }, { containerPort: 8080 }],
+              command: ['/bin/sh', '-c'],
+              args: [
+                'set -e; \
+                /usr/local/bin/start_ipfs config profile apply lowpower; \
+                /usr/local/bin/start_ipfs config --json Gateway.PublicGateways \'{"localhost": null }\'; \
+                /usr/local/bin/start_ipfs config Datastore.StorageMax 200GB; \
+                /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true',
+              ],
+              volumeMounts: [
+                {
+                  name: 'ipfs-data',
+                  mountPath: '/data/ipfs',
+                },
+              ],
+            },
+            // {
+            //   name: 'httpd',
+            //   image: 'crccheck/hello-world',
+            //   ports: [{ name: 'hello-world', containerPort: 8000 }],
+            // },
+            {
+              name: 'caddy',
+              image: 'caddy',
+              ports: [
+                { name: 'caddy-http', containerPort: 80 },
+                { name: 'caddy-https', containerPort: 443 },
+              ],
+              volumeMounts: caddyVolumeMounts,
+            },
+            {
+              name: 'colossus',
+              image: colossusImage,
+              env: [
+                {
+                  name: 'WS_PROVIDER_ENDPOINT_URI',
+                  // example 'wss://18.209.241.63.nip.io/'
+                  value: wsProviderEndpointURI,
+                },
+                {
+                  name: 'DEBUG',
+                  value: 'joystream:*',
+                },
+              ],
+              volumeMounts,
+              command: [
+                'yarn',
+                'colossus',
+                '--ws-provider',
+                wsProviderEndpointURI,
+                '--ipfs-host',
+                'ipfs',
+                ...additionalParams,
+              ],
+              ports: [{ containerPort: colossusPort }],
+            },
+          ],
+          volumes,
+        },
+      },
+    },
+  },
+  {
+    provider: cluster.provider,
+  }
+)
+
+// Export the Deployment name
+export const deploymentName = deployment.metadata.name

devops/infrastructure/storage-node/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "eks-cluster",
+  "devDependencies": {
+    "@types/node": "^10.0.0"
+  },
+  "dependencies": {
+    "@pulumi/aws": "^4.0.0",
+    "@pulumi/awsx": "^0.30.0",
+    "@pulumi/eks": "^0.31.0",
+    "@pulumi/kubernetes": "^3.0.0",
+    "@pulumi/pulumi": "^3.0.0"
+  }
+}

devops/infrastructure/storage-node/tsconfig.json

@@ -0,0 +1,18 @@
+{
+    "compilerOptions": {
+        "strict": true,
+        "outDir": "bin",
+        "target": "es2016",
+        "module": "commonjs",
+        "moduleResolution": "node",
+        "sourceMap": true,
+        "experimentalDecorators": true,
+        "pretty": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitReturns": true,
+        "forceConsistentCasingInFileNames": true
+    },
+    "files": [
+        "index.ts"
+    ]
+}

joystream-node.Dockerfile

@@ -1,11 +1,13 @@
-FROM liuchong/rustup:nightly AS rustup
-RUN rustup install nightly-2021-03-24
+FROM rust:1.52.1-buster AS rust
+RUN rustup self update
+RUN rustup install nightly-2021-03-24 --force
 RUN rustup default nightly-2021-03-24
 RUN rustup target add wasm32-unknown-unknown --toolchain nightly-2021-03-24
+RUN rustup component add --toolchain nightly-2021-03-24 clippy
 RUN apt-get update && \
   apt-get install -y curl git gcc xz-utils sudo pkg-config unzip clang llvm libc6-dev
 
-FROM rustup AS builder
+FROM rust AS builder
 LABEL description="Compiles all workspace artifacts"
 WORKDIR /joystream
 COPY . /joystream
@@ -17,7 +19,7 @@ RUN BUILD_DUMMY_WASM_BINARY=1 cargo clippy --release --all -- -D warnings && \
     cargo test --release --all && \
     cargo build --release
 
-FROM debian:buster
+FROM ubuntu:21.04
 LABEL description="Joystream node"
 WORKDIR /joystream
 COPY --from=builder /joystream/target/release/joystream-node /joystream/node

node/Cargo.toml

@@ -3,7 +3,7 @@ authors = ['Joystream contributors']
 build = 'build.rs'
 edition = '2018'
 name = 'joystream-node'
-version = '5.5.0'
+version = '5.6.0'
 default-run = "joystream-node"
 
 [[bin]]

pioneer/packages/apps/src/SideBar/index.tsx

@@ -14,6 +14,7 @@ import NetworkModal from '../modals/Network';
 import { useTranslation } from '../translate';
 import ChainInfo from './ChainInfo';
 import Item from './Item';
+import SidebarBanner from '../SidebarBanner';
 
 interface Props {
   className?: string;
@@ -100,6 +101,7 @@ function SideBar ({ className = '', collapse, handleResize, isCollapsed, isMenuO
                 )
             ))}
             <Menu.Divider hidden />
+            <SidebarBanner isSidebarCollapsed={isCollapsed}/>
           </div>
           <div className={`apps--SideBar-collapse ${isCollapsed ? 'collapsed' : 'expanded'}`}>
             <Button

pioneer/packages/apps/src/SidebarBanner.tsx

@@ -0,0 +1,325 @@
+import React, { useState, useEffect } from 'react';
+import usePromise from '@polkadot/joy-utils/react/hooks/usePromise';
+import styled from 'styled-components';
+import { Segment, Loader, Button } from 'semantic-ui-react';
+
+const COUNTER_BORDER_RADIUS_VALUE = 2;
+
+const BannerContainer = styled.div<{ isCollapsed?: boolean }>`
+  ${({ isCollapsed }) => isCollapsed ? `
+    min-height: 222px;
+    max-height: 222px;
+  ` : `
+    min-height: 322px;
+    max-height: 322px;
+    padding: 16px;
+  `}
+  width: 100%;
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+  background-color: #4038FF;
+`;
+
+const BannerTitle = styled.h1`
+  padding-right: 1px;
+  font-family: Lato;
+  font-size: 16px;
+  font-weight: 800;
+  line-height: 20px;
+  letter-spacing: 0em;
+  color: white;
+`;
+
+const BannerSubtitle = styled.h2`
+  margin-top: 16px;
+  font-family: Lato;
+  font-size: 14px;
+  font-weight: 400;
+  line-height: 18px;
+  letter-spacing: 0em;
+  color: #E0E1FF;
+`;
+
+const BannerLink = styled.a`
+  margin-top: 8px;
+  font-size: 12px;
+  font-weight: 600;
+  line-height: 16px;
+  letter-spacing: 0em;
+  text-align: center;
+  text-decoration: underline;
+  color: #B4BBFF !important;
+`;
+
+const BannerButton = styled(Button)`
+  width: 100% !important;
+  margin-top: 8px !important;
+`;
+
+const ProgressContainer = styled.div<{ isCollapsed ?: boolean }>`
+  width: 100%;
+  ${({ isCollapsed }) => isCollapsed ? `
+    margin-top: 3px;
+  ` : `
+    margin-top: 8px;
+  `}
+`;
+
+const CounterContainer = styled.div<{ isCollapsed ?: boolean }>`
+  width: 100%;
+  ${({ isCollapsed }) => isCollapsed ? `
+    height: 120px;
+    flex-direction: column;
+  ` : `
+    height: 64px;
+  `}
+  padding: 8px;
+  display: flex;
+  align-items: center;
+  justify-content: ${({ children }) => children && children > 1 ? 'space-between' : 'center'};
+  background-color: #261EE4;
+  border-top-left-radius: ${COUNTER_BORDER_RADIUS_VALUE}px;
+  border-top-right-radius: ${COUNTER_BORDER_RADIUS_VALUE}px;
+`;
+
+const CounterItem = styled.div<{ isCollapsed ?: boolean }>`
+  ${({ isCollapsed }) => isCollapsed ? `
+    width: 43px;
+  ` : `
+    width: 56px;
+  `}
+  height: 48px;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: space-between;
+`;
+
+const CounterItemNumber = styled.p`
+  margin: 0;
+  font-size: 32px;
+  font-weight: 700;
+  line-height: 32px;
+  letter-spacing: 0em;
+  color: white;
+`;
+
+const CounterItemText = styled.p`
+  margin: 0;
+  font-size: 10px;
+  font-weight: 600;
+  line-height: 16px;
+  letter-spacing: 0em;
+  color: white;
+`;
+
+const Progress = styled.div<{ isCollapsed?: boolean }>`
+  width: 100%;
+  height: 6px;
+  background-color: #5252FF;
+  ${({ isCollapsed }) => !isCollapsed && `
+    border-bottom-left-radius: ${COUNTER_BORDER_RADIUS_VALUE}px;
+    border-bottom-right-radius: ${COUNTER_BORDER_RADIUS_VALUE}px;
+  `}
+`;
+
+const ProgressBar = styled.div<{ isCollapsed?: boolean }>`
+  width: 0%;
+  height: 100%;
+  background-color: white;
+  ${({ isCollapsed }) => !isCollapsed && `
+    border-bottom-left-radius: ${COUNTER_BORDER_RADIUS_VALUE}px;
+    border-bottom-right-radius: ${COUNTER_BORDER_RADIUS_VALUE}px;
+  `}
+`;
+
+const ErrorText = styled.h1`
+  font-size: 14px;
+  letter-spacing: 0em;
+  font-weight: 600;
+  color: white;
+`;
+
+const DatesContainer = styled.div`
+  width: 100%;
+  display: flex;
+  justify-content: space-between;
+  margin-top: 8px;
+`;
+
+const DateText = styled.p<{ isCollapsed?: boolean }>`
+  font-size: 12px;
+  line-height: 16px;
+  letter-spacing: 0em;
+  color: #E0E1FF;
+  ${({ isCollapsed }) => isCollapsed && `
+    margin-top: 8px;
+  `}
+`;
+
+const StyledLoader = styled(Loader)`
+  ::before {
+    border-color: rgba(255,255,255,.15) !important;
+  }
+
+  ::after {
+    border-color: white transparent transparent !important;
+  }
+`;
+
+const FM_DATA_URL = 'https://raw.githubusercontent.com/Joystream/founding-members/main/data/fm-info.json';
+const MILLISECONDS_TO_DAYS = 1000 * 60 * 60 * 24;
+
+type FoundingMembersData = {
+  scoringPeriodsFull: {
+    currentScoringPeriod: {
+      started: string;
+      ends: string;
+    }
+  }
+}
+
+const numberToDateString = (number: number) => {
+  const remainingTime: Array<[number, string]> = [];
+
+  const weeks = Math.floor(number / 7);
+  const days = Math.floor(number - (weeks * 7));
+  const hours = Math.floor((number - ((weeks * 7) + days)) * 24);
+
+  if (weeks) {
+    remainingTime.push([weeks, weeks === 1 ? 'WEEK' : 'WEEKS']);
+
+    if (days) {
+      remainingTime.push([days, days === 1 ? 'DAY' : 'DAYS']);
+    }
+
+    return remainingTime;
+  }
+
+  if (days) {
+    remainingTime.push([days, days === 1 ? 'DAY' : 'DAYS']);
+  }
+
+  if (hours) {
+    remainingTime.push([hours, hours === 1 ? 'HOUR' : 'HOURS']);
+  }
+
+  return remainingTime;
+};
+
+const SidebarBanner = ({ isSidebarCollapsed } : { isSidebarCollapsed: boolean}) => {
+  const [foundingMembersData, foundingMembersDataError] = usePromise<FoundingMembersData | undefined>(
+    () => fetch(FM_DATA_URL).then((res) => res.json().then((data) => data as FoundingMembersData)), undefined, []
+  );
+  const [dates, setDates] = useState<{ started: Date, ends: Date }>();
+  const [progress, setProgress] = useState<number>(0);
+  const [remainingTime, setRemainingTime] = useState<number>();
+
+  useEffect(() => {
+    if (foundingMembersData && !foundingMembersDataError) {
+      const scoringPeriodStartedDate = new Date(foundingMembersData.scoringPeriodsFull.currentScoringPeriod.started);
+      const scoringPeriodEndedDate = new Date(foundingMembersData.scoringPeriodsFull.currentScoringPeriod.ends);
+      const now = new Date();
+
+      // calculate the elapsed time from start of scoring period until now
+      const timeDifferenceBetweenDates = Math.abs(scoringPeriodEndedDate.getTime() - scoringPeriodStartedDate.getTime()) / MILLISECONDS_TO_DAYS;
+      const timePassedUntilNow = Math.abs(now.getTime() - scoringPeriodStartedDate.getTime()) / MILLISECONDS_TO_DAYS;
+      const progressPercentage = (timePassedUntilNow / timeDifferenceBetweenDates) * 100;
+
+      // calculate the amount of days remaining until the end of the scoring period
+      const remainingTime = Math.abs(scoringPeriodEndedDate.getTime() - now.getTime()) / MILLISECONDS_TO_DAYS;
+
+      setRemainingTime(remainingTime);
+
+      setDates({
+        started: scoringPeriodStartedDate,
+        ends: scoringPeriodEndedDate
+      });
+
+      setProgress(progressPercentage > 100 ? 100 : progressPercentage);
+    }
+  }, [foundingMembersData]);
+
+  const Loading = ({ isCollapsed } : { isCollapsed ?: boolean}) => (
+    <Segment>
+      <StyledLoader active size={isCollapsed ? 'small' : 'medium'} />
+    </Segment>
+  );
+
+  const Error = () => (
+    <ErrorText> Error.. </ErrorText>
+  );
+
+  if (isSidebarCollapsed) {
+    return (
+      <BannerContainer isCollapsed={true}>
+        <BannerSubtitle>Scoring period ends in:</BannerSubtitle>
+        <ProgressContainer isCollapsed={true}>
+          <CounterContainer isCollapsed={true}>
+            {remainingTime
+              ? numberToDateString(remainingTime).map(([amountOfTime, timePeriodString], index) => (
+                <CounterItem key={`${index}-${amountOfTime}-${timePeriodString}`}>
+                  <CounterItemNumber>{amountOfTime}</CounterItemNumber>
+                  <CounterItemText>{timePeriodString}</CounterItemText>
+                </CounterItem>
+              ))
+              : <Loading isCollapsed={true}/>
+            }
+            {!remainingTime && foundingMembersDataError ? <Error /> : null}
+          </CounterContainer>
+          <Progress isCollapsed={true}>
+            <ProgressBar isCollapsed={true} style={{ width: `${progress}%` }}/>
+          </Progress>
+        </ProgressContainer>
+        <DateText isCollapsed={true} >{dates?.ends.toLocaleString('default', { month: 'short' })} {dates?.ends.getDate()}</DateText>
+      </BannerContainer>
+    );
+  }
+
+  return (
+    <BannerContainer>
+      <BannerTitle>Report your activity to earn FM points</BannerTitle>
+      <BannerSubtitle>Current scoring period ends in:</BannerSubtitle>
+      <ProgressContainer>
+        <CounterContainer>
+          {remainingTime
+            ? numberToDateString(remainingTime).map(([amountOfTime, timePeriodString], index) => (
+              <CounterItem key={`${index}-${amountOfTime}-${timePeriodString}`}>
+                <CounterItemNumber>{amountOfTime}</CounterItemNumber>
+                <CounterItemText>{timePeriodString}</CounterItemText>
+              </CounterItem>
+            ))
+            : <Loading />
+          }
+          {!remainingTime && foundingMembersDataError ? <Error /> : null}
+        </CounterContainer>
+        <Progress>
+          <ProgressBar style={{ width: `${progress}%` }}/>
+        </Progress>
+        <DatesContainer>
+          <DateText>{dates?.started.toLocaleString('default', { month: 'short' })} {dates?.started.getDate()}</DateText>
+          <DateText>{dates?.ends.toLocaleString('default', { month: 'short' })} {dates?.ends.getDate()}</DateText>
+        </DatesContainer>
+      </ProgressContainer>
+      <BannerButton
+        color='black'
+        href='https://www.joystream.org/founding-members/form/'
+        target='_blank'
+        rel='noopener noreferrer'
+      >
+        Report now
+      </BannerButton>
+      <BannerLink
+        href='https://github.com/Joystream/founding-members/blob/main/SUBMISSION-GUIDELINES.md'
+        target='_blank'
+        rel='noopener noreferrer'
+      >
+        Learn more...
+      </BannerLink>
+    </BannerContainer>
+  );
+};
+
+export default SidebarBanner;

pioneer/packages/joy-election/src/index.tsx

@@ -22,6 +22,7 @@ import Reveals from './Reveals';
 import { queryToProp } from '@polkadot/joy-utils/functions/misc';
 import { Seat } from '@joystream/types/council';
 import { ApiProps } from '@polkadot/react-api/types';
+import FMReminderBanner from '@polkadot/joy-utils/react/components/FMReminderBanner';
 
 const ElectionMain = styled.main`${style}`;
 
@@ -67,6 +68,7 @@ class App extends React.PureComponent<Props, State> {
 
     return (
       <ElectionMain className='election--App'>
+        <FMReminderBanner contextualTitle='Council'/>
         <header>
           <Tabs basePath={basePath} items={tabs} />
         </header>

pioneer/packages/joy-forum/src/index.tsx

@@ -16,6 +16,7 @@ import { CategoryList, ViewCategoryById } from './CategoryList';
 import { ViewThreadById } from './ViewThread';
 import { LegacyPagingRedirect } from './LegacyPagingRedirect';
 import ForumRoot from './ForumRoot';
+import FMReminderBanner from '@polkadot/joy-utils/react/components/FMReminderBanner';
 
 const ForumMain = styled.main`${style}`;
 
@@ -29,6 +30,7 @@ class App extends React.PureComponent<Props> {
       <ForumProvider>
         <ForumSudoProvider>
           <ForumMain className='forum--App'>
+            <FMReminderBanner contextualTitle='Forum'/>
             <Switch>
               <Route path={`${basePath}/categories/new`} component={NewCategory} />
               {/* routes for handling legacy format of forum paging within the routing path */}

pioneer/packages/joy-forum/src/style.ts

@@ -1,12 +1,14 @@
 import { css } from 'styled-components';
 
 export default css`
-  padding-top: 1.5rem;
-
   .ui.segment {
     background-color: #fff;
   }
 
+  .ui.breadcrumb {
+    margin-top: 2rem;
+  }
+
   .ForumPageTitle {
     display: flex;
     margin-top: 1rem;

pioneer/packages/joy-media/src/index.tsx

@@ -9,6 +9,7 @@ import translate from './translate';
 import { Button, Grid, Message, Icon, Image } from 'semantic-ui-react';
 
 import AtlasScreenShot from './assets/atlas-screenshot.jpg';
+import JoystreamStudio from './assets/joystream-studio-screenshot.png';
 
 const MediaMain = styled.main`
   display: flex;
@@ -49,6 +50,10 @@ const Screenshot = styled(Image)`
   :hover { opacity: 0.7; }
 `;
 
+const StyledList = styled(Message.List)`
+  margin-top: 0.5em !important;
+`;
+
 interface Props extends AppMainRouteProps, I18nProps {}
 
 const App: React.FC<Props> = () => {
@@ -57,10 +62,10 @@ const App: React.FC<Props> = () => {
       <Header>
         <h1>Hello there!</h1>
         <p>
-          We have now upgraded to the Babylon chain.
+          We have now upgraded to the Sumer chain.
         </p>
         <p>
-          Pioneer consequently <b>no longer supports</b> media uploads and consumption.
+          Pioneer <b>no longer supports</b> media uploads and consumption.
         </p>
       </Header>
       <Grid stackable>
@@ -93,18 +98,28 @@ const App: React.FC<Props> = () => {
             <StyledMessage>
               <Message.Header>Uploading content</Message.Header>
               <Message.Content>
-                Uploading has been migrated over to the Joystream CLI.
-                Instructions on how to use the CLI can be found in our helpdesk.
+                With Sumer, the content uploading process has been streamlined and made accessible through Joystream Studio. To upload a video:
+                <StyledList>
+                  <Message.Item>Go to Joystream Studio</Message.Item>
+                  <Message.Item>Create/connect your membership</Message.Item>
+                  <Message.Item>Create a channel</Message.Item>
+                  <Message.Item>Publish content</Message.Item>
+                </StyledList>
+                <Screenshot
+                  src={JoystreamStudio as string}
+                  href='https://play.joystream.org/studio'
+                  target='_blank'
+                  rel='noopener noreferrer'/>
               </Message.Content>
               <Button
                 size='big'
                 primary
-                href='https://github.com/Joystream/helpdesk/tree/master/roles/content-creators'
+                href='https://play.joystream.org/studio'
                 icon
                 labelPosition='right'
                 target='_blank'
                 rel='noopener noreferrer'>
-                Explore Joystream CLI
+                Explore Joystream Studio
                 <Icon name='arrow right' />
               </Button>
             </StyledMessage>

pioneer/packages/joy-proposals/src/index.tsx

@@ -27,6 +27,7 @@ import { SignalForm,
 import { RouteProps as AppMainRouteProps } from '@polkadot/apps-routing/types';
 import style from './style';
 import { HistoricalProposalFromId } from './Proposal/ProposalFromId';
+import FMReminderBanner from '@polkadot/joy-utils/react/components/FMReminderBanner';
 
 const ProposalsMain = styled.main`${style}`;
 
@@ -58,6 +59,7 @@ function App (props: Props): React.ReactElement<Props> {
 
   return (
     <ProposalsMain className='proposal--App'>
+      <FMReminderBanner contextualTitle='Proposals'/>
       <StyledHeader>
         <Tabs
           basePath={basePath}

pioneer/packages/joy-roles/src/index.tsx

@@ -17,6 +17,7 @@ import { OpportunityController, OpportunityView } from './tabs/Opportunity.contr
 import { OpportunitiesController, OpportunitiesView } from './tabs/Opportunities.controller';
 import { ApplyController, ApplyView } from './flows/apply.controller';
 import { MyRolesController, MyRolesView } from './tabs/MyRoles.controller';
+import FMReminderBanner from '@polkadot/joy-utils/react/components/FMReminderBanner';
 
 import './index.sass';
 
@@ -74,6 +75,7 @@ export const App: React.FC<Props> = (props: Props) => {
 
   return (
     <main className='roles--App'>
+      <FMReminderBanner contextualTitle='Working Groups'/>
       <header>
         <Tabs
           basePath={basePath}

pioneer/packages/joy-tokenomics/src/Overview/OverviewTable.tsx

@@ -93,7 +93,7 @@ const OverviewTable: React.FC<{data?: TokenomicsData; statusData?: StatusServerD
         />
         <OverviewTableRow
           item='Weekly Top Ups'
-          value={displayStatusData((Number(statusData?.dollarPool.replenishAmount) / 2).toFixed(2) || '', 'USD')}
+          value={displayStatusData(Number(statusData?.dollarPool.replenishAmount).toFixed(2) || '', 'USD')}
           help={'The current weekly \'Fiat Pool\' replenishment amount. Does not include KPIs, or other potential top ups.'}
         />
       </Table.Body>

pioneer/packages/joy-utils/src/react/components/FMReminderBanner.tsx

@@ -0,0 +1,138 @@
+import React from 'react';
+import styled from 'styled-components';
+import { Button, Icon } from 'semantic-ui-react';
+import CoinIllustration from '../../assets/coin-illustration.png';
+import CoinIllustrationSmall from '../../assets/coin-illustration1.png';
+
+const Container = styled.div`
+  height: auto;
+  margin: 2em 0 0 0;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+`;
+
+const Banner = styled.div`
+  height: 89px;
+  width: 100%;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  padding: 1.5em;
+  background-color: #262626;
+  box-shadow: inset 0px 0px 0px 1px rgba(34, 36, 38, 0.22);
+  border-radius: 4px;
+  background-image: url(${CoinIllustration});
+  background-position: 90% 0;
+  background-repeat: no-repeat;
+  background-size: contain;
+
+  @media(max-width: 1450px){
+    height: 109px;
+  }
+
+  @media(max-width: 1200px){
+    background-image: none;
+  }
+
+  @media(max-width: 800px){
+    flex-direction: column;
+    align-items: initial;
+    height: auto;
+  }
+
+  @media (max-width: 425px){
+    background-image: url(${CoinIllustrationSmall});
+    padding-top: 7em;
+    background-position: left 0;
+    background-size: 200px;
+  }
+`;
+
+const TextContainer = styled.div``;
+
+const BannerTitle = styled.h1`
+  font-family: Lato;
+  font-size: 16px;
+  font-style: normal;
+  font-weight: 900;
+  line-height: 20px;
+  letter-spacing: 0em;
+  color: white;
+  margin-bottom: 7px;
+`;
+
+const BannerText = styled.p`
+  font-size: 14px;
+  font-style: normal;
+  font-weight: 400;
+  line-height: 20px;
+  letter-spacing: 0.0033em;
+  color: #FFFFFFDE;
+
+  a {
+    text-decoration: underline;
+    color: inherit;
+  }
+`;
+
+const BannerButton = styled(Button)`
+  background-color: #4038FF !important;
+  color: white !important;
+  min-width: 155px !important;
+  width: 155px !important;
+  min-height: 36px !important;
+  height: 36px !important;
+
+  .icon {
+    background-color: #3D35F2 !important;
+  }
+
+  margin-left: 260px !important;
+
+  @media(max-width: 1200px){
+    margin-left: 30px !important;
+  }
+
+  @media(max-width: 800px){
+    margin: 20px 0 0 0 !important;
+  }
+`;
+
+interface Props {
+  contextualTitle: 'Council' | 'Working Groups' | 'Proposals' | 'Forum';
+}
+
+const FMReminderBanner = ({ contextualTitle } : Props) => {
+  return (
+    <Container>
+      <Banner>
+        <TextContainer>
+          <BannerTitle>Report your {contextualTitle} activity to earn Founding Members points!</BannerTitle>
+          <BannerText>
+            Only activity that&apos;s been reported is eligible for earning FM points.
+            <a
+              href='https://github.com/Joystream/founding-members/blob/main/SUBMISSION-GUIDELINES.md'
+              target='_blank'
+              rel='noopener noreferrer'
+            >
+              Learn more about reporting your activity...
+            </a>
+          </BannerText>
+        </TextContainer>
+        <BannerButton
+          icon
+          labelPosition='right'
+          href='https://www.joystream.org/founding-members/form/'
+          target='_blank'
+          rel='noopener noreferrer'
+        >
+            Report Now
+          <Icon name='arrow right' />
+        </BannerButton>
+      </Banner>
+    </Container>
+  );
+};
+
+export default FMReminderBanner;

runtime-modules/content/src/lib.rs

@@ -1363,20 +1363,6 @@ impl<T: Trait> Module<T> {
     }
 }
 
-// Some initial config for the module on runtime upgrade
-impl<T: Trait> Module<T> {
-    pub fn on_runtime_upgrade() {
-        <NextChannelCategoryId<T>>::put(T::ChannelCategoryId::one());
-        <NextVideoCategoryId<T>>::put(T::VideoCategoryId::one());
-        <NextVideoId<T>>::put(T::VideoId::one());
-        <NextChannelId<T>>::put(T::ChannelId::one());
-        <NextPlaylistId<T>>::put(T::PlaylistId::one());
-        <NextSeriesId<T>>::put(T::SeriesId::one());
-        <NextPersonId<T>>::put(T::PersonId::one());
-        <NextChannelOwnershipTransferRequestId<T>>::put(T::ChannelOwnershipTransferRequestId::one());
-    }
-}
-
 decl_event!(
     pub enum Event<T>
     where

runtime/CHANGELOG.md

@@ -1,3 +1,16 @@
+### Version 9.7.0 - Sumer - runtime upgrade - May 27 2021
+- Introduced new content pallet the new content directory
+- Improved data_directory pallet
+  - Any storage provider to handle uploads of new content
+  - Integration with new content directory
+  - Introduction of quota vouchers
+  - Reset data directory
+- Added new working group instance for Operations
+
+### Version 9.3.0 - Antioch - new chain - April 7 2021
+- Following chain failure due to a debug in older version of substrate (v2.0.0-rc4) updated to substrate v2.0.1
+- Same runtime features as babylon
+
 ### Version 7.9.0 - Babylon - runtime upgrade - December 21 2020
 - Introduction of new and improved content directory
 

runtime/Cargo.toml

@@ -4,7 +4,7 @@ edition = '2018'
 name = 'joystream-node-runtime'
 # Follow convention: https://github.com/Joystream/substrate-runtime-joystream/issues/1
 # {Authoring}.{Spec}.{Impl} of the RuntimeVersion
-version = '9.7.0'
+version = '9.8.0'
 
 [dependencies]
 # Third-party dependencies

runtime/src/integration/working_group.rs

@@ -1,17 +1,20 @@
 use frame_support::StorageMap;
 use sp_std::marker::PhantomData;
 
-use crate::{ContentDirectoryWorkingGroupInstance, StorageWorkingGroupInstance};
+use crate::{
+    ContentDirectoryWorkingGroupInstance, GatewayWorkingGroupInstance,
+    OperationsWorkingGroupInstance, StorageWorkingGroupInstance,
+};
 use stake::{BalanceOf, NegativeImbalance};
 
 // Will be removed in the next releases.
 #[allow(clippy::upper_case_acronyms)]
-pub struct ContentDirectoryWGStakingEventsHandler<T> {
+pub struct ContentDirectoryWgStakingEventsHandler<T> {
     pub marker: PhantomData<T>,
 }
 
 impl<T: stake::Trait + working_group::Trait<ContentDirectoryWorkingGroupInstance>>
-    stake::StakingEventsHandler<T> for ContentDirectoryWGStakingEventsHandler<T>
+    stake::StakingEventsHandler<T> for ContentDirectoryWgStakingEventsHandler<T>
 {
     /// Unstake remaining sum back to the source_account_id
     fn unstaked(
@@ -93,3 +96,91 @@ impl<T: stake::Trait + working_group::Trait<StorageWorkingGroupInstance>>
         remaining_imbalance
     }
 }
+
+pub struct OperationsWgStakingEventsHandler<T> {
+    pub marker: PhantomData<T>,
+}
+
+impl<T: stake::Trait + working_group::Trait<OperationsWorkingGroupInstance>>
+    stake::StakingEventsHandler<T> for OperationsWgStakingEventsHandler<T>
+{
+    /// Unstake remaining sum back to the source_account_id
+    fn unstaked(
+        stake_id: &<T as stake::Trait>::StakeId,
+        _unstaked_amount: BalanceOf<T>,
+        remaining_imbalance: NegativeImbalance<T>,
+    ) -> NegativeImbalance<T> {
+        // Stake not related to a staked role managed by the hiring module.
+        if !hiring::ApplicationIdByStakingId::<T>::contains_key(*stake_id) {
+            return remaining_imbalance;
+        }
+
+        let hiring_application_id = hiring::ApplicationIdByStakingId::<T>::get(*stake_id);
+
+        if working_group::MemberIdByHiringApplicationId::<T, OperationsWorkingGroupInstance>::contains_key(
+            hiring_application_id,
+        ) {
+            return <working_group::Module<T, OperationsWorkingGroupInstance>>::refund_working_group_stake(
+				*stake_id,
+				remaining_imbalance,
+			);
+        }
+
+        remaining_imbalance
+    }
+
+    /// Empty handler for the slashing.
+    fn slashed(
+        _: &<T as stake::Trait>::StakeId,
+        _: Option<<T as stake::Trait>::SlashId>,
+        _: BalanceOf<T>,
+        _: BalanceOf<T>,
+        remaining_imbalance: NegativeImbalance<T>,
+    ) -> NegativeImbalance<T> {
+        remaining_imbalance
+    }
+}
+
+pub struct GatewayWgStakingEventsHandler<T> {
+    pub marker: PhantomData<T>,
+}
+
+impl<T: stake::Trait + working_group::Trait<GatewayWorkingGroupInstance>>
+    stake::StakingEventsHandler<T> for GatewayWgStakingEventsHandler<T>
+{
+    /// Unstake remaining sum back to the source_account_id
+    fn unstaked(
+        stake_id: &<T as stake::Trait>::StakeId,
+        _unstaked_amount: BalanceOf<T>,
+        remaining_imbalance: NegativeImbalance<T>,
+    ) -> NegativeImbalance<T> {
+        // Stake not related to a staked role managed by the hiring module.
+        if !hiring::ApplicationIdByStakingId::<T>::contains_key(*stake_id) {
+            return remaining_imbalance;
+        }
+
+        let hiring_application_id = hiring::ApplicationIdByStakingId::<T>::get(*stake_id);
+
+        if working_group::MemberIdByHiringApplicationId::<T, GatewayWorkingGroupInstance>::contains_key(
+            hiring_application_id,
+        ) {
+            return <working_group::Module<T, GatewayWorkingGroupInstance>>::refund_working_group_stake(
+				*stake_id,
+				remaining_imbalance,
+			);
+        }
+
+        remaining_imbalance
+    }
+
+    /// Empty handler for the slashing.
+    fn slashed(
+        _: &<T as stake::Trait>::StakeId,
+        _: Option<<T as stake::Trait>::SlashId>,
+        _: BalanceOf<T>,
+        _: BalanceOf<T>,
+        remaining_imbalance: NegativeImbalance<T>,
+    ) -> NegativeImbalance<T> {
+        remaining_imbalance
+    }
+}

runtime/src/lib.rs

@@ -85,7 +85,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
     spec_name: create_runtime_str!("joystream-node"),
     impl_name: create_runtime_str!("joystream-node"),
     authoring_version: 9,
-    spec_version: 7,
+    spec_version: 8,
     impl_version: 0,
     apis: crate::runtime_api::EXPORTED_RUNTIME_API_VERSIONS,
     transaction_version: 1,
@@ -469,14 +469,21 @@ parameter_types! {
     pub const StakePoolId: [u8; 8] = *b"joystake";
 }
 
+#[allow(clippy::type_complexity)]
 impl stake::Trait for Runtime {
     type Currency = <Self as common::currency::GovernanceCurrency>::Currency;
     type StakePoolId = StakePoolId;
     type StakingEventsHandler = (
         crate::integration::proposals::StakingEventsHandler<Self>,
         (
-            crate::integration::working_group::ContentDirectoryWGStakingEventsHandler<Self>,
-            crate::integration::working_group::StorageWgStakingEventsHandler<Self>,
+            (
+                crate::integration::working_group::ContentDirectoryWgStakingEventsHandler<Self>,
+                crate::integration::working_group::StorageWgStakingEventsHandler<Self>,
+            ),
+            (
+                crate::integration::working_group::OperationsWgStakingEventsHandler<Self>,
+                crate::integration::working_group::GatewayWgStakingEventsHandler<Self>,
+            ),
         ),
     );
     type StakeId = u64;

runtime/src/runtime_api.rs

@@ -10,16 +10,11 @@ use sp_runtime::traits::{BlakeTwo256, Block as BlockT, NumberFor};
 use sp_runtime::{generic, ApplyExtrinsicResult};
 use sp_std::vec::Vec;
 
-use crate::{
-    ContentDirectoryWorkingGroupInstance, DataDirectory, GatewayWorkingGroupInstance,
-    OperationsWorkingGroupInstance, StorageWorkingGroupInstance,
-};
-
 use crate::constants::PRIMARY_PROBABILITY;
 
 use crate::{
-    content, data_directory, AccountId, AuthorityDiscoveryId, Balance, BlockNumber, EpochDuration,
-    GrandpaAuthorityList, GrandpaId, Hash, Index, RuntimeVersion, Signature, VERSION,
+    AccountId, AuthorityDiscoveryId, Balance, BlockNumber, EpochDuration, GrandpaAuthorityList,
+    GrandpaId, Hash, Index, RuntimeVersion, Signature, VERSION,
 };
 use crate::{
     AllModules, AuthorityDiscovery, Babe, Call, Grandpa, Historical, InherentDataExt,
@@ -59,72 +54,10 @@ pub type BlockId = generic::BlockId<Block>;
 /// Unchecked extrinsic type as expected by this runtime.
 pub type UncheckedExtrinsic = generic::UncheckedExtrinsic<AccountId, Call, Signature, SignedExtra>;
 
-// Default Executive type without the RuntimeUpgrade
-// pub type Executive =
-//     frame_executive::Executive<Runtime, Block, frame_system::ChainContext<Runtime>, Runtime, AllModules>;
-
-// Alias for the builder working group
-pub(crate) type OperationsWorkingGroup<T> =
-    working_group::Module<T, OperationsWorkingGroupInstance>;
-
-// Alias for the gateway working group
-pub(crate) type GatewayWorkingGroup<T> = working_group::Module<T, GatewayWorkingGroupInstance>;
-
-// Alias for the storage working group
-pub(crate) type StorageWorkingGroup<T> = working_group::Module<T, StorageWorkingGroupInstance>;
-
-// Alias for the content working group
-pub(crate) type ContentDirectoryWorkingGroup<T> =
-    working_group::Module<T, ContentDirectoryWorkingGroupInstance>;
-
 /// Custom runtime upgrade handler.
 pub struct CustomOnRuntimeUpgrade;
 impl OnRuntimeUpgrade for CustomOnRuntimeUpgrade {
     fn on_runtime_upgrade() -> Weight {
-        content::Module::<Runtime>::on_runtime_upgrade();
-
-        let default_text_constraint = crate::working_group::default_text_constraint();
-
-        let default_storage_size_constraint =
-            crate::working_group::default_storage_size_constraint();
-
-        let default_content_working_group_mint_capacity = 0;
-
-        // Initialize new groups
-        OperationsWorkingGroup::<Runtime>::initialize_working_group(
-            default_text_constraint,
-            default_text_constraint,
-            default_text_constraint,
-            default_storage_size_constraint,
-            default_content_working_group_mint_capacity,
-        );
-
-        GatewayWorkingGroup::<Runtime>::initialize_working_group(
-            default_text_constraint,
-            default_text_constraint,
-            default_text_constraint,
-            default_storage_size_constraint,
-            default_content_working_group_mint_capacity,
-        );
-
-        DataDirectory::initialize_data_directory(
-            Vec::new(),
-            data_directory::DEFAULT_VOUCHER_SIZE_LIMIT_UPPER_BOUND,
-            data_directory::DEFAULT_VOUCHER_OBJECTS_LIMIT_UPPER_BOUND,
-            data_directory::DEFAULT_GLOBAL_VOUCHER,
-            data_directory::DEFAULT_VOUCHER,
-            data_directory::DEFAULT_UPLOADING_BLOCKED_STATUS,
-        );
-
-        // Initialize existing groups
-        StorageWorkingGroup::<Runtime>::set_worker_storage_size_constraint(
-            default_storage_size_constraint,
-        );
-
-        ContentDirectoryWorkingGroup::<Runtime>::set_worker_storage_size_constraint(
-            default_storage_size_constraint,
-        );
-
         10_000_000 // TODO: adjust weight
     }
 }

setup.sh

@@ -17,6 +17,8 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
     brew update
     brew install coreutils gnu-tar jq curl
     echo "It is recommended to setup Docker desktop from: https://www.docker.com/products/docker-desktop"
+    echo "It is also recommended to install qemu emulators with following command:"
+    echo "docker run --privileged --rm tonistiigi/binfmt --install all"
 fi
 
 # If OS is supported will install build tools for rust and substrate.

storage-node/packages/colossus/paths/asset/v0/{id}.js

@@ -25,7 +25,10 @@ const assert = require('assert')
 
 function errorHandler(response, err, code) {
   debug(err)
-  response.status(err.code || code || 500).send({ message: err.toString() })
+  // Some err types don't have a valid http status code such as one that come from ipfs node for example
+  const statusCode = typeof err.code === 'number' ? err.code : code
+  response.status(statusCode || 500).send({ message: err.toString() })
+  response.end()
 }
 
 // The maximum total estimated balance that will be spent submitting transactions
@@ -143,7 +146,7 @@ module.exports = function (storage, runtime, ipfsHttpGatewayUrl, anonymous) {
           }
         })
 
-        stream.on('finish', async () => {
+        stream.on('end', async () => {
           if (!aborted) {
             try {
               // try to get file info and compute ipfs hash before committing the stream to ifps node.
@@ -206,7 +209,11 @@ module.exports = function (storage, runtime, ipfsHttpGatewayUrl, anonymous) {
           }
         })
 
-        stream.on('error', (err) => errorHandler(res, err))
+        stream.on('error', (err) => {
+          stream.end()
+          stream.cleanup()
+          errorHandler(res, err)
+        })
         req.pipe(stream)
       } catch (err) {
         errorHandler(res, err)

storage-node/packages/storage/storage.js

@@ -90,6 +90,10 @@ class StorageWriteStream extends Transform {
 
     // Create temp target.
     this.temp = temp.createWriteStream()
+    this.temp.on('error', (err) => this.emit('error', err))
+
+    // Small temporary buffer storing first fileType.minimumBytes of stream
+    // used for early file type detection
     this.buf = Buffer.alloc(0)
   }
 
@@ -99,13 +103,11 @@ class StorageWriteStream extends Transform {
       chunk = Buffer.from(chunk)
     }
 
-    this.temp.write(chunk)
-
     // Try to detect file type during streaming.
     if (!this.fileInfo && this.buf.byteLength <= fileType.minimumBytes) {
       this.buf = Buffer.concat([this.buf, chunk])
 
-      if (this.buf >= fileType.minimumBytes) {
+      if (this.buf.byteLength >= fileType.minimumBytes) {
         const info = fileType(this.buf)
         // No info? We will try again at the end of the stream.
         if (info) {
@@ -115,13 +117,26 @@ class StorageWriteStream extends Transform {
       }
     }
 
-    callback(null)
+    // Always waiting for write flush can be slow..
+    // this.temp.write(chunk, (err) => {
+    //   callback(err)
+    // })
+
+    // Respect backpressure and handle write error
+    if (!this.temp.write(chunk)) {
+      this.temp.once('drain', () => callback(null))
+    } else {
+      process.nextTick(() => callback(null))
+    }
   }
 
   _flush(callback) {
     debug('Flushing temporary stream:', this.temp.path)
-    this.temp.end()
-    callback(null)
+    this.temp.end(() => {
+      debug('flushed!')
+      callback(null)
+      this.emit('end')
+    })
   }
 
   /*
@@ -183,6 +198,8 @@ class StorageWriteStream extends Transform {
    * Clean up temporary data.
    */
   cleanup() {
+    // Make it safe to call cleanup more than once
+    if (!this.temp) return
     debug('Cleaning up temporary file: ', this.temp.path)
     fs.unlink(this.temp.path, () => {
       /* Ignore errors. */
@@ -333,22 +350,15 @@ class Storage {
 
     // Write stream
     if (mode === 'w') {
-      return await this.createWriteStream(contentId, timeout)
+      return this.createWriteStream(contentId, timeout)
     }
 
     // Read stream - with file type detection
     return await this.createReadStream(contentId, timeout)
   }
 
-  async createWriteStream() {
-    // IPFS wants us to just dump a stream into its storage, then returns a
-    // content ID (of its own).
-    // We need to instead return a stream immediately, that we eventually
-    // decorate with the content ID when that's available.
-    return new Promise((resolve) => {
-      const stream = new StorageWriteStream(this)
-      resolve(stream)
-    })
+  createWriteStream() {
+    return new StorageWriteStream(this)
   }
 
   async createReadStream(contentId, timeout) {

storage-node/packages/storage/test/storage.js

@@ -34,7 +34,7 @@ function write(store, contentId, contents, callback) {
   store
     .open(contentId, 'w')
     .then((stream) => {
-      stream.on('finish', () => {
+      stream.on('end', () => {
         stream.commit()
       })
       stream.on('committed', callback)
@@ -90,39 +90,44 @@ describe('storage/storage', () => {
       })
     })
 
-    // it('detects the MIME type of a write stream', (done) => {
-    // 	const contents = fs.readFileSync('../../storage-node_new.svg')
-    // 	storage
-    // 		.open('mime-test', 'w')
-    // 		.then((stream) => {
-    // 			let fileInfo
-    // 			stream.on('fileInfo', (info) => {
-    // 				// Could filter & abort here now, but we're just going to set this,
-    // 				// and expect it to be set later...
-    // 				fileInfo = info
-    // 			})
-    //
-    // 			stream.on('finish', () => {
-    // 				stream.commit()
-    // 			})
-    //
-    // 			stream.on('committed', () => {
-    // 				// ... if fileInfo is not set here, there's an issue.
-    // 				expect(fileInfo).to.have.property('mimeType', 'application/xml')
-    // 				expect(fileInfo).to.have.property('ext', 'xml')
-    // 				done()
-    // 			})
-    //
-    // 			if (!stream.write(contents)) {
-    // 				stream.once('drain', () => stream.end())
-    // 			} else {
-    // 				process.nextTick(() => stream.end())
-    // 			}
-    // 		})
-    // 		.catch((err) => {
-    // 			expect.fail(err)
-    // 		})
-    // })
+    it('detects the MIME type of a write stream', (done) => {
+      const contents = fs.readFileSync('../../storage-node_new.svg')
+      storage
+        .open('mime-test', 'w')
+        .then((stream) => {
+          let fileInfo
+          stream.on('fileInfo', (info) => {
+            // Could filter & abort here now, but we're just going to set this,
+            // and expect it to be set later...
+            fileInfo = info
+          })
+
+          stream.on('end', () => {
+            stream.info()
+          })
+
+          stream.once('info', async (info) => {
+            fileInfo = info
+            stream.commit()
+          })
+
+          stream.on('committed', () => {
+            // ... if fileInfo is not set here, there's an issue.
+            expect(fileInfo).to.have.property('mimeType', 'application/xml')
+            expect(fileInfo).to.have.property('ext', 'xml')
+            done()
+          })
+
+          if (!stream.write(contents)) {
+            stream.once('drain', () => stream.end())
+          } else {
+            process.nextTick(() => stream.end())
+          }
+        })
+        .catch((err) => {
+          expect.fail(err)
+        })
+    })
 
     it('can read a stream', (done) => {
       const contents = 'test-for-reading'