start docker-compose refactoring

.env

@@ -1,5 +1,5 @@
 COMPOSE_PROJECT_NAME=joystream
-PROJECT_NAME=query_node
+PROJECT_NAME=joystream
 
 # We will use a single postgres service with multiple databases
 # The env variables below are by default used by all services and should be
@@ -38,10 +38,5 @@ WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
 # If running joystream-node on host machine you can use following address to reach it instead
 # WS_PROVIDER_ENDPOINT_URI=ws://host.docker.internal:9944/
 
-######################
-#    Storage Node    #
-######################
-COLOSSUS_PORT=3333
-QUERY_NODE_HOST=${GRAPHQL_SERVER_HOST}:${GRAPHQL_SERVER_PORT}
-WORKER_ID=0
-ACCOUNT_URI=//Alice
+# Storage Node
+COLOSSUS_CONTAINER_PORT=3333

README.md

@@ -28,11 +28,11 @@ After cloning the repo run the following initialization scripts:
 # build local npm packages
 yarn build:packages
 
-# Build joystream/node image
+# Build joystream/node docker image
 yarn build:node:docker
 
-# Build applications docker image
-yarn build:apps:docker
+# Build storage inftrasrucure docker images
+yarn build:storage:docker
 
 # start a local development network
 yarn start

build-npm-packages.sh → build-packages.sh

@@ -9,4 +9,4 @@ yarn workspace query-node-root build
 yarn workspace @joystream/cli build
 yarn workspace storage-node-v2 build
 yarn workspace @joystream/distributor-cli build
-# yarn workspace pioneer build
+yarn workspace pioneer build

build-storage-docker.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+if ! command -v docker-compose &> /dev/null
+then
+  echo "docker-compose not found. Skipping docker image builds."
+  exit 0
+fi
+
+# Build colossus docker image
+echo "Building colossus docker image..."
+docker-compose build colossus-1
+
+# Build distributor docker image
+echo "Building distributor docker image..."
+docker-compose build distributor-1

colossus.Dockerfile

@@ -16,7 +16,7 @@ VOLUME ["/data", "/keystore"]
 ENV WS_PROVIDER_ENDPOINT_URI=ws://not-set
 ENV COLOSSUS_PORT=3333
 ENV QUERY_NODE_HOST=not-set
-ENV WORKER_ID=not-set
+ENV WORKER_ID=
 # - set external key file using the `/keystore` volume
 ENV ACCOUNT_KEYFILE=
 ENV ACCOUNT_PWD=
@@ -32,4 +32,9 @@ ENV ACCOUNT_URI=
 EXPOSE ${COLOSSUS_PORT}
 
 WORKDIR /joystream/storage-node-v2
-ENTRYPOINT yarn storage-node server --queryNodeHost ${QUERY_NODE_HOST} --port ${COLOSSUS_PORT} --uploads /data --worker ${WORKER_ID} --apiUrl ${WS_PROVIDER_ENDPOINT_URI} --sync --syncInterval=${SYNC_INTERVAL} --keyFile=${ACCOUNT_KEYFILE} --elasticSearchHost=${ELASTIC_SEARCH_HOST}
+ENTRYPOINT yarn storage-node server --queryNodeHost ${QUERY_NODE_HOST} \
+    --port ${COLOSSUS_PORT} --uploads /data  \
+    --apiUrl ${WS_PROVIDER_ENDPOINT_URI} --sync --syncInterval=${SYNC_INTERVAL} \
+    --elasticSearchHost=${ELASTIC_SEARCH_HOST} \
+    --accountUri=${ACCOUNT_URI} \
+    --worker ${WORKER_ID}

docker-compose.multi-storage.yml

@@ -1,42 +0,0 @@
-version: '3.4'
-services:
-  colossus-2:
-    image: joystream/colossus:latest
-    restart: on-failure
-    build:
-      context: .
-      dockerfile: colossus.Dockerfile
-    depends_on:
-      - graphql-server
-    volumes:
-      - /data
-      - /keystore
-      - ${ACCOUNT_KEYFILE}:/joystream/storage-node-v2/keyfile
-    ports:
-      - '127.0.0.1:${COLOSSUS_PORT_2}:${COLOSSUS_PORT}'
-    env_file:
-      # relative to working directory where docker-compose was run from
-      - .env
-    environment:
-      WORKER_ID: ${WORKER_ID}
-      ACCOUNT_KEYFILE: /joystream/storage-node-v2/keyfile
-
-  distributor-node-2:
-    image: joystream/distributor-node
-    restart: on-failure
-    build:
-      context: .
-      dockerfile: distributor-node.Dockerfile
-    depends_on:
-      - graphql-server
-    volumes:
-      - /data
-      - /cache
-    ports:
-      - 127.0.0.1:${DISTRIBUTOR_PORT_2}:3334
-    environment:
-      JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE: http://${QUERY_NODE_HOST}/graphql
-      JOYSTREAM_DISTRIBUTOR__KEYS: ${KEYS}
-      JOYSTREAM_DISTRIBUTOR__BUCKETS: ${BUCKETS}
-      JOYSTREAM_DISTRIBUTOR__WORKER_ID: ${WORKER_ID}
-    command: ['start']

docker-compose.yml

@@ -18,8 +18,9 @@ services:
       - '127.0.0.1:9944:9944'
       - '127.0.0.1:9933:9933'
 
-  colossus:
+  colossus-1:
     image: joystream/colossus:latest
+    container_name: colossus-1
     restart: on-failure
     build:
       context: .
@@ -30,20 +31,21 @@ services:
       - /data
       - /keystore
     ports:
-      - '127.0.0.1:3333:${COLOSSUS_PORT}'
+      - '127.0.0.1:3333:3333'
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
     environment:
       - COLOSSUS_PORT=3333
-      - QUERY_NODE_HOST=graphql-server-mnt:${GRAPHQL_SERVER_PORT}
+      - QUERY_NODE_HOST=graphql-server:${GRAPHQL_SERVER_PORT}
       - WORKER_ID=0
-      - ACCOUNT_URI=//Alice
+      - ACCOUNT_URI=//testing//worker//Storage//0
       # enable ElasticSearch server
       # - ELASTIC_SEARCH_HOST=host.docker.internal:9200
 
-  distributor-node:
+  distributor-1:
     image: joystream/distributor-node
+    container_name: distributor-1
     restart: on-failure
     build:
       context: .
@@ -53,12 +55,75 @@ services:
     volumes:
       - /data
       - /cache
+      - /logs
     ports:
       - 127.0.0.1:3334:3334
     # Node configuration can be overriden via env, for exampe:
     environment:
       # JOYSTREAM_DISTRIBUTOR__ID: node-id
       JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE: http://${GRAPHQL_SERVER_HOST}:${GRAPHQL_SERVER_PORT}/graphql
+      JOYSTREAM_DISTRIBUTOR__KEYS: "[{\"suri\":\"//testing//workers//Distribution//0\"}]"
+      JOYSTREAM_DISTRIBUTOR__WORKER_ID: 0
+      JOYSTREAM_DISTRIBUTOR__PORT: 3334
+    #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__JOYSTREAM_NODE_WS: sn-endpoint
+    #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__ELASTIC_SEARCH: es-endpoint
+    #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__ASSETS: assets-dir
+    #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__CACHE_STATE: cache-state-dir
+    #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__LOGS: logs-dir
+    #   JOYSTREAM_DISTRIBUTOR__LOG__CONSOLE: "off"
+    #   JOYSTREAM_DISTRIBUTOR__LOG__FILE: "off"
+    #   JOYSTREAM_DISTRIBUTOR__LOG__ELASTIC: "off"
+    #   JOYSTREAM_DISTRIBUTOR__LIMITS__STORAGE: 50G
+    #   JOYSTREAM_DISTRIBUTOR__BUCKETS: "[1,2]"
+    command: ["start"]
+
+  colossus-2:
+    image: joystream/colossus:latest
+    container_name: colossus-2
+    restart: on-failure
+    build:
+      context: .
+      dockerfile: colossus.Dockerfile
+    depends_on:
+      - graphql-server
+    volumes:
+      - /data
+      - /keystore
+    ports:
+      - '127.0.0.1:3335:3333'
+    env_file:
+      # relative to working directory where docker-compose was run from
+      - .env
+    environment:
+      - COLOSSUS_PORT=3333
+      - QUERY_NODE_HOST=graphql-server:${GRAPHQL_SERVER_PORT}
+      - WORKER_ID=1
+      - ACCOUNT_URI=//testing//worker//Storage//1
+      # enable ElasticSearch server
+      # - ELASTIC_SEARCH_HOST=host.docker.internal:9200
+
+  distributor-2:
+    image: joystream/distributor-node
+    container_name: distributor-2
+    restart: on-failure
+    build:
+      context: .
+      dockerfile: distributor-node.Dockerfile
+    depends_on:
+      - graphql-server
+    volumes:
+      - /data
+      - /cache
+      - /logs
+    ports:
+      - 127.0.0.1:3336:3334
+    # Node configuration can be overriden via env, for exampe:
+    environment:
+      # JOYSTREAM_DISTRIBUTOR__ID: node-id
+      JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE: http://${GRAPHQL_SERVER_HOST}:${GRAPHQL_SERVER_PORT}/graphql
+      JOYSTREAM_DISTRIBUTOR__KEYS: "[{\"suri\":\"//testing//workers//Distribution//1\"}]"
+      JOYSTREAM_DISTRIBUTOR__WORKER_ID: 1
+      JOYSTREAM_DISTRIBUTOR__PORT: 3334
     #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__JOYSTREAM_NODE_WS: sn-endpoint
     #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__ELASTIC_SEARCH: es-endpoint
     #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__ASSETS: assets-dir
@@ -68,14 +133,12 @@ services:
     #   JOYSTREAM_DISTRIBUTOR__LOG__FILE: "off"
     #   JOYSTREAM_DISTRIBUTOR__LOG__ELASTIC: "off"
     #   JOYSTREAM_DISTRIBUTOR__LIMITS__STORAGE: 50G
-    #   JOYSTREAM_DISTRIBUTOR__PORT: 1234
-    #   JOYSTREAM_DISTRIBUTOR__KEYS="[{\"suri\":\"//Bob\"}]"
     #   JOYSTREAM_DISTRIBUTOR__BUCKETS: "[1,2]"
-    #   JOYSTREAM_DISTRIBUTOR__WORKER_ID: 0
     command: ["start"]
 
   db:
     image: postgres:12
+    container_name: db
     restart: always
     ports:
       - '127.0.0.1:${DB_PORT}:5432'
@@ -90,24 +153,8 @@ services:
       POSTGRES_DB: ${INDEXER_DB_NAME}
 
   graphql-server:
-    image: joystream/apps
-    restart: unless-stopped
-    build:
-      context: .
-      dockerfile: apps.Dockerfile
-      args:
-        - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    env_file:
-      # relative to working directory where docker-compose was run from
-      - .env
-    ports:
-      - '127.0.0.1:8081:${GRAPHQL_SERVER_PORT}'
-    depends_on:
-      - db
-    command: ['workspace', 'query-node-root', 'query-node:start:prod']
-
-  graphql-server-mnt:
     image: node:14
+    container_name: graphql-server
     restart: unless-stopped
     env_file:
       # relative to working directory where docker-compose was run from
@@ -124,29 +171,8 @@ services:
     command: ['yarn', 'workspace', 'query-node-root', 'query-node:start:prod']
 
   processor:
-    image: joystream/apps
-    restart: unless-stopped
-    build:
-      context: .
-      dockerfile: apps.Dockerfile
-      args:
-        - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    env_file:
-      # relative to working directory where docker-compose was run from
-      - .env
-    environment:
-      - INDEXER_ENDPOINT_URL=http://hydra-indexer-gateway:${WARTHOG_APP_PORT}/graphql
-      - TYPEORM_HOST=${DB_HOST}
-      - TYPEORM_DATABASE=${DB_NAME}
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    volumes:
-      - ./types/augment/all/defs.json:/joystream/query-node/mappings/lib/generated/types/typedefs.json
-    depends_on:
-      - hydra-indexer-gateway
-    command: ['workspace', 'query-node-root', 'processor:start']
-
-  processor-mnt:
     image: node:14
+    container_name: processor
     restart: unless-stopped
     env_file:
       # relative to working directory where docker-compose was run from
@@ -166,6 +192,7 @@ services:
 
   indexer:
     image: joystream/hydra-indexer:3.0.0
+    container_name: indexer
     restart: unless-stopped
     env_file:
       # relative to working directory where docker-compose was run from
@@ -185,6 +212,7 @@ services:
 
   hydra-indexer-gateway:
     image: joystream/hydra-indexer-gateway:3.0.0
+    container_name: hydra-indexer-gateway
     restart: unless-stopped
     env_file:
       # relative to working directory where docker-compose was run from
@@ -208,14 +236,17 @@ services:
 
   redis:
     image: redis:6.0-alpine
+    container_name: redis
     restart: always
     ports:
       - '127.0.0.1:6379:6379'
 
   pioneer:
-    image: joystream/pioneer
-    build:
-      context: .
-      dockerfile: pioneer.Dockerfile
+    image: nginx
+    container_name: pioneer
+    volumes:
+      - ./pioneer/packages/apps/build:/usr/share/nginx/html
     ports:
-      - '127.0.0.1:3000:80'
+      - "127.0.0.1:3000:80"
+    environment:
+      - NGINX_PORT=80

multi-storage.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# Script to run a second storage node and distributor node on local
-# Make sure to run yarn start prior to running this script
-
-set -a
-. .env
-set +a
-
-export COLOSSUS_PORT_2=3335
-export DISTRIBUTOR_PORT_2=3336
-export KEYS=[//Alice]
-export BUCKETS='all'
-export WORKER_ID=2
-export ACCOUNT_KEYFILE="./types/augment/all/defs.json"
-
-function down()
-{
-    # Stop containers and clear volumes
-    docker-compose -f docker-compose.yml -f docker-compose.multi-storage.yml rm -vsf distributor-node-2
-    docker-compose -f docker-compose.yml -f docker-compose.multi-storage.yml rm -vsf colossus-2
-}
-
-down
-
-trap down EXIT
-
-docker-compose -f docker-compose.yml -f docker-compose.multi-storage.yml run -d --name colossus-2 colossus-2
-
-docker-compose -f docker-compose.yml -f docker-compose.multi-storage.yml run -d --name distributor-node-2 distributor-node-2
-
-echo "use Ctrl+C to shutdown the development network."
-
-while true; do 
-  read
-done

package.json

@@ -5,8 +5,8 @@
   "license": "GPL-3.0-only",
   "scripts": {
     "build:node:docker": "./build-node-docker.sh",
-    "build:apps:docker": "./build-apps-docker.sh",
-    "build:packages": "./build-npm-packages.sh",
+    "build:storage:docker": "./build-storage-docker.sh",
+    "build:packages": "./build-packages.sh",
     "setup": "./setup.sh",
     "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",

start.sh

@@ -23,20 +23,24 @@ trap down EXIT
 # Run a local development chain
 docker-compose up -d joystream-node
 
+WS_PROVIDER_ENDPOINT_URI=ws://localhost:9944 SKIP_MOCK_CONTENT=true \
+  ./tests/network-tests/run-test-scenario.sh setup-new-chain
+
+# Set sudo as the membership screening authority
+yarn workspace api-scripts set-sudo-as-screening-auth
+
 ## Query Node Infrastructure
+
 # Initialize a new database for the query node infrastructure
 docker-compose up -d db
 
 # Override DB_HOST for db setup
-export DB_HOST=localhost
-
 # Make sure we use dev config for db migrations (prevents "Cannot create database..." and some other errors)
+export DB_HOST=localhost
 yarn workspace query-node config:dev
-
 # Migrate the databases
 yarn workspace query-node-root db:prepare
 yarn workspace query-node-root db:migrate
-
 # Set DB_HOST back to docker-service one
 export DB_HOST=db
 
@@ -45,18 +49,16 @@ docker-compose up -d processor
 docker-compose up -d graphql-server
 
 ## Storage Infrastructure
-docker-compose run -d --name colossus --entrypoint sh colossus -c "yarn storage-node dev:init --apiUrl ${WS_PROVIDER_ENDPOINT_URI} && \
-          yarn storage-node server --queryNodeHost ${QUERY_NODE_HOST} --port ${COLOSSUS_PORT} \
-          --uploads /data --worker ${WORKER_ID} --apiUrl ${WS_PROVIDER_ENDPOINT_URI} --sync --syncInterval=1 \
-          --keyFile=${ACCOUNT_KEYFILE} --elasticSearchHost=${ELASTIC_SEARCH_HOST}"
-
-docker-compose up -d distributor-node
-
-# Create a new content directory lead
-yarn workspace api-scripts initialize-content-lead
-
-# Set sudo as the membership screening authority
-yarn workspace api-scripts set-sudo-as-screening-auth
+docker-compose run -d --name colossus-1 --entrypoint sh colossus-1 -c "\
+  yarn storage-node server --queryNodeHost ${GRAPHQL_SERVER_HOST}:${GRAPHQL_SERVER_PORT} \
+  --port ${COLOSSUS_CONTAINER_PORT} \
+  --uploads /data --worker 0 \
+  --accountUri=//testing//worker//Storage//0 \
+  --apiUrl ${WS_PROVIDER_ENDPOINT_URI} \
+  --sync --syncInterval=1 \
+  --elasticSearchHost=${ELASTIC_SEARCH_HOST}"
+
+docker-compose up -d distributor-1
 
 docker-compose up -d pioneer
 

tests/network-tests/src/scenarios/setup-new-chain.ts

@@ -16,8 +16,10 @@ scenario(async ({ job }) => {
 
   const updateWorkerAccounts = job('Update worker accounts', updateAccountsFlow).after(leads)
 
-  // Create some mock content in content directory - without assets or any real metadata
-  job('Create Mock Content', mockContentFlow).after(updateWorkerAccounts)
+  if (!process.env.SKIP_MOCK_CONTENT) {
+    // Create some mock content in content directory - without assets or any real metadata
+    job('Create Mock Content', mockContentFlow).after(updateWorkerAccounts)
+  }
 
   // assign members known accounts?
   // assign council known accounts?