docker-compose start storage services with bind mount

.env

@@ -1,5 +1,5 @@
 COMPOSE_PROJECT_NAME=joystream
-PROJECT_NAME=joystream
+PROJECT_NAME=query_node
 
 # We will use a single postgres service with multiple databases
 # The env variables below are by default used by all services and should be
@@ -26,17 +26,11 @@ BLOCK_HEIGHT=0
 #    Processor GraphQL API    #
 ###############################
 
-GRAPHQL_SERVER_PORT=4002
+GRAPHQL_SERVER_PORT=4001
 GRAPHQL_SERVER_HOST=graphql-server
 
 WARTHOG_APP_PORT=4002
 WARTHOG_APP_HOST=hydra-indexer-gateway
 
-# Default configuration is to use the docker container
+# Websocket RPC endpoint hydra indexer will use
 WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
-
-# If running joystream-node on host machine you can use following address to reach it instead
-# WS_PROVIDER_ENDPOINT_URI=ws://host.docker.internal:9944/
-
-# Storage Node
-COLOSSUS_CONTAINER_PORT=3333

README.md

@@ -31,9 +31,6 @@ yarn build:packages
 # Build joystream/node docker image
 yarn build:node:docker
 
-# Build storage inftrasrucure docker images
-yarn build:storage:docker
-
 # start a local development network
 yarn start
 ```

docker-compose.yml

@@ -19,43 +19,44 @@ services:
       - '127.0.0.1:9933:9933'
 
   colossus-1:
-    image: joystream/colossus:latest
+    image: node:14
     container_name: colossus-1
     restart: on-failure
-    build:
-      context: .
-      dockerfile: colossus.Dockerfile
-    depends_on:
-      - graphql-server
     volumes:
       - /data
       - /keystore
+      - type: bind
+        source: .
+        target: /joystream
+    working_dir: /joystream/storage-node-v2
     ports:
       - '127.0.0.1:3333:3333'
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
     environment:
-      - COLOSSUS_PORT=3333
-      - QUERY_NODE_HOST=graphql-server:${GRAPHQL_SERVER_PORT}
-      - WORKER_ID=0
+      # ACCOUNT_URI overrides command line arg --accountUri
       - ACCOUNT_URI=//testing//worker//Storage//0
-      # enable ElasticSearch server
-      # - ELASTIC_SEARCH_HOST=host.docker.internal:9200
+    command: [
+      'yarn', 'storage-node', 'server', '--worker=1', '--port=3333', '--uploads=/data',
+      '--sync', '--syncInterval=1',
+      '--queryNodeHost=graphql-server:${GRAPHQL_SERVER_PORT}',
+      '--apiUrl=ws://joystream-node:9944/'
+    ]
 
   distributor-1:
-    image: joystream/distributor-node
+    image: node:14
     container_name: distributor-1
     restart: on-failure
-    build:
-      context: .
-      dockerfile: distributor-node.Dockerfile
-    depends_on:
-      - graphql-server
     volumes:
       - /data
       - /cache
       - /logs
+      - type: bind
+        source: .
+        target: /joystream
+    # let the working_dir be the distributor node to pickup the config.yml file
+    working_dir: /joystream/distributor-node
     ports:
       - 127.0.0.1:3334:3334
     # Node configuration can be overriden via env, for exampe:
@@ -65,7 +66,7 @@ services:
       JOYSTREAM_DISTRIBUTOR__KEYS: "[{\"suri\":\"//testing//workers//Distribution//0\"}]"
       JOYSTREAM_DISTRIBUTOR__WORKER_ID: 0
       JOYSTREAM_DISTRIBUTOR__PORT: 3334
-    #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__JOYSTREAM_NODE_WS: sn-endpoint
+      JOYSTREAM_DISTRIBUTOR__ENDPOINTS__JOYSTREAM_NODE_WS: ws://joystream-node:9944/
     #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__ELASTIC_SEARCH: es-endpoint
     #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__ASSETS: assets-dir
     #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__CACHE_STATE: cache-state-dir
@@ -75,46 +76,47 @@ services:
     #   JOYSTREAM_DISTRIBUTOR__LOG__ELASTIC: "off"
     #   JOYSTREAM_DISTRIBUTOR__LIMITS__STORAGE: 50G
     #   JOYSTREAM_DISTRIBUTOR__BUCKETS: "[1,2]"
-    command: ["start"]
+    command: ['yarn', 'joystream-distributor']
 
   colossus-2:
-    image: joystream/colossus:latest
+    image: node:14
     container_name: colossus-2
     restart: on-failure
-    build:
-      context: .
-      dockerfile: colossus.Dockerfile
-    depends_on:
-      - graphql-server
     volumes:
       - /data
       - /keystore
+      - type: bind
+        source: .
+        target: /joystream
+    working_dir: /joystream/storage-node-v2
     ports:
       - '127.0.0.1:3335:3333'
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
     environment:
-      - COLOSSUS_PORT=3333
-      - QUERY_NODE_HOST=graphql-server:${GRAPHQL_SERVER_PORT}
-      - WORKER_ID=1
+      # ACCOUNT_URI overrides command line arg --accountUri
       - ACCOUNT_URI=//testing//worker//Storage//1
-      # enable ElasticSearch server
-      # - ELASTIC_SEARCH_HOST=host.docker.internal:9200
+    command: [
+      'yarn', 'storage-node', 'server', '--worker=1', '--port=3333', '--uploads=/data',
+      '--sync', '--syncInterval=1',
+      '--queryNodeHost=graphql-server:${GRAPHQL_SERVER_PORT}',
+      '--apiUrl=ws://joystream-node:9944/'
+    ]
 
   distributor-2:
-    image: joystream/distributor-node
+    image: node:14
     container_name: distributor-2
     restart: on-failure
-    build:
-      context: .
-      dockerfile: distributor-node.Dockerfile
-    depends_on:
-      - graphql-server
     volumes:
       - /data
       - /cache
       - /logs
+      - type: bind
+        source: .
+        target: /joystream
+    # let the working_dir be the distributor node to pickup the config.yml file
+    working_dir: /joystream/distributor-node
     ports:
       - 127.0.0.1:3336:3334
     # Node configuration can be overriden via env, for exampe:
@@ -124,7 +126,7 @@ services:
       JOYSTREAM_DISTRIBUTOR__KEYS: "[{\"suri\":\"//testing//workers//Distribution//1\"}]"
       JOYSTREAM_DISTRIBUTOR__WORKER_ID: 1
       JOYSTREAM_DISTRIBUTOR__PORT: 3334
-    #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__JOYSTREAM_NODE_WS: sn-endpoint
+      JOYSTREAM_DISTRIBUTOR__ENDPOINTS__JOYSTREAM_NODE_WS: ws://joystream-node:9944/
     #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__ELASTIC_SEARCH: es-endpoint
     #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__ASSETS: assets-dir
     #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__CACHE_STATE: cache-state-dir
@@ -134,7 +136,7 @@ services:
     #   JOYSTREAM_DISTRIBUTOR__LOG__ELASTIC: "off"
     #   JOYSTREAM_DISTRIBUTOR__LIMITS__STORAGE: 50G
     #   JOYSTREAM_DISTRIBUTOR__BUCKETS: "[1,2]"
-    command: ["start"]
+    command: ['yarn', 'joystream-distributor']
 
   db:
     image: postgres:12
@@ -219,7 +221,7 @@ services:
       - .env
     environment:
       - WARTHOG_STARTER_DB_DATABASE=${INDEXER_DB_NAME}
-      - WARTHOG_STARTER_DB_HOST=db
+      - WARTHOG_STARTER_DB_HOST=${DB_HOST}
       - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
       - WARTHOG_STARTER_DB_PORT=${DB_PORT}
       - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
@@ -250,3 +252,15 @@ services:
       - "127.0.0.1:3000:80"
     environment:
       - NGINX_PORT=80
+  
+  # Used to for building native npm packages that will work in the container
+  # yarn cache folder isn't mounted so packages will be downloaded
+  # yarn-build-packages:
+  #   image: node:14
+  #   restart: never
+  #   volumes:
+  #     - type: bind
+  #       source: .
+  #       target: /joystream
+  #   working_dir: /joystream
+  #   command: ['yarn', 'build:packages']

start.sh

@@ -20,13 +20,13 @@ function down()
 
 trap down EXIT
 
-# Run a local development chain
+## Run a local development chain
 docker-compose up -d joystream-node
 
 WS_PROVIDER_ENDPOINT_URI=ws://localhost:9944 SKIP_MOCK_CONTENT=true \
   ./tests/network-tests/run-test-scenario.sh setup-new-chain
 
-# Set sudo as the membership screening authority
+## Set sudo as the membership screening authority
 yarn workspace api-scripts set-sudo-as-screening-auth
 
 ## Query Node Infrastructure
@@ -49,17 +49,10 @@ docker-compose up -d processor
 docker-compose up -d graphql-server
 
 ## Storage Infrastructure
-docker-compose run -d --name colossus-1 --entrypoint sh colossus-1 -c "\
-  yarn storage-node server --queryNodeHost ${GRAPHQL_SERVER_HOST}:${GRAPHQL_SERVER_PORT} \
-  --port ${COLOSSUS_CONTAINER_PORT} \
-  --uploads /data --worker 0 \
-  --accountUri=//testing//worker//Storage//0 \
-  --apiUrl ${WS_PROVIDER_ENDPOINT_URI} \
-  --sync --syncInterval=1 \
-  --elasticSearchHost=${ELASTIC_SEARCH_HOST}"
-
+docker-compose up -d colossus-1
 docker-compose up -d distributor-1
 
+## Pioneer UI
 docker-compose up -d pioneer
 
 echo "use Ctrl+C to shutdown the development network."