Merge pull request #2683 from ahhda/storage_v2_devops_updates

DevOps - Storage v2 environment support

.env

@@ -37,3 +37,11 @@ WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
 
 # If running joystream-node on host machine you can use following address to reach it instead
 # WS_PROVIDER_ENDPOINT_URI=ws://host.docker.internal:9944/
+
+######################
+#    Storage Node    #
+######################
+COLOSSUS_PORT=3333
+QUERY_NODE_HOST=${GRAPHQL_SERVER_HOST}:${GRAPHQL_SERVER_PORT}
+WORKER_ID=0
+ACCOUNT_URI=//Alice

.github/workflows/joystream-cli.yml

@@ -9,22 +9,22 @@ jobs:
       matrix:
         node-version: [14.x]
     steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: checks
-      run: |
-        yarn install --frozen-lockfile
-        yarn workspace @joystream/types build
-        yarn workspace @joystream/metadata-protobuf build
-        yarn workspace @joystream/cli checks --quiet
-    - name: yarn pack test
-      run: |
-        yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
-        tar zxvf ./cli/cli-pack-test.tgz -C cli
-        cd ./cli/package && yarn link
+      - uses: actions/checkout@v1
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: checks
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/metadata-protobuf build
+          yarn workspace @joystream/cli checks --quiet
+      - name: yarn pack test
+        run: |
+          yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
+          tar zxvf ./cli/cli-pack-test.tgz -C cli
+          cd ./cli/package && yarn link
 
   cli_build_osx:
     name: MacOS Checks
@@ -33,19 +33,19 @@ jobs:
       matrix:
         node-version: [14.x]
     steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: checks
-      run: |
-        yarn install --frozen-lockfile --network-timeout 120000
-        yarn workspace @joystream/types build
-        yarn workspace @joystream/metadata-protobuf build
-        yarn workspace @joystream/cli checks --quiet
-    - name: yarn pack test
-      run: |
-        yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
-        tar zxvf ./cli/cli-pack-test.tgz -C cli
-        cd ./cli/package && yarn link
+      - uses: actions/checkout@v1
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: checks
+        run: |
+          yarn install --frozen-lockfile --network-timeout 120000
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/metadata-protobuf build
+          yarn workspace @joystream/cli checks --quiet
+      - name: yarn pack test
+        run: |
+          yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
+          tar zxvf ./cli/cli-pack-test.tgz -C cli
+          cd ./cli/package && yarn link

build-docker-images.sh

@@ -28,10 +28,33 @@ else
   fi
 fi
 
-# Build joystream/apps docker image
-echo "Building 'joystream/apps' docker image..."
+# Build colossus docker image
+echo "Building colossus docker image..."
 docker-compose build colossus
 
+# Build distributor docker image
+echo "Building distributor docker image..."
+docker-compose build distributor-node
+
+if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    IP_ADDRESS=$(ip addr show | grep "\binet\b.*\bdocker0\b" | awk '{print $2}' | cut -d '/' -f 1)
+    # Run a local development chain
+    docker-compose -f docker-compose.linux-gnu-build.yml up -d joystream-node
+
+    # Build processor/graphql-server docker image
+    echo "Building joystream/apps docker image..."
+    WS_PROVIDER_ENDPOINT_URI=ws://${IP_ADDRESS}:9944/ docker-compose build graphql-server
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+    # Run a local development chain
+    docker-compose up -d joystream-node
+
+    # Build processor/graphql-server docker image
+    echo "Building joystream/apps docker image..."
+    WS_PROVIDER_ENDPOINT_URI=ws://host.docker.internal:9944/ docker-compose build graphql-server
+fi
+
+docker-compose down
+
 # Build the pioneer docker image
-echo "Building pioneer docker image"
-docker-compose build pioneer
+# echo "Building pioneer docker image"
+# docker-compose build pioneer

build-npm-packages.sh

@@ -6,7 +6,7 @@ yarn
 yarn workspace @joystream/types build
 yarn workspace @joystream/metadata-protobuf build
 yarn workspace query-node-root build
-yarn workspace @joystream/cli build
-yarn workspace storage-node build
+# yarn workspace @joystream/cli build
 yarn workspace storage-node-v2 build
-yarn workspace pioneer build
+yarn workspace @joystream/distributor-cli build
+# yarn workspace pioneer build

distributor-node.Dockerfile

@@ -10,7 +10,7 @@ COPY ./package.json package.json
 EXPOSE 3334
 
 # Build & cleanup
-# (must be inside a signle "RUN", see: https://stackoverflow.com/questions/40212836/docker-image-larger-than-its-filesystem)
+# (must be inside a single "RUN", see: https://stackoverflow.com/questions/40212836/docker-image-larger-than-its-filesystem)
 RUN \
   yarn --frozen-lockfile &&\
   yarn workspace @joystream/types build &&\

docker-compose.linux-gnu-build.yml

@@ -0,0 +1,14 @@
+version: '3.4'
+services:
+  joystream-node:
+    image: joystream/node:latest
+    build:
+      # context is relative to the compose file
+      context: .
+      # dockerfile is relative to the context
+      dockerfile: joystream-node.Dockerfile
+    container_name: joystream-node
+    volumes:
+      - /data
+    command: --dev --alice --validator --unsafe-ws-external --unsafe-rpc-external --rpc-methods Unsafe --rpc-cors=all --log runtime --base-path /data
+    network_mode: host

docker-compose.yml

@@ -1,7 +1,7 @@
 # Compiles new joystream/node and joystream/apps images if local images not found
 # and runs a complete joystream development network
 # To prevent build of docker images run docker-compose with "--no-build" arg
-version: "3.4"
+version: '3.4'
 services:
   joystream-node:
     image: joystream/node:latest
@@ -15,8 +15,8 @@ services:
       - /data
     command: --dev --alice --validator --unsafe-ws-external --unsafe-rpc-external --rpc-methods Unsafe --rpc-cors=all --log runtime --base-path /data
     ports:
-      - "127.0.0.1:9944:9944"
-      - "127.0.0.1:9933:9933"
+      - '127.0.0.1:9944:9944'
+      - '127.0.0.1:9933:9933'
 
   colossus:
     image: joystream/colossus:latest
@@ -25,20 +25,15 @@ services:
       context: .
       dockerfile: colossus.Dockerfile
     depends_on:
-      - graphql-server-mnt
+      - graphql-server
     volumes:
       - /data
       - /keystore
     ports:
-      - '127.0.0.1:3333:3333'
+      - '127.0.0.1:3333:${COLOSSUS_PORT}'
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
-    environment:
-      - COLOSSUS_PORT=3333
-      - QUERY_NODE_HOST=graphql-server-mnt:4002
-      - WORKER_ID=0
-      - ACCOUNT_URI=//Alice
 
   distributor-node:
     image: joystream/distributor-node
@@ -46,15 +41,17 @@ services:
     build:
       context: .
       dockerfile: distributor-node.Dockerfile
+    depends_on:
+      - graphql-server
     volumes:
       - /data
       - /cache
     ports:
       - 127.0.0.1:3334:3334
     # Node configuration can be overriden via env, for exampe:
-    # environment:
-    #   JOYSTREAM_DISTRIBUTOR__ID: node-id
-    #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE: qn-endpoint
+    environment:
+      # JOYSTREAM_DISTRIBUTOR__ID: node-id
+      JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE: http://${GRAPHQL_SERVER_HOST}:${GRAPHQL_SERVER_PORT}/graphql
     #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__SUBSTRATE_NODE: sn-endpoint
     #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__ELASTIC_SEARCH: es-endpoint
     #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__DATA: data-dir
@@ -68,13 +65,13 @@ services:
     #   JOYSTREAM_DISTRIBUTOR__KEYS: "[\"//Bob\"]"
     #   JOYSTREAM_DISTRIBUTOR__BUCKETS: "[1,2]"
     #   JOYSTREAM_DISTRIBUTOR__WORKER_ID: 0
-    command: ["start"]
+    command: ['start']
 
   db:
     image: postgres:12
     restart: always
     ports:
-      - "127.0.0.1:${DB_PORT}:5432"
+      - '127.0.0.1:${DB_PORT}:5432'
     volumes:
       - /var/lib/postgresql/data
     env_file:
@@ -91,17 +88,16 @@ services:
     build:
       context: .
       dockerfile: apps.Dockerfile
-      network: joystream_default
       args:
         - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
     ports:
-      - "127.0.0.1:8081:${GRAPHQL_SERVER_PORT}"
+      - '127.0.0.1:8081:${GRAPHQL_SERVER_PORT}'
     depends_on:
       - db
-    command: ["workspace", "query-node-root", "query-node:start:prod"]
+    command: ['workspace', 'query-node-root', 'query-node:start:prod']
 
   graphql-server-mnt:
     image: node:14
@@ -110,7 +106,7 @@ services:
       # relative to working directory where docker-compose was run from
       - .env
     ports:
-      - "127.0.0.1:8081:${GRAPHQL_SERVER_PORT}"
+      - '127.0.0.1:8081:${GRAPHQL_SERVER_PORT}'
     depends_on:
       - db
     volumes:
@@ -118,7 +114,7 @@ services:
         source: .
         target: /joystream
     working_dir: /joystream
-    command: ["yarn", "workspace", "query-node-root", "query-node:start:prod"]
+    command: ['yarn', 'workspace', 'query-node-root', 'query-node:start:prod']
 
   processor:
     image: joystream/apps
@@ -126,7 +122,6 @@ services:
     build:
       context: .
       dockerfile: apps.Dockerfile
-      network: joystream_default
       args:
         - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
     env_file:
@@ -141,7 +136,7 @@ services:
       - ./types/augment/all/defs.json:/joystream/query-node/mappings/lib/generated/types/typedefs.json
     depends_on:
       - hydra-indexer-gateway
-    command: ["workspace", "query-node-root", "processor:start"]
+    command: ['workspace', 'query-node-root', 'processor:start']
 
   processor-mnt:
     image: node:14
@@ -160,7 +155,7 @@ services:
         source: .
         target: /joystream
     working_dir: /joystream
-    command: ["yarn", "workspace", "query-node-root", "processor:start"]
+    command: ['yarn', 'workspace', 'query-node-root', 'processor:start']
 
   indexer:
     image: joystream/hydra-indexer:3.0.0
@@ -198,7 +193,7 @@ services:
       - PORT=${WARTHOG_APP_PORT}
       - DEBUG=*
     ports:
-      - "127.0.0.1:4000:${WARTHOG_APP_PORT}"
+      - '127.0.0.1:4000:${WARTHOG_APP_PORT}'
     depends_on:
       - redis
       - db
@@ -208,7 +203,7 @@ services:
     image: redis:6.0-alpine
     restart: always
     ports:
-      - "127.0.0.1:6379:6379"
+      - '127.0.0.1:6379:6379'
 
   pioneer:
     image: joystream/pioneer
@@ -216,4 +211,4 @@ services:
       context: .
       dockerfile: pioneer.Dockerfile
     ports:
-      - "127.0.0.1:3000:80"
+      - '127.0.0.1:3000:80'

start.sh

@@ -23,28 +23,42 @@ trap down EXIT
 # Run a local development chain
 docker-compose up -d joystream-node
 
-## Storage Infrastructure
-# Configure a dev storage node and start storage node
-DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
-docker-compose up -d colossus
-# Create a new content directory lead
-yarn workspace api-scripts initialize-content-lead
-
-# Set sudo as the membership screening authority
-yarn workspace api-scripts set-sudo-as-screening-auth
-
 ## Query Node Infrastructure
 # Initialize a new database for the query node infrastructure
 docker-compose up -d db
+
+# Override DB_HOST for db setup
+export DB_HOST=localhost
+
+# Make sure we use dev config for db migrations (prevents "Cannot create database..." and some other errors)
+yarn workspace query-node config:dev
+
+# Migrate the databases
 yarn workspace query-node-root db:prepare
 yarn workspace query-node-root db:migrate
 
-# Startup all query-node infrastructure services
-export WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944
-docker-compose up -d graphql-server
+# Set DB_HOST back to docker-service one
+export DB_HOST=db
+
+# Start processor and graphql server
 docker-compose up -d processor
+docker-compose up -d graphql-server
+
+## Storage Infrastructure
+docker-compose run -d --name colossus --entrypoint sh colossus -c "yarn storage-node dev:init --apiUrl ${WS_PROVIDER_ENDPOINT_URI} && \
+          yarn storage-node server --queryNodeHost ${QUERY_NODE_HOST} --port ${COLOSSUS_PORT} \
+          --uploads /data --worker ${WORKER_ID} --apiUrl ${WS_PROVIDER_ENDPOINT_URI} --sync --syncInterval=1 \
+          --keyFile=${ACCOUNT_KEYFILE} --elasticSearchHost=${ELASTIC_SEARCH_HOST}"
+
+docker-compose up -d distributor-node
+
+# # Create a new content directory lead
+# yarn workspace api-scripts initialize-content-lead
+
+# # Set sudo as the membership screening authority
+# yarn workspace api-scripts set-sudo-as-screening-auth
 
-docker-compose up -d pioneer
+# docker-compose up -d pioneer
 
 echo "use Ctrl+C to shutdown the development network."