update readme and instructions for building and starting local dev network

README.md

@@ -14,7 +14,7 @@ The following tools are required for building, testing and contributing to this
 - [Rust](https://www.rust-lang.org/tools/install) toolchain - _required_
 - [nodejs](https://nodejs.org/) v14.x - _required_
 - [yarn classic](https://classic.yarnpkg.com/en/docs/install) package manager v1.22.x- _required_
-- [docker](https://www.docker.com/get-started) and docker-compose - _optional_
+- [docker](https://www.docker.com/get-started) and docker-compose - _required_
 - [ansible](https://www.ansible.com/) - _optional_
 
 If you use VSCode as your code editor we recommend using the workspace [settings](devops/vscode/settings.json) for recommend eslint plugin to function properly.
@@ -25,8 +25,24 @@ After cloning the repo run the following initialization scripts:
 # Install rust toolchain
 ./setup.sh
 
-# Install npm package dependencies, build packages and docker images
-yarn build
+# Install npm dependencies
+yarn
+
+# Build joystream/node image
+yarn build:node:docker
+
+# Build applications docker image
+yarn build:apps:docker
+
+# start local dev joystream/node (required when building query-node package)
+# or optionally build native binary and run dev chain with ./script/run-dev-chain.sh
+docker-compose up -d joystream/node
+
+# build local npm packages
+yarn build:packages
+
+# stop dev joystream/node container
+docker-compose down -v
 
 # start a local development network
 yarn start

build-docker-images.sh → build-apps-docker.sh

@@ -8,34 +8,6 @@ then
   exit 0
 fi
 
-# Build or fetch cached joystream/node docker image
-if [[ "$SKIP_JOYSTREAM_NODE" = 1 || "$SKIP_JOYSTREAM_NODE" = "true" ]]; then
-  echo "Skipping build of joystream/node docker image."
-else
-  # Fetch a cached joystream/node image if one is found matching code shasum instead of building
-  CODE_HASH=`scripts/runtime-code-shasum.sh`
-  IMAGE=joystream/node:${CODE_HASH}
-  echo "Trying to fetch cached ${IMAGE} image"
-  docker pull ${IMAGE} || :
-
-  if ! docker inspect ${IMAGE} > /dev/null;
-  then
-    echo "Fetch failed, building image locally"
-    docker-compose build joystream-node
-  else
-    echo "Tagging cached image as 'latest'"
-    docker image tag ${IMAGE} joystream/node:latest
-  fi
-fi
-
-# Build colossus docker image
-echo "Building colossus docker image..."
-docker-compose build colossus
-
-# Build distributor docker image
-echo "Building distributor docker image..."
-docker-compose build distributor-node
-
 if [[ "$OSTYPE" == "linux-gnu" ]]; then
     IP_ADDRESS=$(ip addr show | grep "\binet\b.*\bdocker0\b" | awk '{print $2}' | cut -d '/' -f 1)
     # Run a local development chain
@@ -44,6 +16,7 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
     # Build processor/graphql-server docker image
     echo "Building joystream/apps docker image..."
     WS_PROVIDER_ENDPOINT_URI=ws://${IP_ADDRESS}:9944/ docker-compose build graphql-server
+    docker-compose down
 elif [[ "$OSTYPE" == "darwin"* ]]; then
     # Run a local development chain
     docker-compose up -d joystream-node
@@ -51,10 +24,17 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
     # Build processor/graphql-server docker image
     echo "Building joystream/apps docker image..."
     WS_PROVIDER_ENDPOINT_URI=ws://host.docker.internal:9944/ docker-compose build graphql-server
+    docker-compose down
 fi
 
-docker-compose down
+# Build colossus docker image
+echo "Building colossus docker image..."
+docker-compose build colossus
+
+# Build distributor docker image
+echo "Building distributor docker image..."
+docker-compose build distributor-node
 
 # Build the pioneer docker image
-# echo "Building pioneer docker image"
-# docker-compose build pioneer
+echo "Building pioneer docker image"
+docker-compose build pioneer

build-node-docker.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -e
+
+if ! command -v docker-compose &> /dev/null
+then
+  echo "docker-compose not found. Skipping docker image builds."
+  exit 0
+fi
+
+# Fetch a cached joystream/node image if one is found matching code shasum instead of building
+CODE_HASH=`scripts/runtime-code-shasum.sh`
+IMAGE=joystream/node:${CODE_HASH}
+echo "Trying to fetch cached ${IMAGE} image"
+docker pull ${IMAGE} || :
+
+if ! docker inspect ${IMAGE} > /dev/null;
+then
+  echo "Fetch failed, building image locally"
+  docker-compose build joystream-node
+else
+  echo "Tagging cached image as 'latest'"
+  docker image tag ${IMAGE} joystream/node:latest
+fi

build-npm-packages.sh

@@ -5,8 +5,11 @@ set -e
 yarn
 yarn workspace @joystream/types build
 yarn workspace @joystream/metadata-protobuf build
+# A joystream-node is expected to be running locally for query-node build to work
+# either start a container with: docker-compose up -d joystream-node
+# or run native binary with: ./scripts/run-dev-chain.sh
 yarn workspace query-node-root build
-# yarn workspace @joystream/cli build
+yarn workspace @joystream/cli build
 yarn workspace storage-node-v2 build
 yarn workspace @joystream/distributor-cli build
 # yarn workspace pioneer build

package.json

@@ -4,9 +4,9 @@
   "version": "1.0.0",
   "license": "GPL-3.0-only",
   "scripts": {
-    "build": "./build-npm-packages.sh && ./build-docker-images.sh",
+    "build:node:docker": "./build-node-docker.sh",
+    "build:apps:docker": "./build-apps-docker.sh",
     "build:packages": "./build-npm-packages.sh",
-    "build:docker": "./build-docker-images.sh",
     "setup": "./setup.sh",
     "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",

start.sh

@@ -52,13 +52,13 @@ docker-compose run -d --name colossus --entrypoint sh colossus -c "yarn storage-
 
 docker-compose up -d distributor-node
 
-# # Create a new content directory lead
-# yarn workspace api-scripts initialize-content-lead
+# Create a new content directory lead
+yarn workspace api-scripts initialize-content-lead
 
-# # Set sudo as the membership screening authority
-# yarn workspace api-scripts set-sudo-as-screening-auth
+# Set sudo as the membership screening authority
+yarn workspace api-scripts set-sudo-as-screening-auth
 
-# docker-compose up -d pioneer
+docker-compose up -d pioneer
 
 echo "use Ctrl+C to shutdown the development network."