separate building of npm packages, docker images and cargo crates

.dockerignore

@@ -5,3 +5,5 @@ target/
 query-node/generated
 query-node/**/dist
 query-node/lib
+cli/
+tests/

.github/workflows/content-directory-schemas.yml

@@ -17,4 +17,7 @@ jobs:
     - name: validate
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace @joystream/cd-schemas checks --quiet

.github/workflows/joystream-cli.yml

@@ -17,6 +17,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace @joystream/cli checks --quiet
     - name: yarn pack test
       run: |
@@ -39,6 +42,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build        
         yarn workspace @joystream/cli checks --quiet
     - name: yarn pack test
       run: |

.github/workflows/joystream-types.yml

@@ -17,6 +17,7 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace @joystream/types checks --quiet
     - name: npm pack test
       run: |
@@ -40,6 +41,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace @joystream/types checks --quiet
     - name: npm pack test
       run: |

.github/workflows/network-tests.yml

@@ -17,6 +17,10 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
+        yarn workspace @joystream/cli build
         yarn workspace network-tests checks --quiet
 
   network_build_osx:

.github/workflows/pioneer.yml

@@ -17,6 +17,7 @@ jobs:
     - name: build
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace pioneer build
 
   pioneer_build_osx:
@@ -34,6 +35,7 @@ jobs:
     - name: build
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace pioneer build
 
   pioneer_lint_ubuntu:
@@ -51,6 +53,7 @@ jobs:
     - name: lint
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace pioneer lint --quiet
 
   pioneer_lint_osx:
@@ -68,4 +71,5 @@ jobs:
     - name: lint
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace pioneer lint --quiet

.github/workflows/run-network-tests.yml

@@ -97,7 +97,9 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn build:packages
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       - name: Execute network tests
@@ -121,7 +123,9 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn build:packages
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       - name: Execute network tests
@@ -145,7 +149,11 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/cd-schemas generate:all
+          yarn workspace @joystream/cd-schemas build
       - name: Ensure tests are runnable
         run: yarn workspace @joystream/cd-schemas checks --quiet
       - name: Start chain
@@ -171,9 +179,13 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
-      - name: Ensure query-node builds
-        run: yarn workspace query-node-root build
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/cd-schemas generate:all
+          yarn workspace @joystream/cd-schemas build
+          yarn workspace query-node-root build
+          yarn workspace @joystream/cli build
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       # Bring up hydra query-node development instance, then run content directory
@@ -199,7 +211,9 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
       - name: Build storage node
         run: yarn workspace storage-node build
       - name: Start Services

.github/workflows/storage-node.yml

@@ -17,7 +17,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace storage-node checks --quiet
+        yarn workspace storage-node build
 
   storage_node_build_osx:
     name: MacOS Checks
@@ -34,4 +36,6 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace storage-node checks --quiet
+        yarn workspace storage-node build

README.md

@@ -25,12 +25,11 @@ After cloning the repo run the following initialization scripts:
 # Install rust toolchain
 ./setup.sh
 
-# Install npm package dependencies
-# Also good habit to run this when switching between branches
-yarn install
+# Install npm package dependencies, build packages and docker images
+yarn build
 
-# run some tests
-yarn cargo-checks
+# start a local development network
+yarn start
 ```
 
 ## Software
@@ -75,6 +74,7 @@ The HEAD of the master branch should always be used for the correct version of t
 ```sh
 git checkout master
 yarn install
+yarn build:packages
 yarn workspace pioneer start
 ```
 

apps.Dockerfile

@@ -2,13 +2,31 @@ FROM node:12 as builder
 
 WORKDIR /joystream
 COPY . /joystream
+RUN  rm -fr /joystream/pioneer
 
 # Do not set NODE_ENV=production until after running yarn install
 # to ensure dev dependencies are installed.
-RUN yarn install --frozen-lockfile
+RUN yarn --forzen-lockfile
 
-RUN yarn workspace pioneer build
-RUN yarn workspace storage-node build
+RUN yarn workspace @joystream/types build
 RUN yarn workspace query-node-root build
+RUN yarn workspace storage-node build
+
+# Second stage to reduce image size, enable it when
+# all packages have correctly identified what is a devDependency and what is not.
+# It will reduce the image size by about 500MB (down from 2.2GB to 1.7GB)
+
+# # Remove files that are not needed after build.
+# # We will re-fetch only dependencies needed for running the apps.
+# RUN rm -fr node_modules/
+# RUN rm -fr .git/
+
+# FROM node:12
+# WORKDIR /joystream
+# COPY --from=builder /joystream/ /joystream/
+
+# # Skip installing devDependencies, since we have already built the packages.
+# ENV NODE_ENV=production
+# RUN yarn install --forzen-lockfile --production
 
 ENTRYPOINT [ "yarn" ]

build-docker-images.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -e
+
+if ! command -v docker-compose &> /dev/null
+then
+  echo "docker-compose not found. Skipping docker image builds."
+  exit 0
+fi
+
+# Build or fetch cached joystream/node docker image
+if [[ "$SKIP_JOYSTREAM_NODE" = 1 || "$SKIP_JOYSTREAM_NODE" = "true" ]]; then
+  echo "Skipping build of joystream/node docker image."
+else
+  # Fetch a cached joystream/node image if one is found matching code shasum instead of building
+  CODE_HASH=`scripts/runtime-code-shasum.sh`
+  IMAGE=joystream/node:${CODE_HASH}
+  echo "Trying to fetch cashed ${IMAGE} image"
+  docker pull ${IMAGE} || :
+
+  if ! docker inspect ${IMAGE} > /dev/null;
+  then
+    echo "Fetch failed, building image locally"
+    docker-compose build joystream-node
+  else
+    echo "Tagging cached image as 'latest'"
+    docker image tag ${IMAGE} joystream/node:latest
+  fi
+fi
+
+# Build joystream/apps docker image
+echo "Building 'joystream/apps' docker image..."
+docker-compose build colossus
+
+# Build the pioneer docker image
+echo "Building pioneer docker image"
+docker-compose build pioneer

build-npm-packages.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+yarn
+yarn workspace @joystream/types build
+yarn workspace @joystream/cd-schemas generate:all
+yarn workspace @joystream/cd-schemas build
+yarn workspace query-node-root build
+yarn workspace @joystream/cli build
+yarn workspace storage-node build
+yarn workspace pioneer build

docker-compose.yml

@@ -151,10 +151,9 @@ services:
       - "127.0.0.1:6379:6379"
 
   pioneer:
-    image: joystream/apps
+    image: joystream/pioneer
     build:
       context: .
-      dockerfile: apps.Dockerfile
+      dockerfile: pioneer.Dockerfile
     ports:
-      - "127.0.0.1:3000:3000"
-    command: workspace pioneer start
+      - "127.0.0.1:3000:80"

package.json

@@ -4,8 +4,10 @@
   "version": "1.0.0",
   "license": "GPL-3.0-only",
   "scripts": {
-    "postinstall": "yarn workspace @joystream/types build && yarn workspace @joystream/cd-schemas generate:all && yarn workspace @joystream/cd-schemas build && yarn workspace @joystream/cli build",
-    "build": "./build.sh",
+    "build": "./build-npm-packages.sh && ./build-docker-images.sh",
+    "build:packages": "./build-npm-packages.sh",
+    "build:docker": "./build-docker-images.sh",
+    "setup": "./setup.sh",
     "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",
     "cargo-build": "scripts/cargo-build.sh"

pioneer.Dockerfile

@@ -0,0 +1,18 @@
+FROM node:12 as builder
+
+WORKDIR /joystream
+COPY . /joystream
+
+RUN yarn
+RUN NODE_ENV=production yarn workspace @joystream/types build
+RUN NODE_ENV=production yarn workspace pioneer build
+
+FROM ubuntu:18.04
+
+RUN apt-get update && apt-get -y install nginx
+
+COPY --from=builder /joystream/pioneer/packages/apps/build /var/www/html
+
+EXPOSE 80
+
+CMD ["nginx", "-g", "daemon off;"]

pioneer/README.md

@@ -37,7 +37,7 @@ To start off, this repo (along with others in the [@polkadot](https://github.com
 
 To get started -
 
-1. Clone the repo locally, via `git clone https://github.com/joystream/apps <optional local path>`
+1. Clone the repo locally, via `git clone https://github.com/joystream/joystream <optional local path>`
 2. Ensure that you have a recent LTS version of Node.js, for development purposes [Node >=10.13.0](https://nodejs.org/en/) is recommended.
 3. Ensure that you have a recent version of Yarn, for development purposes [Yarn >=1.10.1](https://yarnpkg.com/docs/install) is required.
 4. Install the dependencies by running `yarn`

start.sh

@@ -38,7 +38,10 @@ yarn workspace query-node-root db:migrate
 docker-compose up -d graphql-server
 docker-compose up -d processor
 
-echo "press Ctrl+C to shutdown"
+docker-compose up -d pioneer
 
-# Start a dev instance of pioneer and wait for exit
-docker-compose up pioneer
+echo "use Ctrl+C to shutdown the development network."
+
+while true; do 
+  read
+done