refactoring: relocating Dockerfiles and compose files, removed ansible scripts

.github/workflows/run-network-tests.yml

@@ -21,6 +21,6 @@ jobs:
       - name: Configure block production to 2000ms (fatser tests)
         run: ./scripts/alter-block-creation-time.sh
       - name: Build and start chain
-        run: docker-compose --file devops/dockerfiles/node-and-runtime/docker-compose.yml up -d
+        run: docker-compose up -d
       - name: Execute network tests
         run: yarn workspace network-tests test

README.md

@@ -18,7 +18,7 @@ The following tools are required for building, testing and contributing to this
 - [Rust](https://www.rust-lang.org/tools/install) toolchain - _required_
 - [nodejs](https://nodejs.org/) v12.x - _required_
 - [yarn classic](https://classic.yarnpkg.com/en/docs/install) package manager v1.22.x- _required_
-- [docker](https://www.docker.com/get-started) - _optional_
+- [docker](https://www.docker.com/get-started) and docker-compose - _optional_
 - [ansible](https://www.ansible.com/) - _optional_
 
 If you use VSCode as your code editor we recommend using the workspace [settings](devops/vscode/settings.json) for recommend eslint plugin to function properly.

devops/ansible/build-and-run-tests-exported-chainspec-playbook.yml

@@ -1,22 +0,0 @@
-- hosts: 127.0.0.1
-  user: root
-  become: yes
-  become_method: sudo
-
-  tasks:
-
-    - name: install dependencies
-      include_role:
-        name: install_dependencies
-
-    - name: alter block creation time
-      include_role:
-        name: alter_block_creation_time
-
-    - name: build node
-      include_role:
-        name: build_docker_image
-
-    - name: run tests
-      include_role:
-        name: run_tests_exported_chainspec

devops/ansible/build-and-run-tests-single-node-playbook.yml

@@ -1,22 +0,0 @@
-- hosts: 127.0.0.1
-  user: root
-  become: yes
-  become_method: sudo
-
-  tasks:
-
-    - name: install dependencies
-      include_role:
-        name: install_dependencies
-
-    - name: alter block creation time
-      include_role:
-        name: alter_block_creation_time
-
-    - name: build node
-      include_role:
-        name: build_docker_image
-
-    - name: run tests
-      include_role:
-        name: run_tests_single_node

devops/ansible/build-and-run-tests-two-nodes-playbook.yml

@@ -1,22 +0,0 @@
-- hosts: 127.0.0.1
-  user: root
-  become: yes
-  become_method: sudo
-
-  tasks:
-
-    - name: install dependencies
-      include_role:
-        name: install_dependencies
-
-    - name: alter block creation time
-      include_role:
-        name: alter_block_creation_time
-
-    - name: build node
-      include_role:
-        name: build_docker_image
-
-    - name: run tests
-      include_role:
-        name: run_tests_two_nodes

devops/ansible/build-image-playbook.yml

@@ -1,18 +0,0 @@
-- hosts: 127.0.0.1
-  user: root
-  become: yes
-  become_method: sudo
-
-  tasks:
-
-    - name: install dependencies
-      include_role:
-        name: install_dependencies
-
-    - name: alter block creation time
-      include_role:
-        name: alter_block_creation_time
-
-    - name: build node
-      include_role:
-        name: build_docker_image

devops/ansible/docker-compose.yml

@@ -1,34 +0,0 @@
-version: "3"
-services:
-  node_alice:
-    image: joystream/node-testing
-    container_name: alice
-    entrypoint: ./node --dev --alice --validator --unsafe-ws-external --rpc-cors=all
-    ports:
-      - "30333:30333"
-      - "9933:9933"
-      - "9944:9944"
-    networks:
-      testing_net:
-        ipv4_address: 172.28.1.1
-
-  node_bob:
-    image: joystream/node-testing
-    container_name: bob
-    entrypoint: ./node --dev --bob --validator --unsafe-ws-external --rpc-cors=all
-    ports:
-      - "30335:30333"
-      - "9935:9933"
-      - "9945:9944"
-    links:
-      - "node_alice:alice"
-    networks:
-      testing_net:
-        ipv4_address: 172.28.1.2
-
-networks:
-  testing_net:
-    ipam:
-      driver: default
-      config:
-        - subnet: 172.28.0.0/16

devops/ansible/hosts

@@ -1,2 +0,0 @@
-[sites]
-127.0.0.1 ansible_connection=local

devops/ansible/roles/alter_block_creation_time/tasks/main.yml

@@ -1,4 +0,0 @@
-- name: alter block creation time
-  shell: ./scripts/alter-block-creation-time.sh
-  args:
-    chdir: ../../

devops/ansible/roles/build_docker_image/tasks/main.yml

@@ -1,4 +0,0 @@
-- name: create testing node docker image
-  shell: ./scripts/build-joystream-node-docker-image.sh
-  args:
-    chdir: ../../

devops/ansible/roles/install_dependencies/tasks/main.yml

@@ -1,46 +0,0 @@
-- name: install pip and npm on Debian
-  block:
-    - name: create temporary folder
-      file:
-        path: ../../.tmp
-        state: directory
-    - name: install pip using apt
-      apt: name=python-pip state=present
-    - name: install npm using apt
-      apt: name=npm state=present
-  when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
-
-- name: install pip on Mac
-  block:
-    - name: create temporary folder
-      file:
-        path: ../../.tmp
-        state: directory
-    - name: get pip installer using curl
-      get_url:
-        url: https://bootstrap.pypa.io/get-pip.py
-        dest: ../../.tmp/get-pip.py
-    - name: install pip
-      shell: python ../../.tmp/get-pip.py
-  when: ansible_distribution == 'MacOSX'
-  always:
-    - name: remove pip installer script
-      file:
-        path: ../../.tmp/get-pip.py
-        state: absent
-
-- name: install docker
-  pip: name=docker
-
-- name: Install yarn with npm
-  npm:
-    name: yarn
-    global: yes
-
-- name: Install pyrsistent
-  pip:
-    name: pyrsistent==0.16.0
-
-- name: Install docker compose
-  pip:
-    name: docker-compose==1.26.2

devops/ansible/roles/run_tests_exported_chainspec/tasks/main.yml

@@ -1,38 +0,0 @@
-- name: run network
-  block:
-    - name: yarn install for joystream types
-      shell: yarn workspace @joystream/types install
-
-    - name: yarn build for joystream types
-      shell: yarn workspace @joystream/types build
-
-    - name: yarn install for network tests
-      shell: yarn workspace joystream-testing install
-
-    - name: run docker container
-      docker_container:
-        name: "joystream-node"
-        image: "joystream/node"
-        ports:
-          - "9944:9944"
-        mounts:
-          - target: /testnet-state
-            source: "{{ playbook_dir }}/../../testnets/nicaea-exported-state"
-            type: bind
-            read_only: yes
-        entrypoint: ./node --chain ../testnet-state/raw_chain_spec.json --alice --validator --unsafe-ws-external --rpc-cors=all
-        state: started
-
-    - name: execute network tests
-      shell: yarn test >> ../../.tmp/tests.log
-      args:
-        chdir: ../../tests/network-tests/
-
-  always:
-    - name: display tests log
-      shell: cat ../../.tmp/tests.log
-
-    - name: stop docker container
-      docker_container:
-        name: "joystream-node-testing"
-        state: absent

devops/ansible/roles/run_tests_single_node/tasks/main.yml

@@ -1,33 +0,0 @@
-- name: run network
-  block:
-    - name: yarn install for joystream types
-      shell: yarn workspace @joystream/types install
-
-    - name: yarn build for joystream types
-      shell: yarn workspace @joystream/types build
-
-    - name: yarn install for network tests
-      shell: yarn workspace joystream-testing install
-
-    - name: run docker container
-      docker_container:
-        name: "joystream-node"
-        image: "joystream/node"
-        ports:
-          - "9944:9944"
-        entrypoint: ./node --dev --alice --validator --unsafe-ws-external --rpc-cors=all
-        state: started
-
-    - name: execute network tests
-      shell: yarn test >> ../../.tmp/tests.log
-      args:
-        chdir: ../../tests/network-tests/
-
-  always:
-    - name: display tests log
-      shell: cat ../../.tmp/tests.log
-
-    - name: stop docker container
-      docker_container:
-        name: "joystream-node-testing"
-        state: absent

devops/ansible/roles/run_tests_two_nodes/tasks/main.yml

@@ -1,18 +0,0 @@
-- name: run network
-  block:
-
-    - name: run two nodes containerized network
-      docker_compose:
-        project_src: ./
-        state: present
-
-    - name: execute network tests
-      shell: yarn test >> ../../.tmp/tests.log
-      args:
-        chdir: ../../tests/network-tests/
-        
-  always:
-    - name: stop containers
-      docker_compose:
-        project_src: ./
-        state: absent

devops/dockerfiles/node-and-runtime/Dockerfile_experimental

@@ -1,30 +0,0 @@
-# syntax=docker/dockerfile:experimental
-# must enable experimental features in docker daemon and set DOCKER_BUILDKIT=1 env variable
-# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md
-FROM joystream/rust-builder AS builder
-LABEL description="compiles and caches dependencies, artifacts and node"
-WORKDIR /joystream
-COPY . /joystream
-RUN mkdir /build-output
-
-RUN --mount=type=cache,target=/joystream/target \
-    --mount=type=cache,target=/root/.cargo/git \
-    --mount=type=cache,target=/root/.cargo/registry \
-    cargo build --release \
-    && cp ./target/release/joystream-node /build-output/joystream-node
-# copy in last part could be done with nightly option --out-dir
-
-FROM debian:stretch
-LABEL description="Joystream node"
-WORKDIR /joystream
-COPY --from=builder /build-output/joystream-node /joystream/node
-
-# Use these volumes to persits chain state and keystore, eg.:
-# --base-path /data
-# optionally separate keystore (otherwise it will be stored in the base path)
-# --keystore-path /keystore
-# if base-path isn't specified, chain state is stored inside container in ~/.local/share/joystream-node/
-# which is not ideal
-VOLUME ["/data", "/keystore"]
-
-ENTRYPOINT ["/joystream/node"]

devops/dockerfiles/node-and-runtime/docker-compose.yml

@@ -1,16 +0,0 @@
-version: "3"
-services:
-  node:
-    image: joystream/node
-    # Build image if not found. To prevent build run docker-compose with "--no-build" arg
-    build:
-      # context is relative to the compose file
-      context: ../../../
-      # dockerfile is relative to the context. thats retarted!
-      # ideally docker and compose file should be placed 
-      dockerfile: devops/dockerfiles/node-and-runtime/Dockerfile
-    container_name: joystream-node
-    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all
-    ports:
-      - "9944:9944"
-  

devops/dockerfiles/node-and-runtime/docker-compose-no-build.yml → docker-compose-no-build.yml

@@ -1,7 +1,9 @@
+# Run joystream local development chain
+# Will use an local image if found, otherwise image will be pulled from:
+# https://hub.docker.com/r/joystream/node
 version: "3"
 services:
   node:
-    # if image is missing it will be pulled from remote repo
     image: joystream/node
     container_name: joystream-node
     command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all

docker-compose.yml

@@ -0,0 +1,17 @@
+# Compiles new joystream node image if local image not found,
+# and runs local development chain.
+# To prevent build run docker-compose with "--no-build" arg
+version: "3"
+services:
+  joystream-node:
+    image: joystream/node
+    build:
+      # context is relative to the compose file
+      context: .
+      # dockerfile is relative to the context
+      dockerfile: joystream-node.Dockerfile
+    container_name: joystream-node
+    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all
+    ports:
+      - "9944:9944"
+  

docs/using-docker/building-node-and-runtime.md

@@ -1,40 +0,0 @@
-### Docker
-
-#### Building localy
-
-A joystream-node can be compiled with given [Dockerfile](./Dockerfile) file:
-
-```bash
-# Build and tag a new image, which will compile joystream-node from source
-docker build . -t joystream-node
-
-# run a development chain with the image just created publishing the websocket port
-docker run -p 9944:9944 joystream-node --dev --ws-external
-```
-
-#### Downloading joystream pre-built images from Docker Hub
-
-```bash
-docker pull joystream/node
-```
-
-#### Running a public node as a service
-
-Create a working directory to store the node's data and where you will need to place the chain file.
-
-```bash
-mkdir ${HOME}/joystream-node-data/
-
-cp rome-testnet.json ${HOME}/joystream-node-data/
-
-docker run -d -p 30333:30333 \
-    -v ${HOME}/joystream-node-data/:/data \
-    --name my-node \
-    joystream/node --base-path /data --chain /data/rome-testnet.json
-
-# check status
-docker ps
-
-# monitor logs
-docker logs --tail 100 -f my-node
-```

scripts/build-joystream-node-docker-image.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-# Build the node and runtime image
-docker build --tag joystream/node \
-    --file ./devops/dockerfiles/node-and-runtime/Dockerfile \
-    .
-

scripts/build-rust-builder-docker-image.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-# Build the joystream/rust-builder image
-docker build --tag joystream/rust-builder \
-    --file ./devops/dockerfiles/rust-builder/Dockerfile \
-    .
-

scripts/run-dev-chain-as-container.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-docker run -p 9944:9944 joystream/node --dev --ws-external

scripts/storage-node-dev/start.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -e
+
+# Avoid pulling joystream/node from docker hub. It is most likely
+# not the version that we want to work with. Either you should
+# build it locally or pull it down manually if you that is what you want
+if ! docker inspect joystream/node:latest > /dev/null 2>&1;
+then
+  echo "Didn't find local joystream/node:latest docker image."
+  exit 1
+fi
+
+# SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+
+# stop prior run and clear volumes
+docker-compose down -v
+
+# Run a development joystream-node chain and ipfs daemon in the background
+docker-compose up -d
+
+# configure the dev chain
+DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
+
+# Run the tests
+# Tests sometimes fail, so skip for now
+# yarn workspace storage-node test
+
+# Start Colossus storage-node
+DEBUG=joystream:* yarn colossus --dev

scripts/storage-node-dev/stop.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -e
+
+# stop prior run and clear volumes
+docker-compose down -v

storage-node/.travis.yml

@@ -1,14 +0,0 @@
-language: node_js
-
-node_js:
-  - 10
-  - 12
-  - 13
-
-services:
-  - docker
-
-script:
-  - docker-compose -f ./scripts/compose/devchain-and-ipfs-node/docker-compose.yaml up -d
-  - yarn test
-  - docker-compose -f ./scripts/compose/devchain-and-ipfs-node/docker-compose.yaml stop

storage-node/README.md

@@ -77,7 +77,6 @@ yarn colossus --dev
 Start pioneer ui:
 ``sh
 yarn workspace pioneer start
-
 ````
 
 Browse pioneer on http://localhost:3000/
@@ -89,16 +88,3 @@ in pioneer.
 ## Detailed Setup and Configuration Guide
 For details on how to setup a storage node on the Joystream network, follow this [step by step guide](https://github.com/Joystream/helpdesk/tree/master/roles/storage-providers).
 
-## Running a development instance with Docker
-If you have docker installed you can run a fresh development setup using the utility scripts provided:
-
-First build a docker image of joystream-node from the root:
-
-```sh
-# Build the latest image
-scripts/build-joystream-node-docker-image.sh
-
-# Run docker-compose to start an ipfs node, and joystream-node in development mode
-# initializes the chain with development settings, and start a storage-node
-storage-node/scripts/run-dev-instance.sh
-````

storage-node/scripts/run-dev-instance.sh

@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# Avoid pulling joystream/node from docker hub. It is most likely
-# not the version that we want to work with. Either you should
-# build it locally or pull it down manually if you that is what you want
-if ! docker inspect joystream/node:latest > /dev/null 2>&1;
-then
-  echo "Didn't find local joystream/node:latest docker image."
-  exit 1
-fi
-
-SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
-
-# stop prior run and clear volumes
-docker-compose -f ${SCRIPT_PATH}/compose/devchain-and-ipfs-node/docker-compose.yaml down -v
-
-# Run a development joystream-node chain and ipfs daemon in the background
-# Will use latest joystream/node images,
-# and will fetch from dockerhub if not found, so build them locally if
-# you need the version from the current branch
-docker-compose -f ${SCRIPT_PATH}/compose/devchain-and-ipfs-node/docker-compose.yaml up -d
-
-# configure the dev chain
-DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
-
-# Run the tests
-# Tests sometimes fail, so skip for now
-# yarn workspace storage-node test
-
-# Run the server in background
-# DEBUG=joystream:storage* yarn colossus --dev > ${SCRIPT_PATH}/colossus.log 2>&1 &
-# PID= ???
-# echo "Development storage node is running in the background process id: ${PID}""
-# prompt for pressing ctrl-c..
-# kill colossus and docker containers...
-# docker-compose -f ${SCRIPT_PATH}/compose/devchain-and-ipfs-node/docker-compose.yaml down -v
-
-# Run the server, without logging ranges debug info, too verbose
-DEBUG=joystream:*,-joystream:util:ranges yarn colossus --dev

storage-node/scripts/stop-dev-instance.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-script_path="$(dirname "${BASH_SOURCE[0]}")"
-
-# stop prior run and clear volumes
-docker-compose -f ${script_path}/compose/devchain-and-ipfs-node/docker-compose.yaml down -v