dockerize storage-node and update to work with intermittent connection to chain

.github/workflows/run-network-tests.yml

@@ -154,7 +154,10 @@ jobs:
           yarn workspace storage-node build
       - name: Build storage node
         run: yarn workspace storage-node build
-      - name: Start chain
-        run: docker-compose up -d
-      - name: Execute tests
-        run: DEBUG=* yarn storage-cli dev-init
+      - name: Start Services
+        run: docker-compose --file docker-compose-with-storage.yml up -d
+      - name: Add development storage node and initialize content directory
+        run: DEBUG=* yarn storage-cli dev-init
+      # - name: Execute tests
+      #   run: Upload/Download assets to/from storage node
+      #        Test querying discovery

apps.Dockerfile

@@ -1,29 +1,38 @@
-# FROM node:12
-FROM ubuntu:18.04 as builder
-
-# Install any needed packages
-RUN apt-get update && apt-get install -y curl git gnupg
-
-# install nodejs
-RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
-RUN apt-get install -y nodejs
+FROM node:12 as builder
 
 WORKDIR /joystream
-RUN npm install -g yarn
-
 COPY . /joystream
 
-RUN NODE_ENV=production yarn install --frozen-lockfile
+# Do not set NODE_ENV=production until after running yarn install
+# to ensure dev dependencies are installed.
+RUN yarn install --frozen-lockfile
 
-# install globally - used to build translations in pioneer but for some reason
-# insn't installed into node_modules/.bin or pioneer/node_modules/.bin after running
-# yarn install
-RUN npm install -g i18next-scanner
-
-# RUN yarn workspace pioneer build:code
+# ENV NODE_ENV=production
 RUN yarn workspace pioneer build
 RUN yarn workspace @joystream/cli build
 RUN yarn workspace storage-node build
 
-ENV PATH="${PATH}:/joystream/node_modules/.bin"
+# The image is huge ~ 3GB!
+# npm package Pruning.. getting rid of all devDependencies.
+# ... to significantly reduce image size (by multiple GBs): It may  cause problems in case:
+# 1. There are package dependencies which were added (incorrectly) as dev dependencies..
+# 2. 'ts-node' is used and added as a non-dev dep -> ends up pulling in dependencies which
+# would normally just be dev dependencies?
+# some packages that are big: @types, babel, electron, prettier, eslint, test frameworks
+# RUN npm prune --production
+# I think it works for a simple pacakge but npm prune doesn't recognize yarn workspaces
+# so it ends up removing too much and things break
+
+# "yarn pruning"
+# RUN cp -R node_modules/.bin somewhere.. ?
+# But don't yarn workspace packages also go into node_modules/ ?
+# How to keep those?
+# RUN rm -fr node_modules/
+# RUN yarn install --production  // should not have post install build steps that depend on devDependencies
+# // RUN yarn cache clean
+# Drops to 1.6GB but still too big!
+# FROM node:12
+# COPY --from=builder /joystream /joystream
+# WORKDIR /joystream
+
 ENTRYPOINT [ "yarn" ]

docker-compose-with-storage.yml

@@ -25,6 +25,25 @@ services:
     volumes:
       - chain-data:/data
     command: --dev --ws-external --base-path /data
+
+  # colossus exits immediately if it cannot connect to joystream node
+  # some workarounds... https://docs.docker.com/compose/startup-order/
+  # Better option is to have colossus retry when starting up
+  colossus:
+    image: joystream/apps
+    restart: on-failure
+    depends_on:
+      - "chain"
+      - "ipfs"
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - '127.0.0.1:3001:3001'
+    command: colossus --dev --ws-provider ws://chain:9944 --ipfs-host chain
+    environment:
+      - DEBUG=*
+
 volumes:
   ipfs-data:
     driver: local

storage-node/packages/colossus/bin/cli.js

@@ -11,6 +11,7 @@ const meow = require('meow')
 const chalk = require('chalk')
 const figlet = require('figlet')
 const _ = require('lodash')
+const sleep = require('@joystream/storage-utils/sleep')
 
 const debug = require('debug')('joystream:colossus')
 
@@ -60,6 +61,10 @@ const FLAG_DEFINITIONS = {
       return !flags.dev && serverCmd
     },
   },
+  ipfsHost: {
+    type: 'string',
+    default: 'localhost',
+  },
 }
 
 const cli = meow(
@@ -82,6 +87,7 @@ const cli = meow(
     --passphrase            Optional passphrase to use to decrypt the key-file.
     --port=PORT, -p PORT    Port number to listen on, defaults to 3000.
     --ws-provider WS_URL    Joystream-node websocket provider, defaults to ws://localhost:9944
+    --ipfs-host   hostname  ipfs host to use, default to 'localhost'. Default port 5001 is always used
   `,
   { flags: FLAG_DEFINITIONS }
 )
@@ -122,7 +128,7 @@ function startDiscoveryService({ api, port }) {
 }
 
 // Get an initialized storage instance
-function getStorage(runtimeApi) {
+function getStorage(runtimeApi, { ipfsHost }) {
   // TODO at some point, we can figure out what backend-specific connection
   // options make sense. For now, just don't use any configuration.
   const { Storage } = require('@joystream/storage-node-backend')
@@ -137,6 +143,7 @@ function getStorage(runtimeApi) {
       // if obj.liaison_judgement !== Accepted .. throw ?
       return obj.unwrap().ipfs_content_id.toString()
     },
+    ipfsHost,
   }
 
   return Storage.create(options)
@@ -146,14 +153,6 @@ async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }
   // Load key information
   const { RuntimeApi } = require('@joystream/storage-runtime-api')
 
-  if (!keyFile) {
-    throw new Error('Must specify a --key-file argument for running a storage node.')
-  }
-
-  if (providerId === undefined) {
-    throw new Error('Must specify a --provider-id argument for running a storage node')
-  }
-
   const api = await RuntimeApi.create({
     account_file: keyFile,
     passphrase,
@@ -167,19 +166,19 @@ async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }
 
   await api.untilChainIsSynced()
 
-  if (!(await api.workers.isRoleAccountOfStorageProvider(api.storageProviderId, api.identities.key.address))) {
-    throw new Error('storage provider role account and storageProviderId are not associated with a worker')
+  // We allow the node to startup without correct provider id and account, but syncing and
+  // publishing of identity will be skipped.
+  if (!(await api.providerIsActiveWorker())) {
+    debug('storage provider role account and storageProviderId are not associated with a worker')
   }
 
   return api
 }
 
-async function initApiDevelopment() {
+async function initApiDevelopment({ wsProvider }) {
   // Load key information
   const { RuntimeApi } = require('@joystream/storage-runtime-api')
 
-  const wsProvider = 'ws://localhost:9944'
-
   const api = await RuntimeApi.create({
     provider_url: wsProvider,
   })
@@ -188,7 +187,17 @@ async function initApiDevelopment() {
 
   api.identities.useKeyPair(dev.roleKeyPair(api))
 
-  api.storageProviderId = await dev.check(api)
+  // Wait until dev provider is added to role
+  while (true) {
+    try {
+      api.storageProviderId = await dev.check(api)
+      break
+    } catch (err) {
+      debug(err)
+    }
+
+    await sleep(10000)
+  }
 
   return api
 }
@@ -221,6 +230,12 @@ async function announcePublicUrl(api, publicUrl) {
     return reannounce(10 * 60 * 1000)
   }
 
+  // postpone if provider not active
+  if (!(await api.providerIsActiveWorker())) {
+    debug('storage provider role account and storageProviderId are not associated with a worker')
+    return reannounce(10 * 60 * 1000)
+  }
+
   const sufficientBalance = await api.providerHasMinimumBalance(1)
   if (!sufficientBalance) {
     debug('Provider role account does not have sufficient balance. Postponing announcing public url.')
@@ -262,7 +277,7 @@ if (!command) {
 
 async function startColossus({ api, publicUrl, port }) {
   // TODO: check valid url, and valid port number
-  const store = getStorage(api)
+  const store = getStorage(api, cli.flags)
   banner()
   const { startSyncing } = require('../lib/sync')
   startSyncing(api, { syncPeriod: SYNC_PERIOD_MS }, store)
@@ -276,7 +291,7 @@ const commands = {
 
     if (cli.flags.dev) {
       const dev = require('../../cli/dist/commands/dev')
-      api = await initApiDevelopment()
+      api = await initApiDevelopment(cli.flags)
       port = dev.developmentPort()
       publicUrl = `http://localhost:${port}/`
     } else {

storage-node/packages/colossus/lib/sync.js

@@ -132,6 +132,14 @@ async function syncPeriodic({ api, flags, storage, contentBeingSynced, contentCo
       return retry()
     }
 
+    // Retry later if provider is not active
+    if (!(await api.providerIsActiveWorker())) {
+      debug(
+        'storage provider role account and storageProviderId are not associated with a worker. Postponing sync run.'
+      )
+      return retry()
+    }
+
     const recommendedBalance = await api.providerHasMinimumBalance(300)
     if (!recommendedBalance) {
       debug('Warning: Provider role account is running low on balance.')

storage-node/packages/runtime-api/index.js

@@ -57,7 +57,17 @@ class RuntimeApi {
     const provider = new WsProvider(options.provider_url || 'ws://localhost:9944')
 
     // Create the API instrance
-    this.api = await ApiPromise.create({ provider, types: types })
+    while (true) {
+      try {
+        this.api = await ApiPromise.create({ provider, types: types })
+        break
+      } catch (err) {
+        debug('connecting to node failed, will retry..')
+      }
+      await sleep(5000)
+    }
+
+    await this.api.isReady
 
     this.asyncLock = new AsyncLock()
 
@@ -104,6 +114,10 @@ class RuntimeApi {
     return this.balances.hasMinimumBalanceOf(providerAccountId, minimumBalance)
   }
 
+  async providerIsActiveWorker() {
+    return this.workers.isRoleAccountOfStorageProvider(this.storageProviderId, this.identities.key.address)
+  }
+
   executeWithAccountLock(accountId, func) {
     return this.asyncLock.acquire(`${accountId}`, func)
   }

storage-node/packages/storage/storage.js

@@ -215,7 +215,7 @@ class Storage {
     this._timeout = this.options.timeout || DEFAULT_TIMEOUT
     this._resolve_content_id = this.options.resolve_content_id || DEFAULT_RESOLVE_CONTENT_ID
 
-    this.ipfs = ipfsClient(this.options.ipfs.connect_options)
+    this.ipfs = ipfsClient(this.options.ipfsHost || 'localhost', '5001', { protocol: 'http' })
 
     this.pinned = {}
     this.pinning = {}