storage-node: refactor colossus bin

storage-node/packages/cli/bin/dev.js

@@ -72,7 +72,7 @@ const init = async (api) => {
   } else { throw new Error('Setup Failed') }
 
   // set localhost colossus as discovery provider on default port
-  const bootstrapTx = api.discovery.setBootstrapEndpoints(['http://localhost:3001/'])
+  const bootstrapTx = api.discovery.setBootstrapEndpoints(['http://localhost:3000/'])
   await api.signAndSend(alice, bootstrapTx)
 }
 

storage-node/packages/colossus/bin/cli.js

@@ -1,55 +1,59 @@
 #!/usr/bin/env node
-'use strict';
+'use strict'
 
 // Node requires
-const path = require('path');
+const path = require('path')
 
 // npm requires
-const meow = require('meow');
-const configstore = require('configstore');
-const chalk = require('chalk');
-const figlet = require('figlet');
-const _ = require('lodash');
+const meow = require('meow')
+const configstore = require('configstore')
+const chalk = require('chalk')
+const figlet = require('figlet')
+const _ = require('lodash')
 
-const debug = require('debug')('joystream:cli');
+const debug = require('debug')('joystream:cli')
 
 // Project root
-const PROJECT_ROOT = path.resolve(__dirname, '..');
+const PROJECT_ROOT = path.resolve(__dirname, '..')
 
 // Configuration (default)
-const pkg = require(path.resolve(PROJECT_ROOT, 'package.json'));
-const default_config = new configstore(pkg.name);
+const pkg = require(path.resolve(PROJECT_ROOT, 'package.json'))
+const default_config = new configstore(pkg.name)
 
 // Parse CLI
 const FLAG_DEFINITIONS = {
   port: {
     type: 'integer',
     alias: 'p',
-    _default: 3000,
+    _default: 3000
   },
-  'syncPeriod': {
+  syncPeriod: {
     type: 'integer',
-    _default: 120000,
+    _default: 120000
   },
   keyFile: {
-    type: 'string',
+    type: 'string'
   },
   config: {
     type: 'string',
-    alias: 'c',
+    alias: 'c'
   },
-  'publicUrl': {
+  publicUrl: {
     type: 'string',
     alias: 'u'
   },
-  'passphrase': {
+  passphrase: {
     type: 'string'
   },
-  'wsProvider': {
+  wsProvider: {
     type: 'string',
     _default: 'ws://localhost:9944'
+  },
+  providerId: {
+    type: 'integer',
+    alias: 'i'
   }
-};
+}
 
 const cli = meow(`
   Usage:
@@ -57,204 +61,142 @@ const cli = meow(`
 
   Commands:
     server [default]  Run a server instance with the given configuration.
-    signup            Sign up as a storage provider. Requires that you provide
-                      a JSON account file of an account that is a member, and has
-                      sufficient balance for staking as a storage provider.
-                      Writes a new account file that should be used to run the
-                      storage node.
-    down              Signal to network that all services are down. Running
-                      the server will signal that services as online again.
     discovery         Run the discovery service only.
 
   Options:
     --config=PATH, -c PATH  Configuration file path. Defaults to
-                            "${default_config.path}".
+                            '${default_config.path}'.
     --port=PORT, -p PORT    Port number to listen on, defaults to 3000.
     --sync-period           Number of milliseconds to wait between synchronization
                             runs. Defaults to 30,000 (30s).
-    --key-file              JSON key export file to use as the storage provider.
+    --provider-id           StorageProviderId assigned to you in working group.
+    --key-file              JSON key export file to use as the storage provider (role account).
     --passphrase            Optional passphrase to use to decrypt the key-file (if its encrypted).
     --public-url            API Public URL to announce. No URL will be announced if not specified.
-    --ws-provider           Joystream Node websocket provider url, eg: "ws://127.0.0.1:9944"
+    --ws-provider           Joystream Node websocket provider url, eg: 'ws://127.0.0.1:9944'
   `,
-  { flags: FLAG_DEFINITIONS });
+  { flags: FLAG_DEFINITIONS })
 
 // Create configuration
-function create_config(pkgname, flags)
-{
+function create_config (pkgname, flags) {
   // Create defaults from flag definitions
-  const defaults = {};
-  for (var key in FLAG_DEFINITIONS) {
-    const defs = FLAG_DEFINITIONS[key];
+  const defaults = {}
+  var key
+  for (key in FLAG_DEFINITIONS) {
+    const defs = FLAG_DEFINITIONS[key]
     if (defs._default) {
-      defaults[key] = defs._default;
+      defaults[key] = defs._default
     }
   }
 
   // Provide flags as defaults. Anything stored in the config overrides.
-  var config = new configstore(pkgname, defaults, { configPath: flags.config });
+  var config = new configstore(pkgname, defaults, { configPath: flags.config })
 
   // But we want the flags to also override what's stored in the config, so
   // set them all.
-  for (var key in flags) {
+  for (key in flags) {
     // Skip aliases and self-referential config flag
-    if (key.length == 1 || key === 'config') continue;
+    if (key.length == 1 || key === 'config') continue
     // Skip sensitive flags
-    if (key == 'passphrase') continue;
+    if (key == 'passphrase') continue
     // Skip unset flags
-    if (!flags[key]) continue;
+    if (!flags[key]) continue
     // Otherwise set.
-    config.set(key, flags[key]);
+    config.set(key, flags[key])
   }
 
-  debug('Configuration at', config.path, config.all);
-  return config;
+  debug('Configuration at', config.path, config.all)
+  return config
 }
 
 // All-important banner!
-function banner()
-{
-  console.log(chalk.blue(figlet.textSync('joystream', 'Speed')));
+function banner () {
+  console.log(chalk.blue(figlet.textSync('joystream', 'Speed')))
 }
 
 function start_express_app(app, port) {
-  const http = require('http');
-  const server = http.createServer(app);
+  const http = require('http')
+  const server = http.createServer(app)
 
   return new Promise((resolve, reject) => {
-    server.on('error', reject);
+    server.on('error', reject)
     server.on('close', (...args) => {
-      console.log('Server closed, shutting down...');
-      resolve(...args);
-    });
+      console.log('Server closed, shutting down...')
+      resolve(...args)
+    })
     server.on('listening', () => {
-      console.log('API server started.', server.address());
-    });
-    server.listen(port, '::');
-    console.log('Starting API server...');
-  });
+      console.log('API server started.', server.address())
+    })
+    server.listen(port, '::')
+    console.log('Starting API server...')
+  })
 }
 // Start app
-function start_all_services(store, api, config)
-{
-  const app = require('../lib/app')(PROJECT_ROOT, store, api, config);
-  const port = config.get('port');
-  return start_express_app(app, port);
+function start_all_services (store, api, config) {
+  const app = require('../lib/app')(PROJECT_ROOT, store, api, config)
+  const port = config.get('port')
+  return start_express_app(app, port)
 }
 
 // Start discovery service app
-function start_discovery_service(api, config)
-{
-  const app = require('../lib/discovery')(PROJECT_ROOT, api, config);
-  const port = config.get('port');
-  return start_express_app(app, port);
+function start_discovery_service (api, config) {
+  const app = require('../lib/discovery')(PROJECT_ROOT, api, config)
+  const port = config.get('port')
+  return start_express_app(app, port)
 }
 
 // Get an initialized storage instance
-function get_storage(runtime_api, config)
-{
+function get_storage (runtime_api, config) {
   // TODO at some point, we can figure out what backend-specific connection
   // options make sense. For now, just don't use any configuration.
-  const { Storage } = require('@joystream/storage');
+  const { Storage } = require('@joystream/storage')
 
   const options = {
     resolve_content_id: async (content_id) => {
       // Resolve via API
-      const obj = await runtime_api.assets.getDataObject(content_id);
+      const obj = await runtime_api.assets.getDataObject(content_id)
       if (!obj || obj.isNone) {
-        return;
+        return
       }
 
-      return obj.unwrap().ipfs_content_id.toString();
-    },
-  };
-
-  return Storage.create(options);
-}
-
-async function run_signup(account_file, provider_url)
-{
-  if (!account_file) {
-    console.log('Cannot proceed without keyfile');
-    return
-  }
-
-  const { RuntimeApi } = require('@joystream/runtime-api');
-  const api = await RuntimeApi.create({account_file, canPromptForPassphrase: true, provider_url});
-
-  if (!api.identities.key) {
-    console.log('Cannot proceed without a member account');
-    return
-  }
-
-  // Check there is an opening
-  let availableSlots = await api.roles.availableSlotsForRole(api.roles.ROLE_STORAGE);
-
-  if (availableSlots == 0) {
-    console.log(`
-      There are no open storage provider slots available at this time.
-      Please try again later.
-    `);
-    return;
-  } else {
-    console.log(`There are still ${availableSlots} slots available, proceeding`);
-  }
-
-  const member_address = api.identities.key.address;
-
-  // Check if account works
-  const min = await api.roles.requiredBalanceForRoleStaking(api.roles.ROLE_STORAGE);
-  console.log(`Account needs to be a member and have a minimum balance of ${min.toString()}`);
-  const check = await api.roles.checkAccountForStaking(member_address);
-  if (check) {
-    console.log('Account is working for staking, proceeding.');
+      return obj.unwrap().ipfs_content_id.toString()
+    }
   }
 
-  // Create a role key
-  const role_key = await api.identities.createRoleKey(member_address);
-  const role_address = role_key.address;
-  console.log('Generated', role_address, '- this is going to be exported to a JSON file.\n',
-    ' You can provide an empty passphrase to make starting the server easier,\n',
-    ' but you must keep the file very safe, then.');
-  const filename = await api.identities.writeKeyPairExport(role_address);
-  console.log('Identity stored in', filename);
-
-  // Ok, transfer for staking.
-  await api.roles.transferForStaking(member_address, role_address, api.roles.ROLE_STORAGE);
-  console.log('Funds transferred.');
-
-  // Now apply for the role
-  await api.roles.applyForRole(role_address, api.roles.ROLE_STORAGE, member_address);
-  console.log('Role application sent.\nNow visit Roles > My Requests in the app.');
+  return Storage.create(options)
 }
 
-async function wait_for_role(config)
-{
+async function init_api_as_storage_provider (config) {
   // Load key information
-  const { RuntimeApi } = require('@joystream/runtime-api');
-  const keyFile = config.get('keyFile');
+  const { RuntimeApi } = require('@joystream/runtime-api')
+  const keyFile = config.get('keyFile')
+  const providerId = config.get('providerId')
+
   if (!keyFile) {
-    throw new Error("Must specify a key file for running a storage node! Sign up for the role; see `colussus --help' for details.");
+    throw new Error('Must specify a key file for running a storage node.')
   }
-  const wsProvider = config.get('wsProvider');
+
+  const wsProvider = config.get('wsProvider')
 
   const api = await RuntimeApi.create({
     account_file: keyFile,
     passphrase: cli.flags.passphrase,
     provider_url: wsProvider,
-  });
+    storageProviderId: providerId
+  })
 
   if (!api.identities.key) {
-    throw new Error('Failed to unlock storage provider account');
+    throw new Error('Failed to unlock storage provider account')
   }
 
-  // Wait for the account role to be finalized
-  console.log('Waiting for the account to be staked as a storage provider role...');
-  const result = await api.roles.waitForRole(api.identities.key.address, api.roles.ROLE_STORAGE);
-  return [result, api];
+  if (!await api.workers.isRoleAccountOfStorageProvider(api.storageProviderId, api.identities.key.address)) {
+    throw new Error('storage provider role account and storageProviderId are not associated with a worker')
+  }
+
+  return api
 }
 
-function get_service_information(config) {
+function get_service_information (config) {
   // For now assume we run all services on the same endpoint
   return({
     asset: {
@@ -268,32 +210,28 @@ function get_service_information(config) {
   })
 }
 
-async function announce_public_url(api, config) {
+async function announce_public_url (api, config) {
   // re-announce in future
   const reannounce = function (timeoutMs) {
-    setTimeout(announce_public_url, timeoutMs, api, config);
+    setTimeout(announce_public_url, timeoutMs, api, config)
   }
 
   debug('announcing public url')
   const { publish } = require('@joystream/discovery')
 
-  const accountId = api.identities.key.address
-
   try {
     const serviceInformation = get_service_information(config)
 
-    let keyId = await publish.publish(serviceInformation);
+    let keyId = await publish.publish(serviceInformation)
 
-    const expiresInBlocks = 600; // ~ 1 hour (6s block interval)
-    await api.discovery.setAccountInfo(accountId, keyId, expiresInBlocks);
+    await api.discovery.setAccountInfo(keyId)
 
     debug('publishing complete, scheduling next update')
 
 // >> sometimes after tx is finalized.. we are not reaching here!
 
     // Reannounce before expiery
-    reannounce(50 * 60 * 1000); // in 50 minutes
-
+    reannounce(50 * 60 * 1000) // in 50 minutes
   } catch (err) {
     debug(`announcing public url failed: ${err.stack}`)
 
@@ -303,28 +241,24 @@ async function announce_public_url(api, config) {
   }
 }
 
-function go_offline(api) {
-  return api.discovery.unsetAccountInfo(api.identities.key.address)
+function go_offline (api) {
+  return api.discovery.unsetAccountInfo()
 }
 
 // Simple CLI commands
-var command = cli.input[0];
+var command = cli.input[0]
 if (!command) {
-  command = 'server';
+  command = 'server'
 }
 
 const commands = {
   'server': async () => {
-    const cfg = create_config(pkg.name, cli.flags);
+    const cfg = create_config(pkg.name, cli.flags)
 
     // Load key information
-    const values = await wait_for_role(cfg);
-    const result = values[0]
-    const api = values[1];
-    if (!result) {
-      throw new Error(`Not staked as storage role.`);
-    }
-    console.log('Staked, proceeding.');
+    const api = await init_api_as_storage_provider(cfg)
+
+    console.log('Storage Provider identity initialized, proceeding.')
 
     // Make sure a public URL is configured
     if (!cfg.get('publicUrl')) {
@@ -332,66 +266,54 @@ const commands = {
     }
 
     // Continue with server setup
-    const store = get_storage(api, cfg);
-    banner();
+    const store = get_storage(api, cfg)
+    banner()
 
-    const { start_syncing } = require('../lib/sync');
-    start_syncing(api, cfg, store);
+    const { start_syncing } = require('../lib/sync')
+    start_syncing(api, cfg, store)
 
-    announce_public_url(api, cfg);
-    await start_all_services(store, api, cfg);
-  },
-  'signup': async (account_file) => {
-    const cfg = create_config(pkg.name, cli.flags);
-    await run_signup(account_file, cfg.get('wsProvider'));
+    announce_public_url(api, cfg)
+    await start_all_services(store, api, cfg)
   },
   'down': async () => {
-    const cfg = create_config(pkg.name, cli.flags);
+    const cfg = create_config(pkg.name, cli.flags)
 
-    const values = await wait_for_role(cfg);
-    const result = values[0]
-    const api = values[1];
-    if (!result) {
-      throw new Error(`Not staked as storage role.`);
-    }
+    const api = await init_api_as_storage_provider(cfg)
 
     await go_offline(api)
   },
   'discovery': async () => {
-    debug("Starting Joystream Discovery Service")
+    debug('Starting Joystream Discovery Service')
     const { RuntimeApi } = require('@joystream/runtime-api')
     const cfg = create_config(pkg.name, cli.flags)
-    const wsProvider = cfg.get('wsProvider');
-    const api = await RuntimeApi.create({ provider_url: wsProvider });
+    const wsProvider = cfg.get('wsProvider')
+    const api = await RuntimeApi.create({ provider_url: wsProvider })
     await start_discovery_service(api, cfg)
   }
-};
-
+}
 
-async function main()
-{
+async function main () {
   // Simple CLI commands
-  var command = cli.input[0];
+  var command = cli.input[0]
   if (!command) {
-    command = 'server';
+    command = 'server'
   }
 
   if (commands.hasOwnProperty(command)) {
     // Command recognized
-    const args = _.clone(cli.input).slice(1);
-    await commands[command](...args);
-  }
-  else {
-    throw new Error(`Command "${command}" not recognized, aborting!`);
+    const args = _.clone(cli.input).slice(1)
+    await commands[command](...args)
+  } else {
+    throw new Error(`Command '${command}' not recognized, aborting!`)
   }
 }
 
 main()
   .then(() => {
-    console.log('Process exiting gracefully.');
-    process.exit(0);
+    console.log('Process exiting gracefully.')
+    process.exit(0)
   })
   .catch((err) => {
-    console.error(chalk.red(err.stack));
-    process.exit(-1);
-  });
+    console.error(chalk.red(err.stack))
+    process.exit(-1)
+  })

storage-node/packages/runtime-api/discovery.js

@@ -34,24 +34,28 @@ class DiscoveryApi {
   }
 
   /*
-   * Set AccountInfo of an accountId
+   * Set AccountInfo of our storage provider
    */
-  async setAccountInfo (storageProviderId, ipnsId) {
+  async setAccountInfo (ipnsId) {
+    const roleAccountId = this.base.identities.key.address
+    const storageProviderId = this.base.storageProviderId
     const isProvider = await this.base.workers.isStorageProvider(storageProviderId)
     if (isProvider) {
       const tx = this.base.api.tx.discovery.setIpnsId(storageProviderId, ipnsId)
-      return this.base.signAndSend(this.base.identities.key.address, tx)
+      return this.base.signAndSend(roleAccountId, tx)
     } else {
       throw new Error('Cannot set AccountInfo, id is not a storage provider')
     }
   }
 
   /*
-   * Clear AccountInfo of an accountId
+   * Clear AccountInfo of our storage provider
    */
-  async unsetAccountInfo (storageProviderId) {
+  async unsetAccountInfo () {
+    const roleAccountId = this.base.identities.key.address
+    const storageProviderId = this.base.storageProviderId
     var tx = this.base.api.tx.discovery.unsetIpnsId(storageProviderId)
-    return this.base.signAndSend(this.base.identities.key.address, tx)
+    return this.base.signAndSend(roleAccountId, tx)
   }
 }
 

storage-node/packages/runtime-api/index.js

@@ -58,6 +58,9 @@ class RuntimeApi {
     // Keep track locally of account nonces.
     this.nonces = {}
 
+    // The storage provider id to use
+    this.storageProviderId = parseInt(options.storageProviderId) // u64 instead ?
+
     // Ok, create individual APIs
     this.identities = await IdentitiesApi.create(this, {
       account_file: options.account_file,