storage-node: drop ipns and discovery

.github/workflows/run-network-tests.yml

@@ -219,18 +219,9 @@ jobs:
           docker-compose up -d colossus
       - name: Test uploading
         run: |
-          WAIT_TIME=90
+          sleep 6
           export DEBUG=joystream:*
-          for i in {1..4}; do
-            [ "$i" == "4" ] && exit -1
-            echo "Waiting for ipfs name registration"
-            sleep ${WAIT_TIME}
-            if yarn storage-cli upload ./pioneer/packages/apps/public/images/default-thumbnail.png 1 0; then
-              break
-            else
-              echo "Upload test failed, will retry"
-            fi
-          done
+          yarn storage-cli upload ./pioneer/packages/apps/public/images/default-thumbnail.png 1 0
           # Wait for storage-node to set status Accepted on uploaded content
           sleep 6
           cd utils/api-scripts/

package.json

@@ -7,6 +7,7 @@
     "build": "./build-npm-packages.sh && ./build-docker-images.sh",
     "build:packages": "./build-npm-packages.sh",
     "build:docker": "./build-docker-images.sh",
+    "setup": "./setup.sh",
     "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",
     "cargo-build": "scripts/cargo-build.sh"

storage-node/packages/cli/package.json

@@ -45,7 +45,6 @@
   },
   "dependencies": {
     "@joystream/storage-runtime-api": "^0.1.0",
-    "@joystream/service-discovery": "^0.1.0",
     "@joystream/storage-utils": "^0.1.0",
     "@joystream/types": "^0.16.0",
     "axios": "^0.21.1",

storage-node/packages/cli/src/commands/dev.ts

@@ -102,13 +102,6 @@ const init = async (api: RuntimeApi): Promise<any> => {
     debug('Alice is already a member.')
   }
 
-  // set localhost colossus as discovery provider
-  // assuming pioneer dev server is running on port 3000 we should run
-  // the storage dev server on a different port than the default for colossus which is also
-  // 3000
-  debug('Setting Local development node as bootstrap endpoint.')
-  await api.discovery.setBootstrapEndpoints(alice, [`http://localhost:${developmentPort()}/`])
-
   debug('Transferring tokens to storage role account.')
   // Give role account some tokens to work with
   api.balances.transfer(alice, roleAccount, 100000)

storage-node/packages/cli/src/commands/upload.ts

@@ -4,7 +4,6 @@ import ipfsHash from 'ipfs-only-hash'
 import { ContentId, DataObject } from '@joystream/types/storage'
 import BN from 'bn.js'
 import { BaseCommand } from './base'
-import { DiscoveryClient } from '@joystream/service-discovery'
 import Debug from 'debug'
 import chalk from 'chalk'
 import { aliceKeyPair } from './dev'
@@ -28,7 +27,6 @@ export class UploadCommand extends BaseCommand {
   private readonly keyFile: string
   private readonly passPhrase: string
   private readonly memberId: string
-  private readonly discoveryClient: DiscoveryClient
 
   constructor(
     api: any,
@@ -41,7 +39,6 @@ export class UploadCommand extends BaseCommand {
     super()
 
     this.api = api
-    this.discoveryClient = new DiscoveryClient({ api })
     this.mediaSourceFilePath = mediaSourceFilePath
     this.dataObjectTypeId = dataObjectTypeId
     this.memberId = memberId
@@ -159,19 +156,13 @@ export class UploadCommand extends BaseCommand {
   // Requests the runtime and obtains the storage node endpoint URL.
   private async discoverStorageProviderEndpoint(storageProviderId: string): Promise<string> {
     try {
-      const serviceInfo = await this.discoveryClient.discover(storageProviderId)
+      const endpoint = await this.api.workers.getWorkerStorageValue(storageProviderId)
 
-      if (serviceInfo === null) {
-        this.fail('Storage node discovery failed.')
-      }
-      debug(`Discovered service info object: ${serviceInfo}`)
-
-      const dataWrapper = JSON.parse(serviceInfo)
-      const assetWrapper = JSON.parse(dataWrapper.serialized)
+      debug(`Resolved endpoint: ${endpoint}`)
 
-      return assetWrapper.asset.endpoint
+      return endpoint
     } catch (err) {
-      this.fail(`Could not get asset endpoint: ${err}`)
+      this.fail(`Could not get provider endpoint: ${err}`)
     }
   }
 

storage-node/packages/colossus/bin/cli.js

@@ -83,9 +83,7 @@ const cli = meow(
     $ colossus [command] [arguments]
 
   Commands:
-    server        Runs a production server instance. (discovery and storage services)
-                  This is the default command if not specified.
-    discovery     Run the discovery service only.
+    server        Runs a production server instance
 
   Arguments (required for with server command, unless --dev or --anonymous args are used):
     --provider-id ID, -i ID     StorageProviderId assigned to you in working group.
@@ -128,14 +126,8 @@ function startExpressApp(app, port) {
 }
 
 // Start app
-function startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl, anonymous }) {
-  const app = require('../lib/app')(PROJECT_ROOT, store, api, discoveryClient, ipfsHttpGatewayUrl, anonymous)
-  return startExpressApp(app, port)
-}
-
-// Start discovery service app only
-function startDiscoveryService({ port, discoveryClient }) {
-  const app = require('../lib/discovery')(PROJECT_ROOT, discoveryClient)
+function startAllServices({ store, api, port, ipfsHttpGatewayUrl, anonymous }) {
+  const app = require('../lib/app')(PROJECT_ROOT, store, api, ipfsHttpGatewayUrl, anonymous)
   return startExpressApp(app, port)
 }
 
@@ -214,26 +206,12 @@ async function initApiDevelopment({ wsProvider }) {
   return api
 }
 
-function getServiceInformation(publicUrl) {
-  // For now assume we run all services on the same endpoint
-  return {
-    asset: {
-      version: 1, // spec version
-      endpoint: publicUrl,
-    },
-    discover: {
-      version: 1, // spec version
-      endpoint: publicUrl,
-    },
-  }
-}
-
 // TODO: instead of recursion use while/async-await and use promise/setTimout based sleep
 // or cleaner code with generators?
-async function announcePublicUrl(api, publicUrl, publisherClient) {
+async function announcePublicUrl(api, publicUrl) {
   // re-announce in future
   const reannounce = function (timeoutMs) {
-    setTimeout(announcePublicUrl, timeoutMs, api, publicUrl, publisherClient)
+    setTimeout(announcePublicUrl, timeoutMs, api, publicUrl)
   }
 
   const chainIsSyncing = await api.chainIsSyncing()
@@ -257,20 +235,9 @@ async function announcePublicUrl(api, publicUrl, publisherClient) {
   debug('announcing public url')
 
   try {
-    const serviceInformation = getServiceInformation(publicUrl)
-
-    const keyId = await publisherClient.publish(serviceInformation)
-
-    await api.discovery.setAccountInfo(keyId)
+    await api.workers.setWorkerStorageValue(publicUrl)
 
     debug('publishing complete, scheduling next update')
-
-    // >> sometimes after tx is finalized.. we are not reaching here!
-
-    // Reannounce before expiery. Here we are concerned primarily
-    // with keeping the account information refreshed and 'available' in
-    // the ipfs network. our record on chain is valid for 24hr
-    reannounce(50 * 60 * 1000) // in 50 minutes
   } catch (err) {
     debug(`announcing public url failed: ${err.stack}`)
 
@@ -306,34 +273,16 @@ const commands = {
     const store = getStorage(api, cli.flags)
 
     const ipfsHost = cli.flags.ipfsHost
-    const ipfs = require('ipfs-http-client')(ipfsHost, '5001', { protocol: 'http' })
     const ipfsHttpGatewayUrl = `http://${ipfsHost}:8080/`
 
     const { startSyncing } = require('../lib/sync')
     startSyncing(api, { syncPeriod: SYNC_PERIOD_MS, anonymous: cli.flags.anonymous }, store)
 
     if (!cli.flags.anonymous) {
-      const { PublisherClient } = require('@joystream/service-discovery')
-      announcePublicUrl(api, publicUrl, new PublisherClient(ipfs))
+      announcePublicUrl(api, publicUrl)
     }
 
-    const { DiscoveryClient } = require('@joystream/service-discovery')
-    const discoveryClient = new DiscoveryClient({ ipfs, api })
-    return startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl, anonymous: cli.flags.anonymous })
-  },
-  discovery: async () => {
-    banner()
-    debug('Starting Joystream Discovery Service')
-    const { RuntimeApi } = require('@joystream/storage-runtime-api')
-    const wsProvider = cli.flags.wsProvider
-    const api = await RuntimeApi.create({ provider_url: wsProvider })
-    const port = cli.flags.port
-    const ipfsHost = cli.flags.ipfsHost
-    const ipfs = require('ipfs-http-client')(ipfsHost, '5001', { protocol: 'http' })
-    const { DiscoveryClient } = require('@joystream/service-discovery')
-    const discoveryClient = new DiscoveryClient({ ipfs, api })
-    await api.untilChainIsSynced()
-    await startDiscoveryService({ api, port, discoveryClient })
+    return startAllServices({ store, api, port, ipfsHttpGatewayUrl, anonymous: cli.flags.anonymous })
   },
 }
 

storage-node/packages/colossus/lib/app.js

@@ -35,7 +35,7 @@ const fileUploads = require('./middleware/file_uploads')
 const pagination = require('@joystream/storage-utils/pagination')
 
 // Configure app
-function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatewayUrl, anonymous) {
+function createApp(projectRoot, storage, runtime, ipfsHttpGatewayUrl, anonymous) {
   const app = express()
   app.use(cors())
   app.use(bodyParser.json())
@@ -59,7 +59,6 @@ function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatew
     dependencies: {
       storage,
       runtime,
-      discoveryClient,
       ipfsHttpGatewayUrl,
       anonymous,
     },

storage-node/packages/colossus/lib/discovery.js

@@ -1,70 +0,0 @@
-/*
- * This file is part of the storage node for the Joystream project.
- * Copyright (C) 2019 Joystream Contributors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-'use strict'
-
-// npm requires
-const express = require('express')
-const openapi = require('express-openapi')
-const bodyParser = require('body-parser')
-const cors = require('cors')
-const yaml = require('js-yaml')
-
-// Node requires
-const fs = require('fs')
-const path = require('path')
-
-// Project requires
-const validateResponses = require('./middleware/validate_responses')
-
-// Configure app
-function createApp(projectRoot, discoveryClient) {
-  const app = express()
-  app.use(cors())
-  app.use(bodyParser.json())
-  // FIXME app.use(bodyParser.urlencoded({ extended: true }));
-
-  // Load & extend/configure API docs
-  const api = yaml.safeLoad(fs.readFileSync(path.resolve(projectRoot, 'api-base.yml')))
-  api['x-express-openapi-additional-middleware'] = [validateResponses]
-  api['x-express-openapi-validation-strict'] = true
-
-  openapi.initialize({
-    apiDoc: api,
-    app,
-    // paths: path.resolve(projectRoot, 'discovery_app_paths'),
-    paths: {
-      path: '/discover/v0/{id}',
-      module: require('../paths/discover/v0/{id}'),
-    },
-    docsPath: '/swagger.json',
-    dependencies: {
-      discoveryClient,
-    },
-  })
-
-  // If no other handler gets triggered (errors), respond with the
-  // error serialized to JSON.
-  app.use(function (err, req, res) {
-    res.status(err.status).json(err)
-  })
-
-  return app
-}
-
-module.exports = createApp

storage-node/packages/colossus/paths/discover/v0/{id}.js

@@ -1,95 +0,0 @@
-const debug = require('debug')('joystream:colossus:api:discovery')
-
-const MAX_CACHE_AGE = 30 * 60 * 1000
-const USE_CACHE = true
-
-module.exports = function (discoveryClient) {
-  const doc = {
-    // parameters for all operations in this path
-    parameters: [
-      {
-        name: 'id',
-        in: 'path',
-        required: true,
-        description: 'Storage Provider Id',
-        schema: {
-          type: 'string', // integer ?
-        },
-      },
-    ],
-
-    // Resolve Service Information
-    async get(req, res) {
-      let parsedId
-      try {
-        parsedId = parseInt(req.params.id)
-      } catch (err) {
-        return res.status(400).end()
-      }
-
-      const id = parsedId
-      let cacheMaxAge = req.query.max_age
-
-      if (cacheMaxAge) {
-        try {
-          cacheMaxAge = parseInt(cacheMaxAge)
-        } catch (err) {
-          cacheMaxAge = MAX_CACHE_AGE
-        }
-      } else {
-        cacheMaxAge = 0
-      }
-
-      // todo - validate id before querying
-
-      try {
-        debug(`resolving ${id}`)
-        // Storage providers discoveryClient must use ipfs client and not rely
-        // on joystream http discovery to avoid potentially an infinite request loop
-        // back to our own api endpoint.
-        if (!discoveryClient.ipfs) {
-          return res.status(500)
-        }
-        const info = await discoveryClient.discover(id, USE_CACHE, cacheMaxAge)
-        if (info === null) {
-          debug('info not found')
-          res.status(404).end()
-        } else {
-          res.status(200).send(info)
-        }
-      } catch (err) {
-        debug(`${err}`)
-        res.status(404).end()
-      }
-    },
-  }
-
-  // OpenAPI specs
-  doc.get.apiDoc = {
-    description: 'Resolve Service Information',
-    operationId: 'discover',
-    // tags: ['asset', 'data'],
-    responses: {
-      200: {
-        description: 'Wrapped JSON Service Information',
-        content: {
-          'application/json': {
-            schema: {
-              required: ['serialized'],
-              properties: {
-                serialized: {
-                  type: 'string',
-                },
-                signature: {
-                  type: 'string',
-                },
-              },
-            },
-          },
-        },
-      },
-    },
-  }
-
-  return doc
-}

storage-node/packages/discovery/README.md

@@ -1,120 +0,0 @@
-# Discovery
-
-The `@joystream/service-discovery` package provides an API for role services to publish
-discovery information about themselves, and for consumers to resolve this
-information.
-
-In the Joystream network, services are provided by having members stake for a
-role. The role is identified by a worker id. Resolving service information
-associated with the worker id is the main purpose of this module.
-
-This implementation is based on [IPNS](https://docs.ipfs.io/guides/concepts/ipns/)
-as well as runtime information.
-
-## Discovery Workflow
-
-The discovery workflow provides worker id to the `discover()` function, which
-will eventually return structured data.
-
-Under the hood, `discover()` the bootstrap nodes from the runtime are
-used in a browser environment, or the local ipfs node otherwise.
-
-There is a distinction in the discovery workflow:
-
-1. If run in the browser environment, a HTTP request to a participating node
-   is performed to discover nodes.
-2. If run in a node.js process, instead:
-
-- A trusted (local) IPFS node must be configured.
-- The chain is queried to resolve a worker id to an IPNS id.
-- The trusted IPFS node is used to resolve the IPNS id to an IPFS
-  file.
-- The IPFS file is fetched; this contains the structured data.
-
-Web services providing the HTTP endpoint used in the first approach will
-themselves use the second approach for fulfilling queries.
-
-## Publishing Workflow
-
-The publishing workflow is a little more involved, and requires more interaction
-with the runtime and the trusted IPFS node.
-
-1. A service information file is created.
-1. The file is published on IPFS, using the IPNS self key of the local node.
-1. The IPNS name of the trusted IPFS node is updated to refer to the published
-   file.
-1. The runtime mapping from the worker ID to the IPNS name is updated.
-
-## Published Information
-
-Any JSON data can theoretically be published with this system; however, the
-following structure is currently imposed:
-
-- The JSON must be an Object at the top-level, not an Array.
-- Each key must correspond to a [service spec](../../docs/json-signing/README.md).
-
-## Service Info Specifications
-
-Service specifications are JSON Objects, not Arrays. All service specifications
-come with their own `version` field which should be intepreted by clients making
-use of the information.
-
-Additionally, some services may only provide an `endpoint` value, as defined
-here:
-
-- `version`: A numeric version identifier for the service info field.
-- `endpoint`: A publicly accessible base URL for a service API.
-
-The `endpoint` should include a scheme and full authority, such that appending
-`swagger.json` to the path resolves the OpenAPI definition of the API served
-at this endpoint.
-
-The OpenAPI definition must include a top level path component corresponding
-to the service name, followed by an API version component. The remainder of the
-provided paths are dependent on the specific version of the API provided.
-
-For example, for an endpoint value of `https://user:password@host:port/` the
-following must hold:
-
-- `https://user:password@host:port/swagger.json` resolves to the OpenAPI
-  definition of the API(s) provided by this endpoint.
-- The OpenAPI definitions include paths prefixed by
-  `https://user:password@host:port/XXX/vYYY` where
-  - `XXX` is the service name, identical to the field name of the service spec
-    in the published service information.
-  - `YYY` the version identifier for the published service API.
-
-**Note:** The `version` field in the spec indicates the version of the spec.
-The `YYY` path component above indicates the version of the published OpenAPI.
-
-### Discovery Service
-
-Publishes `version` and `endpoint` as above; the `version` field is currently
-always `1`.
-
-### Asset Service
-
-Publishes `version` and `endpoint` as above; the `version` field is currently
-always `1`.
-
-### Example
-
-```json
-{
-  "asset": {
-    "version": 1,
-    "endpoint": "https://foo.bar/"
-  },
-  "discovery": {
-    "version": 1,
-    "endpoint": "http://quux.io/"
-  }
-}
-```
-
-Here, the following must be true:
-
-- `https://foo.bar/swagger.json` must include paths beginning with `https://foo.bar/asset/vXXX`
-  where `XXX` is the API version of the asset API.
-- `https://quux.io/swagger.json` must include paths beginning with `https://foo.bar/discovery/vYYY`
-  where `XXX` is the API version of the asset API.

storage-node/packages/discovery/discover.js

@@ -1,276 +0,0 @@
-const axios = require('axios')
-const debug = require('debug')('joystream:discovery:discover')
-const stripEndingSlash = require('@joystream/storage-utils/stripEndingSlash')
-
-const BN = require('bn.js')
-const { newExternallyControlledPromise } = require('@joystream/storage-utils/externalPromise')
-
-/**
- * Determines if code is running in a browser by testing for the global window object.
- * @return {boolean} returns result check.
- */
-function inBrowser() {
-  return typeof window !== 'undefined'
-}
-
-/**
- * After what period of time a cached record is considered stale, and would
- * trigger a re-discovery, but only if a query is made for the same provider.
- */
-const CACHE_TTL = 60 * 60 * 1000
-
-class DiscoveryClient {
-  /**
-   * Map storage-provider id to a Promise of a discovery result. The purpose
-   * is to avoid concurrent active discoveries for the same provider.
-   */
-  activeDiscoveries = {}
-
-  /**
-   * Map of storage provider id to string
-   * Cache of past discovery lookup results
-   */
-  accountInfoCache = {}
-
-  /*
-   * @param {RuntimeApi} api - api instance to query the chain
-   * @param {string} ipfsHttpGatewayUrl - optional ipfs http gateway
-   * @param {IpfsClient} ipfs - optinoal instance of an ipfs-http-client
-   */
-  constructor({ api, ipfs, ipfsHttpGatewayUrl }) {
-    this.runtimeApi = api
-    this.ipfs = ipfs
-    this.ipfsHttpGatewayUrl = ipfsHttpGatewayUrl
-  }
-
-  /**
-   * Queries the ipns id (service key) of the storage provider from the blockchain.
-   * If the storage provider is not registered it will return null.
-   * @param {number | BN | u64} storageProviderId - the provider id to lookup
-   * @returns { Promise<string | null> } - ipns multiformat address
-   */
-  async getIpnsIdentity(storageProviderId) {
-    storageProviderId = new BN(storageProviderId)
-    // lookup ipns identity from chain corresponding to storageProviderId
-    const info = await this.runtimeApi.discovery.getAccountInfo(storageProviderId)
-
-    if (info === null) {
-      // no identity found on chain for account
-      return null
-    }
-    return info.identity.toString()
-  }
-
-  /**
-   * Resolves provider id to its service information.
-   * Will use an IPFS HTTP gateway. If caller doesn't provide a url the default gateway on
-   * the local ipfs node will be used.
-   * If the storage provider is not registered it will throw an error
-   * @param {number | BN | u64} storageProviderId - the provider id to lookup
-   * @param {string} ipfsHttpGatewayUrl - optional ipfs http gateway url to perform ipfs queries
-   * @returns { Promise<object> } - the published service information
-   */
-  async discoverOverIpfsHttpGateway(storageProviderId, ipfsHttpGatewayUrl) {
-    let gateway = ipfsHttpGatewayUrl || this.ipfsHttpGatewayUrl || 'http://localhost:8080'
-    storageProviderId = new BN(storageProviderId)
-    const isProvider = await this.runtimeApi.workers.isStorageProvider(storageProviderId)
-
-    if (!isProvider) {
-      throw new Error('Cannot discover non storage providers')
-    }
-
-    const identity = await this.getIpnsIdentity(storageProviderId)
-
-    if (identity === null) {
-      // dont waste time trying to resolve if no identity was found
-      throw new Error('no identity to resolve')
-    }
-
-    gateway = stripEndingSlash(gateway)
-
-    const url = `${gateway}/ipns/${identity}`
-
-    const response = await axios.get(url)
-
-    return response.data
-  }
-
-  /**
-   * Resolves id of provider to its service information.
-   * Will use the provided colossus discovery api endpoint. If no api endpoint
-   * is provided it attempts to use the configured endpoints from the chain.
-   * If the storage provider is not registered it will throw an error
-   * @param {number | BN | u64 } storageProviderId - provider id to lookup
-   * @param {string} discoverApiEndpoint - url for a colossus discovery api endpoint
-   * @returns { Promise<object> } - the published service information
-   */
-  async discoverOverJoystreamDiscoveryService(storageProviderId, discoverApiEndpoint) {
-    storageProviderId = new BN(storageProviderId)
-    const isProvider = await this.runtimeApi.workers.isStorageProvider(storageProviderId)
-
-    if (!isProvider) {
-      throw new Error('Cannot discover non storage providers')
-    }
-
-    const identity = await this.getIpnsIdentity(storageProviderId)
-
-    // dont waste time trying to resolve if no identity was found
-    if (identity === null) {
-      throw new Error('no identity to resolve')
-    }
-
-    if (!discoverApiEndpoint) {
-      // Use bootstrap nodes
-      const discoveryBootstrapNodes = await this.runtimeApi.discovery.getBootstrapEndpoints()
-
-      if (discoveryBootstrapNodes.length) {
-        discoverApiEndpoint = stripEndingSlash(discoveryBootstrapNodes[0].toString())
-      } else {
-        throw new Error('No known discovery bootstrap nodes found on network')
-      }
-    }
-
-    const url = `${discoverApiEndpoint}/discover/v0/${storageProviderId.toNumber()}`
-
-    // should have parsed if data was json?
-    const response = await axios.get(url)
-
-    return response.data
-  }
-
-  /**
-   * Resolves id of provider to its service information.
-   * Will use the local IPFS node over RPC interface.
-   * If the storage provider is not registered it will throw an error.
-   * @param {number | BN | u64 } storageProviderId - provider id to lookup
-   * @returns { Promise<object> } - the published service information
-   */
-  async discoverOverLocalIpfsNode(storageProviderId) {
-    storageProviderId = new BN(storageProviderId)
-    const isProvider = await this.runtimeApi.workers.isStorageProvider(storageProviderId)
-
-    if (!isProvider) {
-      throw new Error('Cannot discover non storage providers')
-    }
-
-    const identity = await this.getIpnsIdentity(storageProviderId)
-
-    if (identity === null) {
-      // dont waste time trying to resolve if no identity was found
-      throw new Error('no identity to resolve')
-    }
-
-    const ipnsAddress = `/ipns/${identity}/`
-
-    debug('resolved ipns to ipfs object')
-    // Can this call hang forever!? can/should we set a timeout?
-    const ipfsName = await this.ipfs.name.resolve(ipnsAddress, {
-      // don't recurse, there should only be one indirection to the service info file
-      recursive: false,
-      nocache: false,
-    })
-
-    debug('getting ipfs object', ipfsName)
-    const data = await this.ipfs.get(ipfsName) // this can sometimes hang forever!?! can we set a timeout?
-
-    // there should only be one file published under the resolved path
-    const content = data[0].content
-
-    return JSON.parse(content)
-  }
-
-  /**
-   * Internal method that handles concurrent discoveries and caching of results. Will
-   * select the appropriate discovery protocol based on browser environment or not,
-   * and if an ipfs client was passed in the constructor.
-   * @param {number | BN | u64} storageProviderId - ID of the storage provider
-   * @returns { Promise<object | null> } - the published service information
-   */
-  async _discover(storageProviderId) {
-    storageProviderId = new BN(storageProviderId)
-    const id = storageProviderId.toNumber()
-
-    const discoveryResult = this.activeDiscoveries[id]
-    if (discoveryResult) {
-      debug('discovery in progress waiting for result for', id)
-      return discoveryResult
-    }
-
-    debug('starting new discovery for', id)
-    const deferredDiscovery = newExternallyControlledPromise()
-    this.activeDiscoveries[id] = deferredDiscovery.promise
-
-    let result
-    try {
-      if (inBrowser() || !this.ipfs) {
-        result = await this.discoverOverJoystreamDiscoveryService(storageProviderId)
-      } else {
-        result = await this.discoverOverLocalIpfsNode(storageProviderId)
-      }
-
-      debug(result)
-      result = JSON.stringify(result)
-      this.accountInfoCache[id] = {
-        value: result,
-        updated: Date.now(),
-      }
-
-      deferredDiscovery.resolve(result)
-      delete this.activeDiscoveries[id]
-      return result
-    } catch (err) {
-      // we catch the error so we can update all callers
-      // and throw again to inform the first caller.
-      debug(err.message)
-      delete this.activeDiscoveries[id]
-      // deferredDiscovery.reject(err)
-      deferredDiscovery.resolve(null) // resolve to null until we figure out the issue below
-      // throw err // <-- throwing but this isn't being
-      // caught correctly in express server! Is it because there is an uncaught promise somewhere
-      // in the prior .reject() call ?
-      // I've only seen this behaviour when error is from ipfs-client
-      // ... is this unique to errors thrown from ipfs-client?
-      // Problem is its crashing the node so just return null for now
-      return null
-    }
-  }
-
-  /**
-   * Cached discovery of storage provider service information. If useCachedValue is
-   * set to true, will always return the cached result if found. New discovery will be triggered
-   * if record is found to be stale. If a stale record is not desired (CACHE_TTL old) pass a non zero
-   * value for maxCacheAge, which will force a new discovery and return the new resolved value.
-   * This method in turn calls _discovery which handles concurrent discoveries and selects the appropriate
-   * protocol to perform the query.
-   * If the storage provider is not registered it will resolve to null
-   * @param {number | BN | u64} storageProviderId - provider to discover
-   * @param {bool} useCachedValue - optionaly use chached queries
-   * @param {number} maxCacheAge - maximum age of a cached query that triggers automatic re-discovery
-   * @returns { Promise<object | null> } - the published service information
-   */
-  async discover(storageProviderId, useCachedValue = false, maxCacheAge = 0) {
-    storageProviderId = new BN(storageProviderId)
-    const id = storageProviderId.toNumber()
-    const cached = this.accountInfoCache[id]
-
-    if (cached && useCachedValue) {
-      if (maxCacheAge > 0) {
-        // get latest value
-        if (Date.now() > cached.updated + maxCacheAge) {
-          return this._discover(storageProviderId)
-        }
-      }
-      // refresh if cache if stale, new value returned on next cached query
-      if (Date.now() > cached.updated + CACHE_TTL) {
-        this._discover(storageProviderId)
-      }
-      // return best known value
-      return cached.value
-    }
-    return this._discover(storageProviderId)
-  }
-}
-
-module.exports = {
-  DiscoveryClient,
-}

storage-node/packages/discovery/index.js

@@ -1,7 +0,0 @@
-const { PublisherClient } = require('./publish')
-const { DiscoveryClient } = require('./discover')
-
-module.exports = {
-  PublisherClient,
-  DiscoveryClient,
-}

storage-node/packages/discovery/package.json

@@ -1,63 +0,0 @@
-{
-  "name": "@joystream/service-discovery",
-  "private": true,
-  "version": "0.1.0",
-  "description": "Service Discovery - Joystream Storage Node",
-  "author": "Joystream",
-  "homepage": "https://github.com/Joystream/joystream",
-  "bugs": {
-    "url": "https://github.com/Joystream/joystream/issues"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/Joystream/joystream.git"
-  },
-  "license": "GPL-3.0-only",
-  "contributors": [
-    {
-      "name": "Joystream",
-      "url": "https://joystream.org"
-    }
-  ],
-  "keywords": [
-    "joystream",
-    "storage",
-    "node"
-  ],
-  "os": [
-    "darwin",
-    "linux"
-  ],
-  "engines": {
-    "node": ">=12.18.0"
-  },
-  "volta": {
-    "extends": "../../package.json"
-  },
-  "main": "./index.js",
-  "scripts": {
-    "test": "mocha 'test/**/*.js'",
-    "lint": "eslint 'paths/**/*.js' 'lib/**/*.js'"
-  },
-  "devDependencies": {
-    "chai": "^4.2.0",
-    "eslint": "^7.6.0",
-    "mocha": "^5.2.0",
-    "supertest": "^3.4.2",
-    "temp": "^0.9.0"
-  },
-  "dependencies": {
-    "@joystream/storage-runtime-api": "^0.1.0",
-    "@joystream/storage-utils": "^0.1.0",
-    "async-lock": "^1.2.0",
-    "axios": "^0.18.0",
-    "chalk": "^2.4.2",
-    "configstore": "^4.0.0",
-    "figlet": "^1.2.1",
-    "ipfs-http-client": "^32.0.1",
-    "js-yaml": "^3.13.1",
-    "meow": "^5.0.0",
-    "multer": "^1.4.1",
-    "si-prefix": "^0.2.0"
-  }
-}

storage-node/packages/discovery/publish.js

@@ -1,93 +0,0 @@
-const debug = require('debug')('joystream:discovery:publish')
-
-/**
- * The name of the key used for publishing. We use same key used by the ipfs node
- * for the network identitiy, to make it possible to identify the ipfs node of the storage
- * provider and use `ipfs ping` to check on the uptime of a particular node.
- */
-const PUBLISH_KEY = 'self'
-
-/**
- * Applies JSON serialization on the data object and converts the utf-8
- * string to a Buffer.
- * @param {object} data - json object
- * @returns {Buffer} returns buffer from UTF-8 json
- */
-function bufferFrom(data) {
-  return Buffer.from(JSON.stringify(data), 'utf-8')
-}
-
-/**
- * Encodes the service info into a standard format see. /storage-node/docs/json-signing.md
- * To be able to add a signature over the json data. Signing is not currently implemented.
- * @param {object} info - json object
- * @returns {Buffer} return buffer.
- */
-function encodeServiceInfo(info) {
-  return bufferFrom({
-    serialized: JSON.stringify(info),
-  })
-}
-/**
- * A PublisherClient is used to store a JSON serializable piece of "service information" in the ipfs network
- * using the `self` key of the ipfs node. This makes looking up that information available through IPNS.
- */
-class PublisherClient {
-  /**
-   * Create an instance of a PublisherClient, taking an optional ipfs client instance. If not provided
-   * a default client using default localhost node will be used.
-   * @param {IpfsClient} ipfs - optional instance of an ipfs-http-client.
-   */
-  constructor(ipfs) {
-    this.ipfs = ipfs || require('ipfs-http-client')('localhost', '5001', { protocol: 'http' })
-  }
-
-  /**
-   * Publishes the service information, encoded using the standard defined in encodeServiceInfo()
-   * to ipfs, using the local ipfs node's PUBLISH_KEY, and returns the key id used to publish.
-   * What we refer to as the ipns id.
-   * @param {object} serviceInfo - the service information to publish
-   * @return {string} - the ipns id
-   */
-  async publish(serviceInfo) {
-    const keys = await this.ipfs.key.list()
-    let servicesKey = keys.find((key) => key.name === PUBLISH_KEY)
-
-    // An ipfs node will always have the self key.
-    // If the publish key is specified as anything else and it doesn't exist
-    // we create it.
-    if (PUBLISH_KEY !== 'self' && !servicesKey) {
-      debug('generating ipns services key')
-      servicesKey = await this.ipfs.key.gen(PUBLISH_KEY, {
-        type: 'rsa',
-        size: 2048,
-      })
-    }
-
-    if (!servicesKey) {
-      throw new Error('No IPFS publishing key available!')
-    }
-
-    debug('adding service info file to node')
-    const files = await this.ipfs.add(encodeServiceInfo(serviceInfo))
-
-    debug('publishing...')
-    const { name, value } = await this.ipfs.name.publish(files[0].hash, {
-      key: PUBLISH_KEY,
-      resolve: false,
-      // lifetime: // string - Time duration of the record. Default: 24h
-      // ttl:      // string - Time duration this record should be cached
-    })
-
-    debug(`published ipns name: ${name} -> ${value}`)
-
-    // Return the key id under which the content was published. Which is used
-    // to lookup the actual ipfs content id of the published service information
-    // Note: name === servicesKey.id
-    return servicesKey.id
-  }
-}
-
-module.exports = {
-  PublisherClient,
-}

storage-node/packages/discovery/test/index.js

@@ -1 +0,0 @@
-// Add Tests!

storage-node/packages/helios/bin/cli.js

@@ -2,7 +2,6 @@
 
 const { RuntimeApi } = require('@joystream/storage-runtime-api')
 const { encodeAddress } = require('@polkadot/keyring')
-const { DiscoveryClient } = require('@joystream/service-discovery')
 const axios = require('axios')
 const stripEndingSlash = require('@joystream/storage-utils/stripEndingSlash')
 
@@ -124,23 +123,13 @@ async function main() {
     })
   )
 
-  const discoveryClient = new DiscoveryClient({ api: runtime })
-
   // Resolve IPNS identities of providers
   console.log('\nResolving live provider API Endpoints...')
   const endpoints = await Promise.all(
     providersStatuses.map(async ({ providerId }) => {
       try {
-        const serviceInfo = await discoveryClient.discoverOverJoystreamDiscoveryService(providerId)
-
-        if (serviceInfo === null) {
-          console.log(`provider ${providerId} has not published service information`)
-          return { providerId, endpoint: null }
-        }
-
-        const info = JSON.parse(serviceInfo.serialized)
-        console.log(`${providerId} -> ${info.asset.endpoint}`)
-        return { providerId, endpoint: info.asset.endpoint }
+        const endpoint = (await api.workers.getWorkerStorageValue(providerId)).toString()
+        return { providerId, endpoint }
       } catch (err) {
         console.log('resolve failed for id', providerId, err.message)
         return { providerId, endpoint: null }

storage-node/packages/runtime-api/discovery.js

@@ -1,72 +0,0 @@
-'use strict'
-
-const debug = require('debug')('joystream:runtime:discovery')
-
-/*
- * Add discovery related functionality to the substrate API.
- */
-class DiscoveryApi {
-  static async create(base) {
-    const ret = new DiscoveryApi()
-    ret.base = base
-    await DiscoveryApi.init()
-    return ret
-  }
-
-  static async init() {
-    debug('Init')
-  }
-
-  /*
-   * Get Bootstrap endpoints
-   */
-  async getBootstrapEndpoints() {
-    return this.base.api.query.discovery.bootstrapEndpoints()
-  }
-
-  /*
-   * Set Bootstrap endpoints, requires the sudo account to be provided and unlocked
-   */
-  async setBootstrapEndpoints(sudoAccount, endpoints) {
-    const tx = this.base.api.tx.discovery.setBootstrapEndpoints(endpoints)
-    // make sudo call
-    return this.base.signAndSend(sudoAccount, this.base.api.tx.sudo.sudo(tx))
-  }
-
-  /*
-   * Get AccountInfo of a storage provider
-   */
-  async getAccountInfo(storageProviderId) {
-    const info = await this.base.api.query.discovery.accountInfoByStorageProviderId(storageProviderId)
-    // Not an Option so we use default value check to know if info was found
-    return info.expires_at.eq(0) ? null : info
-  }
-
-  /*
-   * Set AccountInfo of our storage provider
-   */
-  async setAccountInfo(ipnsId) {
-    const roleAccountId = this.base.identities.key.address
-    const storageProviderId = this.base.storageProviderId
-    const isProvider = await this.base.workers.isStorageProvider(storageProviderId)
-    if (isProvider) {
-      const tx = this.base.api.tx.discovery.setIpnsId(storageProviderId, ipnsId)
-      return this.base.signAndSend(roleAccountId, tx)
-    }
-    throw new Error('Cannot set AccountInfo, id is not a storage provider')
-  }
-
-  /*
-   * Clear AccountInfo of our storage provider
-   */
-  async unsetAccountInfo() {
-    const roleAccountId = this.base.identities.key.address
-    const storageProviderId = this.base.storageProviderId
-    const tx = this.base.api.tx.discovery.unsetIpnsId(storageProviderId)
-    return this.base.signAndSend(roleAccountId, tx)
-  }
-}
-
-module.exports = {
-  DiscoveryApi,
-}

storage-node/packages/runtime-api/index.js

@@ -27,7 +27,6 @@ const { IdentitiesApi } = require('@joystream/storage-runtime-api/identities')
 const { BalancesApi } = require('@joystream/storage-runtime-api/balances')
 const { WorkersApi } = require('@joystream/storage-runtime-api/workers')
 const { AssetsApi } = require('@joystream/storage-runtime-api/assets')
-const { DiscoveryApi } = require('@joystream/storage-runtime-api/discovery')
 const { SystemApi } = require('@joystream/storage-runtime-api/system')
 const AsyncLock = require('async-lock')
 const Promise = require('bluebird')
@@ -89,7 +88,6 @@ class RuntimeApi {
     this.balances = await BalancesApi.create(this)
     this.workers = await WorkersApi.create(this)
     this.assets = await AssetsApi.create(this)
-    this.discovery = await DiscoveryApi.create(this)
     this.system = await SystemApi.create(this)
   }
 

storage-node/packages/runtime-api/workers.js

@@ -20,7 +20,7 @@
 
 const debug = require('debug')('joystream:runtime:roles')
 const BN = require('bn.js')
-const { Worker } = require('@joystream/types/working-group')
+const { Text } = require('@polkadot/types')
 
 /*
  * Finds assigned worker id corresponding to the application id from the resulting
@@ -101,6 +101,24 @@ class WorkersApi {
     return providers[id.toNumber()] || null
   }
 
+  /*
+   * Returns storage provider's general purpose storage value from chain
+   */
+  async getWorkerStorageValue(id) {
+    const value = await this.base.api.query.storageWorkingGroup.workerStorage(id)
+    return new Text(this.base.api.registry, value).toString()
+  }
+
+  /*
+   * Set storage provider's general purpose storage value on chain
+   */
+  async setWorkerStorageValue(value) {
+    const id = this.base.storageProviderId
+    const tx = this.base.api.tx.storageWorkingGroup.updateRoleStorage(id, value)
+    const senderAccount = await this.storageProviderRoleAccount(id)
+    return this.base.signAndSend(senderAccount, tx)
+  }
+
   /*
    * Returns the the first found provider id with a role account or null if not found
    */

tests/network-tests/run-storage-node-tests.sh

@@ -35,8 +35,8 @@ docker-compose up -d graphql-server
 docker-compose up -d processor
 
 # Fixes Error: No active storage providers available
-echo "Waiting for ipfs name registration"
-sleep 120
+echo "Wait for colossus to announce public url"
+sleep 6
 
 echo "Creating channel..."
 yarn joystream-cli media:createChannel \

yarn.lock

@@ -9672,18 +9672,6 @@ configstore@^3.0.0:
     write-file-atomic "^2.0.0"
     xdg-basedir "^3.0.0"
 
-configstore@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/configstore/-/configstore-4.0.0.tgz#5933311e95d3687efb592c528b922d9262d227e7"
-  integrity sha512-CmquAXFBocrzaSM8mtGPMM/HiWmyIpr4CcJl/rgY2uCObZ/S7cKU0silxslqJejl+t/T9HS8E0PUNQD81JGUEQ==
-  dependencies:
-    dot-prop "^4.1.0"
-    graceful-fs "^4.1.2"
-    make-dir "^1.0.0"
-    unique-string "^1.0.0"
-    write-file-atomic "^2.0.0"
-    xdg-basedir "^3.0.0"
-
 configstore@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96"