storage-cli: update to work with any storage provider

.github/workflows/run-network-tests.yml

@@ -221,10 +221,10 @@ jobs:
         run: |
           sleep 6
           export DEBUG=joystream:*
-          yarn storage-cli upload ./pioneer/packages/apps/public/images/default-thumbnail.png 1 0
+          yarn storage-cli upload ./tests/network-tests/assets/joystream.MOV 1 0
           # Wait for storage-node to set status Accepted on uploaded content
           sleep 6
           cd utils/api-scripts/
           # Assume only one accepted data object was created
           CONTENT_ID=`yarn --silent script get-first-content-id | tail -n2 | head -n1`
-          yarn storage-cli download http://localhost:3001 ${CONTENT_ID} ./download.png
+          yarn storage-cli download ${CONTENT_ID} ./joystream.mov

storage-node/packages/cli/src/cli.ts

@@ -41,8 +41,7 @@ const usage = `
     upload            Upload a file to the Joystream Network. Requires a
                       source file path to upload, data object ID, member ID and account key file with
                       pass phrase to unlock it.
-    download          Retrieve a file. Requires a storage node URL and a content
-                      ID, as well as an output filename.
+    download          Retrieve a file. Requires a content and an output filename.
     head              Send a HEAD request for a file, and print headers.
                       Requires a storage node URL and a content ID.
 
@@ -88,11 +87,8 @@ const commands = {
 
     await uploadCmd.run()
   },
-  // needs to be updated to take a content id and resolve it a potential set
-  // of providers that has it, and select one (possibly try more than one provider)
-  // to fetch it from the get api url of a provider..
-  download: async (api: any, url: string, contentId: string, filePath: string) => {
-    const downloadCmd = new DownloadCommand(api, url, contentId, filePath)
+  download: async (api: any, contentId: string, filePath: string) => {
+    const downloadCmd = new DownloadCommand(api, contentId, filePath)
 
     await downloadCmd.run()
   },

storage-node/packages/cli/src/commands/base.ts

@@ -1,9 +1,17 @@
 import chalk from 'chalk'
 import removeEndingForwardSlash from '@joystream/storage-utils/stripEndingSlash'
 import { ContentId } from '@joystream/types/storage'
+import Debug from 'debug'
+const debug = Debug('joystream:storage-cli:base')
 
 // Commands base abstract class. Contains reusable methods.
 export abstract class BaseCommand {
+  protected readonly api: any
+
+  constructor(api: any) {
+    this.api = api
+  }
+
   // Creates the Colossus asset URL and logs it.
   protected createAndLogAssetUrl(url: string, contentId: string | ContentId): string {
     let normalizedContentId: string
@@ -50,4 +58,36 @@ export abstract class BaseCommand {
     // Maximum content length for the assets (files)
     return 2000 * 1024 * 1024
   }
+
+  // Requests the runtime and obtains the storage node endpoint URL.
+  protected async getStorageProviderEndpoint(storageProviderId: string): Promise<string> {
+    try {
+      const endpoint = await this.api.workers.getWorkerStorageValue(storageProviderId)
+
+      debug(`Resolved endpoint: ${endpoint}`)
+
+      return endpoint
+    } catch (err) {
+      this.fail(`Could not get provider endpoint: ${err}`)
+    }
+  }
+
+  protected async getAnyProviderEndpoint(): Promise<string> {
+    try {
+      const providers = await this.api.workers.getAllProviders()
+
+      debug(`Available Providers: ${providers}`)
+      // select first provider
+      do {
+        const id = providers.ids.pop()
+        const endpoint = await this.getStorageProviderEndpoint(id)
+        if (endpoint) {
+          return endpoint
+        }
+      } while (providers.ids.length)
+      throw new Error('No Providers registered endpoint')
+    } catch (err) {
+      this.fail(`Could not get provider endpoint: ${err}`)
+    }
+  }
 }

storage-node/packages/cli/src/commands/download.ts

@@ -5,38 +5,27 @@ import { BaseCommand } from './base'
 
 // Download command class. Validates input parameters and execute the logic for asset downloading.
 export class DownloadCommand extends BaseCommand {
-  private readonly api: any
-  private readonly storageNodeUrl: string
   private readonly contentId: string
   private readonly outputFilePath: string
 
-  constructor(api: any, storageNodeUrl: string, contentId: string, outputFilePath: string) {
-    super()
+  constructor(api: any, contentId: string, outputFilePath: string) {
+    super(api)
 
-    this.api = api
-    this.storageNodeUrl = storageNodeUrl
     this.contentId = contentId
     this.outputFilePath = outputFilePath
   }
 
   // Provides parameter validation. Overrides the abstract method from the base class.
   protected validateParameters(): boolean {
-    return (
-      this.storageNodeUrl &&
-      this.storageNodeUrl !== '' &&
-      this.contentId &&
-      this.contentId !== '' &&
-      this.outputFilePath &&
-      this.outputFilePath !== ''
-    )
+    return this.contentId && this.contentId !== '' && this.outputFilePath && this.outputFilePath !== ''
   }
 
   // Shows command usage. Overrides the abstract method from the base class.
   protected showUsage() {
     console.log(
       chalk.yellow(`
-        Usage:   storage-cli download colossusURL contentID filePath
-        Example: storage-cli download http://localhost:3001 0x7a6ba7e9157e5fba190dc146fe1baa8180e29728a5c76779ed99655500cff795 ./movie.mp4
+        Usage:   storage-cli download contentID filePath
+        Example: storage-cli download 0x7a6ba7e9157e5fba190dc146fe1baa8180e29728a5c76779ed99655500cff795 ./movie.mp4
       `)
     )
   }
@@ -46,7 +35,9 @@ export class DownloadCommand extends BaseCommand {
     // Checks for input parameters, shows usage if they are invalid.
     if (!this.assertParameters()) return
 
-    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, this.contentId)
+    const storageNodeUrl = await this.getAnyProviderEndpoint()
+
+    const assetUrl = this.createAndLogAssetUrl(storageNodeUrl, this.contentId)
     console.log(chalk.yellow('File path:', this.outputFilePath))
 
     // Create file write stream and set error handler.

storage-node/packages/cli/src/commands/head.ts

@@ -4,14 +4,12 @@ import { BaseCommand } from './base'
 
 // Head command class. Validates input parameters and obtains the asset headers.
 export class HeadCommand extends BaseCommand {
-  private readonly api: any
   private readonly storageNodeUrl: string
   private readonly contentId: string
 
   constructor(api: any, storageNodeUrl: string, contentId: string) {
-    super()
+    super(api)
 
-    this.api = api
     this.storageNodeUrl = storageNodeUrl
     this.contentId = contentId
   }

storage-node/packages/cli/src/commands/upload.ts

@@ -21,7 +21,6 @@ interface AddContentParams {
 
 // Upload command class. Validates input parameters and uploads the asset to the storage node and runtime.
 export class UploadCommand extends BaseCommand {
-  private readonly api: any
   private readonly mediaSourceFilePath: string
   private readonly dataObjectTypeId: string
   private readonly keyFile: string
@@ -36,9 +35,8 @@ export class UploadCommand extends BaseCommand {
     keyFile: string,
     passPhrase: string
   ) {
-    super()
+    super(api)
 
-    this.api = api
     this.mediaSourceFilePath = mediaSourceFilePath
     this.dataObjectTypeId = dataObjectTypeId
     this.memberId = memberId
@@ -153,19 +151,6 @@ export class UploadCommand extends BaseCommand {
     }
   }
 
-  // Requests the runtime and obtains the storage node endpoint URL.
-  private async discoverStorageProviderEndpoint(storageProviderId: string): Promise<string> {
-    try {
-      const endpoint = await this.api.workers.getWorkerStorageValue(storageProviderId)
-
-      debug(`Resolved endpoint: ${endpoint}`)
-
-      return endpoint
-    } catch (err) {
-      this.fail(`Could not get provider endpoint: ${err}`)
-    }
-  }
-
   // Loads and unlocks the runtime identity using the key file and pass phrase.
   private async loadIdentity(): Promise<any> {
     const noKeyFileProvided = !this.keyFile || this.keyFile === ''
@@ -208,7 +193,7 @@ export class UploadCommand extends BaseCommand {
     const dataObject = await this.createContent(addContentParams)
     debug(`Received data object: ${dataObject.toString()}`)
 
-    const colossusEndpoint = await this.discoverStorageProviderEndpoint(dataObject.liaison.toString())
+    const colossusEndpoint = await this.getAnyProviderEndpoint()
     debug(`Discovered storage node endpoint: ${colossusEndpoint}`)
 
     const assetUrl = this.createAndLogAssetUrl(colossusEndpoint, addContentParams.contentId)