Merge pull request #1871 from metmirr/storage-node-integration-tests

Add storage node integration tests

tests/network-tests/.env

@@ -54,3 +54,5 @@ SLASH_AMOUNT = 2
 STAKE_DECREMENT = 3
 # Mint capacity increment value for working gorup mint capacity test
 MINT_CAPACITY_INCREMENT = 1000
+# Storage node address to download content from
+STORAGE_NODE_URL = http://localhost:3001/asset/v0

tests/network-tests/assets/TestChannel.json

@@ -0,0 +1,9 @@
+{
+  "handle": "Storage node channel",
+  "description": "Storage node channel",
+  "language": { "existing": { "code": "EN" } },
+  "coverPhotoUrl": "",
+  "avatarPhotoUrl": "",
+  "isPublic": true,
+  "isCensored": false
+}

tests/network-tests/assets/TestVideo.json

@@ -0,0 +1,22 @@
+{
+  "title": "Storage node test",
+  "description": "Storage node test",
+  "thumbnailUrl": "",
+  "isPublic": true,
+  "isExplicit": true,
+  "hasMarketing": true,
+  "language": { "existing": { "code": "EN" } },
+  "category": { "existing": { "name": "Entertainment" } },
+  "license": {
+    "new": {
+      "knownLicense": {
+        "existing": { "code": "CC_BY" }
+      }
+    }
+  },
+  "media": {
+    "new": {
+      "encoding": { "existing": { "name": "H.264_MP4" } }
+    }
+  }
+}

tests/network-tests/run-storage-node-tests.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -e
+
+# SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+# cd $SCRIPT_PATH
+
+function cleanup() {
+    # Show tail end of logs for the processor and indexer containers to
+    # see any possible errors
+    (echo "## Processor Logs ##" && docker logs joystream_processor_1 --tail 50) || :
+    (echo "## Indexer Logs ##" && docker logs joystream_indexer_1 --tail 50) || :
+    docker-compose down -v
+}
+
+trap cleanup EXIT
+
+export WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
+
+# Only run codegen if no generated files found
+[ ! -d "query-node/generated/" ] && yarn workspace query-node-root build
+
+# Make sure typeorm is available.. it get removed again when yarn is run again
+# typeorm commandline is used by db:migrate step below.
+ln -s ../../../../../node_modules/typeorm/cli.js query-node/generated/graphql-server/node_modules/.bin/typeorm || :
+
+# clean start
+docker-compose down -v
+
+docker-compose up -d joystream-node
+
+# Storage node
+DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
+docker-compose up -d colossus
+
+docker-compose up -d db
+yarn workspace query-node-root db:migrate
+docker-compose up -d graphql-server
+# Starting up processor will bring up all services it depends on
+docker-compose up -d processor
+
+yarn workspace @joystream/cd-schemas initialize:dev
+
+# Fixes Error: No active storage providers available
+sleep 1m 
+
+echo "Creating channel..."
+yarn joystream-cli media:createChannel \
+  --input ./tests/network-tests/assets/TestChannel.json --confirm
+
+echo "Uploading video..."
+yes | yarn joystream-cli media:uploadVideo ./tests/network-tests/assets/joystream.MOV \
+  --input ./tests/network-tests/assets/TestVideo.json \
+  --confirm 
+
+time DEBUG=* yarn workspace network-tests test-run src/scenarios/storage-node.ts

tests/network-tests/src/Api.ts

@@ -35,6 +35,7 @@ import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 import { initializeContentDir, InputParser, ExtrinsicsHelper } from '@joystream/cd-schemas'
 import { OperationType } from '@joystream/types/content-directory'
 import { gql, ApolloClient, ApolloQueryResult, NormalizedCacheObject } from '@apollo/client'
+import { ContentId, DataObject } from '@joystream/types/media'
 
 import Debugger from 'debug'
 const debug = Debugger('api')
@@ -1987,7 +1988,12 @@ export class Api {
     return await this.sendContentDirectoryTransaction(updateOperations)
   }
 
-  public async initializeContentDirectory(leadKeyPair: KeyringPair) {
+  async getDataObjectByContentId(contentId: ContentId): Promise<DataObject | null> {
+    const dataObject = await this.api.query.dataDirectory.dataObjectByContentId<Option<DataObject>>(contentId)
+    return dataObject.unwrapOr(null)
+  }
+
+  public async initializeContentDirectory(leadKeyPair: KeyringPair): Promise<void> {
     await initializeContentDir(this.api, leadKeyPair)
   }
 }
@@ -2092,4 +2098,22 @@ export class QueryNodeApi extends Api {
 
     return await this.queryNodeProvider.query({ query: FULL_TEXT_SEARCH_ON_VIDEO_TITLE, variables: { text } })
   }
+
+  public async performWhereQueryByVideoTitle(title: string): Promise<ApolloQueryResult<any>> {
+    const WHERE_QUERY_ON_VIDEO_TITLE = gql`
+      query($title: String!) {
+        videos(where: { title_eq: $title }) {
+          media {
+            location {
+              __typename
+              ... on JoystreamMediaLocation {
+                dataObjectId
+              }
+            }
+          }
+        }
+      }
+    `
+    return await this.queryNodeProvider.query({ query: WHERE_QUERY_ON_VIDEO_TITLE, variables: { title } })
+  }
 }

tests/network-tests/src/flows/storageNode/getContentFromStorageNode.ts

@@ -0,0 +1,39 @@
+import axios from 'axios'
+import { assert } from 'chai'
+import { ContentId } from '@joystream/types/media'
+import { registry } from '@joystream/types'
+
+import { QueryNodeApi } from '../../Api'
+import { Utils } from '../../utils'
+
+export default async function getContentFromStorageNode(api: QueryNodeApi): Promise<void> {
+  const videoTitle = 'Storage node test'
+
+  // Temporary solution (wait 1 minute)
+  await Utils.wait(60000)
+
+  // Query video by title with where expression
+  const videoWhereQueryResult = await api.performWhereQueryByVideoTitle(videoTitle)
+
+  assert.equal(1, videoWhereQueryResult.data.videos.length, 'Should fetch only one video')
+
+  // Get dataObjectId from the queried video's media location
+  const dataObjectId = videoWhereQueryResult.data.videos[0].media.location.dataObjectId
+
+  assert(dataObjectId.length > 2, 'dataObjectId should not be empty')
+
+  const contentId = ContentId.decode(registry, dataObjectId)
+
+  // Decode data object
+  const dataObject = await api.getDataObjectByContentId(contentId)
+
+  assert(dataObject, 'dataObject should not be null')
+
+  const response = await axios.get(`${process.env.STORAGE_NODE_URL}/${dataObjectId}`)
+
+  assert(response.headers['content-length'], 'Should have some value')
+
+  const contentLenght = Number.parseInt(response.headers['content-length'])
+
+  assert.equal(contentLenght, dataObject!.size_in_bytes.toJSON(), 'Content should be same size')
+}

tests/network-tests/src/scenarios/storage-node.ts

@@ -0,0 +1,33 @@
+import { config } from 'dotenv'
+import { WsProvider } from '@polkadot/api'
+import { ApolloClient, InMemoryCache } from '@apollo/client'
+
+import { QueryNodeApi } from '../Api'
+import getContentFromStorageNode from '../flows/storageNode/getContentFromStorageNode'
+
+const scenario = async () => {
+  // Load env variables
+  config()
+  const env = process.env
+
+  const queryNodeProvider = new ApolloClient({
+    uri: env.QUERY_NODE_URL,
+    cache: new InMemoryCache(),
+    defaultOptions: { query: { fetchPolicy: 'no-cache', errorPolicy: 'all' } },
+  })
+
+  const api: QueryNodeApi = await QueryNodeApi.new(
+    new WsProvider(env.NODE_URL),
+    queryNodeProvider,
+    env.TREASURY_ACCOUNT_URI || '//Alice',
+    env.SUDO_ACCOUNT_URI || '//Alice'
+  )
+
+  await getContentFromStorageNode(api)
+
+  // Note: disconnecting and then reconnecting to the chain in the same process
+  // doesn't seem to work!
+  api.close()
+}
+
+scenario()