CLI - initial update for Giza (workingGroups, content)

cli/content-test.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+echo "{}" > ~/tmp/empty.json
+
+export AUTO_CONFIRM=true
+
+# Init content lead
+GROUP=contentDirectoryWorkingGroup yarn workspace api-scripts initialize-content-lead
+# Test create/update/remove category
+yarn joystream-cli content:createVideoCategory -i ./examples/content/CreateCategory.json
+yarn joystream-cli content:createVideoCategory -i ./examples/content/CreateCategory.json
+yarn joystream-cli content:createVideoCategory -i ./examples/content/CreateCategory.json
+yarn joystream-cli content:createChannelCategory -i ./examples/content/CreateCategory.json
+yarn joystream-cli content:createChannelCategory -i ./examples/content/CreateCategory.json
+yarn joystream-cli content:createChannelCategory -i ./examples/content/CreateCategory.json
+yarn joystream-cli content:updateVideoCategory -i ./examples/content/UpdateCategory.json 2
+yarn joystream-cli content:updateChannelCategory -i ./examples/content/UpdateCategory.json 2
+yarn joystream-cli content:deleteChannelCategory 3
+yarn joystream-cli content:deleteVideoCategory 3
+# Group 1 - a valid group
+yarn joystream-cli content:createCuratorGroup
+yarn joystream-cli content:setCuratorGroupStatus 1 1
+yarn joystream-cli content:addCuratorToGroup 1 0
+# Group 2 - test removeCuratorFromGroup
+yarn joystream-cli content:createCuratorGroup
+yarn joystream-cli content:addCuratorToGroup 2 0
+yarn joystream-cli content:removeCuratorFromGroup 2 0
+# Create/update channel
+yarn joystream-cli content:createChannel -i ./examples/content/CreateChannel.json --context Member || true
+yarn joystream-cli content:createChannel -i ./examples/content/CreateChannel.json --context Curator || true
+yarn joystream-cli content:createChannel -i ~/tmp/empty.json --context Member || true
+yarn joystream-cli content:updateChannel -i ./examples/content/UpdateChannel.json 1 || true
+# Create/update video
+yarn joystream-cli content:createVideo -i ./examples/content/CreateVideo.json -c 1 || true
+yarn joystream-cli content:createVideo -i ./examples/content/CreateVideo.json -c 2 || true
+yarn joystream-cli content:createVideo -i ~/tmp/empty.json -c 2 || true
+yarn joystream-cli content:updateVideo -i ./examples/content/UpdateVideo.json 1 || true
+# Set featured videos
+yarn joystream-cli content:setFeaturedVideos 1,2
+yarn joystream-cli content:setFeaturedVideos 2,3
+# Update channel censorship status
+yarn joystream-cli content:updateChannelCensorshipStatus 1 1 --rationale "Test"
+yarn joystream-cli content:updateVideoCensorshipStatus 1 1 --rationale "Test"
+# Display-only commands
+yarn joystream-cli content:videos
+yarn joystream-cli content:video 1
+yarn joystream-cli content:channels
+yarn joystream-cli content:channel 1
+yarn joystream-cli content:curatorGroups
+yarn joystream-cli content:curatorGroup 1

cli/package.json

@@ -11,7 +11,7 @@
     "@apidevtools/json-schema-ref-parser": "^9.0.6",
     "@ffprobe-installer/ffprobe": "^1.1.0",
     "@joystream/metadata-protobuf": "^1.0.0",
-    "@joystream/types": "^0.16.1",
+    "@joystream/types": "^0.17.0",
     "@oclif/command": "^1.5.19",
     "@oclif/config": "^1.14.0",
     "@oclif/plugin-autocomplete": "^0.2.0",

cli/src/Api.ts

@@ -59,7 +59,6 @@ import {
   ChannelCategoryId,
   VideoCategoryId,
 } from '@joystream/types/content'
-import { ContentId, DataObject } from '@joystream/types/storage'
 import _ from 'lodash'
 
 export const DEFAULT_API_URI = 'ws://localhost:9944/'
@@ -67,9 +66,10 @@ export const DEFAULT_API_URI = 'ws://localhost:9944/'
 // Mapping of working group to api module
 export const apiModuleByGroup: { [key in WorkingGroups]: string } = {
   [WorkingGroups.StorageProviders]: 'storageWorkingGroup',
-  [WorkingGroups.Curators]: 'contentDirectoryWorkingGroup',
+  [WorkingGroups.Curators]: 'contentWorkingGroup',
   [WorkingGroups.Operations]: 'operationsWorkingGroup',
   [WorkingGroups.Gateway]: 'gatewayWorkingGroup',
+  [WorkingGroups.Distribution]: 'distributionWorkingGroup',
 }
 
 // Api wrapper for handling most common api calls and allowing easy API implementation switch in the future
@@ -573,15 +573,6 @@ export default class Api {
     )
   }
 
-  async dataObjectsByContentIds(contentIds: ContentId[]): Promise<DataObject[]> {
-    const dataObjects = await this._api.query.dataDirectory.dataByContentId.multi<DataObject>(contentIds)
-    const notFoundIndex = dataObjects.findIndex((o) => o.isEmpty)
-    if (notFoundIndex !== -1) {
-      throw new CLIError(`DataObject not found by id ${contentIds[notFoundIndex].toString()}`)
-    }
-    return dataObjects
-  }
-
   async getRandomBootstrapEndpoint(): Promise<string | null> {
     const endpoints = await this._api.query.discovery.bootstrapEndpoints<Vec<Url>>()
     const randomEndpoint = _.sample(endpoints.toArray())

cli/src/Types.ts

@@ -9,15 +9,15 @@ import { WorkerId, OpeningType } from '@joystream/types/working-group'
 import { Membership, MemberId } from '@joystream/types/members'
 import { Opening, StakingPolicy, ApplicationStageKeys } from '@joystream/types/hiring'
 import { Validator } from 'inquirer'
-import {
-  VideoMetadata,
-  ChannelMetadata,
-  ChannelCategoryMetadata,
-  VideoCategoryMetadata,
-} from '@joystream/content-metadata-protobuf'
-import { ContentId, ContentParameters } from '@joystream/types/storage'
+import { ContentId, ContentParameters } from '@joystream/types/content'
 
 import { JSONSchema7, JSONSchema7Definition } from 'json-schema'
+import {
+  IChannelCategoryMetadata,
+  IChannelMetadata,
+  IVideoCategoryMetadata,
+  IVideoMetadata,
+} from '@joystream/metadata-protobuf'
 
 // KeyringPair type extended with mandatory "meta.name"
 // It's used for accounts/keys management within CLI.
@@ -82,6 +82,7 @@ export enum WorkingGroups {
   Curators = 'curators',
   Operations = 'operations',
   Gateway = 'gateway',
+  Distribution = 'distributors',
 }
 
 // In contrast to Pioneer, currently only StorageProviders group is available in CLI
@@ -89,6 +90,8 @@ export const AvailableGroups: readonly WorkingGroups[] = [
   WorkingGroups.StorageProviders,
   WorkingGroups.Curators,
   WorkingGroups.Operations,
+  WorkingGroups.Gateway,
+  WorkingGroups.Distribution,
 ] as const
 
 export type Reward = {
@@ -234,47 +237,49 @@ export type VideoFileMetadata = VideoFFProbeMetadata & {
   mimeType: string
 }
 
-export type VideoInputParameters = Omit<VideoMetadata.AsObject, 'video' | 'thumbnailPhoto'> & {
+export type VideoInputParameters = Omit<IVideoMetadata, 'video' | 'thumbnailPhoto'> & {
   videoPath?: string
   thumbnailPhotoPath?: string
 }
 
-export type ChannelInputParameters = Omit<ChannelMetadata.AsObject, 'coverPhoto' | 'avatarPhoto'> & {
+export type ChannelInputParameters = Omit<IChannelMetadata, 'coverPhoto' | 'avatarPhoto'> & {
   coverPhotoPath?: string
   avatarPhotoPath?: string
   rewardAccount?: string
 }
 
-export type ChannelCategoryInputParameters = ChannelCategoryMetadata.AsObject
+export type ChannelCategoryInputParameters = IChannelCategoryMetadata
 
-export type VideoCategoryInputParameters = VideoCategoryMetadata.AsObject
+export type VideoCategoryInputParameters = IVideoCategoryMetadata
+
+type AnyNonObject = string | number | boolean | any[] | Long
 
 // JSONSchema utility types
 export type JSONTypeName<T> = T extends string
   ? 'string' | ['string', 'null']
   : T extends number
   ? 'number' | ['number', 'null']
-  : T extends any[]
-  ? 'array' | ['array', 'null']
-  : T extends Record<string, unknown>
-  ? 'object' | ['object', 'null']
   : T extends boolean
   ? 'boolean' | ['boolean', 'null']
-  : never
+  : T extends any[]
+  ? 'array' | ['array', 'null']
+  : T extends Long
+  ? 'number' | ['number', 'null']
+  : 'object' | ['object', 'null']
 
 export type PropertySchema<P> = Omit<
   JSONSchema7Definition & {
     type: JSONTypeName<P>
-    properties: P extends Record<string, unknown> ? JsonSchemaProperties<P> : never
+    properties: P extends AnyNonObject ? never : JsonSchemaProperties<P>
   },
-  P extends Record<string, unknown> ? '' : 'properties'
+  P extends AnyNonObject ? 'properties' : ''
 >
 
-export type JsonSchemaProperties<T extends Record<string, unknown>> = {
+export type JsonSchemaProperties<T> = {
   [K in keyof Required<T>]: PropertySchema<Required<T>[K]>
 }
 
-export type JsonSchema<T extends Record<string, unknown>> = JSONSchema7 & {
+export type JsonSchema<T> = JSONSchema7 & {
   type: 'object'
   properties: JsonSchemaProperties<T>
 }

cli/src/base/UploadCommandBase.ts

@@ -1,18 +1,18 @@
 import ContentDirectoryCommandBase from './ContentDirectoryCommandBase'
-import { VideoFFProbeMetadata, VideoFileMetadata, AssetType, InputAsset, InputAssetDetails } from '../Types'
-import { ContentId, ContentParameters } from '@joystream/types/storage'
+import { InputAsset, InputAssetDetails, VideoFFProbeMetadata, VideoFileMetadata } from '../Types'
 import { MultiBar, Options, SingleBar } from 'cli-progress'
-import { Assets } from '../json-schemas/typings/Assets.schema'
 import ExitCodes from '../ExitCodes'
 import ipfsHash from 'ipfs-only-hash'
 import fs from 'fs'
 import _ from 'lodash'
-import axios, { AxiosRequestConfig } from 'axios'
+import axios from 'axios'
 import ffprobeInstaller from '@ffprobe-installer/ffprobe'
 import ffmpeg from 'fluent-ffmpeg'
 import path from 'path'
-import chalk from 'chalk'
 import mimeTypes from 'mime-types'
+import { ContentId } from '../../../types/content'
+import { Assets } from '../json-schemas/typings/Assets.schema'
+import chalk from 'chalk'
 
 ffmpeg.setFfprobePath(ffprobeInstaller.path)
 
@@ -132,9 +132,12 @@ export default abstract class UploadCommandBase extends ContentDirectoryCommandB
     }
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   assetUrl(endpointRoot: string, contentId: ContentId): string {
-    // This will also make sure the resulting url is a valid url
-    return new URL(`asset/v0/${contentId.encode()}`, this.normalizeEndpoint(endpointRoot)).toString()
+    return ''
+    // TODO: Update for StorageV2
+    // // This will also make sure the resulting url is a valid url
+    // return new URL(`asset/v0/${contentId.encode()}`, this.normalizeEndpoint(endpointRoot)).toString()
   }
 
   async getRandomProviderEndpoint(): Promise<string | null> {
@@ -152,69 +155,74 @@ export default abstract class UploadCommandBase extends ContentDirectoryCommandB
     return null
   }
 
-  async generateContentParameters(filePath: string, type: AssetType): Promise<ContentParameters> {
-    return this.createType('ContentParameters', {
-      content_id: ContentId.generate(this.getTypesRegistry()),
-      type_id: type,
-      size: this.getFileSize(filePath),
-      ipfs_content_id: await this.calculateFileIpfsHash(filePath),
-    })
-  }
+  // TODO: Update for StorageV2
+  // async generateContentParameters(filePath: string, type: AssetType): Promise<ContentParameters> {
+  //   return this.createType('ContentParameters', {
+  //     content_id: ContentId.generate(this.getTypesRegistry()),
+  //     type_id: type,
+  //     size: this.getFileSize(filePath),
+  //     ipfs_content_id: await this.calculateFileIpfsHash(filePath),
+  //   })
+  // }
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async prepareInputAssets(paths: string[], basePath?: string): Promise<InputAssetDetails[]> {
-    // Resolve assets
-    if (basePath) {
-      paths = paths.map((p) => basePath && path.resolve(path.dirname(basePath), p))
-    }
-    // Validate assets
-    paths.forEach((p) => this.validateFile(p))
+    return []
+    // TODO: Update for StorageV2
+    // // Resolve assets
+    // if (basePath) {
+    //   paths = paths.map((p) => basePath && path.resolve(path.dirname(basePath), p))
+    // }
+    // // Validate assets
+    // paths.forEach((p) => this.validateFile(p))
 
-    // Return data
-    return await Promise.all(
-      paths.map(async (path) => {
-        const parameters = await this.generateContentParameters(path, AssetType.AnyAsset)
-        return {
-          path,
-          contentId: parameters.content_id,
-          parameters,
-        }
-      })
-    )
+    // // Return data
+    // return await Promise.all(
+    //   paths.map(async (path) => {
+    //     const parameters = await this.generateContentParameters(path, AssetType.AnyAsset)
+    //     return {
+    //       path,
+    //       contentId: parameters.content_id,
+    //       parameters,
+    //     }
+    //   })
+    // )
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async uploadAsset(contentId: ContentId, filePath: string, endpoint?: string, multiBar?: MultiBar): Promise<void> {
-    const providerEndpoint = endpoint || (await this.getRandomProviderEndpoint())
-    if (!providerEndpoint) {
-      this.error('No active provider found!', { exit: ExitCodes.ActionCurrentlyUnavailable })
-    }
-    const uploadUrl = this.assetUrl(providerEndpoint, contentId)
-    const fileSize = this.getFileSize(filePath)
-    const { fileStream, progressBar } = this.createReadStreamWithProgressBar(
-      filePath,
-      `Uploading ${contentId.encode()}`,
-      multiBar
-    )
-    fileStream.on('end', () => {
-      // Temporarly disable because with Promise.all it breaks the UI
-      // cli.action.start('Waiting for the file to be processed...')
-    })
-
-    try {
-      const config: AxiosRequestConfig = {
-        headers: {
-          'Content-Type': '', // https://github.com/Joystream/storage-node-joystream/issues/16
-          'Content-Length': fileSize.toString(),
-        },
-        maxBodyLength: fileSize,
-      }
-      await axios.put(uploadUrl, fileStream, config)
-    } catch (e) {
-      progressBar.stop()
-      const msg = (e.response && e.response.data && e.response.data.message) || e.message || e
-      this.error(`Unexpected error when trying to upload a file: ${msg}`, {
-        exit: ExitCodes.ExternalInfrastructureError,
-      })
-    }
+    // TODO: Update for StorageV2
+    // const providerEndpoint = endpoint || (await this.getRandomProviderEndpoint())
+    // if (!providerEndpoint) {
+    //   this.error('No active provider found!', { exit: ExitCodes.ActionCurrentlyUnavailable })
+    // }
+    // const uploadUrl = this.assetUrl(providerEndpoint, contentId)
+    // const fileSize = this.getFileSize(filePath)
+    // const { fileStream, progressBar } = this.createReadStreamWithProgressBar(
+    //   filePath,
+    //   `Uploading ${contentId.encode()}`,
+    //   multiBar
+    // )
+    // fileStream.on('end', () => {
+    //   // Temporarly disable because with Promise.all it breaks the UI
+    //   // cli.action.start('Waiting for the file to be processed...')
+    // })
+    // try {
+    //   const config: AxiosRequestConfig = {
+    //     headers: {
+    //       'Content-Type': '', // https://github.com/Joystream/storage-node-joystream/issues/16
+    //       'Content-Length': fileSize.toString(),
+    //     },
+    //     maxBodyLength: fileSize,
+    //   }
+    //   await axios.put(uploadUrl, fileStream, config)
+    // } catch (e) {
+    //   progressBar.stop()
+    //   const msg = (e.response && e.response.data && e.response.data.message) || e.message || e
+    //   this.error(`Unexpected error when trying to upload a file: ${msg}`, {
+    //     exit: ExitCodes.ExternalInfrastructureError,
+    //   })
+    // }
   }
 
   async uploadAssets(
@@ -249,6 +257,11 @@ export default abstract class UploadCommandBase extends ContentDirectoryCommandB
     multiBar.stop()
   }
 
+  public assetsIndexes(originalPaths: (string | undefined)[], filteredPaths: string[]): (number | undefined)[] {
+    let lastIndex = -1
+    return originalPaths.map((path) => (filteredPaths.includes(path as string) ? ++lastIndex : undefined))
+  }
+
   private handleRejectedUploads(
     assets: InputAsset[],
     results: boolean[],
@@ -259,7 +272,7 @@ export default abstract class UploadCommandBase extends ContentDirectoryCommandB
     const rejectedAssetsOutput: Assets = []
     results.forEach(
       (r, i) =>
-        r === false && rejectedAssetsOutput.push({ contentId: assets[i].contentId.encode(), path: assets[i].path })
+        r === false && rejectedAssetsOutput.push({ contentId: assets[i].contentId.toString(), path: assets[i].path })
     )
     if (rejectedAssetsOutput.length) {
       this.warn(

cli/src/commands/content/createChannel.ts

@@ -1,6 +1,6 @@
 import { getInputJson } from '../../helpers/InputOutput'
 import { ChannelInputParameters } from '../../Types'
-import { metadataToBytes, channelMetadataFromInput } from '../../helpers/serialization'
+import { asValidatedMetadata, metadataToBytes } from '../../helpers/serialization'
 import { flags } from '@oclif/command'
 import { CreateInterface } from '@joystream/types'
 import { ChannelCreationParameters } from '@joystream/types/content'
@@ -8,6 +8,7 @@ import { ChannelInputSchema } from '../../json-schemas/ContentDirectory'
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
 import UploadCommandBase from '../../base/UploadCommandBase'
 import chalk from 'chalk'
+import { ChannelMetadata } from '@joystream/metadata-protobuf'
 
 export default class CreateChannelCommand extends UploadCommandBase {
   static description = 'Create channel inside content directory.'
@@ -32,27 +33,24 @@ export default class CreateChannelCommand extends UploadCommandBase {
     await this.requestAccountDecoding(account)
 
     const channelInput = await getInputJson<ChannelInputParameters>(input, ChannelInputSchema)
+    const meta = asValidatedMetadata(ChannelMetadata, channelInput)
 
-    const meta = channelMetadataFromInput(channelInput)
     const { coverPhotoPath, avatarPhotoPath } = channelInput
     const assetsPaths = [coverPhotoPath, avatarPhotoPath].filter((v) => v !== undefined) as string[]
     const inputAssets = await this.prepareInputAssets(assetsPaths, input)
     const assets = inputAssets.map(({ parameters }) => ({ Upload: parameters }))
     // Set assets indexes in the metadata
-    if (coverPhotoPath) {
-      meta.setCoverPhoto(0)
-    }
-    if (avatarPhotoPath) {
-      meta.setAvatarPhoto(coverPhotoPath ? 1 : 0)
-    }
+    const [coverPhotoIndex, avatarPhotoIndex] = this.assetsIndexes([coverPhotoPath, avatarPhotoPath], assetsPaths)
+    meta.coverPhoto = coverPhotoIndex
+    meta.avatarPhoto = avatarPhotoIndex
 
     const channelCreationParameters: CreateInterface<ChannelCreationParameters> = {
       assets,
-      meta: metadataToBytes(meta),
+      meta: metadataToBytes(ChannelMetadata, meta),
       reward_account: channelInput.rewardAccount,
     }
 
-    this.jsonPrettyPrint(JSON.stringify({ assets, metadata: meta.toObject() }))
+    this.jsonPrettyPrint(JSON.stringify({ assets, metadata: meta }))
 
     await this.requireConfirmation('Do you confirm the provided input?', true)
 

cli/src/commands/content/createChannelCategory.ts

@@ -1,12 +1,13 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
 import { getInputJson } from '../../helpers/InputOutput'
 import { ChannelCategoryInputParameters } from '../../Types'
-import { channelCategoryMetadataFromInput, metadataToBytes } from '../../helpers/serialization'
+import { asValidatedMetadata, metadataToBytes } from '../../helpers/serialization'
 import { flags } from '@oclif/command'
 import { CreateInterface } from '@joystream/types'
 import { ChannelCategoryCreationParameters } from '@joystream/types/content'
 import { ChannelCategoryInputSchema } from '../../json-schemas/ContentDirectory'
 import chalk from 'chalk'
+import { ChannelCategoryMetadata } from '@joystream/metadata-protobuf'
 
 export default class CreateChannelCategoryCommand extends ContentDirectoryCommandBase {
   static description = 'Create channel category inside content directory.'
@@ -28,11 +29,10 @@ export default class CreateChannelCategoryCommand extends ContentDirectoryComman
     const actor = context ? await this.getActor(context) : await this.getCategoryManagementActor()
 
     const channelCategoryInput = await getInputJson<ChannelCategoryInputParameters>(input, ChannelCategoryInputSchema)
-
-    const meta = channelCategoryMetadataFromInput(channelCategoryInput)
+    const meta = asValidatedMetadata(ChannelCategoryMetadata, channelCategoryInput)
 
     const channelCategoryCreationParameters: CreateInterface<ChannelCategoryCreationParameters> = {
-      meta: metadataToBytes(meta),
+      meta: metadataToBytes(ChannelCategoryMetadata, meta),
     }
 
     this.jsonPrettyPrint(JSON.stringify(channelCategoryInput))

cli/src/commands/content/createVideo.ts

@@ -1,12 +1,13 @@
 import UploadCommandBase from '../../base/UploadCommandBase'
 import { getInputJson } from '../../helpers/InputOutput'
-import { videoMetadataFromInput, metadataToBytes } from '../../helpers/serialization'
+import { asValidatedMetadata, metadataToBytes } from '../../helpers/serialization'
 import { VideoInputParameters, VideoFileMetadata } from '../../Types'
 import { CreateInterface } from '@joystream/types'
 import { flags } from '@oclif/command'
 import { VideoCreationParameters } from '@joystream/types/content'
-import { MediaType, VideoMetadata } from '@joystream/content-metadata-protobuf'
+import { IVideoMetadata, VideoMetadata } from '@joystream/metadata-protobuf'
 import { VideoInputSchema } from '../../json-schemas/ContentDirectory'
+import { integrateMeta } from '@joystream/metadata-protobuf/utils'
 import chalk from 'chalk'
 
 export default class CreateVideoCommand extends UploadCommandBase {
@@ -24,17 +25,19 @@ export default class CreateVideoCommand extends UploadCommandBase {
     }),
   }
 
-  setVideoMetadataDefaults(metadata: VideoMetadata, videoFileMetadata: VideoFileMetadata) {
-    const metaObj = metadata.toObject()
-    metadata.setDuration((metaObj.duration || videoFileMetadata.duration) as number)
-    metadata.setMediaPixelWidth((metaObj.mediaPixelWidth || videoFileMetadata.width) as number)
-    metadata.setMediaPixelHeight((metaObj.mediaPixelHeight || videoFileMetadata.height) as number)
-
-    const fileMediaType = new MediaType()
-    fileMediaType.setCodecName(videoFileMetadata.codecName as string)
-    fileMediaType.setContainer(videoFileMetadata.container)
-    fileMediaType.setMimeMediaType(videoFileMetadata.mimeType)
-    metadata.setMediaType(metadata.getMediaType() || fileMediaType)
+  setVideoMetadataDefaults(metadata: IVideoMetadata, videoFileMetadata: VideoFileMetadata): void {
+    const videoMetaToIntegrate = {
+      duration: videoFileMetadata.duration,
+      mediaPixelWidth: videoFileMetadata.width,
+      mediaPixelHeight: videoFileMetadata.height,
+    }
+    const mediaTypeMetaToIntegrate = {
+      codecName: videoFileMetadata.codecName,
+      container: videoFileMetadata.container,
+      mimeMediaType: videoFileMetadata.mimeType,
+    }
+    integrateMeta(metadata, videoMetaToIntegrate, ['duration', 'mediaPixelWidth', 'mediaPixelHeight'])
+    integrateMeta(metadata.mediaType || {}, mediaTypeMetaToIntegrate, ['codecName', 'container', 'mimeMediaType'])
   }
 
   async run() {
@@ -48,8 +51,7 @@ export default class CreateVideoCommand extends UploadCommandBase {
 
     // Get input from file
     const videoCreationParametersInput = await getInputJson<VideoInputParameters>(input, VideoInputSchema)
-
-    const meta = videoMetadataFromInput(videoCreationParametersInput)
+    const meta = asValidatedMetadata(VideoMetadata, videoCreationParametersInput)
 
     // Assets
     const { videoPath, thumbnailPhotoPath } = videoCreationParametersInput
@@ -57,25 +59,24 @@ export default class CreateVideoCommand extends UploadCommandBase {
     const inputAssets = await this.prepareInputAssets(assetsPaths, input)
     const assets = inputAssets.map(({ parameters }) => ({ Upload: parameters }))
     // Set assets indexes in the metadata
-    if (videoPath) {
-      meta.setVideo(0)
-    }
-    if (thumbnailPhotoPath) {
-      meta.setThumbnailPhoto(videoPath ? 1 : 0)
-    }
+    const [videoIndex, thumbnailPhotoIndex] = this.assetsIndexes([videoPath, thumbnailPhotoPath], assetsPaths)
+    meta.video = videoIndex
+    meta.thumbnailPhoto = thumbnailPhotoIndex
 
     // Try to get video file metadata
-    const videoFileMetadata = await this.getVideoFileMetadata(inputAssets[0].path)
-    this.log('Video media file parameters established:', videoFileMetadata)
-    this.setVideoMetadataDefaults(meta, videoFileMetadata)
+    if (videoIndex !== undefined) {
+      const videoFileMetadata = await this.getVideoFileMetadata(inputAssets[videoIndex].path)
+      this.log('Video media file parameters established:', videoFileMetadata)
+      this.setVideoMetadataDefaults(meta, videoFileMetadata)
+    }
 
     // Create final extrinsic params and send the extrinsic
     const videoCreationParameters: CreateInterface<VideoCreationParameters> = {
       assets,
-      meta: metadataToBytes(meta),
+      meta: metadataToBytes(VideoMetadata, meta),
     }
 
-    this.jsonPrettyPrint(JSON.stringify({ assets, metadata: meta.toObject() }))
+    this.jsonPrettyPrint(JSON.stringify({ assets, metadata: meta }))
 
     await this.requireConfirmation('Do you confirm the provided input?', true)
 

cli/src/commands/content/createVideoCategory.ts

@@ -1,12 +1,13 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
 import { getInputJson } from '../../helpers/InputOutput'
 import { VideoCategoryInputParameters } from '../../Types'
-import { metadataToBytes, videoCategoryMetadataFromInput } from '../../helpers/serialization'
+import { asValidatedMetadata, metadataToBytes } from '../../helpers/serialization'
 import { flags } from '@oclif/command'
 import { CreateInterface } from '@joystream/types'
 import { VideoCategoryCreationParameters } from '@joystream/types/content'
 import { VideoCategoryInputSchema } from '../../json-schemas/ContentDirectory'
 import chalk from 'chalk'
+import { VideoCategoryMetadata } from '@joystream/metadata-protobuf'
 
 export default class CreateVideoCategoryCommand extends ContentDirectoryCommandBase {
   static description = 'Create video category inside content directory.'
@@ -28,11 +29,10 @@ export default class CreateVideoCategoryCommand extends ContentDirectoryCommandB
     const actor = context ? await this.getActor(context) : await this.getCategoryManagementActor()
 
     const videoCategoryInput = await getInputJson<VideoCategoryInputParameters>(input, VideoCategoryInputSchema)
-
-    const meta = videoCategoryMetadataFromInput(videoCategoryInput)
+    const meta = asValidatedMetadata(VideoCategoryMetadata, videoCategoryInput)
 
     const videoCategoryCreationParameters: CreateInterface<VideoCategoryCreationParameters> = {
-      meta: metadataToBytes(meta),
+      meta: metadataToBytes(VideoCategoryMetadata, meta),
     }
 
     this.jsonPrettyPrint(JSON.stringify(videoCategoryInput))

cli/src/commands/content/reuploadAssets.ts

@@ -3,7 +3,7 @@ import { getInputJson } from '../../helpers/InputOutput'
 import AssetsSchema from '../../json-schemas/Assets.schema.json'
 import { Assets as AssetsInput } from '../../json-schemas/typings/Assets.schema'
 import { flags } from '@oclif/command'
-import { ContentId } from '@joystream/types/storage'
+import { ContentId } from '@joystream/types/content'
 
 export default class ReuploadVideoAssetsCommand extends UploadCommandBase {
   static description = 'Allows reuploading assets that were not successfully uploaded during channel/video creation'
@@ -26,7 +26,7 @@ export default class ReuploadVideoAssetsCommand extends UploadCommandBase {
     // Get input from file
     const inputData = await getInputJson<AssetsInput>(input, AssetsSchema)
     const inputAssets = inputData.map(({ contentId, path }) => ({
-      contentId: ContentId.decode(this.getTypesRegistry(), contentId),
+      contentId: new ContentId(this.getTypesRegistry(), contentId),
       path,
     }))
 

cli/src/commands/content/updateChannel.ts

@@ -1,11 +1,12 @@
 import { getInputJson } from '../../helpers/InputOutput'
-import { channelMetadataFromInput, metadataToBytes } from '../../helpers/serialization'
+import { asValidatedMetadata, metadataToBytes } from '../../helpers/serialization'
 import { ChannelInputParameters } from '../../Types'
 import { flags } from '@oclif/command'
 import UploadCommandBase from '../../base/UploadCommandBase'
 import { CreateInterface } from '@joystream/types'
 import { ChannelUpdateParameters } from '@joystream/types/content'
 import { ChannelInputSchema } from '../../json-schemas/ContentDirectory'
+import { ChannelMetadata } from '@joystream/metadata-protobuf'
 
 export default class UpdateChannelCommand extends UploadCommandBase {
   static description = 'Update existing content directory channel.'
@@ -51,28 +52,28 @@ export default class UpdateChannelCommand extends UploadCommandBase {
     await this.requestAccountDecoding(currentAccount)
 
     const channelInput = await getInputJson<ChannelInputParameters>(input, ChannelInputSchema)
-
-    const meta = channelMetadataFromInput(channelInput)
+    const meta = asValidatedMetadata(ChannelMetadata, channelInput)
 
     const { coverPhotoPath, avatarPhotoPath, rewardAccount } = channelInput
     const inputPaths = [coverPhotoPath, avatarPhotoPath].filter((p) => p !== undefined) as string[]
     const inputAssets = await this.prepareInputAssets(inputPaths, input)
     const assets = inputAssets.map(({ parameters }) => ({ Upload: parameters }))
     // Set assets indexes in the metadata
-    if (coverPhotoPath) {
-      meta.setCoverPhoto(0)
+    const [coverPhotoIndex, avatarPhotoIndex] = this.assetsIndexes([coverPhotoPath, avatarPhotoPath], inputPaths)
+    if (coverPhotoIndex !== undefined) {
+      meta.coverPhoto = coverPhotoIndex
     }
-    if (avatarPhotoPath) {
-      meta.setAvatarPhoto(coverPhotoPath ? 1 : 0)
+    if (avatarPhotoIndex !== undefined) {
+      meta.avatarPhoto = avatarPhotoIndex
     }
 
     const channelUpdateParameters: CreateInterface<ChannelUpdateParameters> = {
       assets,
-      new_meta: metadataToBytes(meta),
+      new_meta: metadataToBytes(ChannelMetadata, meta),
       reward_account: this.parseRewardAccountInput(rewardAccount),
     }
 
-    this.jsonPrettyPrint(JSON.stringify({ assets, metadata: meta.toObject(), rewardAccount }))
+    this.jsonPrettyPrint(JSON.stringify({ assets, metadata: meta, rewardAccount }))
 
     await this.requireConfirmation('Do you confirm the provided input?', true)
 

cli/src/commands/content/updateChannelCategory.ts

@@ -1,11 +1,12 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
 import { getInputJson } from '../../helpers/InputOutput'
 import { ChannelCategoryInputParameters } from '../../Types'
-import { channelCategoryMetadataFromInput, metadataToBytes } from '../../helpers/serialization'
+import { asValidatedMetadata, metadataToBytes } from '../../helpers/serialization'
 import { CreateInterface } from '@joystream/types'
 import { ChannelCategoryUpdateParameters } from '@joystream/types/content'
 import { flags } from '@oclif/command'
 import { ChannelCategoryInputSchema } from '../../json-schemas/ContentDirectory'
+import { ChannelCategoryMetadata } from '@joystream/metadata-protobuf'
 export default class UpdateChannelCategoryCommand extends ContentDirectoryCommandBase {
   static description = 'Update channel category inside content directory.'
   static flags = {
@@ -36,11 +37,10 @@ export default class UpdateChannelCategoryCommand extends ContentDirectoryComman
     const actor = context ? await this.getActor(context) : await this.getCategoryManagementActor()
 
     const channelCategoryInput = await getInputJson<ChannelCategoryInputParameters>(input, ChannelCategoryInputSchema)
-
-    const meta = channelCategoryMetadataFromInput(channelCategoryInput)
+    const meta = asValidatedMetadata(ChannelCategoryMetadata, channelCategoryInput)
 
     const channelCategoryUpdateParameters: CreateInterface<ChannelCategoryUpdateParameters> = {
-      new_meta: metadataToBytes(meta),
+      new_meta: metadataToBytes(ChannelCategoryMetadata, meta),
     }
 
     this.jsonPrettyPrint(JSON.stringify(channelCategoryInput))

cli/src/commands/content/updateVideo.ts

@@ -1,11 +1,12 @@
 import { getInputJson } from '../../helpers/InputOutput'
 import { VideoInputParameters } from '../../Types'
-import { metadataToBytes, videoMetadataFromInput } from '../../helpers/serialization'
+import { asValidatedMetadata, metadataToBytes } from '../../helpers/serialization'
 import UploadCommandBase from '../../base/UploadCommandBase'
 import { flags } from '@oclif/command'
 import { CreateInterface } from '@joystream/types'
 import { VideoUpdateParameters } from '@joystream/types/content'
 import { VideoInputSchema } from '../../json-schemas/ContentDirectory'
+import { VideoMetadata } from '@joystream/metadata-protobuf'
 
 export default class UpdateVideoCommand extends UploadCommandBase {
   static description = 'Update video under specific id.'
@@ -39,26 +40,27 @@ export default class UpdateVideoCommand extends UploadCommandBase {
     await this.requestAccountDecoding(currentAccount)
 
     const videoInput = await getInputJson<VideoInputParameters>(input, VideoInputSchema)
+    const meta = asValidatedMetadata(VideoMetadata, videoInput)
 
-    const meta = videoMetadataFromInput(videoInput)
     const { videoPath, thumbnailPhotoPath } = videoInput
     const inputPaths = [videoPath, thumbnailPhotoPath].filter((p) => p !== undefined) as string[]
     const inputAssets = await this.prepareInputAssets(inputPaths, input)
     const assets = inputAssets.map(({ parameters }) => ({ Upload: parameters }))
     // Set assets indexes in the metadata
-    if (videoPath) {
-      meta.setVideo(0)
+    const [videoIndex, thumbnailPhotoIndex] = this.assetsIndexes([videoPath, thumbnailPhotoPath], inputPaths)
+    if (videoIndex !== undefined) {
+      meta.video = videoIndex
     }
-    if (thumbnailPhotoPath) {
-      meta.setThumbnailPhoto(videoPath ? 1 : 0)
+    if (thumbnailPhotoIndex !== undefined) {
+      meta.thumbnailPhoto = thumbnailPhotoIndex
     }
 
     const videoUpdateParameters: CreateInterface<VideoUpdateParameters> = {
       assets,
-      new_meta: metadataToBytes(meta),
+      new_meta: metadataToBytes(VideoMetadata, meta),
     }
 
-    this.jsonPrettyPrint(JSON.stringify({ assets, newMetadata: meta.toObject() }))
+    this.jsonPrettyPrint(JSON.stringify({ assets, newMetadata: meta }))
 
     await this.requireConfirmation('Do you confirm the provided input?', true)
 

cli/src/commands/content/updateVideoCategory.ts

@@ -1,11 +1,12 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
 import { getInputJson } from '../../helpers/InputOutput'
 import { VideoCategoryInputParameters } from '../../Types'
-import { metadataToBytes, videoCategoryMetadataFromInput } from '../../helpers/serialization'
+import { asValidatedMetadata, metadataToBytes } from '../../helpers/serialization'
 import { flags } from '@oclif/command'
 import { CreateInterface } from '@joystream/types'
 import { VideoCategoryUpdateParameters } from '@joystream/types/content'
 import { VideoCategoryInputSchema } from '../../json-schemas/ContentDirectory'
+import { VideoCategoryMetadata } from '@joystream/metadata-protobuf'
 
 export default class UpdateVideoCategoryCommand extends ContentDirectoryCommandBase {
   static description = 'Update video category inside content directory.'
@@ -37,11 +38,10 @@ export default class UpdateVideoCategoryCommand extends ContentDirectoryCommandB
     const actor = context ? await this.getActor(context) : await this.getCategoryManagementActor()
 
     const videoCategoryInput = await getInputJson<VideoCategoryInputParameters>(input, VideoCategoryInputSchema)
-
-    const meta = videoCategoryMetadataFromInput(videoCategoryInput)
+    const meta = asValidatedMetadata(VideoCategoryMetadata, videoCategoryInput)
 
     const videoCategoryUpdateParameters: CreateInterface<VideoCategoryUpdateParameters> = {
-      new_meta: metadataToBytes(meta),
+      new_meta: metadataToBytes(VideoCategoryMetadata, meta),
     }
 
     this.jsonPrettyPrint(JSON.stringify(videoCategoryInput))

cli/src/helpers/serialization.ts

@@ -1,98 +1,22 @@
-import {
-  VideoMetadata,
-  PublishedBeforeJoystream,
-  License,
-  MediaType,
-  ChannelMetadata,
-  ChannelCategoryMetadata,
-  VideoCategoryMetadata,
-} from '@joystream/content-metadata-protobuf'
-import {
-  ChannelCategoryInputParameters,
-  ChannelInputParameters,
-  VideoCategoryInputParameters,
-  VideoInputParameters,
-} from '../Types'
+import { AnyMetadataClass, DecodedMetadataObject } from '@joystream/metadata-protobuf/types'
 import { Bytes } from '@polkadot/types/primitive'
 import { createType } from '@joystream/types'
+import { CLIError } from '@oclif/errors'
+import ExitCodes from '../ExitCodes'
+import { metaToObject } from '@joystream/metadata-protobuf/utils'
 
-type AnyMetadata = {
-  serializeBinary(): Uint8Array
+export function metadataToBytes<T>(metaClass: AnyMetadataClass<T>, obj: T): Bytes {
+  return createType('Bytes', '0x' + Buffer.from(metaClass.encode(obj).finish()).toString('hex'))
 }
 
-export function metadataToBytes(metadata: AnyMetadata): Bytes {
-  const bytes = createType('Bytes', '0x' + Buffer.from(metadata.serializeBinary()).toString('hex'))
-  console.log('Metadata as Bytes:', bytes.toString())
-  return bytes
+export function metadataFromBytes<T>(metaClass: AnyMetadataClass<T>, bytes: Bytes): DecodedMetadataObject<T> {
+  return metaToObject(metaClass, metaClass.decode(bytes.toU8a(true)))
 }
 
-// TODO: If "fromObject()" was generated for the protobuffs we could avoid having to create separate converters for each metadata
-
-export function videoMetadataFromInput(videoParametersInput: VideoInputParameters): VideoMetadata {
-  const videoMetadata = new VideoMetadata()
-  videoMetadata.setTitle(videoParametersInput.title as string)
-  videoMetadata.setDescription(videoParametersInput.description as string)
-  videoMetadata.setDuration(videoParametersInput.duration as number)
-  videoMetadata.setMediaPixelHeight(videoParametersInput.mediaPixelHeight as number)
-  videoMetadata.setMediaPixelWidth(videoParametersInput.mediaPixelWidth as number)
-  videoMetadata.setLanguage(videoParametersInput.language as string)
-  videoMetadata.setHasMarketing(videoParametersInput.hasMarketing as boolean)
-  videoMetadata.setIsPublic(videoParametersInput.isPublic as boolean)
-  videoMetadata.setIsExplicit(videoParametersInput.isExplicit as boolean)
-  videoMetadata.setPersonsList(videoParametersInput.personsList as number[])
-  videoMetadata.setCategory(videoParametersInput.category as number)
-
-  if (videoParametersInput.mediaType) {
-    const mediaType = new MediaType()
-    mediaType.setCodecName(videoParametersInput.mediaType.codecName as string)
-    mediaType.setContainer(videoParametersInput.mediaType.container as string)
-    mediaType.setMimeMediaType(videoParametersInput.mediaType.mimeMediaType as string)
-    videoMetadata.setMediaType(mediaType)
-  }
-
-  if (videoParametersInput.publishedBeforeJoystream) {
-    const publishedBeforeJoystream = new PublishedBeforeJoystream()
-    publishedBeforeJoystream.setIsPublished(videoParametersInput.publishedBeforeJoystream.isPublished as boolean)
-    publishedBeforeJoystream.setDate(videoParametersInput.publishedBeforeJoystream.date as string)
-    videoMetadata.setPublishedBeforeJoystream(publishedBeforeJoystream)
+export function asValidatedMetadata<T>(metaClass: AnyMetadataClass<T>, anyObject: any): T {
+  const error = metaClass.verify(anyObject)
+  if (error) {
+    throw new CLIError(`Invalid metadata: ${error}`, { exit: ExitCodes.InvalidInput })
   }
-
-  if (videoParametersInput.license) {
-    const license = new License()
-    license.setCode(videoParametersInput.license.code as number)
-    license.setAttribution(videoParametersInput.license.attribution as string)
-    license.setCustomText(videoParametersInput.license.customText as string)
-    videoMetadata.setLicense(license)
-  }
-
-  return videoMetadata
-}
-
-export function channelMetadataFromInput(channelParametersInput: ChannelInputParameters): ChannelMetadata {
-  const channelMetadata = new ChannelMetadata()
-  channelMetadata.setTitle(channelParametersInput.title as string)
-  channelMetadata.setDescription(channelParametersInput.description as string)
-  channelMetadata.setIsPublic(channelParametersInput.isPublic as boolean)
-  channelMetadata.setLanguage(channelParametersInput.language as string)
-  channelMetadata.setCategory(channelParametersInput.category as number)
-
-  return channelMetadata
-}
-
-export function channelCategoryMetadataFromInput(
-  channelCategoryParametersInput: ChannelCategoryInputParameters
-): ChannelCategoryMetadata {
-  const channelCategoryMetadata = new ChannelCategoryMetadata()
-  channelCategoryMetadata.setName(channelCategoryParametersInput.name as string)
-
-  return channelCategoryMetadata
-}
-
-export function videoCategoryMetadataFromInput(
-  videoCategoryParametersInput: VideoCategoryInputParameters
-): VideoCategoryMetadata {
-  const videoCategoryMetadata = new VideoCategoryMetadata()
-  videoCategoryMetadata.setName(videoCategoryParametersInput.name as string)
-
-  return videoCategoryMetadata
+  return { ...anyObject } as T
 }

cli/src/json-schemas/ContentDirectory.ts

@@ -74,7 +74,7 @@ export const VideoInputSchema: JsonSchema<VideoInputParameters> = {
         },
       },
     },
-    personsList: { type: 'array' },
+    persons: { type: 'array' },
     publishedBeforeJoystream: {
       type: 'object',
       properties: {

utils/api-scripts/package.json

@@ -11,7 +11,7 @@
     "tsnode-strict": "node -r ts-node/register --unhandled-rejections=strict"
   },
   "dependencies": {
-    "@joystream/types": "^0.16.1",
+    "@joystream/types": "^0.17.0",
     "@polkadot/api": "4.2.1",
     "@polkadot/types": "4.2.1",
     "@polkadot/keyring": "^6.0.5",

utils/api-scripts/src/initialize-content-lead.ts

@@ -38,19 +38,19 @@ async function main() {
   }
 
   // Create a new lead opening
-  if ((await api.query.contentDirectoryWorkingGroup.currentLead()).isSome) {
+  if ((await api.query.contentWorkingGroup.currentLead()).isSome) {
     console.log('Curators lead already exists, aborting...')
   } else {
     console.log(`Making member id: ${memberId} the content lead.`)
-    const newOpeningId = (await api.query.contentDirectoryWorkingGroup.nextOpeningId()).toNumber()
-    const newApplicationId = (await api.query.contentDirectoryWorkingGroup.nextApplicationId()).toNumber()
+    const newOpeningId = (await api.query.contentWorkingGroup.nextOpeningId()).toNumber()
+    const newApplicationId = (await api.query.contentWorkingGroup.nextApplicationId()).toNumber()
     // Create curator lead opening
     console.log('Perparing Create Curator Lead Opening extrinsic...')
     await txHelper.sendAndCheck(
       SudoKeyPair,
       [
         sudo(
-          api.tx.contentDirectoryWorkingGroup.addOpening(
+          api.tx.contentWorkingGroup.addOpening(
             { CurrentBlock: null }, // activate_at
             { max_review_period_length: 9999 }, // OpeningPolicyCommitment
             'bootstrap curator opening', // human_readable_text
@@ -66,7 +66,7 @@ async function main() {
     await txHelper.sendAndCheck(
       LeadKeyPair,
       [
-        api.tx.contentDirectoryWorkingGroup.applyOnOpening(
+        api.tx.contentWorkingGroup.applyOnOpening(
           memberId, // member id
           newOpeningId, // opening id
           LeadKeyPair.address, // address
@@ -81,13 +81,13 @@ async function main() {
     const extrinsics: SubmittableExtrinsic<'promise'>[] = []
     // Begin review period
     console.log('Perparing Begin Applicant Review extrinsic...')
-    extrinsics.push(sudo(api.tx.contentDirectoryWorkingGroup.beginApplicantReview(newOpeningId)))
+    extrinsics.push(sudo(api.tx.contentWorkingGroup.beginApplicantReview(newOpeningId)))
 
     // Fill opening
     console.log('Perparing Fill Opening extrinsic...')
     extrinsics.push(
       sudo(
-        api.tx.contentDirectoryWorkingGroup.fillOpening(
+        api.tx.contentWorkingGroup.fillOpening(
           newOpeningId, // opening id
           api.createType('ApplicationIdSet', [newApplicationId]), // succesful applicants
           null // reward policy