Improve the tooling, get rid of class ids and other limitations

content-directory-schemas/README.md

@@ -27,9 +27,10 @@ This will handle:
 
 ### Input files naming
 
-Currently the tooling has some limitations when it comes to naming files inside the `inputs` directory. There is a specific pattern that has to be respected:
+In order to get the full benefit of the tooling, in some cases you may need to respect a specific pattern of file naming:
 
-Each input file name should start with related class id (counting from `1`, since it's assumed that the classes will be created inside an initially empty content directory) followed by and underscore and a class name (for example: `1_Language`), followed by one of the following strings: `Class`, `Schema` or `Batch` (based on the input type, ie. `1_LanguageBatch`)
+Each input file name should end with `Class`, `Schema` or `Batch` (depending on the input type, ie. `LanguageBatch`).
+It is also recommended that each of those file names starts with a class name (currently in `entityBatches` there's no distinction between schemas and classes, as it is assumed there will be a one-to-one relationship between them)
 
 ### `json-schemas` support for json inputs in `VSCode`
 
@@ -70,7 +71,7 @@ For example, in order to describe creating entities as simple as `Language`, whi
 ]
 ```
 
-_(This is the actual content of `inputs/entityBatches/1_LanguageBatch.json`)_
+_(This is the actual content of `inputs/entityBatches/LanguageBatch.json`)_
 
 #### Related entities
 
@@ -91,7 +92,7 @@ We can do it by either using `"new"` or `"existing"` keyword.
 }
 ```
 
-- The `"existing"` keyword allows referencing an entity created as part of some other (**previous!**) batch inside `inputs/entityBatches`. We can do it by specifying the value of **any unique property of the referenced entity**. So, for example to reference a `Language` entity from `VideoBatch.json` file, we use this syntax:
+- The `"existing"` keyword allows referencing an entity created as part of any other batch inside `inputs/entityBatches`. We can do it by specifying the value of **any unique property of the referenced entity**. So, for example to reference a `Language` entity from `VideoBatch.json` file, we use this syntax:
 
 ```
 {
@@ -135,7 +136,7 @@ This command will generate:
 The most obvious use-case of those interfaces currently is that when we're parsing any json files inside `inputs` using a Typescript code, we can assert that the resulting object will be of given type, ie.:
 
 ```
-const createClassInput = JSON.parse(fs.readFileSync('/path/to/inputs/1_LanguageClass.json')) as CreateClass
+const createClassInput = JSON.parse(fs.readFileSync('/path/to/inputs/LanguageClass.json')) as CreateClass
 ```
 
 Besides that, a Typescript code can be written to generate some inputs (ie. using a loop) that can then can be used to create classes/schemas or insert entities into the content directory.
@@ -147,7 +148,5 @@ There are a lot of other potential use-cases, but for the purpose of this docume
 Some limitations that should be dealt with in the nearest future:
 
 - Filename restrictions described in **_Input files naming_** section
-- The requirement of knowing the class id (ie. when defining the references in `inputs/schemas`)
 - Some code runs on the assumption that there is only one schema for each class, which is very limiting
-- Inside `input/entityBatches` we cannot reference entities that are part of a latter batch (batches are ordered by class id)
 - `Vector<Reference>` property type is not yet supported when parsing entity batches

content-directory-schemas/inputs/classes/8_KnownLicenseClass.json → content-directory-schemas/inputs/classes/KnownLicenseClass.json

@@ -1,5 +1,5 @@
 {
-  "name": "KnownLicenseType",
+  "name": "KnownLicense",
   "description": "A commonly recognized license (ie. CC_BY_SA)",
   "maximum_entities_count": 100,
   "default_entity_creation_voucher_upper_bound": 50

content-directory-schemas/inputs/classes/4_MediaLocationClass.json → content-directory-schemas/inputs/classes/MediaLocationClass.json

@@ -1,5 +1,5 @@
 {
-  "name": "MediaLocationClass",
+  "name": "MediaLocation",
   "description": "An object describing how the related media object can be accessed",
   "maximum_entities_count": 400,
   "default_entity_creation_voucher_upper_bound": 50

content-directory-schemas/inputs/entityBatches/12_VideoBatch.json

@@ -1,62 +0,0 @@
-[
-  {
-    "title": "Caminades 2",
-    "description": "Caminandes 2: Gran Dillama",
-    "language": { "existing": { "Code": "EN" } },
-    "category": { "existing": { "Name": "Cartoon" } },
-    "channel": { "existing": { "title": "Joystream Cartoons" } },
-    "duration": 146,
-    "hasMarketing": false,
-    "isCurated": false,
-    "isPublic": true,
-    "media": {
-      "new": {
-        "encoding": { "existing": { "Name": "MPEG4" } },
-        "location": {
-          "new": {
-            "httpMediaLocation": {
-              "new": {
-                "url": "http://www.caminandes.com/download/02_gran_dillama_1080p.zip"
-              }
-            }
-          }
-        },
-        "pixelWidth": 1920,
-        "pixelHeight": 1080
-      }
-    },
-    "thumbnailURL": "http://www.caminandes.com/wp-content/uploads/2016/02/web_header4.png",
-    "isExplicit": false,
-    "license": { "new": { "knownLicense": { "existing": { "code": "CC_BY" } } } }
-  },
-  {
-    "title": "Caminades 3",
-    "description": "Caminandes 3: Llamigos",
-    "language": { "existing": { "Code": "EN" } },
-    "category": { "existing": { "Name": "Cartoon" } },
-    "channel": { "existing": { "title": "Joystream Cartoons" } },
-    "duration": 150,
-    "hasMarketing": false,
-    "isCurated": false,
-    "isPublic": true,
-    "media": {
-      "new": {
-        "encoding": { "existing": { "Name": "MPEG4" } },
-        "location": {
-          "new": {
-            "httpMediaLocation": {
-              "new": {
-                "url": "http://www.caminandes.com/download/03_caminandes_llamigos_1080p.mp4"
-              }
-            }
-          }
-        },
-        "pixelWidth": 1920,
-        "pixelHeight": 1080
-      }
-    },
-    "thumbnailURL": "http://www.caminandes.com/wp-content/uploads/2016/02/web_header4.png",
-    "isExplicit": false,
-    "license": { "new": { "knownLicense": { "existing": { "code": "CC_BY" } } } }
-  }
-]

content-directory-schemas/inputs/entityBatches/1_LanguageBatch.json

@@ -1,5 +0,0 @@
-[
-  { "Code": "EN", "Name": "English" },
-  { "Code": "RU", "Name": "Russian" },
-  { "Code": "DE", "Name": "German" }
-]

content-directory-schemas/inputs/entityBatches/5_ContentCategoryBatch.json

@@ -1,4 +0,0 @@
-[
-  { "Name": "Cartoon", "Description": "Content which is a cartoon or related to cartoons" },
-  { "Name": "Sports", "Description": "Content related to sports" }
-]

content-directory-schemas/inputs/entityBatches/6_ChannelBatch.json

@@ -1,11 +0,0 @@
-[
-  {
-    "title": "Joystream Cartoons",
-    "description": "Joystream Cartoons channel",
-    "language": { "existing": { "Code": "EN" } },
-    "coverPhotoUrl": "https://user-images.githubusercontent.com/4144334/91547902-7e90db00-e91c-11ea-9f5c-45d4921928d5.png",
-    "avatarPhotoURL": "https://user-images.githubusercontent.com/4144334/91546674-ba2aa580-e91a-11ea-96e2-abc7654c0461.png",
-    "isPublic": true,
-    "isCurated": false
-  }
-]

content-directory-schemas/inputs/entityBatches/7_VideoMediaEncodingBatch.json

@@ -1 +0,0 @@
-[{ "Name": "MPEG4" }]

content-directory-schemas/inputs/entityBatches/8_KnownLicenseBatch.json

@@ -1,8 +0,0 @@
-[
-  { "code": "CC_BY" },
-  { "code": "CC_BY_SA" },
-  { "code": "CC_BY_ND" },
-  { "code": "CC_BY_NC" },
-  { "code": "CC_BY_NC_SA" },
-  { "code": "CC_BY_NC_ND" }
-]

content-directory-schemas/inputs/entityBatches/ChannelBatch.json

@@ -0,0 +1,14 @@
+{
+  "className": "Channel",
+  "entries": [
+    {
+      "title": "Joystream Cartoons",
+      "description": "Joystream Cartoons channel",
+      "language": { "existing": { "Code": "EN" } },
+      "coverPhotoUrl": "https://user-images.githubusercontent.com/4144334/91547902-7e90db00-e91c-11ea-9f5c-45d4921928d5.png",
+      "avatarPhotoURL": "https://user-images.githubusercontent.com/4144334/91546674-ba2aa580-e91a-11ea-96e2-abc7654c0461.png",
+      "isPublic": true,
+      "isCurated": false
+    }
+  ]
+}

content-directory-schemas/inputs/entityBatches/ContentCategoryBatch.json

@@ -0,0 +1,7 @@
+{
+  "className": "ContentCategory",
+  "entries": [
+    { "Name": "Cartoon", "Description": "Content which is a cartoon or related to cartoons" },
+    { "Name": "Sports", "Description": "Content related to sports" }
+  ]
+}

content-directory-schemas/inputs/entityBatches/KnownLicenseBatch.json

@@ -0,0 +1,11 @@
+{
+  "className": "KnownLicense",
+  "entries": [
+    { "code": "CC_BY" },
+    { "code": "CC_BY_SA" },
+    { "code": "CC_BY_ND" },
+    { "code": "CC_BY_NC" },
+    { "code": "CC_BY_NC_SA" },
+    { "code": "CC_BY_NC_ND" }
+  ]
+}

content-directory-schemas/inputs/entityBatches/LanguageBatch.json

@@ -0,0 +1,8 @@
+{
+  "className": "Language",
+  "entries": [
+    { "Code": "EN", "Name": "English" },
+    { "Code": "RU", "Name": "Russian" },
+    { "Code": "DE", "Name": "German" }
+  ]
+}

content-directory-schemas/inputs/entityBatches/VideoBatch.json

@@ -0,0 +1,65 @@
+{
+  "className": "Video",
+  "entries": [
+    {
+      "title": "Caminades 2",
+      "description": "Caminandes 2: Gran Dillama",
+      "language": { "existing": { "Code": "EN" } },
+      "category": { "existing": { "Name": "Cartoon" } },
+      "channel": { "existing": { "title": "Joystream Cartoons" } },
+      "duration": 146,
+      "hasMarketing": false,
+      "isCurated": false,
+      "isPublic": true,
+      "media": {
+        "new": {
+          "encoding": { "existing": { "Name": "MPEG4" } },
+          "location": {
+            "new": {
+              "httpMediaLocation": {
+                "new": {
+                  "url": "http://www.caminandes.com/download/02_gran_dillama_1080p.zip"
+                }
+              }
+            }
+          },
+          "pixelWidth": 1920,
+          "pixelHeight": 1080
+        }
+      },
+      "thumbnailURL": "http://www.caminandes.com/wp-content/uploads/2016/02/web_header4.png",
+      "isExplicit": false,
+      "license": { "new": { "knownLicense": { "existing": { "code": "CC_BY" } } } }
+    },
+    {
+      "title": "Caminades 3",
+      "description": "Caminandes 3: Llamigos",
+      "language": { "existing": { "Code": "EN" } },
+      "category": { "existing": { "Name": "Cartoon" } },
+      "channel": { "existing": { "title": "Joystream Cartoons" } },
+      "duration": 150,
+      "hasMarketing": false,
+      "isCurated": false,
+      "isPublic": true,
+      "media": {
+        "new": {
+          "encoding": { "existing": { "Name": "MPEG4" } },
+          "location": {
+            "new": {
+              "httpMediaLocation": {
+                "new": {
+                  "url": "http://www.caminandes.com/download/03_caminandes_llamigos_1080p.mp4"
+                }
+              }
+            }
+          },
+          "pixelWidth": 1920,
+          "pixelHeight": 1080
+        }
+      },
+      "thumbnailURL": "http://www.caminandes.com/wp-content/uploads/2016/02/web_header4.png",
+      "isExplicit": false,
+      "license": { "new": { "knownLicense": { "existing": { "code": "CC_BY" } } } }
+    }
+  ]
+}

content-directory-schemas/inputs/entityBatches/VideoMediaEncodingBatch.json

@@ -0,0 +1,4 @@
+{
+  "className": "VideoMediaEncoding",
+  "entries": [{ "Name": "MPEG4" }]
+}

content-directory-schemas/inputs/schemas/6_ChannelSchema.json → content-directory-schemas/inputs/schemas/ChannelSchema.json

@@ -1,5 +1,5 @@
 {
-  "classId": 6,
+  "className": "Channel",
   "newProperties": [
     {
       "name": "title",
@@ -43,7 +43,7 @@
       "name": "language",
       "description": "The primary langauge of the channel's content",
       "required": false,
-      "property_type": { "Single": { "Reference": [1, false] } }
+      "property_type": { "Single": { "Reference": { "className": "Language" } } }
     }
   ]
 }

content-directory-schemas/inputs/schemas/5_ContentCategorySchema.json → content-directory-schemas/inputs/schemas/ContentCategorySchema.json

@@ -1,5 +1,5 @@
 {
-  "classId": 5,
+  "className": "ContentCategory",
   "newProperties": [
     {
       "name": "Name",

content-directory-schemas/inputs/schemas/2_HttpMediaLocationSchema.json → content-directory-schemas/inputs/schemas/HttpMediaLocationSchema.json

@@ -1,5 +1,5 @@
 {
-  "classId": 2,
+  "className": "HttpMediaLocation",
   "newProperties": [
     {
       "name": "url",

content-directory-schemas/inputs/schemas/3_JoystreamMediaLocationSchema.json → content-directory-schemas/inputs/schemas/JoystreamMediaLocationSchema.json

@@ -1,5 +1,5 @@
 {
-  "classId": 3,
+  "className": "JoystreamMediaLocation",
   "newProperties": [
     {
       "name": "dataObjectId",

content-directory-schemas/inputs/schemas/8_KnownLicenseSchema.json → content-directory-schemas/inputs/schemas/KnownLicenseSchema.json

@@ -1,5 +1,5 @@
 {
-  "classId": 8,
+  "className": "KnownLicense",
   "newProperties": [
     {
       "name": "code",

content-directory-schemas/inputs/schemas/1_LanguageSchema.json → content-directory-schemas/inputs/schemas/LanguageSchema.json

@@ -1,5 +1,5 @@
 {
-  "classId": 1,
+  "className": "Language",
   "newProperties": [
     {
       "name": "Name",

content-directory-schemas/inputs/schemas/10_LicenseSchema.json → content-directory-schemas/inputs/schemas/LicenseSchema.json

@@ -1,17 +1,17 @@
 {
-  "classId": 10,
+  "className": "License",
   "newProperties": [
     {
       "name": "knownLicense",
       "description": "Reference to a known license",
       "required": false,
-      "property_type": { "Single": { "Reference": [8, false] } }
+      "property_type": { "Single": { "Reference": { "className": "KnownLicense" } } }
     },
     {
       "name": "userDefinedLicense",
       "description": "Reference to user-defined license",
       "required": false,
-      "property_type": { "Single": { "Reference": [9, true] } }
+      "property_type": { "Single": { "Reference": { "className": "UserDefinedLicense", "sameOwner": true } } }
     }
   ]
 }

content-directory-schemas/inputs/schemas/4_MediaLocationSchema.json → content-directory-schemas/inputs/schemas/MediaLocationSchema.json

@@ -1,17 +1,17 @@
 {
-  "classId": 4,
+  "className": "MediaLocation",
   "newProperties": [
     {
       "name": "httpMediaLocation",
       "description": "A reference to HttpMediaLocation",
       "required": false,
-      "property_type": { "Single": { "Reference": [2, true] } }
+      "property_type": { "Single": { "Reference": { "className": "HttpMediaLocation", "sameOwner": true } } }
     },
     {
       "name": "joystreamMediaLocation",
       "description": "A reference to JoystreamMediaLocation",
       "required": false,
-      "property_type": { "Single": { "Reference": [3, true] } }
+      "property_type": { "Single": { "Reference": { "className": "JoystreamMediaLocation", "sameOwner": true } } }
     }
   ]
 }

content-directory-schemas/inputs/schemas/9_UserDefinedLicenseSchema.json → content-directory-schemas/inputs/schemas/UserDefinedLicenseSchema.json

@@ -1,5 +1,5 @@
 {
-  "classId": 9,
+  "className": "UserDefinedLicense",
   "newProperties": [
     {
       "name": "content",

content-directory-schemas/inputs/schemas/7_VideoMediaEncodingSchema.json → content-directory-schemas/inputs/schemas/VideoMediaEncodingSchema.json

@@ -1,5 +1,5 @@
 {
-  "classId": 7,
+  "className": "VideoMediaEncoding",
   "newProperties": [
     {
       "name": "Name",

content-directory-schemas/inputs/schemas/11_VideoMediaSchema.json → content-directory-schemas/inputs/schemas/VideoMediaSchema.json

@@ -1,11 +1,11 @@
 {
-  "classId": 11,
+  "className": "VideoMedia",
   "newProperties": [
     {
       "name": "encoding",
       "description": "Encoding of the video media object",
       "required": true,
-      "property_type": { "Single": { "Reference": [7, false] } }
+      "property_type": { "Single": { "Reference": { "className": "VideoMediaEncoding" } } }
     },
     {
       "name": "pixelWidth",
@@ -29,7 +29,7 @@
       "name": "location",
       "description": "Location of the video media object",
       "required": true,
-      "property_type": { "Single": { "Reference": [4, true] } }
+      "property_type": { "Single": { "Reference": { "className": "MediaLocation", "sameOwner": true } } }
     }
   ]
 }

content-directory-schemas/inputs/schemas/12_VideoSchema.json → content-directory-schemas/inputs/schemas/VideoSchema.json

@@ -1,17 +1,17 @@
 {
-  "classId": 12,
+  "className": "Video",
   "newProperties": [
     {
       "name": "channel",
       "description": "Reference to member's channel",
       "required": true,
-      "property_type": { "Single": { "Reference": [6, true] } }
+      "property_type": { "Single": { "Reference": { "className": "Channel", "sameOwner": true } } }
     },
     {
       "name": "category",
       "description": "Reference to a video category",
       "required": true,
-      "property_type": { "Single": { "Reference": [5, false] } }
+      "property_type": { "Single": { "Reference": { "className": "ContentCategory" } } }
     },
     {
       "name": "title",
@@ -47,13 +47,13 @@
       "name": "language",
       "description": "Video's main langauge",
       "required": false,
-      "property_type": { "Single": { "Reference": [1, false] } }
+      "property_type": { "Single": { "Reference": { "className": "Language" } } }
     },
     {
       "name": "media",
       "description": "Reference to VideoMedia",
       "required": true,
-      "property_type": { "Single": { "Reference": [11, true] } }
+      "property_type": { "Single": { "Reference": { "className": "VideoMedia", "sameOwner": true } } }
     },
     {
       "name": "hasMarketing",
@@ -91,7 +91,7 @@
       "name": "license",
       "description": "A License the Video is published under",
       "required": true,
-      "property_type": { "Single": { "Reference": [10, true] } }
+      "property_type": { "Single": { "Reference": { "className": "License", "sameOwner": true } } }
     }
   ]
 }

content-directory-schemas/package.json

@@ -20,7 +20,9 @@
     "ajv": "6.12.5",
     "@joystream/prettier-config": "*",
     "@polkadot/api": "1.26.1",
-    "@joystream/types": "^0.14.0"
+    "@polkadot/keyring": "^3.0.1",
+    "@joystream/types": "^0.14.0",
+    "@apidevtools/json-schema-ref-parser": "^9.0.6"
   },
   "devDependencies": {
     "ts-node": "^8.8.2",

content-directory-schemas/schemas/extrinsics/AddClassSchema.schema.json

@@ -5,18 +5,16 @@
   "description": "JSON schema to describe a new schema for a certain class in Joystream network",
   "type": "object",
   "additionalProperties": false,
-  "required": ["classId", "newProperties"],
+  "required": ["className", "newProperties"],
   "properties": {
-    "classId": { "$ref": "#/definitions/ClassId" },
+    "className": { "type": "string" },
     "existingProperties": {
       "type": "array",
-      "minItems": 1,
       "uniqueItems": true,
       "items": { "$ref": "#/definitions/PropertyInSchemIndex" }
     },
     "newProperties": {
       "type": "array",
-      "minItems": 1,
       "uniqueItems": true,
       "items": { "$ref": "#/definitions/Property" }
     }
@@ -61,7 +59,7 @@
     },
     "SinglePropertyType": {
       "oneOf": [
-        { "$ref": "#/definitions/PrimitiveProperty" },
+        { "$ref": "#/definitions/PrimitiveProperty", "description": "Primitive property (bool/integer)" },
         { "$ref": "#/definitions/TextProperty" },
         { "$ref": "#/definitions/HashProperty" },
         { "$ref": "#/definitions/ReferenceProperty" }
@@ -128,13 +126,19 @@
       "required": ["Reference"],
       "properties": {
         "Reference": {
-          "type": "array",
-          "minItems": 2,
-          "additionalItems": false,
-          "items": [
-            { "$ref": "#/definitions/ClassId", "description": "Referenced class ID" },
-            { "$ref": "#/definitions/DefaultBoolean", "description": "Same owner required (flag)" }
-          ]
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["className"],
+          "properties": {
+            "className": {
+              "type": "string",
+              "description": "Referenced class name"
+            },
+            "sameOwner": {
+              "$ref": "#/definitions/DefaultBoolean",
+              "description": "Whether same owner (controller) is required"
+            }
+          }
         }
       }
     },

content-directory-schemas/scripts/devInitAliceLead.ts

@@ -1,6 +1,7 @@
 import { types } from '@joystream/types'
-import { ApiPromise, Keyring, WsProvider } from '@polkadot/api'
+import { ApiPromise, WsProvider } from '@polkadot/api'
 import { SubmittableExtrinsic } from '@polkadot/api/types'
+import { ExtrinsicsHelper, getAlicePair } from './helpers/extrinsics'
 
 async function main() {
   // Init api
@@ -8,14 +9,13 @@ async function main() {
   console.log(`Initializing the api (${WS_URI})...`)
   const provider = new WsProvider(WS_URI)
   const api = await ApiPromise.create({ provider, types })
-  // Init ALICE keypair
-  const keyring = new Keyring({ type: 'sr25519' })
-  keyring.addFromUri('//Alice', { name: 'Alice' })
-  const ALICE = keyring.getPairs()[0]
 
-  let nonce = (await api.query.system.account(ALICE.address)).nonce.toNumber()
-  const stdCall = (tx: SubmittableExtrinsic<'promise'>) => tx.signAndSend(ALICE, { nonce: nonce++ })
-  const sudoCall = (tx: SubmittableExtrinsic<'promise'>) => api.tx.sudo.sudo(tx).signAndSend(ALICE, { nonce: nonce++ })
+  const ALICE = getAlicePair()
+
+  const txHelper = new ExtrinsicsHelper(api)
+
+  const sudo = (tx: SubmittableExtrinsic<'promise'>) => api.tx.sudo.sudo(tx)
+  const extrinsics: SubmittableExtrinsic<'promise'>[] = []
 
   // Create membership if not already created
   let aliceMemberId: number | undefined = (await api.query.members.memberIdsByControllerAccountId(ALICE.address))
@@ -23,9 +23,9 @@ async function main() {
     ?.toNumber()
 
   if (aliceMemberId === undefined) {
-    console.log('Sending Alice member account creation extrinsic...')
+    console.log('Perparing Alice member account creation extrinsic...')
     aliceMemberId = (await api.query.members.nextMemberId()).toNumber()
-    await stdCall(api.tx.members.buyMembership(0, 'alice', null, null))
+    extrinsics.push(api.tx.members.buyMembership(0, 'alice', null, null))
   } else {
     console.log(`Alice member id found: ${aliceMemberId}...`)
   }
@@ -35,19 +35,21 @@ async function main() {
     const newOpeningId = (await api.query.contentDirectoryWorkingGroup.nextOpeningId()).toNumber()
     const newApplicationId = (await api.query.contentDirectoryWorkingGroup.nextApplicationId()).toNumber()
     // Create curator lead opening
-    console.log('Sending Create Curator Lead Opening extrinsic...')
-    await sudoCall(
-      api.tx.contentDirectoryWorkingGroup.addOpening(
-        { CurrentBlock: null }, // activate_at
-        { max_review_period_length: 9999 }, // OpeningPolicyCommitment
-        'api-examples curator opening', // human_readable_text
-        'Leader' // opening_type
+    console.log('Perparing Create Curator Lead Opening extrinsic...')
+    extrinsics.push(
+      sudo(
+        api.tx.contentDirectoryWorkingGroup.addOpening(
+          { CurrentBlock: null }, // activate_at
+          { max_review_period_length: 9999 }, // OpeningPolicyCommitment
+          'api-examples curator opening', // human_readable_text
+          'Leader' // opening_type
+        )
       )
     )
 
     // Apply to lead opening
-    console.log('Sending Apply to Curator Lead Opening as Alice extrinsic...')
-    await stdCall(
+    console.log('Perparing Apply to Curator Lead Opening as Alice extrinsic...')
+    extrinsics.push(
       api.tx.contentDirectoryWorkingGroup.applyOnOpening(
         aliceMemberId, // member id
         newOpeningId, // opening id
@@ -59,18 +61,23 @@ async function main() {
     )
 
     // Begin review period
-    console.log('Sending Begin Applicant Review extrinsic...')
-    await sudoCall(api.tx.contentDirectoryWorkingGroup.beginApplicantReview(newOpeningId))
+    console.log('Perparing Begin Applicant Review extrinsic...')
+    extrinsics.push(sudo(api.tx.contentDirectoryWorkingGroup.beginApplicantReview(newOpeningId)))
 
     // Fill opening
-    console.log('Sending Fill Opening extrinsic...')
-    await sudoCall(
-      api.tx.contentDirectoryWorkingGroup.fillOpening(
-        newOpeningId, // opening id
-        api.createType('ApplicationIdSet', [newApplicationId]), // succesful applicants
-        null // reward policy
+    console.log('Perparing Fill Opening extrinsic...')
+    extrinsics.push(
+      sudo(
+        api.tx.contentDirectoryWorkingGroup.fillOpening(
+          newOpeningId, // opening id
+          api.createType('ApplicationIdSet', [newApplicationId]), // succesful applicants
+          null // reward policy
+        )
       )
     )
+
+    console.log('Sending extrinsics...')
+    await txHelper.sendAndCheck(ALICE, extrinsics, 'Failed to initialize Alice as Content Curators Lead!')
   } else {
     console.log('Curators lead already exists, skipping...')
   }

content-directory-schemas/scripts/helpers/EntityBatchParser.ts

@@ -1,141 +0,0 @@
-import { AddClassSchema } from '../../types/extrinsics/AddClassSchema'
-import { FetchedInput } from './inputs'
-import { createType } from '@joystream/types'
-import { blake2AsHex } from '@polkadot/util-crypto'
-import { OperationType, ParametrizedPropertyValue } from '@joystream/types/content-directory'
-
-// TODO: Make it more typesafe
-type Batch = Record<string, any>[]
-type FetchedBatch = FetchedInput<Batch>
-
-export class EntityBatchesParser {
-  schemaInputs: FetchedInput<AddClassSchema>[]
-  batchInputs: FetchedBatch[]
-  operations: OperationType[] = []
-  entityIndexByUniqueQueryMap = new Map<string, number>()
-
-  constructor(schemaInputs: FetchedInput<AddClassSchema>[], batchInputs: FetchedBatch[]) {
-    this.schemaInputs = schemaInputs
-    this.batchInputs = batchInputs
-  }
-
-  private schemaByEntityBatchFilename(entityBatchFilename: string) {
-    const foundSchema = this.schemaInputs.find(
-      ({ fileName: schemaFilename }) => schemaFilename.replace('Schema.json', 'Batch.json') === entityBatchFilename
-    )
-    if (!foundSchema) {
-      throw new Error(`Related schema not found for entity batch: ${entityBatchFilename}`)
-    }
-
-    return foundSchema.data
-  }
-
-  private schemaByClassId(classId: number) {
-    const foundSchema = this.schemaInputs.find(({ fileName }) => parseInt(fileName.split('_')[0]) === classId)
-    if (!foundSchema) {
-      throw new Error(`Schema not found by class id: ${classId}`)
-    }
-
-    return foundSchema.data
-  }
-
-  private getRefEntitySchema(parentEntitySchema: AddClassSchema, propName: string) {
-    const refProp = parentEntitySchema.newProperties.find((p) => p.name === propName)
-    if (!refProp) {
-      throw new Error(`findRefEntitySchema: Property ${propName} not found in class ${parentEntitySchema.classId}`)
-    }
-    const refClassId = parseInt(
-      createType('PropertyType', refProp.property_type).asType('Single').asType('Reference')[0].toString()
-    )
-    return this.schemaByClassId(refClassId)
-  }
-
-  private getUniqueQueryHash(uniquePropVal: Record<string, any>, classId: number) {
-    return blake2AsHex(JSON.stringify([classId, uniquePropVal]))
-  }
-
-  private findEntityIndexByUniqueQuery(uniquePropVal: Record<string, any>, classId: number) {
-    const hash = this.getUniqueQueryHash(uniquePropVal, classId)
-    const foundIndex = this.entityIndexByUniqueQueryMap.get(hash)
-    if (foundIndex === undefined) {
-      throw new Error(
-        `findEntityIndexByUniqueQuery failed for class ${classId} and query: ${JSON.stringify(uniquePropVal)}`
-      )
-    }
-
-    return foundIndex
-  }
-
-  private parseEntityInput(entityInput: Record<string, any>, schema: AddClassSchema) {
-    const parametrizedPropertyValues = Object.entries(entityInput).map(([propertyName, propertyValue]) => {
-      const schemaPropertyIndex = schema.newProperties.findIndex((p) => p.name === propertyName)
-      const schemaPropertyType = createType('PropertyType', schema.newProperties[schemaPropertyIndex].property_type)
-
-      let value: ParametrizedPropertyValue
-
-      // Handle references
-      if (schemaPropertyType.isOfType('Single') && schemaPropertyType.asType('Single').isOfType('Reference')) {
-        const refEntitySchema = this.getRefEntitySchema(schema, propertyName)
-        let entityIndex: number
-        if (Object.keys(propertyValue).includes('new')) {
-          entityIndex = this.parseEntityInput(propertyValue.new, refEntitySchema)
-        } else if (Object.keys(propertyValue).includes('existing')) {
-          entityIndex = this.findEntityIndexByUniqueQuery(propertyValue.existing, refEntitySchema.classId)
-        } else {
-          throw new Error(`Invalid reference property value: ${JSON.stringify(propertyValue)}`)
-        }
-        value = createType('ParametrizedPropertyValue', { InternalEntityJustAdded: entityIndex })
-      } else {
-        value = createType('ParametrizedPropertyValue', {
-          InputPropertyValue: schemaPropertyType.toInputPropertyValue(propertyValue).toJSON(),
-        })
-      }
-
-      return {
-        in_class_index: schemaPropertyIndex,
-        value: value.toJSON(),
-      }
-    })
-
-    // Add operations
-    const createEntityOperationIndex = this.operations.length
-    this.operations.push(createType('OperationType', { CreateEntity: { class_id: schema.classId } }))
-    this.operations.push(
-      createType('OperationType', {
-        AddSchemaSupportToEntity: {
-          schema_id: 0,
-          entity_id: { InternalEntityJustAdded: createEntityOperationIndex },
-          parametrized_property_values: parametrizedPropertyValues,
-        },
-      })
-    )
-
-    // Add entries to entityIndexByUniqueQueryMap
-    schema.newProperties
-      .filter((p) => p.unique)
-      .forEach(({ name }) => {
-        const hash = this.getUniqueQueryHash({ [name]: entityInput[name] }, schema.classId)
-        this.entityIndexByUniqueQueryMap.set(hash, createEntityOperationIndex)
-      })
-
-    // Return CreateEntity operation index
-    return createEntityOperationIndex
-  }
-
-  private reset() {
-    this.entityIndexByUniqueQueryMap = new Map<string, number>()
-    this.operations = []
-  }
-
-  public getOperations() {
-    this.batchInputs.forEach(({ fileName: batchFilename, data: batch }) => {
-      const entitySchema = this.schemaByEntityBatchFilename(batchFilename)
-      batch.forEach((entityInput) => this.parseEntityInput(entityInput, entitySchema))
-    })
-
-    const operations = this.operations
-    this.reset()
-
-    return operations
-  }
-}

content-directory-schemas/scripts/helpers/InputParser.ts

@@ -0,0 +1,225 @@
+import { AddClassSchema, Property } from '../../types/extrinsics/AddClassSchema'
+import { FetchedInput } from './inputs'
+import { createType } from '@joystream/types'
+import { blake2AsHex } from '@polkadot/util-crypto'
+import {
+  ClassId,
+  OperationType,
+  ParametrizedPropertyValue,
+  PropertyId,
+  PropertyType,
+} from '@joystream/types/content-directory'
+import { isSingle, isReference } from './propertyType'
+import { ApiPromise } from '@polkadot/api'
+import { JoyBTreeSet } from '@joystream/types/common'
+import { CreateClass } from 'types/extrinsics/CreateClass'
+import { EntityBatch } from 'types/EntityBatch'
+
+export class InputParser {
+  private api: ApiPromise
+  private classInputs: FetchedInput<CreateClass>[]
+  private schemaInputs: FetchedInput<AddClassSchema>[]
+  private batchInputs: FetchedInput<EntityBatch>[]
+  private createEntityOperations: OperationType[] = []
+  private addSchemaToEntityOprations: OperationType[] = []
+  private entityIndexByUniqueQueryMap = new Map<string, number>()
+  private entityByUniqueQueryCurrentIndex = 0
+  private classIdByNameMap = new Map<string, number>()
+  private classMapInitialized = false
+
+  constructor(
+    api: ApiPromise,
+    classInputs?: FetchedInput<CreateClass>[],
+    schemaInputs?: FetchedInput<AddClassSchema>[],
+    batchInputs?: FetchedInput<EntityBatch>[]
+  ) {
+    this.api = api
+    this.classInputs = classInputs || []
+    this.schemaInputs = schemaInputs || []
+    this.batchInputs = batchInputs || []
+  }
+
+  private async initializeClassMap() {
+    if (this.classMapInitialized) {
+      return
+    }
+    const classEntries = await this.api.query.contentDirectory.classById.entries()
+    classEntries.forEach(([key, aClass]) => {
+      this.classIdByNameMap.set(aClass.name.toString(), (key.args[0] as ClassId).toNumber())
+    })
+    this.classMapInitialized = true
+  }
+
+  private schemaByClassName(className: string) {
+    const foundSchema = this.schemaInputs.find(({ data }) => data.className === className)
+    if (!foundSchema) {
+      throw new Error(`Schema not found by class name: ${className}`)
+    }
+
+    return foundSchema.data
+  }
+
+  private getUniqueQueryHash(uniquePropVal: Record<string, any>, className: string) {
+    return blake2AsHex(JSON.stringify([className, uniquePropVal]))
+  }
+
+  private findEntityIndexByUniqueQuery(uniquePropVal: Record<string, any>, className: string) {
+    const hash = this.getUniqueQueryHash(uniquePropVal, className)
+    const foundIndex = this.entityIndexByUniqueQueryMap.get(hash)
+    if (foundIndex === undefined) {
+      throw new Error(
+        `findEntityIndexByUniqueQuery failed for class ${className} and query: ${JSON.stringify(uniquePropVal)}`
+      )
+    }
+
+    return foundIndex
+  }
+
+  private getClassIdByName(className: string): number {
+    const classId = this.classIdByNameMap.get(className)
+    if (classId === undefined) {
+      throw new Error(`Could not find class id by name: "${className}"!`)
+    }
+    return classId
+  }
+
+  private parsePropertyType(propertyType: Property['property_type']): PropertyType {
+    if (isSingle(propertyType) && isReference(propertyType.Single)) {
+      const { className, sameOwner } = propertyType.Single.Reference
+      const classId = this.getClassIdByName(className)
+      return createType('PropertyType', { Single: { Reference: [classId, sameOwner] } })
+    }
+    // Types other than reference are fully compatible
+    return createType('PropertyType', propertyType)
+  }
+
+  private includeEntityInputInUniqueQueryMap(entityInput: Record<string, any>, schema: AddClassSchema) {
+    Object.entries(entityInput).forEach(([propertyName, propertyValue]) => {
+      const schemaPropertyType = schema.newProperties.find((p) => p.name === propertyName)!.property_type
+      // Handle entities "nested" via "new"
+      if (isSingle(schemaPropertyType) && isReference(schemaPropertyType.Single)) {
+        const refEntitySchema = this.schemaByClassName(schemaPropertyType.Single.Reference.className)
+        if (Object.keys(propertyValue).includes('new')) {
+          this.includeEntityInputInUniqueQueryMap(propertyValue.new, refEntitySchema)
+        }
+      }
+    })
+    // Add entries to entityIndexByUniqueQueryMap
+    schema.newProperties
+      .filter((p) => p.unique)
+      .forEach(({ name }) => {
+        if (entityInput[name] === undefined) {
+          // Skip empty values (not all unique properties are required)
+          return
+        }
+        const hash = this.getUniqueQueryHash({ [name]: entityInput[name] }, schema.className)
+        this.entityIndexByUniqueQueryMap.set(hash, this.entityByUniqueQueryCurrentIndex)
+      })
+    ++this.entityByUniqueQueryCurrentIndex
+  }
+
+  private parseEntityInput(entityInput: Record<string, any>, schema: AddClassSchema) {
+    const parametrizedPropertyValues = Object.entries(entityInput).map(([propertyName, propertyValue]) => {
+      const schemaPropertyIndex = schema.newProperties.findIndex((p) => p.name === propertyName)
+      const schemaPropertyType = schema.newProperties[schemaPropertyIndex].property_type
+
+      let value: ParametrizedPropertyValue
+
+      // Handle references
+      if (isSingle(schemaPropertyType) && isReference(schemaPropertyType.Single)) {
+        const refEntitySchema = this.schemaByClassName(schemaPropertyType.Single.Reference.className)
+        let entityIndex: number
+        if (Object.keys(propertyValue).includes('new')) {
+          entityIndex = this.parseEntityInput(propertyValue.new, refEntitySchema)
+        } else if (Object.keys(propertyValue).includes('existing')) {
+          entityIndex = this.findEntityIndexByUniqueQuery(propertyValue.existing, refEntitySchema.className)
+        } else {
+          throw new Error(`Invalid reference property value: ${JSON.stringify(propertyValue)}`)
+        }
+        value = createType('ParametrizedPropertyValue', { InternalEntityJustAdded: entityIndex })
+      } else {
+        value = createType('ParametrizedPropertyValue', {
+          InputPropertyValue: this.parsePropertyType(schemaPropertyType).toInputPropertyValue(propertyValue).toJSON(),
+        })
+      }
+
+      return {
+        in_class_index: schemaPropertyIndex,
+        value: value.toJSON(),
+      }
+    })
+
+    // Add operations
+    const createEntityOperationIndex = this.createEntityOperations.length
+    const classId = this.classIdByNameMap.get(schema.className)
+    this.createEntityOperations.push(createType('OperationType', { CreateEntity: { class_id: classId } }))
+    this.addSchemaToEntityOprations.push(
+      createType('OperationType', {
+        AddSchemaSupportToEntity: {
+          schema_id: 0,
+          entity_id: { InternalEntityJustAdded: createEntityOperationIndex },
+          parametrized_property_values: parametrizedPropertyValues,
+        },
+      })
+    )
+
+    // Return CreateEntity operation index
+    return createEntityOperationIndex
+  }
+
+  private reset() {
+    this.entityIndexByUniqueQueryMap = new Map<string, number>()
+    this.classIdByNameMap = new Map<string, number>()
+    this.createEntityOperations = []
+    this.addSchemaToEntityOprations = []
+    this.entityByUniqueQueryCurrentIndex = 0
+  }
+
+  public async getEntityBatchOperations() {
+    await this.initializeClassMap()
+    // First - create entityUniqueQueryMap to allow referencing any entity at any point
+    this.batchInputs.forEach(({ data: batch }) => {
+      const entitySchema = this.schemaByClassName(batch.className)
+      batch.entries.forEach((entityInput) => this.includeEntityInputInUniqueQueryMap(entityInput, entitySchema))
+    })
+    // Then - parse into actual operations
+    this.batchInputs.forEach(({ data: batch }) => {
+      const entitySchema = this.schemaByClassName(batch.className)
+      batch.entries.forEach((entityInput) => this.parseEntityInput(entityInput, entitySchema))
+    })
+
+    const operations = [...this.createEntityOperations, ...this.addSchemaToEntityOprations]
+    this.reset()
+
+    return operations
+  }
+
+  public async getAddSchemaExtrinsics() {
+    await this.initializeClassMap()
+    return this.schemaInputs.map(({ data: schema }) => {
+      const classId = this.getClassIdByName(schema.className)
+      const newProperties = schema.newProperties.map((p) => ({
+        ...p,
+        // Parse different format for Reference (and potentially other propTypes in the future)
+        property_type: this.parsePropertyType(p.property_type).toJSON(),
+      }))
+      return this.api.tx.contentDirectory.addClassSchema(
+        classId,
+        new (JoyBTreeSet(PropertyId))(this.api.registry, schema.existingProperties),
+        newProperties
+      )
+    })
+  }
+
+  public getCreateClassExntrinsics() {
+    return this.classInputs.map(({ data: aClass }) =>
+      this.api.tx.contentDirectory.createClass(
+        aClass.name,
+        aClass.description,
+        aClass.class_permissions || {},
+        aClass.maximum_entities_count,
+        aClass.default_entity_creation_voucher_upper_bound
+      )
+    )
+  }
+}

content-directory-schemas/scripts/helpers/entitySchemas.ts

@@ -1,15 +0,0 @@
-import fs from 'fs'
-import { getInputsLocation } from './inputs'
-
-const schemaInputFilenames = fs.readdirSync(getInputsLocation('schemas'))
-
-type EntitySchemaType = 'Ref' | 'Entity' | 'Batch'
-
-export const schemaFilenameToEntitySchemaName = (inputFilename: string, schemaType?: EntitySchemaType) =>
-  inputFilename.split('_')[1].replace('Schema.json', schemaType || '')
-
-export const classIdToEntitySchemaName = (classId: number, schemaType: EntitySchemaType) =>
-  schemaFilenameToEntitySchemaName(
-    schemaInputFilenames.find((fileName) => fileName.startsWith(`${classId}_`))!,
-    schemaType
-  )

content-directory-schemas/scripts/helpers/extrinsics.ts

@@ -0,0 +1,55 @@
+import { Keyring } from '@polkadot/keyring'
+import { KeyringPair } from '@polkadot/keyring/types'
+import { ApiPromise } from '@polkadot/api'
+import { SubmittableExtrinsic } from '@polkadot/api/types'
+
+export function getAlicePair() {
+  const keyring = new Keyring({ type: 'sr25519' })
+  keyring.addFromUri('//Alice', { name: 'Alice' })
+  const ALICE = keyring.getPairs()[0]
+
+  return ALICE
+}
+
+export class ExtrinsicsHelper {
+  api: ApiPromise
+  noncesByAddress: Map<string, number>
+
+  constructor(api: ApiPromise, initialNonces?: [string, number][]) {
+    this.api = api
+    this.noncesByAddress = new Map<string, number>(initialNonces)
+  }
+
+  private async nextNonce(address: string): Promise<number> {
+    const nonce = this.noncesByAddress.get(address) || (await this.api.query.system.account(address)).nonce.toNumber()
+    this.noncesByAddress.set(address, nonce + 1)
+
+    return nonce
+  }
+
+  async sendAndCheck(sender: KeyringPair, extrinsics: SubmittableExtrinsic<'promise'>[], errorMessage: string) {
+    const promises: Promise<void>[] = []
+    for (const tx of extrinsics) {
+      const nonce = await this.nextNonce(sender.address)
+      promises.push(
+        new Promise((resolve, reject) => {
+          tx.signAndSend(sender, { nonce }, (result) => {
+            if (result.isError) {
+              reject(new Error(errorMessage))
+            }
+            if (result.status.isInBlock) {
+              if (
+                result.events.some(({ event }) => event.section === 'system' && event.method === 'ExtrinsicSuccess')
+              ) {
+                resolve()
+              } else {
+                reject(new Error(errorMessage))
+              }
+            }
+          })
+        })
+      )
+    }
+    await Promise.all(promises)
+  }
+}

content-directory-schemas/scripts/helpers/inputs.ts

@@ -7,20 +7,14 @@ export const INPUT_TYPES = ['classes', 'schemas', 'entityBatches'] as const
 export type InputType = typeof INPUT_TYPES[number]
 export type FetchedInput<Schema = any> = { fileName: string; data: Schema }
 
-export const sortFiles = (filenameA: string, filenameB: string) =>
-  parseInt(filenameA.split('_')[0]) - parseInt(filenameB.split('_')[0])
-
 export const getInputsLocation = (inputType: InputType) => path.join(INPUTS_LOCATION, inputType)
 
 export function getInputs<Schema = any>(inputType: InputType): FetchedInput<Schema>[] {
-  return fs
-    .readdirSync(getInputsLocation(inputType))
-    .sort(sortFiles)
-    .map((fileName) => {
-      const inputJson = fs.readFileSync(path.join(INPUTS_LOCATION, inputType, fileName)).toString()
-      return {
-        fileName,
-        data: JSON.parse(inputJson) as Schema,
-      }
-    })
+  return fs.readdirSync(getInputsLocation(inputType)).map((fileName) => {
+    const inputJson = fs.readFileSync(path.join(INPUTS_LOCATION, inputType, fileName)).toString()
+    return {
+      fileName,
+      data: JSON.parse(inputJson) as Schema,
+    }
+  })
 }

content-directory-schemas/scripts/helpers/propertyType.ts

@@ -0,0 +1,11 @@
+import { Property, ReferenceProperty, SinglePropertyType, SinglePropertyVariant } from 'types/extrinsics/AddClassSchema'
+
+type PropertyType = Property['property_type']
+
+export function isSingle(propertyType: PropertyType): propertyType is SinglePropertyVariant {
+  return (propertyType as SinglePropertyVariant).Single !== undefined
+}
+
+export function isReference(propertySubtype: SinglePropertyType): propertySubtype is ReferenceProperty {
+  return (propertySubtype as ReferenceProperty).Reference !== undefined
+}

content-directory-schemas/scripts/initializeContentDir.ts

@@ -1,25 +1,20 @@
 import { CreateClass } from '../types/extrinsics/CreateClass'
 import { AddClassSchema } from '../types/extrinsics/AddClassSchema'
 import { types } from '@joystream/types'
-import { ApiPromise, Keyring, WsProvider } from '@polkadot/api'
-import { JoyBTreeSet } from '@joystream/types/common'
-import { PropertyId } from '@joystream/types/content-directory'
+import { ApiPromise, WsProvider } from '@polkadot/api'
 import { getInputs } from './helpers/inputs'
-import { SubmittableExtrinsic } from '@polkadot/api/types'
-import { EntityBatchesParser } from './helpers/EntityBatchParser'
 import fs from 'fs'
 import path from 'path'
+import { EntityBatch } from 'types/EntityBatch'
+import { InputParser } from './helpers/InputParser'
+import { ExtrinsicsHelper, getAlicePair } from './helpers/extrinsics'
 
-type DescribedExtrinsic = {
-  type: string
-  inputFilename: string
-  tx: SubmittableExtrinsic<'promise'>
-}
+// Save entity operations output here for easier debugging
+const ENTITY_OPERATIONS_OUTPUT_PATH = path.join(__dirname, '../operations.json')
 
-// Classes
 const classInputs = getInputs<CreateClass>('classes')
 const schemaInputs = getInputs<AddClassSchema>('schemas')
-const entityBatchInputs = getInputs('entityBatches')
+const entityBatchInputs = getInputs<EntityBatch>('entityBatches')
 
 async function main() {
   // Init api
@@ -27,10 +22,8 @@ async function main() {
   console.log(`Initializing the api (${WS_URI})...`)
   const provider = new WsProvider(WS_URI)
   const api = await ApiPromise.create({ provider, types })
-  // Init ALICE keypair
-  const keyring = new Keyring({ type: 'sr25519' })
-  keyring.addFromUri('//Alice', { name: 'Alice' })
-  const ALICE = keyring.getPairs()[0]
+
+  const ALICE = getAlicePair()
 
   // Emptiness check
   if ((await api.query.contentDirectory.nextClassId()).toNumber() > 1) {
@@ -38,54 +31,35 @@ async function main() {
     process.exit()
   }
 
-  const classExtrinsics = classInputs.map(({ data, fileName }) => ({
-    type: 'CreateClass',
-    inputFilename: fileName,
-    tx: api.tx.contentDirectory.createClass(
-      data.name,
-      data.description,
-      data.class_permissions || {},
-      data.maximum_entities_count,
-      data.default_entity_creation_voucher_upper_bound
-    ),
-  }))
-  // Schemas
-  const schemaExtrinsics = schemaInputs.map(({ data, fileName }) => ({
-    type: 'AddClassSchema',
-    inputFilename: fileName,
-    tx: api.tx.contentDirectory.addClassSchema(
-      data.classId,
-      new (JoyBTreeSet(PropertyId))(api.registry, data.existingProperties),
-      data.newProperties || []
-    ),
-  }))
+  const txHelper = new ExtrinsicsHelper(api)
+  const parser = new InputParser(api, classInputs, schemaInputs, entityBatchInputs)
 
-  let nonce = (await api.query.system.account(ALICE.address)).nonce.toNumber()
-  console.log('Initializing classes and schemas...\n')
-  const extrinsics: DescribedExtrinsic[] = [...classExtrinsics, ...schemaExtrinsics]
-  for (const { type, inputFilename, tx } of extrinsics) {
-    console.log(`Sending ${type} extrinsic based on input file: ${inputFilename}`)
-    await tx.signAndSend(ALICE, { nonce: nonce++ })
-  }
+  console.log(`Initializing classes (${classInputs.length} input files found)...\n`)
+  const classExtrinsics = parser.getCreateClassExntrinsics()
+  await txHelper.sendAndCheck(ALICE, classExtrinsics, 'Class initialization failed!')
 
-  console.log('Initializing entities...\n')
-  // Entity batches
-  const entityBatchParser = new EntityBatchesParser(schemaInputs, entityBatchInputs)
-  console.log('Parsing input into operations...')
-  const operations = entityBatchParser.getOperations()
+  console.log(`Initializing schemas (${schemaInputs.length} input files found)...\n`)
+  const schemaExtrinsics = await parser.getAddSchemaExtrinsics()
+  await txHelper.sendAndCheck(ALICE, schemaExtrinsics, 'Schemas initialization failed!')
+
+  console.log(`Initializing entities (${entityBatchInputs.length} input files found)`)
+  const entityOperations = await parser.getEntityBatchOperations()
   // Save operations in operations.json (for reference in case of errors)
-  const outputPath = path.join(__dirname, '../operations.json')
-  console.log(`Saving entity batch operations in ${outputPath}...`)
+  console.log(`Saving entity batch operations in ${ENTITY_OPERATIONS_OUTPUT_PATH}...`)
   fs.writeFileSync(
-    outputPath,
+    ENTITY_OPERATIONS_OUTPUT_PATH,
     JSON.stringify(
-      operations.map((o) => o.toJSON()),
+      entityOperations.map((o) => o.toJSON()),
       null,
       4
     )
   )
   console.log('Sending Transaction extrinsic...')
-  await api.tx.contentDirectory.transaction({ Lead: null }, operations).signAndSend(ALICE, { nonce: nonce++ })
+  await txHelper.sendAndCheck(
+    ALICE,
+    [api.tx.contentDirectory.transaction({ Lead: null }, entityOperations)],
+    'Entity initialization failed!'
+  )
 }
 
 main()

content-directory-schemas/scripts/inputSchemasToEntitySchemas.ts

@@ -9,26 +9,23 @@ import {
   TextProperty,
   VecPropertyVariant,
 } from '../types/extrinsics/AddClassSchema'
-import _ from 'lodash'
-import { schemaFilenameToEntitySchemaName, classIdToEntitySchemaName } from './helpers/entitySchemas'
-
 import PRIMITIVE_PROPERTY_DEFS from '../schemas/propertyValidationDefs.schema.json'
-import { getInputsLocation } from './helpers/inputs'
+import { getInputs } from './helpers/inputs'
+import { JSONSchema7 } from 'json-schema'
 
-const INPUTS_LOCATION = getInputsLocation('schemas')
 const SINGLE_ENTITY_SCHEMAS_LOCATION = path.join(__dirname, '../schemas/entities')
 const BATCH_OF_ENITIES_SCHEMAS_LOCATION = path.join(__dirname, '../schemas/entityBatches')
 const ENTITY_REFERENCE_SCHEMAS_LOCATION = path.join(__dirname, '../schemas/entityReferences')
 
-const inputFilenames = fs.readdirSync(INPUTS_LOCATION)
+const schemaInputs = getInputs<AddClassSchema>('schemas')
 
-const strictObjectDef = (def: Record<string, any>) => ({
+const strictObjectDef = (def: Record<string, any>): JSONSchema7 => ({
   type: 'object',
   additionalProperties: false,
   ...def,
 })
 
-const onePropertyObjectDef = (propertyName: string, propertyDef: Record<string, any>) =>
+const onePropertyObjectDef = (propertyName: string, propertyDef: Record<string, any>): JSONSchema7 =>
   strictObjectDef({
     required: [propertyName],
     properties: {
@@ -36,28 +33,26 @@ const onePropertyObjectDef = (propertyName: string, propertyDef: Record<string,
     },
   })
 
-const TextPropertyDef = ({ Text: maxLength }: TextProperty) => ({
+const TextPropertyDef = ({ Text: maxLength }: TextProperty): JSONSchema7 => ({
   type: 'string',
   maxLength,
 })
 
-const HashPropertyDef = ({ Hash: maxLength }: HashProperty) => ({
+const HashPropertyDef = ({ Hash: maxLength }: HashProperty): JSONSchema7 => ({
   type: 'string',
   maxLength,
 })
 
-const ReferencePropertyDef = ({ Reference: ref }: ReferenceProperty) => ({
+const ReferencePropertyDef = ({ Reference: ref }: ReferenceProperty): JSONSchema7 => ({
   'oneOf': [
-    onePropertyObjectDef('new', { '$ref': `./${classIdToEntitySchemaName(ref[0], 'Entity')}.schema.json` }),
-    onePropertyObjectDef('existing', {
-      '$ref': `../entityReferences/${classIdToEntitySchemaName(ref[0], 'Ref')}.schema.json`,
-    }),
+    onePropertyObjectDef('new', { '$ref': `./${ref.className}Entity.schema.json` }),
+    onePropertyObjectDef('existing', { '$ref': `../entityReferences/${ref.className}Ref.schema.json` }),
   ],
 })
 
-const SinglePropertyDef = ({ Single: singlePropType }: SinglePropertyVariant) => {
+const SinglePropertyDef = ({ Single: singlePropType }: SinglePropertyVariant): JSONSchema7 => {
   if (typeof singlePropType === 'string') {
-    return PRIMITIVE_PROPERTY_DEFS.definitions[singlePropType]
+    return PRIMITIVE_PROPERTY_DEFS.definitions[singlePropType] as JSONSchema7
   } else if ((singlePropType as TextProperty).Text) {
     return TextPropertyDef(singlePropType as TextProperty)
   } else if ((singlePropType as HashProperty).Hash) {
@@ -69,13 +64,13 @@ const SinglePropertyDef = ({ Single: singlePropType }: SinglePropertyVariant) =>
   throw new Error(`Unknown single proprty type: ${JSON.stringify(singlePropType)}`)
 }
 
-const VecPropertyDef = ({ Vector: vec }: VecPropertyVariant) => ({
+const VecPropertyDef = ({ Vector: vec }: VecPropertyVariant): JSONSchema7 => ({
   type: 'array',
   maxItems: vec.max_length,
   'items': SinglePropertyDef({ Single: vec.vec_type }),
 })
 
-const PropertyDef = ({ property_type: propertyType, description }: Property) => ({
+const PropertyDef = ({ property_type: propertyType, description }: Property): JSONSchema7 => ({
   ...((propertyType as SinglePropertyVariant).Single
     ? SinglePropertyDef(propertyType as SinglePropertyVariant)
     : VecPropertyDef(propertyType as VecPropertyVariant)),
@@ -95,12 +90,8 @@ entitySchemasDirs.forEach((dir) => {
 })
 
 // Run schema conversion:
-inputFilenames.forEach((fileName) => {
-  const inputFilePath = path.join(INPUTS_LOCATION, fileName)
-  const inputJson = fs.readFileSync(inputFilePath).toString()
-  const inputData = JSON.parse(inputJson) as AddClassSchema
-
-  const schemaName = schemaFilenameToEntitySchemaName(fileName)
+schemaInputs.forEach(({ fileName, data: inputData }) => {
+  const schemaName = fileName.replace('Schema.json', '')
 
   if (inputData.newProperties && !inputData.existingProperties) {
     const properites = inputData.newProperties
@@ -109,35 +100,43 @@ inputFilenames.forEach((fileName) => {
       return pObj
     }, {} as Record<string, ReturnType<typeof PropertyDef>>)
 
-    const EntitySchema = {
-      '$schema': 'http://json-schema.org/draft-07/schema',
-      '$id': `https://joystream.org/entities/${schemaName}Entity.schema.json`,
-      'title': `${schemaName}Entity`,
-      'description': `JSON schema for entities based on ${schemaName} runtime schema`,
+    const EntitySchema: JSONSchema7 = {
+      $schema: 'http://json-schema.org/draft-07/schema',
+      $id: `https://joystream.org/entities/${schemaName}Entity.schema.json`,
+      title: `${schemaName}Entity`,
+      description: `JSON schema for entities based on ${schemaName} runtime schema`,
       ...strictObjectDef({
         required: properites.filter((p) => p.required).map((p) => p.name),
         properties: propertiesObj,
       }),
     }
 
-    const ReferenceSchema = {
-      '$schema': 'http://json-schema.org/draft-07/schema',
-      '$id': `https://joystream.org/entityReferences/${schemaName}Ref.schema.json`,
-      'title': `${schemaName}Reference`,
-      'description': `JSON schema for reference to ${schemaName} entity based on runtime schema`,
-      'anyOf': [
+    const ReferenceSchema: JSONSchema7 = {
+      $schema: 'http://json-schema.org/draft-07/schema',
+      $id: `https://joystream.org/entityReferences/${schemaName}Ref.schema.json`,
+      title: `${schemaName}Reference`,
+      description: `JSON schema for reference to ${schemaName} entity based on runtime schema`,
+      anyOf: [
         ...properites.filter((p) => p.required && p.unique).map((p) => onePropertyObjectDef(p.name, PropertyDef(p))),
-        PRIMITIVE_PROPERTY_DEFS.definitions.Uint64,
+        PRIMITIVE_PROPERTY_DEFS.definitions.Uint64 as JSONSchema7,
       ],
     }
 
-    const BatchSchema = {
-      '$schema': 'http://json-schema.org/draft-07/schema',
-      '$id': `https://joystream.org/entityBatches/${schemaName}Batch.schema.json`,
-      'title': `${schemaName}Batch`,
-      'description': `JSON schema for batch of entities based on ${schemaName} runtime schema`,
-      'type': 'array',
-      'items': { '$ref': `../entities/${schemaName}Entity.schema.json` },
+    const BatchSchema: JSONSchema7 = {
+      $schema: 'http://json-schema.org/draft-07/schema',
+      $id: `https://joystream.org/entityBatches/${schemaName}Batch.schema.json`,
+      title: `${schemaName}Batch`,
+      description: `JSON schema for batch of entities based on ${schemaName} runtime schema`,
+      ...strictObjectDef({
+        required: ['className', 'entries'],
+        properties: {
+          className: { type: 'string' },
+          entries: {
+            type: 'array',
+            items: { '$ref': `../entities/${schemaName}Entity.schema.json` },
+          },
+        },
+      }),
     }
 
     const entitySchemaPath = path.join(SINGLE_ENTITY_SCHEMAS_LOCATION, `${schemaName}Entity.schema.json`)

content-directory-schemas/scripts/validate.ts

@@ -3,18 +3,11 @@ import Ajv from 'ajv'
 import { FetchedInput, getInputs, InputType, INPUT_TYPES } from './helpers/inputs'
 import path from 'path'
 import fs from 'fs'
+import $RefParser from '@apidevtools/json-schema-ref-parser'
 
 const SCHEMAS_LOCATION = path.join(__dirname, '../schemas')
 
 const ajv = new Ajv({ allErrors: true })
-// Add all existing schemas to Ajv to allow them to reference each other
-fs.readdirSync(SCHEMAS_LOCATION)
-  .filter((subdir) => !subdir.includes('.'))
-  .forEach((subdir) => {
-    fs.readdirSync(path.join(SCHEMAS_LOCATION, subdir)).forEach((schemaFilename) => {
-      ajv.addSchema(JSON.parse(fs.readFileSync(path.join(SCHEMAS_LOCATION, subdir, schemaFilename)).toString()))
-    })
-  })
 
 const validateJsonSchema = (jsonSchemaShortPath: string, jsonSchema: Record<string, unknown>) => {
   if (!ajv.validateSchema(jsonSchema)) {
@@ -51,7 +44,7 @@ const getJsonSchemaForInput = (inputType: InputType, input: FetchedInput) => {
     schemaLocation = path.join(SCHEMAS_LOCATION, 'extrinsics', 'AddClassSchema.schema.json')
   }
   if (inputType === 'entityBatches') {
-    const jsonSchemaFilename = input.fileName.split('_')[1].replace('.json', '.schema.json')
+    const jsonSchemaFilename = input.fileName.replace('.json', '.schema.json')
     schemaLocation = path.join(SCHEMAS_LOCATION, 'entityBatches', jsonSchemaFilename)
   }
 
@@ -61,25 +54,32 @@ const getJsonSchemaForInput = (inputType: InputType, input: FetchedInput) => {
   }
 }
 
-const alreadyValidatedJsonSchemas = new Map<string, boolean>()
-INPUT_TYPES.forEach((inputType) => {
-  console.log(`Validating inputs/${inputType} and related json-schemas...\n`)
-  getInputs(inputType).forEach((input) => {
-    const { jsonSchemaPath, jsonSchema } = getJsonSchemaForInput(inputType, input)
-    const jsonSchemaShortPath = path.relative(path.join(SCHEMAS_LOCATION, '..'), jsonSchemaPath)
-    // Validate the schema itself
-    let isJsonSchemaValid = alreadyValidatedJsonSchemas.get(jsonSchemaShortPath)
-    if (isJsonSchemaValid === undefined) {
-      console.log(`Validating ${jsonSchemaShortPath}...`)
-      isJsonSchemaValid = validateJsonSchema(jsonSchemaShortPath, jsonSchema)
-      alreadyValidatedJsonSchemas.set(jsonSchemaShortPath, isJsonSchemaValid)
+async function main() {
+  const alreadyValidatedJsonSchemas = new Map<string, boolean>()
+  for (const inputType of INPUT_TYPES) {
+    console.log(`Validating inputs/${inputType} and related json-schemas...\n`)
+    for (const input of getInputs(inputType)) {
+      let { jsonSchemaPath, jsonSchema } = getJsonSchemaForInput(inputType, input)
+      jsonSchema = await $RefParser.dereference(jsonSchemaPath.replace(/\/(^\/)+\.schema\.json$/, ''), jsonSchema)
+      const jsonSchemaShortPath = path.relative(path.join(SCHEMAS_LOCATION, '..'), jsonSchemaPath)
+      // Validate the schema itself
+      let isJsonSchemaValid = alreadyValidatedJsonSchemas.get(jsonSchemaShortPath)
+      if (isJsonSchemaValid === undefined) {
+        console.log(`Validating ${jsonSchemaShortPath}...`)
+        isJsonSchemaValid = validateJsonSchema(jsonSchemaShortPath, jsonSchema)
+        alreadyValidatedJsonSchemas.set(jsonSchemaShortPath, isJsonSchemaValid)
+      }
+      if (!isJsonSchemaValid) {
+        return
+      }
+      console.log(`Validating inputs/${inputType}/${input.fileName}...`)
+      validateInputAgainstSchema(input, jsonSchema)
     }
-    if (!isJsonSchemaValid) {
-      return
-    }
-    console.log(`Validating inputs/${inputType}/${input.fileName}...`)
-    validateInputAgainstSchema(input, jsonSchema)
-  })
 
-  console.log('\n\n')
-})
+    console.log('\n\n')
+  }
+}
+
+main()
+  .then(() => process.exit())
+  .catch((e) => console.error(e))

content-directory-schemas/types/EntityBatch.d.ts

@@ -0,0 +1,4 @@
+export interface EntityBatch {
+  className: string
+  entries: Record<string, any>[]
+}

content-directory-schemas/types/extrinsics/AddClassSchema.d.ts

@@ -5,10 +5,12 @@
  * and run json-schema-to-typescript to regenerate this file.
  */
 
-export type ClassId = number
 export type PropertyInSchemIndex = number
-export type SinglePropertyType = PrimitiveProperty | TextProperty | HashProperty | ReferenceProperty
-export type PrimitiveProperty = 'Bool' | 'Uint16' | 'Uint32' | 'Uint64' | 'Int16' | 'Int32' | 'Int64'
+export type SinglePropertyType =
+  | ('Bool' | 'Uint16' | 'Uint32' | 'Uint64' | 'Int16' | 'Int32' | 'Int64')
+  | TextProperty
+  | HashProperty
+  | ReferenceProperty
 export type MaxTextLength = number
 export type MaxVecItems = number
 export type PropertyName = string
@@ -19,9 +21,9 @@ export type DefaultBoolean = boolean
  * JSON schema to describe a new schema for a certain class in Joystream network
  */
 export interface AddClassSchema {
-  classId: ClassId
-  existingProperties?: [PropertyInSchemIndex, ...PropertyInSchemIndex[]]
-  newProperties: [Property, ...Property[]]
+  className: string
+  existingProperties?: PropertyInSchemIndex[]
+  newProperties: Property[]
 }
 export interface Property {
   property_type: SinglePropertyVariant | VecPropertyVariant
@@ -41,7 +43,16 @@ export interface HashProperty {
   Hash: MaxTextLength
 }
 export interface ReferenceProperty {
-  Reference: [number, boolean]
+  Reference: {
+    /**
+     * Referenced class name
+     */
+    className: string
+    /**
+     * Whether same owner (controller) is required
+     */
+    sameOwner?: boolean
+  }
 }
 export interface VecPropertyVariant {
   Vector: VecPropertyType

content-directory-schemas/vscode-recommended.settings.json

@@ -9,51 +9,51 @@
       "url": "/content-directory-schemas/schemas/extrinsics/CreateClass.schema.json"
     },
     {
-      "fileMatch": ["*_LanguageBatch.json"],
+      "fileMatch": ["*/LanguageBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/LanguageBatch.schema.json"
     },
     {
-      "fileMatch": ["*_MediaLocationBatch.json"],
+      "fileMatch": ["*/MediaLocationBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/MediaLocationBatch.schema.json"
     },
     {
-      "fileMatch": ["*_HttpMediaLocationBatch.json"],
+      "fileMatch": ["*/HttpMediaLocationBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/HttpMediaLocationBatch.schema.json"
     },
     {
-      "fileMatch": ["*_JoystreamMediaLocationBatch.json"],
+      "fileMatch": ["*/JoystreamMediaLocationBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/JoystreamMediaLocationBatch.schema.json"
     },
     {
-      "fileMatch": ["*_ContentCategoryBatch.json"],
+      "fileMatch": ["*/ContentCategoryBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/ContentCategoryBatch.schema.json"
     },
     {
-      "fileMatch": ["*_ChannelBatch.json"],
+      "fileMatch": ["*/ChannelBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/ChannelBatch.schema.json"
     },
     {
-      "fileMatch": ["*_VideoMediaEncodingBatch.json"],
+      "fileMatch": ["*/VideoMediaEncodingBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/VideoMediaEncodingBatch.schema.json"
     },
     {
-      "fileMatch": ["*_KnownLicenseBatch.json"],
+      "fileMatch": ["*/KnownLicenseBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/KnownLicenseBatch.schema.json"
     },
     {
-      "fileMatch": ["*_UserDefinedLicenseBatch.json"],
+      "fileMatch": ["*/UserDefinedLicenseBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/UserDefinedLicenseBatch.schema.json"
     },
     {
-      "fileMatch": ["*_LicenseBatch.json"],
+      "fileMatch": ["*/LicenseBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/LicenseBatch.schema.json"
     },
     {
-      "fileMatch": ["*_VideoMediaBatch.json"],
+      "fileMatch": ["*/VideoMediaBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/VideoMediaBatch.schema.json"
     },
     {
-      "fileMatch": ["*_VideoBatch.json"],
+      "fileMatch": ["*/VideoBatch.json"],
       "url": "/content-directory-schemas/schemas/entityBatches/VideoBatch.schema.json"
     }
   ]

yarn.lock

@@ -18,7 +18,7 @@
   version "0.0.0"
   uid ""
 
-"@apidevtools/json-schema-ref-parser@9.0.6":
+"@apidevtools/json-schema-ref-parser@9.0.6", "@apidevtools/json-schema-ref-parser@^9.0.6":
   version "9.0.6"
   resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz#5d9000a3ac1fd25404da886da6b266adcd99cf1c"
   integrity sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==