OffchainState fix: Batch inserts during import to avoid query errors (#131)

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "orion",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "orion",
-      "version": "2.1.0",
+      "version": "2.2.0",
       "hasInstallScript": true,
       "dependencies": {
         "@joystream/js": "^1.4.0",

src/utils/offchainState.ts

@@ -136,7 +136,18 @@ export class OffchainState {
           )
         }
       } else {
-        await em.getRepository(entityName).insert(values)
+        // For inserts we also use batches, but this is because otherwise the query may fail
+        // if the number of entities is very large
+        const batchSize = 1000
+        let batchNumber = 1
+        while (values.length) {
+          ++batchNumber
+          const batch = values.splice(0, batchSize)
+          this.logger.info(
+            `Executing batch #${batchNumber} of ${batch.length} entities (${values.length} entities left)...`
+          )
+          await em.getRepository(entityName).insert(batch)
+        }
       }
       this.logger.info(
         `Done ${type === 'update' ? 'updating' : 'inserting'} ${entityName} entities`