Merge branch 'giza_staging' into giza-staging-update-from-giza-and-master

cli/src/base/UploadCommandBase.ts

@@ -338,16 +338,22 @@ export default abstract class UploadCommandBase extends ContentDirectoryCommandB
 
   async prepareAssetsForExtrinsic(resolvedAssets: ResolvedAsset[]): Promise<StorageAssets | undefined> {
     const feePerMB = await this.getOriginalApi().query.storage.dataObjectPerMegabyteFee()
+    const { dataObjectDeletionPrize } = this.getOriginalApi().consts.storage
     if (resolvedAssets.length) {
       const totalBytes = resolvedAssets
         .reduce((a, b) => {
           return a.add(b.parameters.getField('size'))
         }, new BN(0))
         .toNumber()
-      const totalFee = feePerMB.muln(Math.ceil(totalBytes / 1024 / 1024))
+      const totalStorageFee = feePerMB.muln(Math.ceil(totalBytes / 1024 / 1024))
+      const totalDeletionPrize = dataObjectDeletionPrize.muln(resolvedAssets.length)
       await this.requireConfirmation(
-        `Total fee of ${chalk.cyan(formatBalance(totalFee))} ` +
-          `will have to be paid in order to store the provided assets. Are you sure you want to continue?`
+        `Some additional costs will be associated with this operation:\n` +
+          `Total data storage fee: ${chalk.cyan(formatBalance(totalStorageFee))}\n` +
+          `Total deletion prize: ${chalk.cyan(
+            formatBalance(totalDeletionPrize)
+          )} (recoverable on data object(s) removal)\n` +
+          `Are you sure you want to continue?`
       )
       return createTypeFromConstructor(StorageAssets, {
         expected_data_size_fee: feePerMB,

cli/src/commands/api/setQueryNodeEndpoint.ts

@@ -28,7 +28,7 @@ export default class ApiSetQueryNodeEndpoint extends ApiCommandBase {
     } else {
       newEndpoint = await this.promptForQueryNodeUri()
     }
-    await this.setPreservedState({ queryNodeUri: endpoint })
+    await this.setPreservedState({ queryNodeUri: newEndpoint })
     this.log(
       chalk.greenBright('Query node endpoint successfuly changed! New endpoint: ') + chalk.magentaBright(newEndpoint)
     )

cli/src/graphql/generated/queries.ts

@@ -7,7 +7,7 @@ export type StorageNodeInfoFragment = {
 }
 
 export type GetStorageNodesInfoByBagIdQueryVariables = Types.Exact<{
-  bagId?: Types.Maybe<Types.Scalars['String']>
+  bagId?: Types.Maybe<Types.Scalars['ID']>
 }>
 
 export type GetStorageNodesInfoByBagIdQuery = { storageBuckets: Array<StorageNodeInfoFragment> }
@@ -81,11 +81,11 @@ export const DataObjectInfo = gql`
   }
 `
 export const GetStorageNodesInfoByBagId = gql`
-  query getStorageNodesInfoByBagId($bagId: String) {
+  query getStorageNodesInfoByBagId($bagId: ID) {
     storageBuckets(
       where: {
         operatorStatus_json: { isTypeOf_eq: "StorageBucketOperatorStatusActive" }
-        bagAssignments_some: { storageBagId_eq: $bagId }
+        bags_some: { id_eq: $bagId }
         operatorMetadata: { nodeEndpoint_contains: "http" }
       }
     ) {

cli/src/graphql/queries/storage.graphql

@@ -5,11 +5,11 @@ fragment StorageNodeInfo on StorageBucket {
   }
 }
 
-query getStorageNodesInfoByBagId($bagId: String) {
+query getStorageNodesInfoByBagId($bagId: ID) {
   storageBuckets(
     where: {
       operatorStatus_json: { isTypeOf_eq: "StorageBucketOperatorStatusActive" }
-      bagAssignments_some: { storageBagId_eq: $bagId }
+      bags_some: { id_eq: $bagId }
       operatorMetadata: { nodeEndpoint_contains: "http" }
     }
   ) {

distributor-node/.eslintignore

@@ -1 +1,2 @@
 src/types/generated
+src/services/networking/query-node/generated

distributor-node/.prettierignore

@@ -1,4 +1,3 @@
-/**/generated
 /**/mock.graphql
 lib
 local

distributor-node/README.md

@@ -27,7 +27,9 @@ To determine environment variable name based on a config key, for example `inter
 - replace all dots with `__`: `INTERVALS.CACHE_CLEANUP` => `INTERVALS__CACHE_CLEANUP`
 - add `JOYSTREAM_DISTRIBUTOR__` prefix: `INTERVALS__CACHE_CLEANUP` => `JOYSTREAM_DISTRIBUTOR__INTERVALS__CACHE_CLEANUP`
 
-In case of arrays, the values must be provided as json string, for example `JOYSTREAM_DISTRIBUTOR__KEYS="[{\"suri\":\"//Bob\"}]"`.
+In case of arrays or `oneOf` objects (ie. `keys`), the values must be provided as json string, for example `JOYSTREAM_DISTRIBUTOR__KEYS="[{\"suri\":\"//Bob\"}]"`.
+
+In order to unset a value you can use one of the following strings as env variable value: `"off"` `"null"`, `"undefined"`, for example: `JOYSTREAM_DISTRIBUTOR__LOGS__FILE="off"`.
 
 For more envirnoment variable examples see the `distributor-node` service configuration in [docker-compose.yml](../docker-compose.yml).
 

distributor-node/config.yml

@@ -2,29 +2,39 @@ id: test-node
 endpoints:
   queryNode: http://localhost:8081/graphql
   joystreamNodeWs: ws://localhost:9944
-  # elasticSearch: http://localhost:9200
 directories:
   assets: ./local/data
   cacheState: ./local/cache
-  logs: ./local/logs
-log:
-  file: debug
-  console: verbose
-  # elastic: info
+logs:
+  file:
+    level: debug
+    path: ./local/logs
+    maxFiles: 5
+    maxSize: 1000000
+  console:
+    level: verbose
+  # elastic:
+  #   level: info
+  #   endpoint: http://localhost:9200
 limits:
   storage: 100G
   maxConcurrentStorageNodeDownloads: 100
   maxConcurrentOutboundConnections: 300
-  outboundRequestsTimeout: 5000
+  outboundRequestsTimeoutMs: 5000
+  pendingDownloadTimeoutSec: 3600
+  maxCachedItemSize: 1G
 intervals:
   saveCacheState: 60
   checkStorageNodeResponseTimes: 60
   cacheCleanup: 60
-port: 3334
+publicApi:
+  port: 3334
+operatorApi:
+  port: 3335
+  hmacSecret: this-is-not-so-secret
 keys:
   - suri: //Alice
   # - mnemonic: "escape naive annual throw tragic achieve grunt verify cram note harvest problem"
   #   type: ed25519
   # - keyfile: "/path/to/keyfile.json"
-buckets: 'all'
 workerId: 0

distributor-node/docs/api/operator/index.md

@@ -0,0 +1,375 @@
+---
+title: Distributor node operator API v0.1.0
+language_tabs:
+  - javascript: JavaScript
+  - shell: Shell
+language_clients:
+  - javascript: ""
+  - shell: ""
+toc_footers: []
+includes: []
+search: true
+highlight_theme: darkula
+headingLevel: 2
+
+---
+
+<!-- AUTO-GENERATED-CONTENT:START (TOC) -->
+<!-- AUTO-GENERATED-CONTENT:END -->
+
+<h1 id="distributor-node-operator-api">Distributor node operator API v0.1.0</h1>
+
+> Scroll down for code samples, example requests and responses.
+
+Distributor node operator API
+
+Base URLs:
+
+* <a href="http://localhost:3335/api/v1/">http://localhost:3335/api/v1/</a>
+
+Email: <a href="mailto:info@joystream.org">Support</a> 
+License: <a href="https://spdx.org/licenses/GPL-3.0-only.html">GPL-3.0-only</a>
+
+undefined
+
+<h1 id="distributor-node-operator-api-default">Default</h1>
+
+## operator.stopApi
+
+<a id="opIdoperator.stopApi"></a>
+
+> Code samples
+
+```javascript
+
+const headers = {
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/stop-api',
+{
+  method: 'POST',
+
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/stop-api \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /stop-api`
+
+Turns off the public api.
+
+<h3 id="operator.stopapi-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|409|[Conflict](https://tools.ietf.org/html/rfc7231#section-6.5.8)|Already stopped|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+## operator.startApi
+
+<a id="opIdoperator.startApi"></a>
+
+> Code samples
+
+```javascript
+
+const headers = {
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/start-api',
+{
+  method: 'POST',
+
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/start-api \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /start-api`
+
+Turns on the public api.
+
+<h3 id="operator.startapi-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|409|[Conflict](https://tools.ietf.org/html/rfc7231#section-6.5.8)|Already started|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+## operator.shutdown
+
+<a id="opIdoperator.shutdown"></a>
+
+> Code samples
+
+```javascript
+
+const headers = {
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/shutdown',
+{
+  method: 'POST',
+
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/shutdown \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /shutdown`
+
+Shuts down the node.
+
+<h3 id="operator.shutdown-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|409|[Conflict](https://tools.ietf.org/html/rfc7231#section-6.5.8)|Already shutting down|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+## operator.setWorker
+
+<a id="opIdoperator.setWorker"></a>
+
+> Code samples
+
+```javascript
+const inputBody = '{
+  "workerId": 0
+}';
+const headers = {
+  'Content-Type':'application/json',
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/set-worker',
+{
+  method: 'POST',
+  body: inputBody,
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/set-worker \
+  -H 'Content-Type: application/json' \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /set-worker`
+
+Updates the operator worker id.
+
+> Body parameter
+
+```json
+{
+  "workerId": 0
+}
+```
+
+<h3 id="operator.setworker-parameters">Parameters</h3>
+
+|Name|In|Type|Required|Description|
+|---|---|---|---|---|
+|body|body|[SetWorkerOperation](#schemasetworkeroperation)|false|none|
+
+<h3 id="operator.setworker-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+## operator.setBuckets
+
+<a id="opIdoperator.setBuckets"></a>
+
+> Code samples
+
+```javascript
+const inputBody = '{
+  "buckets": [
+    0
+  ]
+}';
+const headers = {
+  'Content-Type':'application/json',
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/set-buckets',
+{
+  method: 'POST',
+  body: inputBody,
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/set-buckets \
+  -H 'Content-Type: application/json' \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /set-buckets`
+
+Updates buckets supported by the node.
+
+> Body parameter
+
+```json
+{
+  "buckets": [
+    0
+  ]
+}
+```
+
+<h3 id="operator.setbuckets-parameters">Parameters</h3>
+
+|Name|In|Type|Required|Description|
+|---|---|---|---|---|
+|body|body|[SetBucketsOperation](#schemasetbucketsoperation)|false|none|
+
+<h3 id="operator.setbuckets-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+# Schemas
+
+<h2 id="tocS_SetWorkerOperation">SetWorkerOperation</h2>
+
+<a id="schemasetworkeroperation"></a>
+<a id="schema_SetWorkerOperation"></a>
+<a id="tocSsetworkeroperation"></a>
+<a id="tocssetworkeroperation"></a>
+
+```json
+{
+  "workerId": 0
+}
+
+```
+
+### Properties
+
+|Name|Type|Required|Restrictions|Description|
+|---|---|---|---|---|
+|workerId|integer|true|none|none|
+
+<h2 id="tocS_SetBucketsOperation">SetBucketsOperation</h2>
+
+<a id="schemasetbucketsoperation"></a>
+<a id="schema_SetBucketsOperation"></a>
+<a id="tocSsetbucketsoperation"></a>
+<a id="tocssetbucketsoperation"></a>
+
+```json
+{
+  "buckets": [
+    0
+  ]
+}
+
+```
+
+### Properties
+
+|Name|Type|Required|Restrictions|Description|
+|---|---|---|---|---|
+|buckets|[integer]|false|none|Set of bucket ids to be distributed by the node. If not provided - all buckets assigned to currently configured worker will be distributed.|
+
+undefined
+

distributor-node/docs/api/index.md → distributor-node/docs/api/public/index.md

@@ -1,5 +1,5 @@
 ---
-title: Distributor node API v0.1.0
+title: Distributor node public API v0.1.0
 language_tabs:
   - javascript: JavaScript
   - shell: Shell
@@ -8,7 +8,7 @@ language_clients:
   - shell: ""
 toc_footers:
   - <a href="https://github.com/Joystream/joystream/issues/2224">Distributor
-    node API</a>
+    node public API</a>
 includes: []
 search: true
 highlight_theme: darkula
@@ -17,33 +17,13 @@ headingLevel: 2
 ---
 
 <!-- AUTO-GENERATED-CONTENT:START (TOC) -->
-- [public](#public)
-- [public.status](#publicstatus)
-  - [Responses](#responses)
-  - [Responses](#responses-1)
-- [public.buckets](#publicbuckets)
-- [public.assetHead](#publicassethead)
-  - [Parameters](#parameters)
-  - [Responses](#responses-2)
-  - [Response Headers](#response-headers)
-- [public.asset](#publicasset)
-  - [Parameters](#parameters-1)
-  - [Responses](#responses-3)
-- [ErrorResponse](#errorresponse)
-  - [Response Headers](#response-headers-1)
-- [Schemas](#schemas)
-  - [Properties](#properties)
-- [StatusResponse](#statusresponse)
-  - [Properties](#properties-1)
-- [BucketsResponse](#bucketsresponse)
-  - [Properties](#properties-2)
 <!-- AUTO-GENERATED-CONTENT:END -->
 
-<h1 id="distributor-node-api">Distributor node API v0.1.0</h1>
+<h1 id="distributor-node-public-api">Distributor node public API v0.1.0</h1>
 
 > Scroll down for code samples, example requests and responses.
 
-Distributor node API
+Distributor node public API
 
 Base URLs:
 
@@ -52,9 +32,7 @@ Base URLs:
 Email: <a href="mailto:info@joystream.org">Support</a> 
 License: <a href="https://spdx.org/licenses/GPL-3.0-only.html">GPL-3.0-only</a>
 
-<h1 id="distributor-node-api-public">public</h1>
-
-Public distributor node API
+<h1 id="distributor-node-public-api-default">Default</h1>
 
 ## public.status
 
@@ -187,7 +165,7 @@ This operation does not require authentication
 
 ```javascript
 
-fetch('http://localhost:3334/api/v1/asset/{objectId}',
+fetch('http://localhost:3334/api/v1/assets/{objectId}',
 {
   method: 'HEAD'
 
@@ -202,11 +180,11 @@ fetch('http://localhost:3334/api/v1/asset/{objectId}',
 
 ```shell
 # You can also use wget
-curl -X HEAD http://localhost:3334/api/v1/asset/{objectId}
+curl -X HEAD http://localhost:3334/api/v1/assets/{objectId}
 
 ```
 
-`HEAD /asset/{objectId}`
+`HEAD /assets/{objectId}`
 
 Returns asset response headers (cache status, content type and/or length, accepted ranges etc.)
 
@@ -247,7 +225,7 @@ const headers = {
   'Accept':'image/*'
 };
 
-fetch('http://localhost:3334/api/v1/asset/{objectId}',
+fetch('http://localhost:3334/api/v1/assets/{objectId}',
 {
   method: 'GET',
 
@@ -263,12 +241,12 @@ fetch('http://localhost:3334/api/v1/asset/{objectId}',
 
 ```shell
 # You can also use wget
-curl -X GET http://localhost:3334/api/v1/asset/{objectId} \
+curl -X GET http://localhost:3334/api/v1/assets/{objectId} \
   -H 'Accept: image/*'
 
 ```
 
-`GET /asset/{objectId}`
+`GET /assets/{objectId}`
 
 Returns a media file.
 

distributor-node/docs/commands/dev.md

@@ -9,8 +9,6 @@ Developer utility commands
 ## `joystream-distributor dev:batchUpload`
 
 ```
-undefined
-
 USAGE
   $ joystream-distributor dev:batchUpload
 
@@ -33,8 +31,6 @@ _See code: [src/commands/dev/batchUpload.ts](https://github.com/Joystream/joystr
 Initialize development environment. Sets Alice as distributor working group leader.
 
 ```
-Initialize development environment. Sets Alice as distributor working group leader.
-
 USAGE
   $ joystream-distributor dev:init
 

distributor-node/docs/commands/help.md

@@ -10,8 +10,6 @@ display help for joystream-distributor
 display help for joystream-distributor
 
 ```
-display help for <%= config.bin %>
-
 USAGE
   $ joystream-distributor help [COMMAND]
 
@@ -22,4 +20,4 @@ OPTIONS
   --all  see all commands in CLI
 ```
 
-_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v2.2.3/src/commands/help.ts)_
+_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v3.2.2/src/commands/help.ts)_

distributor-node/docs/commands/leader.md

@@ -22,9 +22,6 @@ Commands for performing Distribution Working Group leader on-chain duties (like
 Cancel pending distribution bucket operator invitation.
 
 ```
-Cancel pending distribution bucket operator invitation.
-  Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:cancel-invitation
 
@@ -51,8 +48,6 @@ _See code: [src/commands/leader/cancel-invitation.ts](https://github.com/Joystre
 Create new distribution bucket. Requires distribution working group leader permissions.
 
 ```
-Create new distribution bucket. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:create-bucket
 
@@ -74,8 +69,6 @@ _See code: [src/commands/leader/create-bucket.ts](https://github.com/Joystream/j
 Create new distribution bucket family. Requires distribution working group leader permissions.
 
 ```
-Create new distribution bucket family. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:create-bucket-family
 
@@ -93,8 +86,6 @@ _See code: [src/commands/leader/create-bucket-family.ts](https://github.com/Joys
 Delete distribution bucket. The bucket must have no operators. Requires distribution working group leader permissions.
 
 ```
-Delete distribution bucket. The bucket must have no operators. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:delete-bucket
 
@@ -116,8 +107,6 @@ _See code: [src/commands/leader/delete-bucket.ts](https://github.com/Joystream/j
 Delete distribution bucket family. Requires distribution working group leader permissions.
 
 ```
-Delete distribution bucket family. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:delete-bucket-family
 
@@ -137,10 +126,6 @@ _See code: [src/commands/leader/delete-bucket-family.ts](https://github.com/Joys
 Invite distribution bucket operator (distribution group worker).
 
 ```
-Invite distribution bucket operator (distribution group worker).
-  The specified bucket must not have any operator currently.
-  Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:invite-bucket-operator
 
@@ -168,9 +153,6 @@ _See code: [src/commands/leader/invite-bucket-operator.ts](https://github.com/Jo
 Remove distribution bucket operator (distribution group worker).
 
 ```
-Remove distribution bucket operator (distribution group worker).
-  Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:remove-bucket-operator
 
@@ -198,9 +180,6 @@ _See code: [src/commands/leader/remove-bucket-operator.ts](https://github.com/Jo
 Set/update distribution bucket family metadata.
 
 ```
-Set/update distribution bucket family metadata.
-  Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:set-bucket-family-metadata
 
@@ -225,8 +204,6 @@ _See code: [src/commands/leader/set-bucket-family-metadata.ts](https://github.co
 Set max. distribution buckets per bag limit. Requires distribution working group leader permissions.
 
 ```
-Set max. distribution buckets per bag limit. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:set-buckets-per-bag-limit
 
@@ -246,8 +223,6 @@ _See code: [src/commands/leader/set-buckets-per-bag-limit.ts](https://github.com
 Add/remove distribution buckets from a bag.
 
 ```
-Add/remove distribution buckets from a bag.
-
 USAGE
   $ joystream-distributor leader:update-bag
 
@@ -291,8 +266,6 @@ _See code: [src/commands/leader/update-bag.ts](https://github.com/Joystream/joys
 Update distribution bucket mode ("distributing" flag). Requires distribution working group leader permissions.
 
 ```
-Update distribution bucket mode ("distributing" flag). Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:update-bucket-mode
 
@@ -316,8 +289,6 @@ _See code: [src/commands/leader/update-bucket-mode.ts](https://github.com/Joystr
 Update distribution bucket status ("acceptingNewBags" flag). Requires distribution working group leader permissions.
 
 ```
-Update distribution bucket status ("acceptingNewBags" flag). Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:update-bucket-status
 
@@ -340,9 +311,6 @@ _See code: [src/commands/leader/update-bucket-status.ts](https://github.com/Joys
 Update dynamic bag creation policy (number of buckets by family that should store given dynamic bag type).
 
 ```
-Update dynamic bag creation policy (number of buckets by family that should store given dynamic bag type).
-    Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:update-dynamic-bag-policy
 

distributor-node/docs/commands/node.md

@@ -0,0 +1,120 @@
+`joystream-distributor node`
+============================
+
+Commands for interacting with a running distributor node through OperatorApi
+
+* [`joystream-distributor node:set-buckets`](#joystream-distributor-nodeset-buckets)
+* [`joystream-distributor node:set-worker`](#joystream-distributor-nodeset-worker)
+* [`joystream-distributor node:shutdown`](#joystream-distributor-nodeshutdown)
+* [`joystream-distributor node:start-public-api`](#joystream-distributor-nodestart-public-api)
+* [`joystream-distributor node:stop-public-api`](#joystream-distributor-nodestop-public-api)
+
+## `joystream-distributor node:set-buckets`
+
+Send an api request to change the set of buckets distributed by given distributor node.
+
+```
+USAGE
+  $ joystream-distributor node:set-buckets
+
+OPTIONS
+  -B, --bucketIds=bucketIds    Set of bucket ids to distribute
+  -a, --all                    Distribute all buckets belonging to configured worker
+
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/set-buckets.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/set-buckets.ts)_
+
+## `joystream-distributor node:set-worker`
+
+Send an api request to change workerId assigned to given distributor node instance.
+
+```
+USAGE
+  $ joystream-distributor node:set-worker
+
+OPTIONS
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -w, --workerId=workerId      (required) New workerId to set
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/set-worker.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/set-worker.ts)_
+
+## `joystream-distributor node:shutdown`
+
+Send an api request to shutdown given distributor node.
+
+```
+USAGE
+  $ joystream-distributor node:shutdown
+
+OPTIONS
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/shutdown.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/shutdown.ts)_
+
+## `joystream-distributor node:start-public-api`
+
+Send an api request to start public api of given distributor node.
+
+```
+USAGE
+  $ joystream-distributor node:start-public-api
+
+OPTIONS
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/start-public-api.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/start-public-api.ts)_
+
+## `joystream-distributor node:stop-public-api`
+
+Send an api request to stop public api of given distributor node.
+
+```
+USAGE
+  $ joystream-distributor node:stop-public-api
+
+OPTIONS
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/stop-public-api.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/stop-public-api.ts)_

distributor-node/docs/commands/operator.md

@@ -11,9 +11,6 @@ Commands for performing node operator (Distribution Working Group worker) on-cha
 Accept pending distribution bucket operator invitation.
 
 ```
-Accept pending distribution bucket operator invitation.
-  Requires the invited distribution group worker role key.
-
 USAGE
   $ joystream-distributor operator:accept-invitation
 
@@ -40,9 +37,6 @@ _See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joyst
 Set/update distribution bucket operator metadata.
 
 ```
-Set/update distribution bucket operator metadata.
-  Requires active distribution bucket operator worker role key.
-
 USAGE
   $ joystream-distributor operator:set-metadata
 

distributor-node/docs/commands/start.md

@@ -10,8 +10,6 @@ Start the node
 Start the node
 
 ```
-Start the node
-
 USAGE
   $ joystream-distributor start
 

distributor-node/docs/node/index.md

@@ -1,5 +1,7 @@
 <!-- AUTO-GENERATED-CONTENT:START (TOC:firsth1=true) -->
-- [The API](#the-api)
+- [API](#api)
+  - [Public API](#public-api)
+  - [Operator API](#operator-api)
   - [Requesting assets](#requesting-assets)
     - [Scenario 1 (cache hit)](#scenario-1-cache-hit)
     - [Scenario 2 (pending)](#scenario-2-pending)
@@ -32,19 +34,32 @@
 
 <a name="the-api"></a>
 
-# The API
+# API
 
-The Distributor Node exposes an HTTP api implemented with [ExpressJS](https://expressjs.com/).
+The Distributor Node, depending on the configuration, can expose either one or two HTTP APIs, both implemented with [ExpressJS](https://expressjs.com/).
 
-The api is described by an [OpenAPI](https://swagger.io/specification/) schema located at _[src/api-spec/openapi.yml](../../src/api-spec/openapi.yml)_
+## Public API
 
-**Current, detailed api documentation can be found [here](../api/index.md)**
+Public API is enabled by default and can be used to retrieve assets from the node as well as some basic information about its current status.
+
+Public API is described by an [OpenAPI](https://swagger.io/specification/) schema located at _[src/api-spec/public.yml](../../src/api-spec/public.yml)_
+
+**Full public API documentation can be found [here](../api/public/index.md)**
+
+## Operator API
+
+Secured operator API can be enabled with [`config.operatorApi`](../schema/definition-properties-operatorapi.md).
+Operator API makes it possible to remotely execute some operations on a running node (ie. changing supported buckets).
+
+Operator API is described by an [OpenAPI](https://swagger.io/specification/) schema located at _[src/api-spec/operator.yml](../../src/api-spec/operator.yml)_
+
+**Full operator API documentation can be found [here](../api/operator/index.md)**
 
 <a name="requesting-assets"></a>
 
 ## Requesting assets
 
-The assets are requested from the distributor node by using a `GET` request to [`/asset/{objectId}`](../api/index.md#opIdpublic.asset) endpoint.
+The assets are requested from the distributor node by using a `GET` request to [`/assets/{objectId}`](../api/index.md#opIdpublic.asset) endpoint.
 
 There are multiple scenarios of how a distributor will act upon that request, depending on its current state:
 
@@ -120,7 +135,7 @@ In this case
 
 ## Checking asset status
 
-It is possible to check an asset status without affecting the distributor node state in any way (for example - by triggering the process of [fetching the missing data object](#data-fetching)), by sending a [`HEAD` request to `/asset/{objectId}`](../api/index.md#opIdpublic.assetHead) endpoint.
+It is possible to check an asset status without affecting the distributor node state in any way (for example - by triggering the process of [fetching the missing data object](#data-fetching)), by sending a [`HEAD` request to `/assets/{objectId}`](../api/index.md#opIdpublic.assetHead) endpoint.
 
 If the request is valid, the node will respond with, among others, the `x-cache`, `content-length`, `cache-control` headers.
 
@@ -378,10 +393,10 @@ No-longer-distributed data objects are dropped from the cache periodically every
 
 The distributor node supports detailed logging with [winston](https://www.npmjs.com/package/winston) library. [NPM log levels](https://www.npmjs.com/package/winston#logging-levels) are used to specify the log priority.
 
-The logs can be directed to some of the 3 available outputs, depending on the [`log`](../schema/definition-properties-log.md) configuration settings:
-- console
-- a log file inside [`directories.logs`](../schema/definition-properties-directories.md#logs)
-- an elasticsearch endpoint specified via [`endpoints.elasticsearch`](../schema/definition-properties-endpoints.md#elasticsearch)
+The logs can be directed to some of the 3 available outputs, depending on the [`logs`](../schema/definition-properties-logs.md) configuration settings:
+- console ([`logs.console`](../schema/definition-properties-logs-properties-console.md))
+- log file(s) ([`logs.file`](../schema/definition-properties-logs-properties-file.md))
+- an elasticsearch endpoint ([`logs.elastic`](../schema/definition-properties-logs-properties-elastic.md))
 
 # Query node integration
 

distributor-node/docs/schema/definition-properties-bucket-ids.md

@@ -0,0 +1,9 @@
+## buckets Type
+
+`integer[]`
+
+## buckets Constraints
+
+**minimum number of items**: the minimum number of items for this array is: `1`
+
+**unique items**: all items in this array must be unique. Duplicates are not allowed.

distributor-node/docs/schema/definition-properties-buckets-oneof-all-buckets.md

@@ -1,11 +0,0 @@
-## 1 Type
-
-`string` ([All buckets](definition-properties-buckets-oneof-all-buckets.md))
-
-## 1 Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value   | Explanation |
-| :------ | :---------- |
-| `"all"` |             |

distributor-node/docs/schema/definition-properties-buckets-oneof-bucket-ids.md

@@ -1,7 +0,0 @@
-## 0 Type
-
-`integer[]`
-
-## 0 Constraints
-
-**minimum number of items**: the minimum number of items for this array is: `1`

distributor-node/docs/schema/definition-properties-buckets.md

@@ -1,9 +0,0 @@
-## buckets Type
-
-merged type ([Details](definition-properties-buckets.md))
-
-one (and only one) of
-
-*   [Bucket ids](definition-properties-buckets-oneof-bucket-ids.md "check type definition")
-
-*   [All buckets](definition-properties-buckets-oneof-all-buckets.md "check type definition")

distributor-node/docs/schema/definition-properties-directories-properties-logs.md

@@ -1,3 +0,0 @@
-## logs Type
-
-`string`

distributor-node/docs/schema/definition-properties-directories.md

@@ -4,11 +4,10 @@
 
 # directories Properties
 
-| Property                  | Type     | Required | Nullable       | Defined by                                                                                                                                             |
-| :------------------------ | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [assets](#assets)         | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-directories-properties-assets.md "undefined#/properties/directories/properties/assets")         |
-| [cacheState](#cachestate) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-directories-properties-cachestate.md "undefined#/properties/directories/properties/cacheState") |
-| [logs](#logs)             | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-directories-properties-logs.md "undefined#/properties/directories/properties/logs")             |
+| Property                  | Type     | Required | Nullable       | Defined by                                                                                                                                                                              |
+| :------------------------ | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [assets](#assets)         | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-directories-properties-assets.md "https://joystream.org/schemas/argus/config#/properties/directories/properties/assets")         |
+| [cacheState](#cachestate) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-directories-properties-cachestate.md "https://joystream.org/schemas/argus/config#/properties/directories/properties/cacheState") |
 
 ## assets
 
@@ -22,7 +21,7 @@ Path to a directory where all the cached assets will be stored
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-directories-properties-assets.md "undefined#/properties/directories/properties/assets")
+*   defined in: [Distributor node configuration](definition-properties-directories-properties-assets.md "https://joystream.org/schemas/argus/config#/properties/directories/properties/assets")
 
 ### assets Type
 
@@ -40,26 +39,8 @@ Path to a directory where information about the current cache state will be stor
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-directories-properties-cachestate.md "undefined#/properties/directories/properties/cacheState")
+*   defined in: [Distributor node configuration](definition-properties-directories-properties-cachestate.md "https://joystream.org/schemas/argus/config#/properties/directories/properties/cacheState")
 
 ### cacheState Type
 
 `string`
-
-## logs
-
-Path to a directory where logs will be stored if logging to a file was enabled (via `log.file`).
-
-`logs`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-directories-properties-logs.md "undefined#/properties/directories/properties/logs")
-
-### logs Type
-
-`string`

distributor-node/docs/schema/definition-properties-endpoints-properties-elasticsearch.md

@@ -1,3 +0,0 @@
-## elasticSearch Type
-
-`string`

distributor-node/docs/schema/definition-properties-endpoints.md

@@ -4,11 +4,10 @@
 
 # endpoints Properties
 
-| Property                            | Type     | Required | Nullable       | Defined by                                                                                                                                                   |
-| :---------------------------------- | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [queryNode](#querynode)             | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "undefined#/properties/endpoints/properties/queryNode")             |
-| [joystreamNodeWs](#joystreamnodews) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-joystreamnodews.md "undefined#/properties/endpoints/properties/joystreamNodeWs") |
-| [elasticSearch](#elasticsearch)     | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-elasticsearch.md "undefined#/properties/endpoints/properties/elasticSearch")     |
+| Property                            | Type     | Required | Nullable       | Defined by                                                                                                                                                                                    |
+| :---------------------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [queryNode](#querynode)             | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/queryNode")             |
+| [joystreamNodeWs](#joystreamnodews) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-joystreamnodews.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/joystreamNodeWs") |
 
 ## queryNode
 
@@ -22,7 +21,7 @@ Query node graphql server uri (for example: <http://localhost:8081/graphql>)
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "undefined#/properties/endpoints/properties/queryNode")
+*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/queryNode")
 
 ### queryNode Type
 
@@ -40,26 +39,8 @@ Joystream node websocket api uri (for example: ws\://localhost:9944)
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-joystreamnodews.md "undefined#/properties/endpoints/properties/joystreamNodeWs")
+*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-joystreamnodews.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/joystreamNodeWs")
 
 ### joystreamNodeWs Type
 
 `string`
-
-## elasticSearch
-
-Elasticsearch uri used for submitting the distributor node logs (if enabled via `log.elastic`)
-
-`elasticSearch`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-elasticsearch.md "undefined#/properties/endpoints/properties/elasticSearch")
-
-### elasticSearch Type
-
-`string`

distributor-node/docs/schema/definition-properties-intervals.md

@@ -4,11 +4,11 @@
 
 # intervals Properties
 
-| Property                                                        | Type      | Required | Nullable       | Defined by                                                                                                                                                                               |
-| :-------------------------------------------------------------- | :-------- | :------- | :------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [saveCacheState](#savecachestate)                               | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-savecachestate.md "undefined#/properties/intervals/properties/saveCacheState")                               |
-| [checkStorageNodeResponseTimes](#checkstoragenoderesponsetimes) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-checkstoragenoderesponsetimes.md "undefined#/properties/intervals/properties/checkStorageNodeResponseTimes") |
-| [cacheCleanup](#cachecleanup)                                   | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-cachecleanup.md "undefined#/properties/intervals/properties/cacheCleanup")                                   |
+| Property                                                        | Type      | Required | Nullable       | Defined by                                                                                                                                                                                                                |
+| :-------------------------------------------------------------- | :-------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [saveCacheState](#savecachestate)                               | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-savecachestate.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/saveCacheState")                               |
+| [checkStorageNodeResponseTimes](#checkstoragenoderesponsetimes) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-checkstoragenoderesponsetimes.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/checkStorageNodeResponseTimes") |
+| [cacheCleanup](#cachecleanup)                                   | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-cachecleanup.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/cacheCleanup")                                   |
 
 ## saveCacheState
 
@@ -22,7 +22,7 @@ How often, in seconds, will the cache state be saved in `directories.state`. Ind
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-intervals-properties-savecachestate.md "undefined#/properties/intervals/properties/saveCacheState")
+*   defined in: [Distributor node configuration](definition-properties-intervals-properties-savecachestate.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/saveCacheState")
 
 ### saveCacheState Type
 
@@ -44,7 +44,7 @@ How often, in seconds, will the distributor node attempt to send requests to all
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-intervals-properties-checkstoragenoderesponsetimes.md "undefined#/properties/intervals/properties/checkStorageNodeResponseTimes")
+*   defined in: [Distributor node configuration](definition-properties-intervals-properties-checkstoragenoderesponsetimes.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/checkStorageNodeResponseTimes")
 
 ### checkStorageNodeResponseTimes Type
 
@@ -66,7 +66,7 @@ How often, in seconds, will the distributor node fetch data about all its distri
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-intervals-properties-cachecleanup.md "undefined#/properties/intervals/properties/cacheCleanup")
+*   defined in: [Distributor node configuration](definition-properties-intervals-properties-cachecleanup.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/cacheCleanup")
 
 ### cacheCleanup Type
 

distributor-node/docs/schema/definition-properties-keys-items-oneof-json-backup-file.md

@@ -4,9 +4,9 @@
 
 # 2 Properties
 
-| Property            | Type     | Required | Nullable       | Defined by                                                                                                                                                                    |
-| :------------------ | :------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [keyfile](#keyfile) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-json-backup-file-properties-keyfile.md "undefined#/properties/keys/items/oneOf/2/properties/keyfile") |
+| Property            | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                     |
+| :------------------ | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [keyfile](#keyfile) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-json-backup-file-properties-keyfile.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/2/properties/keyfile") |
 
 ## keyfile
 
@@ -20,7 +20,7 @@
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-json-backup-file-properties-keyfile.md "undefined#/properties/keys/items/oneOf/2/properties/keyfile")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-json-backup-file-properties-keyfile.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/2/properties/keyfile")
 
 ### keyfile Type
 

distributor-node/docs/schema/definition-properties-keys-items-oneof-mnemonic-phrase.md

@@ -4,10 +4,10 @@
 
 # 1 Properties
 
-| Property              | Type     | Required | Nullable       | Defined by                                                                                                                                                                     |
-| :-------------------- | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [type](#type)         | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-type.md "undefined#/properties/keys/items/oneOf/1/properties/type")         |
-| [mnemonic](#mnemonic) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-mnemonic.md "undefined#/properties/keys/items/oneOf/1/properties/mnemonic") |
+| Property              | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                      |
+| :-------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [type](#type)         | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-type.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/1/properties/type")         |
+| [mnemonic](#mnemonic) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-mnemonic.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/1/properties/mnemonic") |
 
 ## type
 
@@ -21,7 +21,7 @@
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-type.md "undefined#/properties/keys/items/oneOf/1/properties/type")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-type.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/1/properties/type")
 
 ### type Type
 
@@ -57,7 +57,7 @@ The default value is:
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-mnemonic.md "undefined#/properties/keys/items/oneOf/1/properties/mnemonic")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-mnemonic.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/1/properties/mnemonic")
 
 ### mnemonic Type
 

distributor-node/docs/schema/definition-properties-keys-items-oneof-substrate-uri.md

@@ -4,10 +4,10 @@
 
 # 0 Properties
 
-| Property      | Type     | Required | Nullable       | Defined by                                                                                                                                                           |
-| :------------ | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [type](#type) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-type.md "undefined#/properties/keys/items/oneOf/0/properties/type") |
-| [suri](#suri) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-suri.md "undefined#/properties/keys/items/oneOf/0/properties/suri") |
+| Property      | Type     | Required | Nullable       | Defined by                                                                                                                                                                                            |
+| :------------ | :------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [type](#type) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-type.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/0/properties/type") |
+| [suri](#suri) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-suri.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/0/properties/suri") |
 
 ## type
 
@@ -21,7 +21,7 @@
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-type.md "undefined#/properties/keys/items/oneOf/0/properties/type")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-type.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/0/properties/type")
 
 ### type Type
 
@@ -57,7 +57,7 @@ The default value is:
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-suri.md "undefined#/properties/keys/items/oneOf/0/properties/suri")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-suri.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/0/properties/suri")
 
 ### suri Type
 

distributor-node/docs/schema/definition-properties-limits-properties-dataobjectsourcebyobjectidttl.md

@@ -0,0 +1,15 @@
+## dataObjectSourceByObjectIdTTL Type
+
+`integer`
+
+## dataObjectSourceByObjectIdTTL Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
+
+## dataObjectSourceByObjectIdTTL Default Value
+
+The default value is:
+
+```json
+60
+```

distributor-node/docs/schema/definition-properties-limits-properties-maxcacheditemsize.md

@@ -0,0 +1,13 @@
+## maxCachedItemSize Type
+
+`string`
+
+## maxCachedItemSize Constraints
+
+**pattern**: the string must match the following regular expression: 
+
+```regexp
+^[0-9]+(B|K|M|G|T)$
+```
+
+[try pattern](https://regexr.com/?expression=%5E%5B0-9%5D%2B\(B%7CK%7CM%7CG%7CT\)%24 "try regular expression with regexr.com")

distributor-node/docs/schema/definition-properties-limits-properties-outboundrequeststimeoutms.md

@@ -0,0 +1,7 @@
+## outboundRequestsTimeoutMs Type
+
+`integer`
+
+## outboundRequestsTimeoutMs Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1000`

distributor-node/docs/schema/definition-properties-limits-properties-pendingdownloadtimeoutsec.md

@@ -0,0 +1,7 @@
+## pendingDownloadTimeoutSec Type
+
+`integer`
+
+## pendingDownloadTimeoutSec Constraints
+
+**minimum**: the value of this number must greater than or equal to: `60`

distributor-node/docs/schema/definition-properties-limits.md

@@ -4,12 +4,15 @@
 
 # limits Properties
 
-| Property                                                                | Type      | Required | Nullable       | Defined by                                                                                                                                                                                 |
-| :---------------------------------------------------------------------- | :-------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [storage](#storage)                                                     | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-storage.md "undefined#/properties/limits/properties/storage")                                                     |
-| [maxConcurrentStorageNodeDownloads](#maxconcurrentstoragenodedownloads) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxconcurrentstoragenodedownloads.md "undefined#/properties/limits/properties/maxConcurrentStorageNodeDownloads") |
-| [maxConcurrentOutboundConnections](#maxconcurrentoutboundconnections)   | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxconcurrentoutboundconnections.md "undefined#/properties/limits/properties/maxConcurrentOutboundConnections")   |
-| [outboundRequestsTimeout](#outboundrequeststimeout)                     | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-outboundrequeststimeout.md "undefined#/properties/limits/properties/outboundRequestsTimeout")                     |
+| Property                                                                | Type      | Required | Nullable       | Defined by                                                                                                                                                                                                                  |
+| :---------------------------------------------------------------------- | :-------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [storage](#storage)                                                     | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-storage.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/storage")                                                     |
+| [maxConcurrentStorageNodeDownloads](#maxconcurrentstoragenodedownloads) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxconcurrentstoragenodedownloads.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxConcurrentStorageNodeDownloads") |
+| [maxConcurrentOutboundConnections](#maxconcurrentoutboundconnections)   | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxconcurrentoutboundconnections.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxConcurrentOutboundConnections")   |
+| [outboundRequestsTimeoutMs](#outboundrequeststimeoutms)                 | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-outboundrequeststimeoutms.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/outboundRequestsTimeoutMs")                 |
+| [pendingDownloadTimeoutSec](#pendingdownloadtimeoutsec)                 | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-pendingdownloadtimeoutsec.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/pendingDownloadTimeoutSec")                 |
+| [maxCachedItemSize](#maxcacheditemsize)                                 | `string`  | Optional | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxcacheditemsize.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxCachedItemSize")                                 |
+| [dataObjectSourceByObjectIdTTL](#dataobjectsourcebyobjectidttl)         | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-limits-properties-dataobjectsourcebyobjectidttl.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/dataObjectSourceByObjectIdTTL")         |
 
 ## storage
 
@@ -23,7 +26,7 @@ Maximum total size of all (cached) assets stored in `directories.assets`
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits-properties-storage.md "undefined#/properties/limits/properties/storage")
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-storage.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/storage")
 
 ### storage Type
 
@@ -51,7 +54,7 @@ Maximum number of concurrent downloads from the storage node(s)
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxconcurrentstoragenodedownloads.md "undefined#/properties/limits/properties/maxConcurrentStorageNodeDownloads")
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxconcurrentstoragenodedownloads.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxConcurrentStorageNodeDownloads")
 
 ### maxConcurrentStorageNodeDownloads Type
 
@@ -63,7 +66,7 @@ Maximum number of concurrent downloads from the storage node(s)
 
 ## maxConcurrentOutboundConnections
 
-Maximum number of total simultaneous outbound connections to storage node(s)
+Maximum number of total simultaneous outbound connections to storage node(s) (excluding proxy connections)
 
 `maxConcurrentOutboundConnections`
 
@@ -73,7 +76,7 @@ Maximum number of total simultaneous outbound connections to storage node(s)
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxconcurrentoutboundconnections.md "undefined#/properties/limits/properties/maxConcurrentOutboundConnections")
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxconcurrentoutboundconnections.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxConcurrentOutboundConnections")
 
 ### maxConcurrentOutboundConnections Type
 
@@ -83,11 +86,11 @@ Maximum number of total simultaneous outbound connections to storage node(s)
 
 **minimum**: the value of this number must greater than or equal to: `1`
 
-## outboundRequestsTimeout
+## outboundRequestsTimeoutMs
 
 Timeout for all outbound storage node http requests in miliseconds
 
-`outboundRequestsTimeout`
+`outboundRequestsTimeoutMs`
 
 *   is required
 
@@ -95,12 +98,92 @@ Timeout for all outbound storage node http requests in miliseconds
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits-properties-outboundrequeststimeout.md "undefined#/properties/limits/properties/outboundRequestsTimeout")
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-outboundrequeststimeoutms.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/outboundRequestsTimeoutMs")
 
-### outboundRequestsTimeout Type
+### outboundRequestsTimeoutMs Type
 
 `integer`
 
-### outboundRequestsTimeout Constraints
+### outboundRequestsTimeoutMs Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1000`
+
+## pendingDownloadTimeoutSec
+
+Timeout for pending storage node downloads in seconds
+
+`pendingDownloadTimeoutSec`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-pendingdownloadtimeoutsec.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/pendingDownloadTimeoutSec")
+
+### pendingDownloadTimeoutSec Type
+
+`integer`
+
+### pendingDownloadTimeoutSec Constraints
+
+**minimum**: the value of this number must greater than or equal to: `60`
+
+## maxCachedItemSize
+
+Maximum size of a data object allowed to be cached by the node
+
+`maxCachedItemSize`
+
+*   is optional
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxcacheditemsize.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxCachedItemSize")
+
+### maxCachedItemSize Type
+
+`string`
+
+### maxCachedItemSize Constraints
+
+**pattern**: the string must match the following regular expression: 
+
+```regexp
+^[0-9]+(B|K|M|G|T)$
+```
+
+[try pattern](https://regexr.com/?expression=%5E%5B0-9%5D%2B\(B%7CK%7CM%7CG%7CT\)%24 "try regular expression with regexr.com")
+
+## dataObjectSourceByObjectIdTTL
+
+TTL (in seconds) for dataObjectSourceByObjectId cache used when proxying objects of size greater than maxCachedItemSize to the right storage node.
+
+`dataObjectSourceByObjectIdTTL`
+
+*   is optional
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-dataobjectsourcebyobjectidttl.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/dataObjectSourceByObjectIdTTL")
+
+### dataObjectSourceByObjectIdTTL Type
+
+`integer`
+
+### dataObjectSourceByObjectIdTTL Constraints
 
 **minimum**: the value of this number must greater than or equal to: `1`
+
+### dataObjectSourceByObjectIdTTL Default Value
+
+The default value is:
+
+```json
+60
+```

distributor-node/docs/schema/definition-properties-log-properties-console.md

@@ -1,18 +0,0 @@
-## console Type
-
-`string`
-
-## console Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |

distributor-node/docs/schema/definition-properties-log-properties-elastic.md

@@ -1,18 +0,0 @@
-## elastic Type
-
-`string`
-
-## elastic Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |

distributor-node/docs/schema/definition-properties-log.md

@@ -1,110 +0,0 @@
-## log Type
-
-`object` ([Details](definition-properties-log.md))
-
-# log Properties
-
-| Property            | Type     | Required | Nullable       | Defined by                                                                                                                       |
-| :------------------ | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------- |
-| [file](#file)       | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-log-properties-file.md "undefined#/properties/log/properties/file")       |
-| [console](#console) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-log-properties-console.md "undefined#/properties/log/properties/console") |
-| [elastic](#elastic) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-log-properties-elastic.md "undefined#/properties/log/properties/elastic") |
-
-## file
-
-Minimum level of logs written to a file specified in `directories.logs`
-
-`file`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-log-properties-file.md "undefined#/properties/log/properties/file")
-
-### file Type
-
-`string`
-
-### file Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |
-
-## console
-
-Minimum level of logs outputted to a console
-
-`console`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-log-properties-console.md "undefined#/properties/log/properties/console")
-
-### console Type
-
-`string`
-
-### console Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |
-
-## elastic
-
-Minimum level of logs sent to elasticsearch endpoint specified in `endpoints.elasticSearch`
-
-`elastic`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-log-properties-elastic.md "undefined#/properties/log/properties/elastic")
-
-### elastic Type
-
-`string`
-
-### elastic Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |

distributor-node/docs/schema/definition-properties-logs-properties-console-logging-options.md

@@ -0,0 +1,41 @@
+## console Type
+
+`object` ([Console logging options](definition-properties-logs-properties-console-logging-options.md))
+
+# console Properties
+
+| Property        | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                         |
+| :-------------- | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [level](#level) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/console/properties/level") |
+
+## level
+
+Minimum level of logs sent to this output
+
+`level`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/console/properties/level")
+
+### level Type
+
+`string`
+
+### level Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"error"`   |             |
+| `"warn"`    |             |
+| `"info"`    |             |
+| `"http"`    |             |
+| `"verbose"` |             |
+| `"debug"`   |             |
+| `"silly"`   |             |

distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md

@@ -0,0 +1,3 @@
+## endpoint Type
+
+`string`

distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md

@@ -0,0 +1,60 @@
+## elastic Type
+
+`object` ([Elasticsearch logging options](definition-properties-logs-properties-elasticsearch-logging-options.md))
+
+# elastic Properties
+
+| Property              | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                                        |
+| :-------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [level](#level)       | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level")                |
+| [endpoint](#endpoint) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint") |
+
+## level
+
+Minimum level of logs sent to this output
+
+`level`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level")
+
+### level Type
+
+`string`
+
+### level Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"error"`   |             |
+| `"warn"`    |             |
+| `"info"`    |             |
+| `"http"`    |             |
+| `"verbose"` |             |
+| `"debug"`   |             |
+| `"silly"`   |             |
+
+## endpoint
+
+Elastichsearch endpoint to push the logs to (for example: <http://localhost:9200>)
+
+`endpoint`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint")
+
+### endpoint Type
+
+`string`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-archive.md

@@ -0,0 +1,3 @@
+## archive Type
+
+`boolean`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-frequency.md

@@ -0,0 +1,22 @@
+## frequency Type
+
+`string`
+
+## frequency Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"yearly"`  |             |
+| `"monthly"` |             |
+| `"daily"`   |             |
+| `"hourly"`  |             |
+
+## frequency Default Value
+
+The default value is:
+
+```json
+"daily"
+```

distributor-node/docs/schema/definition-properties-log-properties-file.md → distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-level.md

@@ -1,8 +1,8 @@
-## file Type
+## level Type
 
 `string`
 
-## file Constraints
+## level Constraints
 
 **enum**: the value of this property must be equal to one of the following values:
 
@@ -15,4 +15,3 @@
 | `"verbose"` |             |
 | `"debug"`   |             |
 | `"silly"`   |             |
-| `"off"`     |             |

distributor-node/docs/schema/definition-properties-limits-properties-outboundrequeststimeout.md → distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-maxfiles.md

@@ -1,7 +1,7 @@
-## outboundRequestsTimeout Type
+## maxFiles Type
 
 `integer`
 
-## outboundRequestsTimeout Constraints
+## maxFiles Constraints
 
 **minimum**: the value of this number must greater than or equal to: `1`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-maxsize.md

@@ -0,0 +1,7 @@
+## maxSize Type
+
+`integer`
+
+## maxSize Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1024`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-path.md

@@ -0,0 +1,3 @@
+## path Type
+
+`string`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options.md

@@ -0,0 +1,163 @@
+## file Type
+
+`object` ([File logging options](definition-properties-logs-properties-file-logging-options.md))
+
+# file Properties
+
+| Property                | Type      | Required | Nullable       | Defined by                                                                                                                                                                                                              |
+| :---------------------- | :-------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [level](#level)         | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/level")         |
+| [path](#path)           | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-path.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/path")           |
+| [maxFiles](#maxfiles)   | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-maxfiles.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/maxFiles")   |
+| [maxSize](#maxsize)     | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-maxsize.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/maxSize")     |
+| [frequency](#frequency) | `string`  | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-frequency.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/frequency") |
+| [archive](#archive)     | `boolean` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-archive.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/archive")     |
+
+## level
+
+Minimum level of logs sent to this output
+
+`level`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/level")
+
+### level Type
+
+`string`
+
+### level Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"error"`   |             |
+| `"warn"`    |             |
+| `"info"`    |             |
+| `"http"`    |             |
+| `"verbose"` |             |
+| `"debug"`   |             |
+| `"silly"`   |             |
+
+## path
+
+Path where the logs will be stored (absolute or relative to config file)
+
+`path`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-path.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/path")
+
+### path Type
+
+`string`
+
+## maxFiles
+
+Maximum number of log files to store
+
+`maxFiles`
+
+*   is optional
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-maxfiles.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/maxFiles")
+
+### maxFiles Type
+
+`integer`
+
+### maxFiles Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
+
+## maxSize
+
+Maximum size of a single log file in bytes
+
+`maxSize`
+
+*   is optional
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-maxsize.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/maxSize")
+
+### maxSize Type
+
+`integer`
+
+### maxSize Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1024`
+
+## frequency
+
+The frequency of creating new log files (regardless of maxSize)
+
+`frequency`
+
+*   is optional
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-frequency.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/frequency")
+
+### frequency Type
+
+`string`
+
+### frequency Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"yearly"`  |             |
+| `"monthly"` |             |
+| `"daily"`   |             |
+| `"hourly"`  |             |
+
+### frequency Default Value
+
+The default value is:
+
+```json
+"daily"
+```
+
+## archive
+
+Whether to archive old logs
+
+`archive`
+
+*   is optional
+
+*   Type: `boolean`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-archive.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/archive")
+
+### archive Type
+
+`boolean`

distributor-node/docs/schema/definition-properties-logs.md

@@ -0,0 +1,65 @@
+## logs Type
+
+`object` ([Details](definition-properties-logs.md))
+
+# logs Properties
+
+| Property            | Type     | Required | Nullable       | Defined by                                                                                                                                                                                |
+| :------------------ | :------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [file](#file)       | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file")             |
+| [console](#console) | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-console-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/console")       |
+| [elastic](#elastic) | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic") |
+
+## file
+
+
+
+`file`
+
+*   is optional
+
+*   Type: `object` ([File logging options](definition-properties-logs-properties-file-logging-options.md))
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file")
+
+### file Type
+
+`object` ([File logging options](definition-properties-logs-properties-file-logging-options.md))
+
+## console
+
+
+
+`console`
+
+*   is optional
+
+*   Type: `object` ([Console logging options](definition-properties-logs-properties-console-logging-options.md))
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-console-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/console")
+
+### console Type
+
+`object` ([Console logging options](definition-properties-logs-properties-console-logging-options.md))
+
+## elastic
+
+
+
+`elastic`
+
+*   is optional
+
+*   Type: `object` ([Elasticsearch logging options](definition-properties-logs-properties-elasticsearch-logging-options.md))
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic")
+
+### elastic Type
+
+`object` ([Elasticsearch logging options](definition-properties-logs-properties-elasticsearch-logging-options.md))

distributor-node/docs/schema/definition-properties-operatorapi-properties-hmacsecret.md

@@ -0,0 +1,3 @@
+## hmacSecret Type
+
+`string`

distributor-node/docs/schema/definition-properties-operatorapi.md

@@ -0,0 +1,50 @@
+## operatorApi Type
+
+`object` ([Details](definition-properties-operatorapi.md))
+
+# operatorApi Properties
+
+| Property                  | Type      | Required | Nullable       | Defined by                                                                                                                                                                              |
+| :------------------------ | :-------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [port](#port)             | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-operatorapi-properties-port.md "https://joystream.org/schemas/argus/config#/properties/operatorApi/properties/port")             |
+| [hmacSecret](#hmacsecret) | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-operatorapi-properties-hmacsecret.md "https://joystream.org/schemas/argus/config#/properties/operatorApi/properties/hmacSecret") |
+
+## port
+
+Distributor node operator api port
+
+`port`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-operatorapi-properties-port.md "https://joystream.org/schemas/argus/config#/properties/operatorApi/properties/port")
+
+### port Type
+
+`integer`
+
+### port Constraints
+
+**minimum**: the value of this number must greater than or equal to: `0`
+
+## hmacSecret
+
+HMAC (HS256) secret key used for JWT authorization
+
+`hmacSecret`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-operatorapi-properties-hmacsecret.md "https://joystream.org/schemas/argus/config#/properties/operatorApi/properties/hmacSecret")
+
+### hmacSecret Type
+
+`string`

distributor-node/docs/schema/definition-properties-publicapi-properties-port.md

@@ -0,0 +1,7 @@
+## port Type
+
+`integer`
+
+## port Constraints
+
+**minimum**: the value of this number must greater than or equal to: `0`

distributor-node/docs/schema/definition-properties-publicapi.md

@@ -0,0 +1,31 @@
+## publicApi Type
+
+`object` ([Details](definition-properties-publicapi.md))
+
+# publicApi Properties
+
+| Property      | Type      | Required | Nullable       | Defined by                                                                                                                                                              |
+| :------------ | :-------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [port](#port) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-publicapi-properties-port.md "https://joystream.org/schemas/argus/config#/properties/publicApi/properties/port") |
+
+## port
+
+Distributor node public api port
+
+`port`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-publicapi-properties-port.md "https://joystream.org/schemas/argus/config#/properties/publicApi/properties/port")
+
+### port Type
+
+`integer`
+
+### port Constraints
+
+**minimum**: the value of this number must greater than or equal to: `0`

distributor-node/docs/schema/definition.md

@@ -4,18 +4,19 @@
 
 # Distributor node configuration Properties
 
-| Property                    | Type      | Required | Nullable       | Defined by                                                                                                 |
-| :-------------------------- | :-------- | :------- | :------------- | :--------------------------------------------------------------------------------------------------------- |
-| [id](#id)                   | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-id.md "undefined#/properties/id")                   |
-| [endpoints](#endpoints)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints.md "undefined#/properties/endpoints")     |
-| [directories](#directories) | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-directories.md "undefined#/properties/directories") |
-| [log](#log)                 | `object`  | Optional | cannot be null | [Distributor node configuration](definition-properties-log.md "undefined#/properties/log")                 |
-| [limits](#limits)           | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-limits.md "undefined#/properties/limits")           |
-| [intervals](#intervals)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-intervals.md "undefined#/properties/intervals")     |
-| [port](#port)               | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-port.md "undefined#/properties/port")               |
-| [keys](#keys)               | `array`   | Required | cannot be null | [Distributor node configuration](definition-properties-keys.md "undefined#/properties/keys")               |
-| [buckets](#buckets)         | Merged    | Required | cannot be null | [Distributor node configuration](definition-properties-buckets.md "undefined#/properties/buckets")         |
-| [workerId](#workerid)       | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-workerid.md "undefined#/properties/workerId")       |
+| Property                    | Type      | Required | Nullable       | Defined by                                                                                                                                  |
+| :-------------------------- | :-------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------ |
+| [id](#id)                   | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-id.md "https://joystream.org/schemas/argus/config#/properties/id")                   |
+| [endpoints](#endpoints)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints.md "https://joystream.org/schemas/argus/config#/properties/endpoints")     |
+| [directories](#directories) | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-directories.md "https://joystream.org/schemas/argus/config#/properties/directories") |
+| [logs](#logs)               | `object`  | Optional | cannot be null | [Distributor node configuration](definition-properties-logs.md "https://joystream.org/schemas/argus/config#/properties/logs")               |
+| [limits](#limits)           | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-limits.md "https://joystream.org/schemas/argus/config#/properties/limits")           |
+| [intervals](#intervals)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-intervals.md "https://joystream.org/schemas/argus/config#/properties/intervals")     |
+| [publicApi](#publicapi)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-publicapi.md "https://joystream.org/schemas/argus/config#/properties/publicApi")     |
+| [operatorApi](#operatorapi) | `object`  | Optional | cannot be null | [Distributor node configuration](definition-properties-operatorapi.md "https://joystream.org/schemas/argus/config#/properties/operatorApi") |
+| [keys](#keys)               | `array`   | Optional | cannot be null | [Distributor node configuration](definition-properties-keys.md "https://joystream.org/schemas/argus/config#/properties/keys")               |
+| [buckets](#buckets)         | `array`   | Optional | cannot be null | [Distributor node configuration](definition-properties-bucket-ids.md "https://joystream.org/schemas/argus/config#/properties/buckets")      |
+| [workerId](#workerid)       | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-workerid.md "https://joystream.org/schemas/argus/config#/properties/workerId")       |
 
 ## id
 
@@ -29,7 +30,7 @@ Node identifier used when sending elasticsearch logs and exposed on /status endp
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-id.md "undefined#/properties/id")
+*   defined in: [Distributor node configuration](definition-properties-id.md "https://joystream.org/schemas/argus/config#/properties/id")
 
 ### id Type
 
@@ -51,7 +52,7 @@ Specifies external endpoints that the distributor node will connect to
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-endpoints.md "undefined#/properties/endpoints")
+*   defined in: [Distributor node configuration](definition-properties-endpoints.md "https://joystream.org/schemas/argus/config#/properties/endpoints")
 
 ### endpoints Type
 
@@ -69,29 +70,29 @@ Specifies paths where node's data will be stored
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-directories.md "undefined#/properties/directories")
+*   defined in: [Distributor node configuration](definition-properties-directories.md "https://joystream.org/schemas/argus/config#/properties/directories")
 
 ### directories Type
 
 `object` ([Details](definition-properties-directories.md))
 
-## log
+## logs
 
-Specifies minimum log levels by supported log outputs
+Specifies the logging configuration
 
-`log`
+`logs`
 
 *   is optional
 
-*   Type: `object` ([Details](definition-properties-log.md))
+*   Type: `object` ([Details](definition-properties-logs.md))
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-log.md "undefined#/properties/log")
+*   defined in: [Distributor node configuration](definition-properties-logs.md "https://joystream.org/schemas/argus/config#/properties/logs")
 
-### log Type
+### logs Type
 
-`object` ([Details](definition-properties-log.md))
+`object` ([Details](definition-properties-logs.md))
 
 ## limits
 
@@ -105,7 +106,7 @@ Specifies node limits w\.r.t. storage, outbound connections etc.
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits.md "undefined#/properties/limits")
+*   defined in: [Distributor node configuration](definition-properties-limits.md "https://joystream.org/schemas/argus/config#/properties/limits")
 
 ### limits Type
 
@@ -123,33 +124,47 @@ Specifies how often periodic tasks (for example cache cleanup) are executed by t
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-intervals.md "undefined#/properties/intervals")
+*   defined in: [Distributor node configuration](definition-properties-intervals.md "https://joystream.org/schemas/argus/config#/properties/intervals")
 
 ### intervals Type
 
 `object` ([Details](definition-properties-intervals.md))
 
-## port
+## publicApi
 
-Distributor node http server port
+Public api configuration
 
-`port`
+`publicApi`
 
 *   is required
 
-*   Type: `integer`
+*   Type: `object` ([Details](definition-properties-publicapi.md))
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-port.md "undefined#/properties/port")
+*   defined in: [Distributor node configuration](definition-properties-publicapi.md "https://joystream.org/schemas/argus/config#/properties/publicApi")
 
-### port Type
+### publicApi Type
 
-`integer`
+`object` ([Details](definition-properties-publicapi.md))
 
-### port Constraints
+## operatorApi
 
-**minimum**: the value of this number must greater than or equal to: `0`
+Operator api configuration
+
+`operatorApi`
+
+*   is optional
+
+*   Type: `object` ([Details](definition-properties-operatorapi.md))
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-operatorapi.md "https://joystream.org/schemas/argus/config#/properties/operatorApi")
+
+### operatorApi Type
+
+`object` ([Details](definition-properties-operatorapi.md))
 
 ## keys
 
@@ -157,13 +172,13 @@ Specifies the keys available within distributor node CLI.
 
 `keys`
 
-*   is required
+*   is optional
 
 *   Type: an array of merged types ([Details](definition-properties-keys-items.md))
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys.md "undefined#/properties/keys")
+*   defined in: [Distributor node configuration](definition-properties-keys.md "https://joystream.org/schemas/argus/config#/properties/keys")
 
 ### keys Type
 
@@ -175,27 +190,27 @@ an array of merged types ([Details](definition-properties-keys-items.md))
 
 ## buckets
 
-Specifies the buckets distributed by the node
+Set of bucket ids distributed by the node. If not specified, all buckets currently assigned to worker specified in `config.workerId` will be distributed.
 
 `buckets`
 
-*   is required
+*   is optional
 
-*   Type: merged type ([Details](definition-properties-buckets.md))
+*   Type: `integer[]`
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-buckets.md "undefined#/properties/buckets")
+*   defined in: [Distributor node configuration](definition-properties-bucket-ids.md "https://joystream.org/schemas/argus/config#/properties/buckets")
 
 ### buckets Type
 
-merged type ([Details](definition-properties-buckets.md))
+`integer[]`
 
-one (and only one) of
+### buckets Constraints
 
-*   [Bucket ids](definition-properties-buckets-oneof-bucket-ids.md "check type definition")
+**minimum number of items**: the minimum number of items for this array is: `1`
 
-*   [All buckets](definition-properties-buckets-oneof-all-buckets.md "check type definition")
+**unique items**: all items in this array must be unique. Duplicates are not allowed.
 
 ## workerId
 
@@ -203,13 +218,13 @@ ID of the node operator (distribution working group worker)
 
 `workerId`
 
-*   is required
+*   is optional
 
 *   Type: `integer`
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-workerid.md "undefined#/properties/workerId")
+*   defined in: [Distributor node configuration](definition-properties-workerid.md "https://joystream.org/schemas/argus/config#/properties/workerId")
 
 ### workerId Type
 

distributor-node/package.json

@@ -14,7 +14,7 @@
     "@joystream/types": "^0.17.0",
     "@oclif/command": "^1",
     "@oclif/config": "^1",
-    "@oclif/plugin-help": "^3.2.4",
+    "@oclif/plugin-help": "^3",
     "ajv": "^7",
     "axios": "^0.21.1",
     "blake3-wasm": "^2.1.5",
@@ -41,6 +41,10 @@
     "tslib": "^1",
     "winston": "^3.3.3",
     "winston-elasticsearch": "^0.15.8",
+    "url-join": "^4.0.1",
+    "@types/url-join": "^4.0.1",
+    "winston-daily-rotate-file": "^4.5.5",
+    "jsonwebtoken": "^8.5.1",
     "yaml": "^1.10.2"
   },
   "devDependencies": {
@@ -76,6 +80,10 @@
   "engines": {
     "node": ">=14.16.1"
   },
+  "volta": {
+    "node": "14.16.1",
+    "yarn": "1.22.4"
+  },
   "files": [
     "/bin",
     "/lib",
@@ -101,6 +109,9 @@
       "operator": {
         "description": "Commands for performing node operator (Distribution Working Group worker) on-chain duties (like accepting bucket invitations, setting node metadata)"
       },
+      "node": {
+        "description": "Commands for interacting with a running distributor node through OperatorApi"
+      },
       "dev": {
         "description": "Developer utility commands"
       }
@@ -118,14 +129,17 @@
     "version": "generate:docs:cli && git add docs/cli/*",
     "generate:types:json-schema": "yarn ts-node ./src/schemas/scripts/generateTypes.ts",
     "generate:types:graphql": "yarn graphql-codegen -c ./src/services/networking/query-node/codegen.yml",
-    "generate:types:openapi": "yarn openapi-typescript ./src/api-spec/openapi.yml -o ./src/types/generated/OpenApi.ts -c ../prettierrc.js",
-    "generate:types:all": "yarn generate:types:json-schema && yarn generate:types:graphql && yarn generate:types:openapi",
+    "generate:types:public-api": "yarn openapi-typescript ./src/api-spec/public.yml -o ./src/types/generated/PublicApi.ts -c ../prettierrc.js",
+    "generate:types:operator-api": "yarn openapi-typescript ./src/api-spec/operator.yml -o ./src/types/generated/OperatorApi.ts -c ../prettierrc.js",
+    "generate:types:api": "yarn generate:types:public-api && yarn generate:types:operator-api",
+    "generate:types:all": "yarn generate:types:json-schema && yarn generate:types:graphql && yarn generate:types:api",
     "generate:api:storage-node": "yarn openapi-generator-cli generate -i ../storage-node-v2/src/api-spec/openapi.yaml -g typescript-axios -o ./src/services/networking/storage-node/generated",
-    "generate:api:distributor-node": "yarn openapi-generator-cli generate -i ./src/api-spec/openapi.yml -g typescript-axios -o ./src/services/networking/distributor-node/generated",
-    "generate:api:all": "yarn generate:api:storage-node && yarn generate:api:distributor-node",
+    "generate:api:all": "yarn generate:api:storage-node",
     "generate:docs:cli": "yarn oclif-dev readme --multi --dir ./docs/commands",
     "generate:docs:config": "yarn ts-node --transpile-only ./src/schemas/scripts/generateConfigDoc.ts",
-    "generate:docs:api": "yarn widdershins ./src/api-spec/openapi.yml --language_tabs javascript:JavaScript shell:Shell -o ./docs/api/index.md -u ./docs/api/templates",
+    "generate:docs:public-api": "yarn widdershins ./src/api-spec/public.yml --language_tabs javascript:JavaScript shell:Shell -o ./docs/api/public/index.md -u ./docs/api/templates",
+    "generate:docs:operator-api": "yarn widdershins ./src/api-spec/operator.yml --language_tabs javascript:JavaScript shell:Shell -o ./docs/api/operator/index.md -u ./docs/api/templates",
+    "generate:docs:api": "yarn generate:docs:public-api && yarn generate:docs:operator-api",
     "generate:docs:toc": "yarn md-magic --path ./docs/**/*.md",
     "generate:docs:all": "yarn generate:docs:cli && yarn generate:docs:config && yarn generate:docs:api && yarn generate:docs:toc",
     "generate:all": "yarn generate:types:all && yarn generate:api:all && yarn generate:docs:all",

distributor-node/scripts/init-bucket.sh

@@ -15,3 +15,6 @@ ${CLI} leader:update-bag -b static:council -f ${FAMILY_ID} -a ${BUCKET_ID}
 ${CLI} leader:update-bucket-mode -f ${FAMILY_ID} -B ${BUCKET_ID} --mode on
 ${CLI} leader:invite-bucket-operator -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} operator:accept-invitation -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
+${CLI} operator:set-metadata -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0 -e http://localhost:3334
+${CLI} leader:update-dynamic-bag-policy -t Channel -p ${FAMILY_ID}:1
+${CLI} leader:update-dynamic-bag-policy -t Member -p ${FAMILY_ID}:1

distributor-node/scripts/test-commands.sh

@@ -21,6 +21,7 @@ ${CLI} leader:update-bag -b static:wg:gateway -f ${FAMILY_ID} -a ${BUCKET_ID}
 ${CLI} leader:update-bag -b static:wg:distribution -f ${FAMILY_ID} -a ${BUCKET_ID}
 ${CLI} leader:update-bucket-status -f ${FAMILY_ID} -B ${BUCKET_ID}  --acceptingBags yes
 ${CLI} leader:update-bucket-mode -f ${FAMILY_ID} -B ${BUCKET_ID} --mode on
+${CLI} leader:update-dynamic-bag-policy -t Channel -p ${FAMILY_ID}:5
 ${CLI} leader:update-dynamic-bag-policy -t Member -p ${FAMILY_ID}:5
 ${CLI} leader:update-dynamic-bag-policy -t Member
 ${CLI} leader:invite-bucket-operator -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
@@ -28,11 +29,13 @@ ${CLI} leader:cancel-invitation -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} leader:invite-bucket-operator -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} operator:accept-invitation -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} operator:set-metadata -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0 -i ./data/operator-metadata.json
-${CLI} leader:remove-bucket-operator -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} leader:set-bucket-family-metadata -f ${FAMILY_ID} -i ./data/family-metadata.json
 
-# Deletion commands tested separately, since bucket operator removal is not yet supported
+# Deletion commands tested separately
 FAMILY_TO_DELETE_ID=`${CLI} leader:create-bucket-family`
 BUCKET_TO_DELETE_ID=`${CLI} leader:create-bucket -f ${FAMILY_TO_DELETE_ID} -a yes`
+${CLI} leader:invite-bucket-operator -f ${FAMILY_TO_DELETE_ID} -B ${BUCKET_TO_DELETE_ID} -w 0
+${CLI} operator:accept-invitation -f ${FAMILY_TO_DELETE_ID} -B ${BUCKET_TO_DELETE_ID} -w 0
+${CLI} leader:remove-bucket-operator -f ${FAMILY_TO_DELETE_ID} -B ${BUCKET_TO_DELETE_ID} -w 0
 ${CLI} leader:delete-bucket -f ${FAMILY_TO_DELETE_ID} -B ${BUCKET_TO_DELETE_ID}
 ${CLI} leader:delete-bucket-family -f ${FAMILY_TO_DELETE_ID}

distributor-node/src/api-spec/operator.yml

@@ -0,0 +1,119 @@
+openapi: 3.0.3
+info:
+  title: Distributor node operator API
+  description: Distributor node operator API
+  contact:
+    email: info@joystream.org
+  license:
+    name: GPL-3.0-only
+    url: https://spdx.org/licenses/GPL-3.0-only.html
+  version: 0.1.0
+servers:
+  - url: http://localhost:3335/api/v1/
+
+paths:
+  /stop-api:
+    post:
+      operationId: operator.stopApi
+      description: Turns off the public api.
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        409:
+          description: Already stopped
+        500:
+          description: Unexpected server error
+  /start-api:
+    post:
+      operationId: operator.startApi
+      description: Turns on the public api.
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        409:
+          description: Already started
+        500:
+          description: Unexpected server error
+  /shutdown:
+    post:
+      operationId: operator.shutdown
+      description: Shuts down the node.
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        409:
+          description: Already shutting down
+        500:
+          description: Unexpected server error
+  /set-worker:
+    post:
+      operationId: operator.setWorker
+      description: Updates the operator worker id.
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/SetWorkerOperation'
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        500:
+          description: Unexpected server error
+  /set-buckets:
+    post:
+      operationId: operator.setBuckets
+      description: Updates buckets supported by the node.
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/SetBucketsOperation'
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        500:
+          description: Unexpected server error
+
+components:
+  securitySchemes:
+    OperatorAuth:
+      type: http
+      scheme: bearer
+      bearerFormat:
+        "JWT signed with HMAC (HS256) secret key specified in distributor node's `config.operator.hmacSecret`.
+        The payload should include:
+        - `reqBody` - content of the request body
+        - `reqUrl` - request url (only pathname + query string, without origin. For example: `/api/v1/set-buckets`)"
+  schemas:
+    SetWorkerOperation:
+      type: object
+      required:
+        - workerId
+      properties:
+        workerId:
+          type: integer
+          minimum: 0
+    SetBucketsOperation:
+      type: object
+      properties:
+        buckets:
+          description: 'Set of bucket ids to be distributed by the node.
+            If not provided - all buckets assigned to currently configured worker will be distributed.'
+          type: array
+          minItems: 1
+          items:
+            type: integer
+            minimum: 0
+
+security:
+  - OperatorAuth: []

distributor-node/src/api-spec/openapi.yml → distributor-node/src/api-spec/public.yml

@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
-  title: Distributor node API
-  description: Distributor node API
+  title: Distributor node public API
+  description: Distributor node public API
   contact:
     email: info@joystream.org
   license:
@@ -9,22 +9,16 @@ info:
     url: https://spdx.org/licenses/GPL-3.0-only.html
   version: 0.1.0
 externalDocs:
-  description: Distributor node API
+  description: Distributor node public API
   url: https://github.com/Joystream/joystream/issues/2224
 servers:
   - url: http://localhost:3334/api/v1/
 
-tags:
-  - name: public
-    description: Public distributor node API
-
 paths:
   /status:
     get:
       operationId: public.status
       description: Returns json object describing current node status.
-      tags:
-        - public
       responses:
         200:
           description: OK
@@ -38,8 +32,6 @@ paths:
     get:
       operationId: public.buckets
       description: Returns list of distributed buckets
-      tags:
-        - public
       responses:
         200:
           description: OK
@@ -49,12 +41,10 @@ paths:
                 $ref: '#/components/schemas/BucketsResponse'
         500:
           description: Unexpected server error
-  /asset/{objectId}:
+  /assets/{objectId}:
     head:
       operationId: public.assetHead
       description: Returns asset response headers (cache status, content type and/or length, accepted ranges etc.)
-      tags:
-        - public
       parameters:
         - $ref: '#/components/parameters/ObjectId'
       responses:
@@ -72,8 +62,6 @@ paths:
     get:
       operationId: public.asset
       description: Returns a media file.
-      tags:
-        - public
       parameters:
         - $ref: '#/components/parameters/ObjectId'
       responses:
@@ -203,7 +191,6 @@ components:
           properties:
             bucketIds:
               type: array
-              minItems: 1
               items:
                 type: integer
                 minimum: 0

distributor-node/src/app/index.ts

@@ -1,31 +1,37 @@
-import { ReadonlyConfig } from '../types'
+import { Config } from '../types'
 import { NetworkingService } from '../services/networking'
 import { LoggingService } from '../services/logging'
 import { StateCacheService } from '../services/cache/StateCacheService'
 import { ContentService } from '../services/content/ContentService'
-import { ServerService } from '../services/server/ServerService'
 import { Logger } from 'winston'
 import fs from 'fs'
 import nodeCleanup from 'node-cleanup'
 import { AppIntervals } from '../types/app'
+import { PublicApiService } from '../services/httpApi/PublicApiService'
+import { OperatorApiService } from '../services/httpApi/OperatorApiService'
 
 export class App {
-  private config: ReadonlyConfig
+  private config: Config
   private content: ContentService
   private stateCache: StateCacheService
   private networking: NetworkingService
-  private server: ServerService
+  private publicApi: PublicApiService
+  private operatorApi: OperatorApiService | undefined
   private logging: LoggingService
   private logger: Logger
   private intervals: AppIntervals | undefined
+  private isStopping = false
 
-  constructor(config: ReadonlyConfig) {
+  constructor(config: Config) {
     this.config = config
     this.logging = LoggingService.withAppConfig(config)
     this.stateCache = new StateCacheService(config, this.logging)
     this.networking = new NetworkingService(config, this.stateCache, this.logging)
     this.content = new ContentService(config, this.logging, this.networking, this.stateCache)
-    this.server = new ServerService(config, this.stateCache, this.content, this.logging, this.networking)
+    this.publicApi = new PublicApiService(config, this.stateCache, this.content, this.logging, this.networking)
+    if (this.config.operatorApi) {
+      this.operatorApi = new OperatorApiService(config, this, this.logging, this.publicApi)
+    }
     this.logger = this.logging.createLogger('App')
   }
 
@@ -46,30 +52,32 @@ export class App {
     }
   }
 
-  private checkConfigDirectories(): void {
-    Object.entries(this.config.directories).forEach(([name, path]) => {
-      if (path === undefined) {
-        return
-      }
-      const dirInfo = `${name} directory (${path})`
-      if (!fs.existsSync(path)) {
-        try {
-          fs.mkdirSync(path, { recursive: true })
-        } catch (e) {
-          throw new Error(`${dirInfo} doesn't exist and cannot be created!`)
-        }
-      }
-      try {
-        fs.accessSync(path, fs.constants.R_OK)
-      } catch (e) {
-        throw new Error(`${dirInfo} is not readable`)
-      }
+  private checkConfigDir(name: string, path: string): void {
+    const dirInfo = `${name} directory (${path})`
+    if (!fs.existsSync(path)) {
       try {
-        fs.accessSync(path, fs.constants.W_OK)
+        fs.mkdirSync(path, { recursive: true })
       } catch (e) {
-        throw new Error(`${dirInfo} is not writable`)
+        throw new Error(`${dirInfo} doesn't exist and cannot be created!`)
       }
-    })
+    }
+    try {
+      fs.accessSync(path, fs.constants.R_OK)
+    } catch (e) {
+      throw new Error(`${dirInfo} is not readable`)
+    }
+    try {
+      fs.accessSync(path, fs.constants.W_OK)
+    } catch (e) {
+      throw new Error(`${dirInfo} is not writable`)
+    }
+  }
+
+  private checkConfigDirectories(): void {
+    Object.entries(this.config.directories).forEach(([name, path]) => this.checkConfigDir(name, path))
+    if (this.config.logs?.file) {
+      this.checkConfigDir('logs.file.path', this.config.logs.file.path)
+    }
   }
 
   public async start(): Promise<void> {
@@ -79,7 +87,8 @@ export class App {
       this.stateCache.load()
       await this.content.startupInit()
       this.setIntervals()
-      this.server.start()
+      this.publicApi.start()
+      this.operatorApi?.start()
     } catch (err) {
       this.logger.error('Node initialization failed!', { err })
       process.exit(-1)
@@ -87,6 +96,21 @@ export class App {
     nodeCleanup(this.exitHandler.bind(this))
   }
 
+  public stop(timeoutSec?: number): boolean {
+    if (this.isStopping) {
+      return false
+    }
+    this.logger.info(`Stopping the app${timeoutSec ? ` in ${timeoutSec} sec...` : ''}`)
+    this.isStopping = true
+    if (timeoutSec) {
+      setTimeout(() => process.kill(process.pid, 'SIGINT'), timeoutSec * 1000)
+    } else {
+      process.kill(process.pid, 'SIGINT')
+    }
+
+    return true
+  }
+
   private async exitGracefully(): Promise<void> {
     // Async exit handler - ideally should not take more than 10 sec
     // We can try to wait until some pending downloads are finished here etc.
@@ -125,10 +149,15 @@ export class App {
     this.logger.info('Exiting...')
     // Clear intervals
     this.clearIntervals()
-    // Stop the server
-    this.server.stop()
+    // Stop the http apis
+    this.publicApi.stop()
+    this.operatorApi?.stop()
     // Save cache
-    this.stateCache.saveSync()
+    try {
+      this.stateCache.saveSync()
+    } catch (err) {
+      this.logger.error('Failed to save the cache state on exit!', { err })
+    }
     if (signal) {
       // Async exit can be executed
       this.exitGracefully()

distributor-node/src/command-base/accounts.ts

@@ -1,7 +1,7 @@
 import ApiCommandBase from './api'
 import { AccountId } from '@polkadot/types/interfaces'
 import { Keyring } from '@polkadot/api'
-import { KeyringInstance, KeyringOptions, KeyringPair } from '@polkadot/keyring/types'
+import { KeyringInstance, KeyringOptions, KeyringPair, KeyringPair$Json } from '@polkadot/keyring/types'
 import { CLIError } from '@oclif/errors'
 import ExitCodes from './ExitCodes'
 import fs from 'fs'
@@ -30,7 +30,7 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
         exit: ExitCodes.InvalidFile,
       })
     }
-    let accountJsonObj: any
+    let accountJsonObj: unknown
     try {
       accountJsonObj = require(jsonBackupFilePath)
     } catch (e) {
@@ -48,8 +48,8 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
     let account: KeyringPair
     try {
       // Try adding and retrieving the keys in order to validate that the backup file is correct
-      keyring.addFromJson(accountJsonObj)
-      account = keyring.getPair(accountJsonObj.address)
+      keyring.addFromJson(accountJsonObj as KeyringPair$Json)
+      account = keyring.getPair((accountJsonObj as KeyringPair$Json).address)
     } catch (e) {
       throw new CLIError(`Keypair backup json file is is not valid: ${jsonBackupFilePath}`, {
         exit: ExitCodes.InvalidFile,
@@ -124,7 +124,7 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
 
   initKeyring(): void {
     this.keyring = new Keyring(KEYRING_OPTIONS)
-    this.appConfig.keys.forEach((keyData) => {
+    this.appConfig.keys?.forEach((keyData) => {
       if ('suri' in keyData) {
         this.keyring.addFromUri(keyData.suri, undefined, keyData.type)
       }

distributor-node/src/command-base/default.ts

@@ -22,8 +22,8 @@ export const flags = {
   }),
   bagId: oclifFlags.build({
     parse: (value: string) => {
-      const parser = new BagIdParserService()
-      return parser.parseBagId(value)
+      const parser = new BagIdParserService(value)
+      return parser.parse()
     },
     description: `Bag ID. Format: {bag_type}:{sub_type}:{id}.
     - Bag types: 'static', 'dynamic'
@@ -61,8 +61,8 @@ export default abstract class DefaultCommandBase extends Command {
 
   async init(): Promise<void> {
     const { configPath, yes } = this.parse(this.constructor as typeof DefaultCommandBase).flags
-    const configParser = new ConfigParserService()
-    this.appConfig = configParser.loadConfing(configPath) as ReadonlyConfig
+    const configParser = new ConfigParserService(configPath)
+    this.appConfig = configParser.parse() as ReadonlyConfig
     this.logging = LoggingService.withCLIConfig()
     this.logger = this.logging.createLogger('CLI')
     this.autoConfirm = !!(process.env.AUTO_CONFIRM === 'true' || parseInt(process.env.AUTO_CONFIRM || '') || yes)
@@ -89,11 +89,11 @@ export default abstract class DefaultCommandBase extends Command {
     }
   }
 
-  async finally(err: any): Promise<void> {
+  async finally(err: unknown): Promise<void> {
     if (!err) this.exit(ExitCodes.OK)
     if (process.env.DEBUG === 'true') {
       console.error(err)
     }
-    super.finally(err)
+    super.finally(err as Error)
   }
 }

distributor-node/src/command-base/node.ts

@@ -0,0 +1,58 @@
+import axios from 'axios'
+import urljoin from 'url-join'
+import DefaultCommandBase, { flags } from './default'
+import jwt from 'jsonwebtoken'
+import ExitCodes from './ExitCodes'
+
+export default abstract class NodeCommandBase extends DefaultCommandBase {
+  static flags = {
+    url: flags.string({
+      char: 'u',
+      description: 'Distributor node operator api base url (ie. http://localhost:3335)',
+      required: true,
+    }),
+    secret: flags.string({
+      char: 's',
+      description: 'HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)',
+      required: false,
+    }),
+    ...DefaultCommandBase.flags,
+  }
+
+  protected abstract reqUrl(): string
+
+  protected reqBody(): Record<string, unknown> {
+    return {}
+  }
+
+  async run(): Promise<void> {
+    const { url, secret } = this.parse(this.constructor as typeof NodeCommandBase).flags
+
+    const hmacSecret = secret || this.appConfig.operatorApi?.hmacSecret
+
+    if (!hmacSecret) {
+      this.error('No --secret was provided and no config.operatorApi.hmacSecret is set!', {
+        exit: ExitCodes.InvalidInput,
+      })
+    }
+
+    const reqUrl = this.reqUrl()
+    const reqBody = this.reqBody()
+    const payload = { reqUrl, reqBody }
+    try {
+      await axios.post(urljoin(url, reqUrl), reqBody, {
+        headers: {
+          authorization: `bearer ${jwt.sign(payload, hmacSecret, { expiresIn: 60 })}`,
+        },
+      })
+      this.log('Request successful')
+    } catch (e) {
+      if (axios.isAxiosError(e)) {
+        this.error(`Request failed: ${e.response ? JSON.stringify(e.response.data) : e.message}`, {
+          exit: ExitCodes.ApiError,
+        })
+      }
+      this.error(e instanceof Error ? e.message : JSON.stringify(e), { exit: ExitCodes.ApiError })
+    }
+  }
+}

distributor-node/src/commands/dev/batchUpload.ts

@@ -1,18 +1,17 @@
 import AccountsCommandBase from '../../command-base/accounts'
 import DefaultCommandBase, { flags } from '../../command-base/default'
-import { hash } from 'blake3-wasm'
 import { FilesApi, Configuration, TokenRequest } from '../../services/networking/storage-node/generated'
 import { u8aToHex } from '@polkadot/util'
-import * as multihash from 'multihashes'
 import FormData from 'form-data'
 import imgGen from 'js-image-generator'
 import { SubmittableExtrinsic } from '@polkadot/api/types'
 import { BagIdParserService } from '../../services/parsers/BagIdParserService'
 import axios from 'axios'
+import { ContentHash } from '../../services/crypto/ContentHash'
 
 async function generateRandomImage(): Promise<Buffer> {
   return new Promise((resolve, reject) => {
-    imgGen.generateImage(10, 10, 80, function (err: any, image: any) {
+    imgGen.generateImage(10, 10, 80, function (err: unknown, image: { data: Buffer }) {
       if (err) {
         reject(err)
       } else {
@@ -61,7 +60,7 @@ export default class DevBatchUpload extends AccountsCommandBase {
       const batch: [SubmittableExtrinsic<'promise'>, Buffer][] = []
       for (let j = 0; j < batchSize; ++j) {
         const dataObject = await generateRandomImage()
-        const dataHash = multihash.toB58String(multihash.encode(hash(dataObject) as Buffer, 'blake3'))
+        const dataHash = new ContentHash().update(dataObject).digest()
         batch.push([
           api.tx.sudo.sudo(
             api.tx.storage.sudoUploadDataObjects({
@@ -73,7 +72,7 @@ export default class DevBatchUpload extends AccountsCommandBase {
                 },
               ],
               expectedDataSizeFee: dataFee,
-              bagId: new BagIdParserService().parseBagId(bagId),
+              bagId: new BagIdParserService(bagId).parse(),
             })
           ),
           dataObject,
@@ -102,7 +101,7 @@ export default class DevBatchUpload extends AccountsCommandBase {
             signature,
           })
           if (!token) {
-            throw new Error('Recieved empty token!')
+            throw new Error('Received empty token!')
           }
 
           const formData = new FormData()

distributor-node/src/commands/leader/update-dynamic-bag-policy.ts

@@ -2,6 +2,7 @@ import { flags } from '@oclif/command'
 import { DynamicBagTypeKey } from '@joystream/types/storage'
 import AccountsCommandBase from '../../command-base/accounts'
 import DefaultCommandBase from '../../command-base/default'
+import { createType } from '@joystream/types'
 
 export default class LeaderUpdateDynamicBagPolicy extends AccountsCommandBase {
   static description = `Update dynamic bag creation policy (number of buckets by family that should store given dynamic bag type).
@@ -41,11 +42,7 @@ export default class LeaderUpdateDynamicBagPolicy extends AccountsCommandBase {
       await this.getDecodedPair(leadKey),
       this.api.tx.storage.updateFamiliesInDynamicBagCreationPolicy(
         type,
-        // FIXME: https://github.com/polkadot-js/api/pull/3789
-        this.api.createType(
-          'DynamicBagCreationPolicyDistributorFamiliesMap',
-          new Map((policy || []).sort(([keyA], [keyB]) => keyA - keyB))
-        )
+        createType('DynamicBagCreationPolicyDistributorFamiliesMap', new Map(policy))
       )
     )
     this.log('Dynamic bag creation policy succesfully updated!')

distributor-node/src/commands/node/set-buckets.ts

@@ -0,0 +1,41 @@
+import { flags } from '@oclif/command'
+import ExitCodes from '../../command-base/ExitCodes'
+import NodeCommandBase from '../../command-base/node'
+import { SetBucketsOperation } from '../../types'
+
+export default class NodeSetBucketsCommand extends NodeCommandBase {
+  static description = `Send an api request to change the set of buckets distributed by given distributor node.`
+
+  static flags = {
+    all: flags.boolean({
+      char: 'a',
+      description: 'Distribute all buckets belonging to configured worker',
+      exclusive: ['bucketIds'],
+    }),
+    bucketIds: flags.integer({
+      char: 'B',
+      description: 'Set of bucket ids to distribute',
+      exclusive: ['all'],
+      multiple: true,
+    }),
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/set-buckets'
+  }
+
+  protected reqBody(): SetBucketsOperation {
+    const {
+      flags: { all, bucketIds },
+    } = this.parse(NodeSetBucketsCommand)
+    if (!all && !bucketIds) {
+      this.error('You must provide either --bucketIds or --all flag!', { exit: ExitCodes.InvalidInput })
+    }
+    return all
+      ? {}
+      : {
+          buckets: bucketIds,
+        }
+  }
+}

distributor-node/src/commands/node/set-worker.ts

@@ -0,0 +1,29 @@
+import { flags } from '@oclif/command'
+import NodeCommandBase from '../../command-base/node'
+import { SetWorkerOperation } from '../../types'
+
+export default class NodeSetWorkerCommand extends NodeCommandBase {
+  static description = `Send an api request to change workerId assigned to given distributor node instance.`
+
+  static flags = {
+    workerId: flags.integer({
+      char: 'w',
+      description: 'New workerId to set',
+      required: true,
+    }),
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/set-worker'
+  }
+
+  protected reqBody(): SetWorkerOperation {
+    const {
+      flags: { workerId },
+    } = this.parse(NodeSetWorkerCommand)
+    return {
+      workerId,
+    }
+  }
+}

distributor-node/src/commands/node/shutdown.ts

@@ -0,0 +1,17 @@
+import NodeCommandBase from '../../command-base/node'
+
+export default class NodeShutdownCommand extends NodeCommandBase {
+  static description = `Send an api request to shutdown given distributor node.`
+
+  static flags = {
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/shutdown'
+  }
+
+  protected reqBody(): Record<string, unknown> {
+    return {}
+  }
+}

distributor-node/src/commands/node/start-public-api.ts

@@ -0,0 +1,17 @@
+import NodeCommandBase from '../../command-base/node'
+
+export default class NodeStartPublicApiCommand extends NodeCommandBase {
+  static description = `Send an api request to start public api of given distributor node.`
+
+  static flags = {
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/start-api'
+  }
+
+  protected reqBody(): Record<string, unknown> {
+    return {}
+  }
+}

distributor-node/src/commands/node/stop-public-api.ts

@@ -0,0 +1,17 @@
+import NodeCommandBase from '../../command-base/node'
+
+export default class NodeStopPublicApiCommand extends NodeCommandBase {
+  static description = `Send an api request to stop public api of given distributor node.`
+
+  static flags = {
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/stop-api'
+  }
+
+  protected reqBody(): Record<string, unknown> {
+    return {}
+  }
+}

distributor-node/src/commands/start.ts

@@ -1,5 +1,6 @@
 import DefaultCommandBase from '../command-base/default'
 import { App } from '../app'
+import { Config } from '../types'
 
 export default class StartNode extends DefaultCommandBase {
   static description = 'Start the node'
@@ -9,7 +10,7 @@ export default class StartNode extends DefaultCommandBase {
   }
 
   async run(): Promise<void> {
-    const app = new App(this.appConfig)
+    const app = new App(this.appConfig as Config)
     await app.start()
   }
 

distributor-node/src/schemas/configSchema.ts

@@ -1,27 +1,30 @@
 import { JSONSchema4 } from 'json-schema'
 import winston from 'winston'
 import { MAX_CONCURRENT_RESPONSE_TIME_CHECKS } from '../services/networking/NetworkingService'
+import { objectSchema } from './utils'
 
 export const bytesizeUnits = ['B', 'K', 'M', 'G', 'T']
 export const bytesizeRegex = new RegExp(`^[0-9]+(${bytesizeUnits.join('|')})$`)
 
-export const configSchema: JSONSchema4 = {
+const logLevelSchema: JSONSchema4 = {
+  description: 'Minimum level of logs sent to this output',
+  type: 'string',
+  enum: [...Object.keys(winston.config.npm.levels)],
+}
+
+export const configSchema: JSONSchema4 = objectSchema({
+  '$id': 'https://joystream.org/schemas/argus/config',
   title: 'Distributor node configuration',
   description: 'Configuration schema for distirubtor CLI and node',
-  type: 'object',
-  required: ['id', 'endpoints', 'directories', 'buckets', 'keys', 'port', 'workerId', 'limits', 'intervals'],
-  additionalProperties: false,
+  required: ['id', 'endpoints', 'directories', 'limits', 'intervals', 'publicApi'],
   properties: {
     id: {
       type: 'string',
       description: 'Node identifier used when sending elasticsearch logs and exposed on /status endpoint',
       minLength: 1,
     },
-    endpoints: {
-      type: 'object',
+    endpoints: objectSchema({
       description: 'Specifies external endpoints that the distributor node will connect to',
-      additionalProperties: false,
-      required: ['queryNode', 'joystreamNodeWs'],
       properties: {
         queryNode: {
           description: 'Query node graphql server uri (for example: http://localhost:8081/graphql)',
@@ -31,16 +34,10 @@ export const configSchema: JSONSchema4 = {
           description: 'Joystream node websocket api uri (for example: ws://localhost:9944)',
           type: 'string',
         },
-        elasticSearch: {
-          description: 'Elasticsearch uri used for submitting the distributor node logs (if enabled via `log.elastic`)',
-          type: 'string',
-        },
       },
-    },
-    directories: {
-      type: 'object',
-      required: ['assets', 'cacheState'],
-      additionalProperties: false,
+      required: ['queryNode', 'joystreamNodeWs'],
+    }),
+    directories: objectSchema({
       description: "Specifies paths where node's data will be stored",
       properties: {
         assets: {
@@ -52,45 +49,65 @@ export const configSchema: JSONSchema4 = {
             'Path to a directory where information about the current cache state will be stored (LRU-SP cache data, stored assets mime types etc.)',
           type: 'string',
         },
-        logs: {
-          description:
-            'Path to a directory where logs will be stored if logging to a file was enabled (via `log.file`).',
-          type: 'string',
-        },
       },
-    },
-    log: {
-      type: 'object',
-      additionalProperties: false,
-      description: 'Specifies minimum log levels by supported log outputs',
+      required: ['assets', 'cacheState'],
+    }),
+    logs: objectSchema({
+      description: 'Specifies the logging configuration',
       properties: {
-        file: {
-          description: 'Minimum level of logs written to a file specified in `directories.logs`',
-          type: 'string',
-          enum: [...Object.keys(winston.config.npm.levels), 'off'],
-        },
-        console: {
-          description: 'Minimum level of logs outputted to a console',
-          type: 'string',
-          enum: [...Object.keys(winston.config.npm.levels), 'off'],
-        },
-        elastic: {
-          description: 'Minimum level of logs sent to elasticsearch endpoint specified in `endpoints.elasticSearch`',
-          type: 'string',
-          enum: [...Object.keys(winston.config.npm.levels), 'off'],
-        },
+        file: objectSchema({
+          title: 'File logging options',
+          properties: {
+            level: logLevelSchema,
+            path: {
+              description: 'Path where the logs will be stored (absolute or relative to config file)',
+              type: 'string',
+            },
+            maxFiles: {
+              description: 'Maximum number of log files to store',
+              type: 'integer',
+              minimum: 1,
+            },
+            maxSize: {
+              description: 'Maximum size of a single log file in bytes',
+              type: 'integer',
+              minimum: 1024,
+            },
+            frequency: {
+              description: 'The frequency of creating new log files (regardless of maxSize)',
+              default: 'daily',
+              type: 'string',
+              enum: ['yearly', 'monthly', 'daily', 'hourly'],
+            },
+            archive: {
+              description: 'Whether to archive old logs',
+              default: false,
+              type: 'boolean',
+            },
+          },
+          required: ['level', 'path'],
+        }),
+        console: objectSchema({
+          title: 'Console logging options',
+          properties: { level: logLevelSchema },
+          required: ['level'],
+        }),
+        elastic: objectSchema({
+          title: 'Elasticsearch logging options',
+          properties: {
+            level: logLevelSchema,
+            endpoint: {
+              description: 'Elastichsearch endpoint to push the logs to (for example: http://localhost:9200)',
+              type: 'string',
+            },
+          },
+          required: ['level', 'endpoint'],
+        }),
       },
-    },
-    limits: {
-      type: 'object',
-      required: [
-        'storage',
-        'maxConcurrentStorageNodeDownloads',
-        'maxConcurrentOutboundConnections',
-        'outboundRequestsTimeout',
-      ],
+      required: [],
+    }),
+    limits: objectSchema({
       description: 'Specifies node limits w.r.t. storage, outbound connections etc.',
-      additionalProperties: false,
       properties: {
         storage: {
           description: 'Maximum total size of all (cached) assets stored in `directories.assets`',
@@ -103,21 +120,43 @@ export const configSchema: JSONSchema4 = {
           minimum: 1,
         },
         maxConcurrentOutboundConnections: {
-          description: 'Maximum number of total simultaneous outbound connections to storage node(s)',
+          description:
+            'Maximum number of total simultaneous outbound connections to storage node(s) (excluding proxy connections)',
           type: 'integer',
           minimum: 1,
         },
-        outboundRequestsTimeout: {
+        outboundRequestsTimeoutMs: {
           description: 'Timeout for all outbound storage node http requests in miliseconds',
           type: 'integer',
+          minimum: 1000,
+        },
+        pendingDownloadTimeoutSec: {
+          description: 'Timeout for pending storage node downloads in seconds',
+          type: 'integer',
+          minimum: 60,
+        },
+        maxCachedItemSize: {
+          description: 'Maximum size of a data object allowed to be cached by the node',
+          type: 'string',
+          pattern: bytesizeRegex.source,
+        },
+        dataObjectSourceByObjectIdTTL: {
+          description:
+            'TTL (in seconds) for dataObjectSourceByObjectId cache used when proxying objects of size greater than maxCachedItemSize to the right storage node.',
+          default: 60,
+          type: 'integer',
           minimum: 1,
         },
       },
-    },
-    intervals: {
-      type: 'object',
-      required: ['saveCacheState', 'checkStorageNodeResponseTimes', 'cacheCleanup'],
-      additionalProperties: false,
+      required: [
+        'storage',
+        'maxConcurrentStorageNodeDownloads',
+        'maxConcurrentOutboundConnections',
+        'outboundRequestsTimeoutMs',
+        'pendingDownloadTimeoutSec',
+      ],
+    }),
+    intervals: objectSchema({
       description: 'Specifies how often periodic tasks (for example cache cleanup) are executed by the node.',
       properties: {
         saveCacheState: {
@@ -143,66 +182,66 @@ export const configSchema: JSONSchema4 = {
           minimum: 1,
         },
       },
-    },
-    port: { description: 'Distributor node http server port', type: 'integer', minimum: 0 },
+      required: ['saveCacheState', 'checkStorageNodeResponseTimes', 'cacheCleanup'],
+    }),
+    publicApi: objectSchema({
+      description: 'Public api configuration',
+      properties: {
+        port: { description: 'Distributor node public api port', type: 'integer', minimum: 0 },
+      },
+      required: ['port'],
+    }),
+    operatorApi: objectSchema({
+      description: 'Operator api configuration',
+      properties: {
+        port: { description: 'Distributor node operator api port', type: 'integer', minimum: 0 },
+        hmacSecret: { description: 'HMAC (HS256) secret key used for JWT authorization', type: 'string' },
+      },
+      required: ['port', 'hmacSecret'],
+    }),
     keys: {
       description: 'Specifies the keys available within distributor node CLI.',
       type: 'array',
       items: {
         oneOf: [
-          {
-            type: 'object',
+          objectSchema({
             title: 'Substrate uri',
             description: "Keypair's substrate uri (for example: //Alice)",
-            required: ['suri'],
-            additionalProperties: false,
             properties: {
               type: { type: 'string', enum: ['ed25519', 'sr25519', 'ecdsa'], default: 'sr25519' },
               suri: { type: 'string' },
             },
-          },
-          {
-            type: 'object',
+            required: ['suri'],
+          }),
+          objectSchema({
             title: 'Mnemonic phrase',
             description: 'Menomonic phrase',
-            required: ['mnemonic'],
-            additionalProperties: false,
             properties: {
               type: { type: 'string', enum: ['ed25519', 'sr25519', 'ecdsa'], default: 'sr25519' },
               mnemonic: { type: 'string' },
             },
-          },
-          {
-            type: 'object',
+            required: ['mnemonic'],
+          }),
+          objectSchema({
             title: 'JSON backup file',
             description: 'Path to JSON backup file from polkadot signer / polakdot/apps (relative to config file path)',
-            required: ['keyfile'],
-            additionalProperties: false,
             properties: {
               keyfile: { type: 'string' },
             },
-          },
+            required: ['keyfile'],
+          }),
         ],
       },
       minItems: 1,
     },
     buckets: {
-      description: 'Specifies the buckets distributed by the node',
-      oneOf: [
-        {
-          title: 'Bucket ids',
-          description: 'List of distribution bucket ids',
-          type: 'array',
-          items: { type: 'integer', minimum: 0 },
-          minItems: 1,
-        },
-        {
-          title: 'All buckets',
-          description: 'Distribute all buckets assigned to worker specified in `workerId`',
-          type: 'string',
-          enum: ['all'],
-        },
-      ],
+      description:
+        'Set of bucket ids distributed by the node. If not specified, all buckets currently assigned to worker specified in `config.workerId` will be distributed.',
+      title: 'Bucket ids',
+      type: 'array',
+      uniqueItems: true,
+      items: { type: 'integer', minimum: 0 },
+      minItems: 1,
     },
     workerId: {
       description: 'ID of the node operator (distribution working group worker)',
@@ -210,6 +249,6 @@ export const configSchema: JSONSchema4 = {
       minimum: 0,
     },
   },
-}
+})
 
 export default configSchema

distributor-node/src/schemas/scripts/generateTypes.ts

@@ -7,7 +7,10 @@ import { schemas } from '..'
 const prettierConfig = require('@joystream/prettier-config')
 
 Object.entries(schemas).forEach(([schemaKey, schema]) => {
-  compile(schema, `${schemaKey}Json`, { style: prettierConfig })
+  compile(schema, `${schemaKey}Json`, {
+    style: prettierConfig,
+    ignoreMinAndMaxItems: true,
+  })
     .then((output) => fs.writeFileSync(path.resolve(__dirname, `../../types/generated/${schemaKey}Json.d.ts`), output))
     .catch(console.error)
 })

distributor-node/src/schemas/utils.ts

@@ -0,0 +1,15 @@
+import { JSONSchema4 } from 'json-schema'
+
+export function objectSchema<P extends NonNullable<JSONSchema4['properties']>>(props: {
+  $id?: string
+  title?: string
+  description?: string
+  properties: P
+  required: Array<keyof P & string>
+}): JSONSchema4 {
+  return {
+    type: 'object',
+    additionalProperties: false,
+    ...props,
+  }
+}

distributor-node/src/services/cache/StateCacheService.ts

@@ -1,22 +1,17 @@
 import { Logger } from 'winston'
-import { ReadonlyConfig, StorageNodeDownloadResponse } from '../../types'
+import { ReadonlyConfig } from '../../types'
 import { LoggingService } from '../logging'
 import _ from 'lodash'
 import fs from 'fs'
+import NodeCache from 'node-cache'
+import { PendingDownload } from '../networking/PendingDownload'
 
 // LRU-SP cache parameters
 // Since size is in KB, these parameters should be enough for grouping objects of size up to 2^24 KB = 16 GB
-// TODO: Intoduce MAX_CACHED_ITEM_SIZE and skip caching for large objects entirely? (ie. 10 GB objects)
 export const CACHE_GROUP_LOG_BASE = 2
 export const CACHE_GROUPS_COUNT = 24
 
-type PendingDownloadStatus = 'Waiting' | 'LookingForSource' | 'Downloading'
-
-export interface PendingDownloadData {
-  objectSize: number
-  status: PendingDownloadStatus
-  promise: Promise<StorageNodeDownloadResponse>
-}
+export const DEFAULT_DATA_OBJECT_SOURCE_CACHE_TTL = 60
 
 export interface StorageNodeEndpointData {
   last10ResponseTimes: number[]
@@ -34,9 +29,12 @@ export class StateCacheService {
   private cacheFilePath: string
 
   private memoryState = {
-    pendingDownloadsByObjectId: new Map<string, PendingDownloadData>(),
+    pendingDownloadsByObjectId: new Map<string, PendingDownload>(),
     storageNodeEndpointDataByEndpoint: new Map<string, StorageNodeEndpointData>(),
     groupNumberByObjectId: new Map<string, number>(),
+    dataObjectSourceByObjectId: new NodeCache({
+      deleteOnExpire: true,
+    }),
   }
 
   private storedState = {
@@ -149,17 +147,8 @@ export class StateCacheService {
     return bestCandidate
   }
 
-  public newPendingDownload(
-    objectId: string,
-    objectSize: number,
-    promise: Promise<StorageNodeDownloadResponse>
-  ): PendingDownloadData {
-    const pendingDownload: PendingDownloadData = {
-      status: 'Waiting',
-      objectSize,
-      promise,
-    }
-    this.memoryState.pendingDownloadsByObjectId.set(objectId, pendingDownload)
+  public addPendingDownload(pendingDownload: PendingDownload): PendingDownload {
+    this.memoryState.pendingDownloadsByObjectId.set(pendingDownload.getObjectId(), pendingDownload)
     return pendingDownload
   }
 
@@ -167,18 +156,22 @@ export class StateCacheService {
     return this.memoryState.pendingDownloadsByObjectId.size
   }
 
-  public getPendingDownload(objectId: string): PendingDownloadData | undefined {
+  public getPendingDownload(objectId: string): PendingDownload | undefined {
     return this.memoryState.pendingDownloadsByObjectId.get(objectId)
   }
 
   public dropPendingDownload(objectId: string): void {
-    this.memoryState.pendingDownloadsByObjectId.delete(objectId)
+    const pendingDownload = this.memoryState.pendingDownloadsByObjectId.get(objectId)
+    if (pendingDownload) {
+      pendingDownload.cleanup()
+      this.memoryState.pendingDownloadsByObjectId.delete(objectId)
+    }
   }
 
   public dropById(objectId: string): void {
     this.logger.debug('Dropping all state by object id', { objectId })
     this.storedState.mimeTypeByObjectId.delete(objectId)
-    this.memoryState.pendingDownloadsByObjectId.delete(objectId)
+    this.dropPendingDownload(objectId)
     const cacheGroupNumber = this.memoryState.groupNumberByObjectId.get(objectId)
     this.logger.debug('Cache group by object id established', { objectId, cacheGroupNumber })
     if (cacheGroupNumber) {
@@ -210,6 +203,26 @@ export class StateCacheService {
     ])
   }
 
+  public cacheDataObjectSource(objectId: string, source: string): void {
+    this.memoryState.dataObjectSourceByObjectId.set<string>(
+      objectId,
+      source,
+      this.config.limits.dataObjectSourceByObjectIdTTL || DEFAULT_DATA_OBJECT_SOURCE_CACHE_TTL
+    )
+  }
+
+  public getCachedDataObjectSource(objectId: string): string | undefined {
+    return this.memoryState.dataObjectSourceByObjectId.get<string | undefined>(objectId)
+  }
+
+  public dropCachedDataObjectSource(objectId: string, expectedSource?: string): void {
+    const cachedSource = this.memoryState.dataObjectSourceByObjectId.get<string | undefined>(objectId)
+    if (!expectedSource || cachedSource === expectedSource) {
+      this.logger.info('Force-dropping cached dataObjectSource', { objectId, cachedSource, expectedSource })
+      this.memoryState.dataObjectSourceByObjectId.del(objectId)
+    }
+  }
+
   private serializeData() {
     const { lruCacheGroups, mimeTypeByObjectId } = this.storedState
     return JSON.stringify(
@@ -218,7 +231,7 @@ export class StateCacheService {
         mimeTypeByObjectId: Array.from(mimeTypeByObjectId.entries()),
       },
       null,
-      2 // TODO: Only for debugging
+      2
     )
   }
 

distributor-node/src/services/content/ContentService.ts

@@ -1,5 +1,5 @@
 import fs from 'fs'
-import { ReadonlyConfig } from '../../types'
+import { ObjectStatus, ObjectStatusType, ReadonlyConfig } from '../../types'
 import { StateCacheService } from '../cache/StateCacheService'
 import { LoggingService } from '../logging'
 import { Logger } from 'winston'
@@ -7,10 +7,11 @@ import { FileContinousReadStream, FileContinousReadStreamOptions } from './FileC
 import FileType from 'file-type'
 import { Readable, pipeline } from 'stream'
 import { NetworkingService } from '../networking'
-import { createHash } from 'blake3-wasm'
-import * as multihash from 'multihashes'
+import { ContentHash } from '../crypto/ContentHash'
+import readChunk from 'read-chunk'
 
 export const DEFAULT_CONTENT_TYPE = 'application/octet-stream'
+export const MIME_TYPE_DETECTION_CHUNK_SIZE = 4100
 
 export class ContentService {
   private config: ReadonlyConfig
@@ -90,6 +91,12 @@ export class ContentService {
         continue
       }
 
+      // Drop files that are missing in the cache
+      if (!this.stateCache.peekContent(objectId)) {
+        this.drop(objectId, 'Missing cache data')
+        continue
+      }
+
       // Compare file size to expected one
       const { size: dataObjectSize } = dataObject
       if (fileSize !== dataObjectSize) {
@@ -122,11 +129,13 @@ export class ContentService {
     })
   }
 
-  public drop(objectId: string, reason?: string): void {
+  public drop(objectId: string, reason?: string, unreserveSpace = true): void {
     if (this.exists(objectId)) {
       const size = this.fileSize(objectId)
       fs.unlinkSync(this.path(objectId))
-      this.contentSizeSum -= size
+      if (unreserveSpace) {
+        this.contentSizeSum -= size
+      }
       this.logger.debug('Dropping content', { objectId, reason, size, contentSizeSum: this.contentSizeSum })
     } else {
       this.logger.warn('Trying to drop content that no loger exists', { objectId, reason })
@@ -159,8 +168,19 @@ export class ContentService {
   }
 
   public async detectMimeType(objectId: string): Promise<string> {
-    const result = await FileType.fromFile(this.path(objectId))
-    return result?.mime || DEFAULT_CONTENT_TYPE
+    const objectPath = this.path(objectId)
+    try {
+      const buffer = await readChunk(objectPath, 0, MIME_TYPE_DETECTION_CHUNK_SIZE)
+      const result = await FileType.fromBuffer(buffer)
+      return result?.mime || DEFAULT_CONTENT_TYPE
+    } catch (err) {
+      this.logger.error(`Error while trying to detect object mimeType: ${err instanceof Error ? err.message : err}`, {
+        err,
+        objectId,
+        objectPath,
+      })
+      return DEFAULT_CONTENT_TYPE
+    }
   }
 
   private async evictCacheUntilFreeSpaceReached(targetFreeSpace: number): Promise<void> {
@@ -203,44 +223,60 @@ export class ContentService {
       newContentSizeSum: this.contentSizeSum,
     })
 
+    const rejectContent = (reason: string, metadata: Record<string, unknown>) => {
+      const msg = `Content rejected: ${reason}`
+      // Drop (without unreserving space, will do that manually)
+      this.drop(objectId, msg, false)
+      // Unreserve reserved space
+      this.contentSizeSum -= expectedSize
+      // Log the error
+      this.logger.error(msg, { ...metadata })
+    }
+
     // Return a promise that resolves when the new file is created
     return new Promise<void>((resolve, reject) => {
       const fileStream = this.createWriteStream(objectId)
 
-      let bytesRecieved = 0
-      const hash = createHash()
+      let bytesReceived = 0
+      const hash = new ContentHash()
+
+      const onData = (chunk: Buffer) => {
+        bytesReceived += chunk.length
+        hash.update(chunk)
+
+        if (bytesReceived > expectedSize) {
+          dataStream.destroy(new Error('Unexpected content size: Too much data received from source!'))
+        }
+      }
 
       pipeline(dataStream, fileStream, async (err) => {
+        dataStream.off('data', onData)
         const { bytesWritten } = fileStream
-        const finalHash = multihash.toB58String(multihash.encode(hash.digest(), 'blake3'))
+        const finalHash = hash.digest()
         const logMetadata = {
           objectId,
           expectedSize,
-          expectedHash,
-          bytesRecieved,
+          bytesReceived,
           bytesWritten,
+          expectedHash,
+          finalHash,
         }
         if (err) {
-          this.logger.error(`Error while processing content data stream`, {
+          rejectContent(`Error while processing content data stream`, {
             err,
             ...logMetadata,
           })
-          this.drop(objectId)
           reject(err)
           return
         }
 
-        if (bytesWritten !== bytesRecieved || bytesWritten !== expectedSize) {
-          this.logger.error('Content rejected: Bytes written/recieved/expected mismatch!', {
-            ...logMetadata,
-          })
-          this.drop(objectId)
+        if (bytesWritten !== bytesReceived || bytesWritten !== expectedSize) {
+          rejectContent('Bytes written/received/expected mismatch!', { ...logMetadata })
           return
         }
 
         if (finalHash !== expectedHash) {
-          this.logger.error('Content rejected: Hash mismatch!', { ...logMetadata })
-          this.drop(objectId)
+          rejectContent('Hash mismatch!', { ...logMetadata })
           return
         }
 
@@ -255,15 +291,35 @@ export class ContentService {
         // Note: The promise is resolved on "ready" event, since that's what's awaited in the current flow
         resolve()
       })
+      dataStream.on('data', onData)
+    })
+  }
 
-      dataStream.on('data', (chunk) => {
-        bytesRecieved += chunk.length
-        hash.update(chunk)
+  public async objectStatus(objectId: string): Promise<ObjectStatus> {
+    const pendingDownload = this.stateCache.getPendingDownload(objectId)
 
-        if (bytesRecieved > expectedSize) {
-          dataStream.destroy(new Error('Unexpected content size: Too much data recieved from source!'))
-        }
-      })
-    })
+    if (!pendingDownload && this.exists(objectId)) {
+      return { type: ObjectStatusType.Available, path: this.path(objectId) }
+    }
+
+    if (pendingDownload) {
+      return { type: ObjectStatusType.PendingDownload, pendingDownload }
+    }
+
+    const objectInfo = await this.networking.dataObjectInfo(objectId)
+    if (!objectInfo.exists) {
+      return { type: ObjectStatusType.NotFound }
+    }
+
+    if (!objectInfo.isSupported) {
+      return { type: ObjectStatusType.NotSupported }
+    }
+
+    const { data: objectData } = objectInfo
+    if (!objectData) {
+      throw new Error('Missing data object data')
+    }
+
+    return { type: ObjectStatusType.Missing, objectData }
   }
 }

distributor-node/src/services/crypto/ContentHash.ts

@@ -0,0 +1,21 @@
+import { createHash, HashInput, NodeHash } from 'blake3-wasm'
+import { HashReader } from 'blake3-wasm/dist/wasm/nodejs/blake3_js'
+import { toB58String, encode } from 'multihashes'
+
+export class ContentHash {
+  private hash: NodeHash<HashReader>
+  public static readonly algorithm = 'blake3'
+
+  constructor() {
+    this.hash = createHash()
+  }
+
+  update(data: HashInput): this {
+    this.hash.update(data)
+    return this
+  }
+
+  digest(): string {
+    return toB58String(encode(this.hash.digest(), ContentHash.algorithm))
+  }
+}

distributor-node/src/services/httpApi/HttpApiBase.ts

@@ -0,0 +1,145 @@
+import express from 'express'
+import * as OpenApiValidator from 'express-openapi-validator'
+import { HttpError, OpenApiValidatorOpts } from 'express-openapi-validator/dist/framework/types'
+import { ReadonlyConfig } from '../../types/config'
+import expressWinston from 'express-winston'
+import { Logger } from 'winston'
+import { Server } from 'http'
+import cors from 'cors'
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type HttpApiRoute = ['get' | 'head' | 'post', string, express.RequestHandler<any>]
+
+export abstract class HttpApiBase {
+  protected abstract port: number
+  protected expressApp: express.Application
+  protected config: ReadonlyConfig
+  protected logger: Logger
+  private httpServer: Server | undefined
+  private isInitialized = false
+  private isOn = false
+
+  protected routeWrapper(handler: express.RequestHandler) {
+    return async (req: express.Request, res: express.Response, next: express.NextFunction): Promise<void> => {
+      // Fix for express-winston in order to also log prematurely closed requests
+      res.on('close', () => {
+        res.locals.prematurelyClosed = !res.writableFinished
+        res.end()
+      })
+      try {
+        await handler(req, res, next)
+      } catch (err) {
+        next(err)
+      }
+    }
+  }
+
+  public constructor(config: ReadonlyConfig, logger: Logger) {
+    this.expressApp = express()
+    this.logger = logger
+    this.config = config
+  }
+
+  protected createRoutes(routes: HttpApiRoute[]): void {
+    routes.forEach(([type, path, handler]) => {
+      this.expressApp[type](path, this.routeWrapper(handler))
+    })
+  }
+
+  protected abstract routes(): HttpApiRoute[]
+
+  protected defaultOpenApiValidatorConfig(): Partial<OpenApiValidatorOpts> {
+    const isProd = process.env.NODE_ENV === 'prod'
+    return {
+      validateResponses: !isProd,
+    }
+  }
+
+  protected abstract openApiValidatorConfig(): OpenApiValidatorOpts
+
+  protected defaultRequestLoggerConfig(): expressWinston.LoggerOptions {
+    return {
+      winstonInstance: this.logger,
+      level: 'http',
+      dynamicMeta: (req, res) => {
+        return { prematurelyClosed: res.locals.prematurelyClosed ?? false }
+      },
+    }
+  }
+
+  protected requestLoggerConfig(): expressWinston.LoggerOptions {
+    return this.defaultRequestLoggerConfig()
+  }
+
+  protected defaultErrorLoggerConfig(): expressWinston.ErrorLoggerOptions {
+    return {
+      winstonInstance: this.logger,
+      level: 'error',
+      metaField: null,
+      exceptionToMeta: (err) => ({ err }),
+    }
+  }
+
+  protected errorLoggerConfig(): expressWinston.ErrorLoggerOptions {
+    return this.defaultErrorLoggerConfig()
+  }
+
+  protected errorHandler() {
+    return (err: HttpError, req: express.Request, res: express.Response, next: express.NextFunction): void => {
+      if (res.headersSent) {
+        return next(err)
+      }
+      if (err.status && err.status >= 400 && err.status < 500) {
+        res
+          .status(err.status)
+          .json({
+            type: 'request_validation',
+            message: err.message,
+            errors: err.errors,
+          })
+          .end()
+      } else {
+        res.status(err.status || 500).json({ type: 'exception', message: err.message })
+      }
+    }
+  }
+
+  protected initApp(): void {
+    if (this.isInitialized) {
+      return
+    }
+    const { expressApp: app } = this
+    app.use(express.json())
+    app.use(cors())
+    app.use(expressWinston.logger(this.requestLoggerConfig()))
+    app.use(OpenApiValidator.middleware(this.openApiValidatorConfig()))
+    this.createRoutes(this.routes())
+    app.use(expressWinston.errorLogger(this.errorLoggerConfig()))
+    app.use(this.errorHandler())
+    this.isInitialized = true
+  }
+
+  public start(): boolean {
+    if (this.isOn) {
+      return false
+    }
+    if (!this.isInitialized) {
+      this.initApp()
+    }
+    this.httpServer = this.expressApp.listen(this.port, () => {
+      this.logger.info(`Express server started listening on port ${this.port}`)
+    })
+    this.isOn = true
+    return true
+  }
+
+  public stop(): boolean {
+    if (!this.isOn) {
+      return false
+    }
+    this.httpServer?.close()
+    this.logger.info(`Express server stopped`)
+    this.isOn = false
+    return true
+  }
+}

distributor-node/src/services/httpApi/OperatorApiService.ts

@@ -0,0 +1,90 @@
+import express from 'express'
+import path from 'path'
+import { OpenApiValidatorOpts } from 'express-openapi-validator/dist/framework/types'
+import { Config } from '../../types/config'
+import { LoggingService } from '../logging'
+import jwt from 'jsonwebtoken'
+import { OperatorApiController } from './controllers/operator'
+import { HttpApiBase, HttpApiRoute } from './HttpApiBase'
+import { PublicApiService } from './PublicApiService'
+import _ from 'lodash'
+import { App } from '../../app'
+
+const OPENAPI_SPEC_PATH = path.join(__dirname, '../../api-spec/operator.yml')
+const JWT_TOKEN_MAX_AGE = '5m'
+
+export class OperatorApiService extends HttpApiBase {
+  protected port: number
+  protected operatorSecretKey: string
+  protected config: Config
+  protected app: App
+  protected publicApi: PublicApiService
+  protected logging: LoggingService
+
+  public constructor(config: Config, app: App, logging: LoggingService, publicApi: PublicApiService) {
+    super(config, logging.createLogger('OperatorApi'))
+    if (!config.operatorApi) {
+      throw new Error('Cannot construct OperatorApiService - missing operatorApi config!')
+    }
+    this.port = config.operatorApi.port
+    this.operatorSecretKey = config.operatorApi.hmacSecret
+    this.config = config
+    this.app = app
+    this.logging = logging
+    this.publicApi = publicApi
+    this.initApp()
+  }
+
+  protected openApiValidatorConfig(): OpenApiValidatorOpts {
+    return {
+      apiSpec: OPENAPI_SPEC_PATH,
+      validateSecurity: {
+        handlers: {
+          OperatorAuth: this.operatorRequestValidator(),
+        },
+      },
+      ...this.defaultOpenApiValidatorConfig(),
+    }
+  }
+
+  protected routes(): HttpApiRoute[] {
+    const controller = new OperatorApiController(this.config, this.app, this.publicApi, this.logging)
+    return [
+      ['post', '/api/v1/stop-api', controller.stopApi.bind(controller)],
+      ['post', '/api/v1/start-api', controller.startApi.bind(controller)],
+      ['post', '/api/v1/shutdown', controller.shutdown.bind(controller)],
+      ['post', '/api/v1/set-worker', controller.setWorker.bind(controller)],
+      ['post', '/api/v1/set-buckets', controller.setBuckets.bind(controller)],
+    ]
+  }
+
+  private operatorRequestValidator() {
+    return (req: express.Request): boolean => {
+      const authHeader = req.headers.authorization
+      if (!authHeader) {
+        throw new Error('Authrorization header missing')
+      }
+
+      const [authType, token] = authHeader.split(' ')
+      if (authType.toLowerCase() !== 'bearer') {
+        throw new Error(`Unexpected authorization type: ${authType}`)
+      }
+
+      if (!token) {
+        throw new Error(`Bearer token missing`)
+      }
+
+      const decoded = jwt.verify(token, this.operatorSecretKey, { maxAge: JWT_TOKEN_MAX_AGE }) as jwt.JwtPayload
+
+      if (!_.isEqual(req.body, decoded.reqBody)) {
+        throw new Error('Invalid token: Request body does not match')
+      }
+
+      if (req.originalUrl !== decoded.reqUrl) {
+        throw new Error('Invalid token: Request url does not match')
+      }
+
+      return true
+    }
+  }
+}

distributor-node/src/services/httpApi/PublicApiService.ts

@@ -0,0 +1,60 @@
+import path from 'path'
+import { ReadonlyConfig } from '../../types/config'
+import { LoggingService } from '../logging'
+import { PublicApiController } from './controllers/public'
+import { StateCacheService } from '../cache/StateCacheService'
+import { NetworkingService } from '../networking'
+import { ContentService } from '../content/ContentService'
+import { HttpApiBase, HttpApiRoute } from './HttpApiBase'
+import { OpenApiValidatorOpts } from 'express-openapi-validator/dist/openapi.validator'
+
+const OPENAPI_SPEC_PATH = path.join(__dirname, '../../api-spec/public.yml')
+
+export class PublicApiService extends HttpApiBase {
+  protected port: number
+
+  private loggingService: LoggingService
+  private networkingService: NetworkingService
+  private stateCache: StateCacheService
+  private contentService: ContentService
+
+  public constructor(
+    config: ReadonlyConfig,
+    stateCache: StateCacheService,
+    content: ContentService,
+    logging: LoggingService,
+    networking: NetworkingService
+  ) {
+    super(config, logging.createLogger('PublicApi'))
+    this.stateCache = stateCache
+    this.loggingService = logging
+    this.networkingService = networking
+    this.contentService = content
+    this.port = config.publicApi.port
+    this.initApp()
+  }
+
+  protected openApiValidatorConfig(): OpenApiValidatorOpts {
+    return {
+      apiSpec: OPENAPI_SPEC_PATH,
+      ...this.defaultOpenApiValidatorConfig(),
+    }
+  }
+
+  protected routes(): HttpApiRoute[] {
+    const publicController = new PublicApiController(
+      this.config,
+      this.loggingService,
+      this.networkingService,
+      this.stateCache,
+      this.contentService
+    )
+
+    return [
+      ['head', '/api/v1/assets/:objectId', publicController.assetHead.bind(publicController)],
+      ['get', '/api/v1/assets/:objectId', publicController.asset.bind(publicController)],
+      ['get', '/api/v1/status', publicController.status.bind(publicController)],
+      ['get', '/api/v1/buckets', publicController.buckets.bind(publicController)],
+    ]
+  }
+}

distributor-node/src/services/httpApi/controllers/operator.ts

@@ -0,0 +1,79 @@
+import { Logger } from 'winston'
+import * as express from 'express'
+import { PublicApiService } from '../PublicApiService'
+import { LoggingService } from '../../logging'
+import { App } from '../../../app'
+import { Config, SetBucketsOperation, SetWorkerOperation } from '../../../types'
+import { ParamsDictionary } from 'express-serve-static-core'
+
+export class OperatorApiController {
+  private config: Config
+  private app: App
+  private publicApi: PublicApiService
+  private logger: Logger
+
+  public constructor(config: Config, app: App, publicApi: PublicApiService, logging: LoggingService) {
+    this.config = config
+    this.app = app
+    this.publicApi = publicApi
+    this.logger = logging.createLogger('OperatorApiController')
+  }
+
+  public async stopApi(req: express.Request, res: express.Response): Promise<void> {
+    this.logger.info(`Stopping public api on operator request from ${req.ip}`, { ip: req.ip })
+    const stopped = this.publicApi.stop()
+    if (!stopped) {
+      res.status(409).json({ message: 'Already stopped' })
+    }
+    res.status(200).send()
+  }
+
+  public async startApi(req: express.Request, res: express.Response): Promise<void> {
+    this.logger.info(`Starting public api on operator request from ${req.ip}`, { ip: req.ip })
+    const started = this.publicApi.start()
+    if (!started) {
+      res.status(409).json({ message: 'Already started' })
+    }
+    res.status(200).send()
+  }
+
+  public async shutdown(req: express.Request, res: express.Response): Promise<void> {
+    this.logger.info(`Shutting down the app on operator request from ${req.ip}`, { ip: req.ip })
+    const shutdown = this.app.stop(5)
+    if (!shutdown) {
+      res.status(409).json({ message: 'Already shutting down' })
+    }
+    res.status(200).send()
+  }
+
+  public async setWorker(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    req: express.Request<ParamsDictionary, any, SetWorkerOperation>,
+    res: express.Response
+  ): Promise<void> {
+    const { workerId } = req.body
+    this.logger.info(`Updating workerId to ${workerId} on operator request from ${req.ip}`, {
+      workerId,
+      ip: req.ip,
+    })
+    this.config.workerId = workerId
+    res.status(200).send()
+  }
+
+  public async setBuckets(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    req: express.Request<ParamsDictionary, any, SetBucketsOperation>,
+    res: express.Response
+  ): Promise<void> {
+    const { buckets } = req.body
+    this.logger.info(
+      `Updating buckets to ${buckets ? JSON.stringify(buckets) : '"all"'} on operator request from ${req.ip}`,
+      {
+        buckets,
+        ip: req.ip,
+      }
+    )
+    this.config.buckets = buckets
+    res.status(200).send()
+  }
+}

distributor-node/src/services/server/controllers/public.ts → distributor-node/src/services/httpApi/controllers/public.ts

@@ -1,13 +1,22 @@
 import * as express from 'express'
 import { Logger } from 'winston'
 import send from 'send'
-import { StateCacheService } from '../../../services/cache/StateCacheService'
-import { NetworkingService } from '../../../services/networking'
-import { AssetRouteParams, BucketsResponse, ErrorResponse, StatusResponse } from '../../../types/api'
+import { StateCacheService } from '../../cache/StateCacheService'
+import { NetworkingService } from '../../networking'
+import {
+  AssetRouteParams,
+  BucketsResponse,
+  ErrorResponse,
+  StatusResponse,
+  DataObjectData,
+  ObjectStatusType,
+  ReadonlyConfig,
+} from '../../../types'
 import { LoggingService } from '../../logging'
 import { ContentService, DEFAULT_CONTENT_TYPE } from '../../content/ContentService'
 import proxy from 'express-http-proxy'
-import { ReadonlyConfig } from '../../../types'
+import { PendingDownloadStatusDownloading, PendingDownloadStatusType } from '../../networking/PendingDownload'
+import urljoin from 'url-join'
 
 const CACHED_MAX_AGE = 31536000
 const PENDING_MAX_AGE = 180
@@ -33,14 +42,75 @@ export class PublicApiController {
     this.content = content
   }
 
+  private createErrorResponse(message: string, type?: string): ErrorResponse {
+    return { type, message }
+  }
+
+  private async proxyAssetRequest(
+    req: express.Request<AssetRouteParams>,
+    res: express.Response,
+    next: express.NextFunction,
+    objectId: string,
+    sourceApiEndpoint: string
+  ) {
+    const sourceObjectUrl = new URL(urljoin(sourceApiEndpoint, `files/${objectId}`))
+    res.setHeader('x-data-source', 'external')
+    this.logger.verbose(`Forwarding request to ${sourceObjectUrl.toString()}`, {
+      objectId,
+      sourceUrl: sourceObjectUrl.href,
+    })
+    return proxy(sourceObjectUrl.origin, {
+      proxyReqPathResolver: () => sourceObjectUrl.pathname,
+      proxyErrorHandler: (err, res, next) => {
+        this.logger.error(`Proxy request to ${sourceObjectUrl} failed!`, {
+          objectId,
+          sourceObjectUrl: sourceObjectUrl.href,
+        })
+        this.stateCache.dropCachedDataObjectSource(objectId, sourceApiEndpoint)
+        next(err)
+      },
+    })(req, res, next)
+  }
+
+  private async serveMissingAsset(
+    req: express.Request<AssetRouteParams>,
+    res: express.Response,
+    next: express.NextFunction,
+    objectData: DataObjectData
+  ): Promise<void> {
+    const { objectId, size, contentHash } = objectData
+
+    const { maxCachedItemSize } = this.config.limits
+    if (maxCachedItemSize && size > maxCachedItemSize) {
+      this.logger.info(`Requested object is above maxCachedItemSize: ${size}/${maxCachedItemSize}`, {
+        objectId,
+        size,
+        maxCachedItemSize,
+      })
+      const source = await this.networking.getDataObjectDownloadSource(objectData)
+      res.setHeader('x-cache', 'miss')
+      return this.proxyAssetRequest(req, res, next, objectId, source)
+    }
+
+    const downloadResponse = await this.networking.downloadDataObject({ objectData })
+
+    if (downloadResponse) {
+      // Note: Await will only wait unil the file is created, so we may serve the response from it
+      await this.content.handleNewContent(objectId, size, contentHash, downloadResponse.data)
+      res.setHeader('x-cache', 'miss')
+    } else {
+      res.setHeader('x-cache', 'pending')
+    }
+    return this.servePendingDownloadAsset(req, res, next, objectId)
+  }
+
   private serveAssetFromFilesystem(
     req: express.Request<AssetRouteParams>,
     res: express.Response,
     next: express.NextFunction,
     objectId: string
   ): void {
-    // TODO: Limit the number of times useContent is trigerred for similar requests?
-    // (for example: same ip, 3 different request within a minute = 1 request)
+    this.logger.verbose('Serving object from filesystem', { objectId })
     this.stateCache.useContent(objectId)
 
     const path = this.content.path(objectId)
@@ -82,12 +152,13 @@ export class PublicApiController {
     if (!pendingDownload) {
       throw new Error('Trying to serve pending download asset that is not pending download!')
     }
+    const status = pendingDownload.getStatus().type
+    this.logger.verbose('Serving object in pending download state', { objectId, status })
 
-    const { promise, objectSize } = pendingDownload
-    const response = await promise
-    const source = new URL(response.config.url!)
-    const contentType = response.headers['content-type'] || DEFAULT_CONTENT_TYPE
-    res.setHeader('content-type', contentType)
+    await pendingDownload.untilStatus(PendingDownloadStatusType.Downloading)
+    const objectSize = pendingDownload.getObjectSize()
+    const { source, contentType } = pendingDownload.getStatus() as PendingDownloadStatusDownloading
+    res.setHeader('content-type', contentType || DEFAULT_CONTENT_TYPE)
     // Allow caching pendingDownload reponse only for very short period of time and requite revalidation,
     // since the data coming from the source may not be valid
     res.setHeader('cache-control', `max-age=${PENDING_MAX_AGE}, must-revalidate`)
@@ -106,9 +177,7 @@ export class PublicApiController {
     }
 
     // Range doesn't start from the beginning of the content or the file was not found - froward request to source storage node
-    this.logger.verbose(`Forwarding request to ${source.href}`, { source: source.href })
-    res.setHeader('x-data-source', 'external')
-    return proxy(source.origin, { proxyReqPathResolver: () => source.pathname })(req, res, next)
+    return this.proxyAssetRequest(req, res, next, objectId, source)
   }
 
   private async servePendingDownloadAssetFromFile(
@@ -134,41 +203,44 @@ export class PublicApiController {
     stream.pipe(res)
     req.on('close', () => {
       stream.destroy()
-      res.destroy()
+      res.end()
     })
   }
 
   public async assetHead(req: express.Request<AssetRouteParams>, res: express.Response): Promise<void> {
-    const objectId = req.params.objectId
-    const pendingDownload = this.stateCache.getPendingDownload(objectId)
+    const { objectId } = req.params
+    const objectStatus = await this.content.objectStatus(objectId)
 
     res.setHeader('timing-allow-origin', '*')
     res.setHeader('accept-ranges', 'bytes')
     res.setHeader('content-disposition', 'inline')
 
-    if (!pendingDownload && this.content.exists(objectId)) {
-      res.status(200)
-      res.setHeader('x-cache', 'hit')
-      res.setHeader('cache-control', `max-age=${CACHED_MAX_AGE}`)
-      res.setHeader('content-type', this.stateCache.getContentMimeType(objectId) || DEFAULT_CONTENT_TYPE)
-      res.setHeader('content-length', this.content.fileSize(objectId))
-    } else if (pendingDownload) {
-      res.status(200)
-      res.setHeader('x-cache', 'pending')
-      res.setHeader('cache-control', `max-age=${PENDING_MAX_AGE}, must-revalidate`)
-      res.setHeader('content-length', pendingDownload.objectSize)
-    } else {
-      const objectInfo = await this.networking.dataObjectInfo(objectId)
-      if (!objectInfo.exists) {
+    switch (objectStatus.type) {
+      case ObjectStatusType.Available:
+        res.status(200)
+        res.setHeader('x-cache', 'hit')
+        res.setHeader('cache-control', `max-age=${CACHED_MAX_AGE}`)
+        res.setHeader('content-type', this.stateCache.getContentMimeType(objectId) || DEFAULT_CONTENT_TYPE)
+        res.setHeader('content-length', this.content.fileSize(objectId))
+        break
+      case ObjectStatusType.PendingDownload:
+        res.status(200)
+        res.setHeader('x-cache', 'pending')
+        res.setHeader('cache-control', `max-age=${PENDING_MAX_AGE}, must-revalidate`)
+        res.setHeader('content-length', objectStatus.pendingDownload.getObjectSize())
+        break
+      case ObjectStatusType.NotFound:
         res.status(404)
-      } else if (!objectInfo.isSupported) {
+        break
+      case ObjectStatusType.NotSupported:
         res.status(421)
-      } else {
+        break
+      case ObjectStatusType.Missing:
         res.status(200)
         res.setHeader('x-cache', 'miss')
         res.setHeader('cache-control', `max-age=${PENDING_MAX_AGE}, must-revalidate`)
-        res.setHeader('content-length', objectInfo.data?.size || 0)
-      }
+        res.setHeader('content-length', objectStatus.objectData.size)
+        break
     }
 
     res.send()
@@ -179,55 +251,30 @@ export class PublicApiController {
     res: express.Response,
     next: express.NextFunction
   ): Promise<void> {
-    const objectId = req.params.objectId
-    const pendingDownload = this.stateCache.getPendingDownload(objectId)
+    const { objectId } = req.params
+    const objectStatus = await this.content.objectStatus(objectId)
 
     this.logger.verbose('Data object requested', {
       objectId,
-      status: pendingDownload && pendingDownload.status,
+      objectStatus,
     })
 
     res.setHeader('timing-allow-origin', '*')
 
-    if (!pendingDownload && this.content.exists(objectId)) {
-      this.logger.verbose('Requested file found in filesystem', { path: this.content.path(objectId) })
-      return this.serveAssetFromFilesystem(req, res, next, objectId)
-    } else if (pendingDownload) {
-      this.logger.verbose('Requested file is in pending download state', { path: this.content.path(objectId) })
-      res.setHeader('x-cache', 'pending')
-      return this.servePendingDownloadAsset(req, res, next, objectId)
-    } else {
-      this.logger.verbose('Requested file not found in filesystem')
-      const objectInfo = await this.networking.dataObjectInfo(objectId)
-      if (!objectInfo.exists) {
-        const errorRes: ErrorResponse = {
-          message: 'Data object does not exist',
-        }
-        res.status(404).json(errorRes)
-      } else if (!objectInfo.isSupported) {
-        const errorRes: ErrorResponse = {
-          message: 'Data object not served by this node',
-        }
-        res.status(421).json(errorRes)
-        // TODO: Try to direct to a node that supports it?
-      } else {
-        const { data: objectData } = objectInfo
-        if (!objectData) {
-          throw new Error('Missing data object data')
-        }
-        const { size, contentHash } = objectData
-
-        const downloadResponse = await this.networking.downloadDataObject({ objectData })
-
-        if (downloadResponse) {
-          // Note: Await will only wait unil the file is created, so we may serve the response from it
-          await this.content.handleNewContent(objectId, size, contentHash, downloadResponse.data)
-          res.setHeader('x-cache', 'miss')
-        } else {
-          res.setHeader('x-cache', 'pending')
-        }
+    switch (objectStatus.type) {
+      case ObjectStatusType.Available:
+        return this.serveAssetFromFilesystem(req, res, next, objectId)
+      case ObjectStatusType.PendingDownload:
+        res.setHeader('x-cache', 'pending')
         return this.servePendingDownloadAsset(req, res, next, objectId)
-      }
+      case ObjectStatusType.NotFound:
+        res.status(404).json(this.createErrorResponse('Data object does not exist'))
+        return
+      case ObjectStatusType.NotSupported:
+        res.status(421).json(this.createErrorResponse('Data object not served by this node'))
+        return
+      case ObjectStatusType.Missing:
+        return this.serveMissingAsset(req, res, next, objectStatus.objectData)
     }
   }
 
@@ -247,9 +294,11 @@ export class PublicApiController {
     res
       .status(200)
       .json(
-        this.config.buckets === 'all'
+        this.config.buckets
+          ? { bucketIds: [...this.config.buckets] }
+          : typeof this.config.workerId === 'number'
           ? { allByWorkerId: this.config.workerId }
-          : { bucketIds: [...this.config.buckets] }
+          : { bucketIds: [] }
       )
   }
 }

distributor-node/src/services/logging/LoggingService.ts

@@ -6,6 +6,8 @@ import { blake2AsHex } from '@polkadot/util-crypto'
 import { Format } from 'logform'
 import stringify from 'fast-safe-stringify'
 import NodeCache from 'node-cache'
+import path from 'path'
+import 'winston-daily-rotate-file'
 
 const cliColors = {
   error: 'red',
@@ -22,7 +24,8 @@ const pausedLogs = new NodeCache({
 })
 
 // Pause log for a specified time period
-const pauseFormat: (opts: { id: string }) => Format = winston.format((info, opts: { id: string }) => {
+type PauseFormatOpts = { id: string }
+const pauseFormat: (opts: PauseFormatOpts) => Format = winston.format((info, opts: PauseFormatOpts) => {
   if (info['@pauseFor']) {
     const messageHash = blake2AsHex(`${opts.id}:${info.level}:${info.message}`)
     if (!pausedLogs.has(messageHash)) {
@@ -37,8 +40,20 @@ const pauseFormat: (opts: { id: string }) => Format = winston.format((info, opts
   return info
 })
 
+// Error format applied to specific log meta field
+type ErrorFormatOpts = { filedName: string }
+const errorFormat: (opts: ErrorFormatOpts) => Format = winston.format((info, opts: ErrorFormatOpts) => {
+  if (!info[opts.filedName]) {
+    return info
+  }
+  const formatter = winston.format.errors({ stack: true })
+  info[opts.filedName] = formatter.transform(info[opts.filedName], formatter.options)
+  return info
+})
+
 const cliFormat = winston.format.combine(
   winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss:ms' }),
+  errorFormat({ filedName: 'err' }),
   winston.format.metadata({ fillExcept: ['label', 'level', 'timestamp', 'message'] }),
   winston.format.colorize({ all: true }),
   winston.format.printf(
@@ -61,39 +76,44 @@ export class LoggingService {
     const transports: winston.LoggerOptions['transports'] = []
 
     let esTransport: ElasticsearchTransport | undefined
-    if (config.log?.elastic && config.log.elastic !== 'off') {
-      if (!config.endpoints.elasticSearch) {
-        throw new Error('config.endpoints.elasticSearch must be provided when elasticSeach logging is enabled!')
-      }
+    if (config.logs?.elastic) {
       esTransport = new ElasticsearchTransport({
-        level: config.log.elastic,
+        index: 'distributor-node',
+        level: config.logs.elastic.level,
         format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
         flushInterval: 5000,
         source: config.id,
         clientOpts: {
           node: {
-            url: new URL(config.endpoints.elasticSearch),
+            url: new URL(config.logs.elastic.endpoint),
           },
         },
       })
       transports.push(esTransport)
     }
 
-    if (config.log?.file && config.log.file !== 'off') {
-      if (!config.directories.logs) {
-        throw new Error('config.directories.logs must be provided when file logging is enabled!')
+    if (config.logs?.file) {
+      const datePatternByFrequency = {
+        yearly: 'YYYY',
+        monthly: 'YYYY-MM',
+        daily: 'YYYY-MM-DD',
+        hourly: 'YYYY-MM-DD-HH',
       }
-      const fileTransport = new winston.transports.File({
-        filename: `${config.directories.logs}/logs.json`,
-        level: config.log.file,
+      const fileTransport = new winston.transports.DailyRotateFile({
+        filename: path.join(config.logs.file.path, 'argus-%DATE%.log'),
+        datePattern: datePatternByFrequency[config.logs.file.frequency || 'daily'],
+        zippedArchive: config.logs.file.archive,
+        maxSize: config.logs.file.maxSize,
+        maxFiles: config.logs.file.maxFiles,
+        level: config.logs.file.level,
         format: winston.format.combine(pauseFormat({ id: 'file' }), escFormat()),
       })
       transports.push(fileTransport)
     }
 
-    if (config.log?.console && config.log.console !== 'off') {
+    if (config.logs?.console) {
       const consoleTransport = new winston.transports.Console({
-        level: config.log.console,
+        level: config.logs.console.level,
         format: winston.format.combine(pauseFormat({ id: 'cli' }), cliFormat),
       })
       transports.push(consoleTransport)

distributor-node/src/services/networking/NetworkingService.ts

@@ -3,9 +3,9 @@ import { QueryNodeApi } from './query-node/api'
 import { Logger } from 'winston'
 import { LoggingService } from '../logging'
 import { StorageNodeApi } from './storage-node/api'
-import { PendingDownloadData, StateCacheService } from '../cache/StateCacheService'
+import { StateCacheService } from '../cache/StateCacheService'
 import { DataObjectDetailsFragment } from './query-node/generated/queries'
-import axios, { AxiosRequestConfig } from 'axios'
+import axios from 'axios'
 import {
   StorageNodeEndpointData,
   DataObjectAccessPoints,
@@ -19,15 +19,15 @@ import { DistributionBucketOperatorStatus } from './query-node/generated/schema'
 import http from 'http'
 import https from 'https'
 import { parseAxiosError } from '../parsers/errors'
+import { PendingDownload, PendingDownloadStatusType } from './PendingDownload'
 
 // Concurrency limits
-export const MAX_CONCURRENT_AVAILABILITY_CHECKS_PER_DOWNLOAD = 10
+export const MAX_CONCURRENT_AVAILABILITY_CHECKS_PER_OBJECT = 10
 export const MAX_CONCURRENT_RESPONSE_TIME_CHECKS = 10
 
 export class NetworkingService {
   private config: ReadonlyConfig
   private queryNodeApi: QueryNodeApi
-  // private runtimeApi: RuntimeApi
   private logging: LoggingService
   private stateCache: StateCacheService
   private logger: Logger
@@ -36,10 +36,10 @@ export class NetworkingService {
   private downloadQueue: queue
 
   constructor(config: ReadonlyConfig, stateCache: StateCacheService, logging: LoggingService) {
-    axios.defaults.timeout = config.limits.outboundRequestsTimeout
+    axios.defaults.timeout = config.limits.outboundRequestsTimeoutMs
     const httpConfig: http.AgentOptions | https.AgentOptions = {
       keepAlive: true,
-      timeout: config.limits.outboundRequestsTimeout,
+      timeout: config.limits.outboundRequestsTimeoutMs,
       maxSockets: config.limits.maxConcurrentOutboundConnections,
     }
     axios.defaults.httpAgent = new http.Agent(httpConfig)
@@ -49,7 +49,6 @@ export class NetworkingService {
     this.stateCache = stateCache
     this.logger = logging.createLogger('NetworkingManager')
     this.queryNodeApi = new QueryNodeApi(config.endpoints.queryNode, this.logging)
-    // this.runtimeApi = new RuntimeApi(config.endpoints.substrateNode)
     void this.checkActiveStorageNodeEndpoints()
     // Queues
     this.testLatencyQueue = queue({ concurrency: MAX_CONCURRENT_RESPONSE_TIME_CHECKS, autostart: true }).on(
@@ -61,6 +60,9 @@ export class NetworkingService {
       }
     )
     this.downloadQueue = queue({ concurrency: config.limits.maxConcurrentStorageNodeDownloads, autostart: true })
+    this.downloadQueue.on('error', (err) => {
+      this.logger.error('Data object download failed', { err })
+    })
   }
 
   private validateNodeEndpoint(endpoint: string): void {
@@ -92,17 +94,13 @@ export class NetworkingService {
   }
 
   private prepareStorageNodeEndpoints(details: DataObjectDetailsFragment) {
-    const endpointsData = details.storageBag.storageAssignments
-      .filter(
-        (a) =>
-          a.storageBucket.operatorStatus.__typename === 'StorageBucketOperatorStatusActive' &&
-          a.storageBucket.operatorMetadata?.nodeEndpoint
-      )
-      .map((a) => {
-        const rootEndpoint = a.storageBucket.operatorMetadata!.nodeEndpoint!
-        const apiEndpoint = this.getApiEndpoint(rootEndpoint)
+    const endpointsData = details.storageBag.storageBuckets
+      .filter((bucket) => bucket.operatorStatus.__typename === 'StorageBucketOperatorStatusActive')
+      .map((bucket) => {
+        const rootEndpoint = bucket.operatorMetadata?.nodeEndpoint
+        const apiEndpoint = rootEndpoint ? this.getApiEndpoint(rootEndpoint) : ''
         return {
-          bucketId: a.storageBucket.id,
+          bucketId: bucket.id,
           endpoint: apiEndpoint,
         }
       })
@@ -116,32 +114,42 @@ export class NetworkingService {
     }
   }
 
+  private getDataObjectActiveDistributorsSet(objectDetails: DataObjectDetailsFragment): Set<number> {
+    const activeDistributorsSet = new Set<number>()
+    const { distributionBuckets } = objectDetails.storageBag
+    for (const bucket of distributionBuckets) {
+      for (const operator of bucket.operators) {
+        if (operator.status === DistributionBucketOperatorStatus.Active) {
+          activeDistributorsSet.add(operator.workerId)
+        }
+      }
+    }
+    return activeDistributorsSet
+  }
+
   public async dataObjectInfo(objectId: string): Promise<DataObjectInfo> {
     const details = await this.queryNodeApi.getDataObjectDetails(objectId)
-    return {
-      exists: !!details,
-      isSupported:
-        (this.config.buckets === 'all' &&
-          details?.storageBag.distirbutionAssignments.some((d) =>
-            d.distributionBucket.operators.some(
-              (o) => o.workerId === this.config.workerId && o.status === DistributionBucketOperatorStatus.Active
-            )
-          )) ||
-        (Array.isArray(this.config.buckets) &&
-          this.config.buckets.some((bucketId) =>
-            details?.storageBag.distirbutionAssignments
-              .map((a) => a.distributionBucket.id)
-              .includes(bucketId.toString())
-          )),
-      data: details
-        ? {
-            objectId,
-            accessPoints: this.parseDataObjectAccessPoints(details),
-            contentHash: details.ipfsHash,
-            size: parseInt(details.size),
-          }
-        : undefined,
+    let exists = false
+    let isSupported = false
+    let data: DataObjectData | undefined
+    if (details) {
+      exists = true
+      if (!this.config.buckets) {
+        const distributors = this.getDataObjectActiveDistributorsSet(details)
+        isSupported = typeof this.config.workerId === 'number' ? distributors.has(this.config.workerId) : false
+      } else {
+        const supportedBucketIds = this.config.buckets.map((id) => id.toString())
+        isSupported = details.storageBag.distributionBuckets.some((b) => supportedBucketIds.includes(b.id))
+      }
+      data = {
+        objectId,
+        accessPoints: this.parseDataObjectAccessPoints(details),
+        contentHash: details.ipfsHash,
+        size: parseInt(details.size),
+      }
     }
+
+    return { exists, isSupported, data }
   }
 
   private sortEndpointsByMeanResponseTime(endpoints: string[]) {
@@ -152,8 +160,93 @@ export class NetworkingService {
     )
   }
 
+  private async checkObjectAvailability(objectId: string, endpoint: string): Promise<void> {
+    const api = new StorageNodeApi(endpoint, this.logging, this.config)
+    const available = await api.isObjectAvailable(objectId)
+    if (!available) {
+      throw new Error('Not available')
+    }
+  }
+
+  private createDataObjectAvailabilityCheckQueue(objectId: string, storageEndpoints: string[]) {
+    const availabilityQueue = queue({
+      concurrency: MAX_CONCURRENT_AVAILABILITY_CHECKS_PER_OBJECT,
+      autostart: true,
+    })
+
+    storageEndpoints.forEach(async (endpoint) => {
+      availabilityQueue.push(async () => {
+        await this.checkObjectAvailability(objectId, endpoint)
+        return endpoint
+      })
+    })
+
+    availabilityQueue.on('error', () => {
+      /*
+      Do nothing.
+      The handler is needed to avoid unhandled promise rejection
+      */
+    })
+
+    return availabilityQueue
+  }
+
+  public async getDataObjectDownloadSource(objectData: DataObjectData): Promise<string> {
+    const { objectId } = objectData
+    const cachedSource = await this.checkCachedDataObjectSource(objectId)
+    if (cachedSource) {
+      this.logger.info(`Found active download source for object ${objectId} in cache`, { objectId, cachedSource })
+      return cachedSource
+    }
+    return this.findDataObjectDownloadSource(objectData)
+  }
+
+  private async checkCachedDataObjectSource(objectId: string): Promise<string | undefined> {
+    const cachedSource = this.stateCache.getCachedDataObjectSource(objectId)
+    if (cachedSource) {
+      try {
+        await this.checkObjectAvailability(objectId, cachedSource)
+      } catch (err) {
+        this.stateCache.dropCachedDataObjectSource(objectId, cachedSource)
+        return undefined
+      }
+      return cachedSource
+    }
+  }
+
+  private findDataObjectDownloadSource({ objectId, accessPoints }: DataObjectData): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const storageEndpoints = this.sortEndpointsByMeanResponseTime(
+        accessPoints?.storageNodes.map((n) => n.endpoint) || []
+      )
+
+      this.logger.info('Looking for data object source', {
+        objectId,
+        possibleSources: storageEndpoints.map((e) => ({
+          endpoint: e,
+          meanResponseTime: this.stateCache.getStorageNodeEndpointMeanResponseTime(e),
+        })),
+      })
+      if (!storageEndpoints.length) {
+        return reject(new Error('No storage endpoints available to download the data object from'))
+      }
+
+      const availabilityQueue = this.createDataObjectAvailabilityCheckQueue(objectId, storageEndpoints)
+
+      availabilityQueue.on('success', (endpoint) => {
+        availabilityQueue.stop()
+        this.stateCache.cacheDataObjectSource(objectId, endpoint)
+        return resolve(endpoint)
+      })
+
+      availabilityQueue.on('end', () => {
+        return reject(new Error('Failed to find data object download source'))
+      })
+    })
+  }
+
   private downloadJob(
-    pendingDownload: PendingDownloadData,
+    pendingDownload: PendingDownload,
     downloadData: DownloadData,
     onSourceFound: (response: StorageNodeDownloadResponse) => void,
     onError: (error: Error) => void,
@@ -164,7 +257,7 @@ export class NetworkingService {
       startAt,
     } = downloadData
 
-    pendingDownload.status = 'LookingForSource'
+    pendingDownload.setStatus({ type: PendingDownloadStatusType.LookingForSource })
 
     return new Promise<void>((resolve, reject) => {
       // Handlers:
@@ -174,9 +267,13 @@ export class NetworkingService {
         reject(new Error(message))
       }
 
-      const sourceFound = (response: StorageNodeDownloadResponse) => {
-        this.logger.info('Download source chosen', { objectId, source: response.config.url })
-        pendingDownload.status = 'Downloading'
+      const sourceFound = (endpoint: string, response: StorageNodeDownloadResponse) => {
+        this.logger.info('Download source chosen', { objectId, source: endpoint })
+        pendingDownload.setStatus({
+          type: PendingDownloadStatusType.Downloading,
+          source: endpoint,
+          contentType: response.headers['content-type'],
+        })
         onSourceFound(response)
       }
 
@@ -197,46 +294,25 @@ export class NetworkingService {
         })),
       })
       if (!storageEndpoints.length) {
-        return fail('No storage endpoints available to download the data object from')
+        return fail(`No storage endpoints available to download the data object: ${objectId}`)
       }
 
-      const availabilityQueue = queue({
-        concurrency: MAX_CONCURRENT_AVAILABILITY_CHECKS_PER_DOWNLOAD,
-        autostart: true,
-      })
+      const availabilityQueue = this.createDataObjectAvailabilityCheckQueue(objectId, storageEndpoints)
       const objectDownloadQueue = queue({ concurrency: 1, autostart: true })
 
-      storageEndpoints.forEach(async (endpoint) => {
-        availabilityQueue.push(async () => {
-          const api = new StorageNodeApi(endpoint, this.logging)
-          const available = await api.isObjectAvailable(objectId)
-          if (!available) {
-            throw new Error('Not avilable')
-          }
-          return endpoint
-        })
-      })
-
       availabilityQueue.on('success', (endpoint) => {
         availabilityQueue.stop()
         const job = async () => {
-          const api = new StorageNodeApi(endpoint, this.logging)
+          const api = new StorageNodeApi(endpoint, this.logging, this.config)
           const response = await api.downloadObject(objectId, startAt)
-          return response
+          return [endpoint, response]
         }
         objectDownloadQueue.push(job)
       })
 
-      availabilityQueue.on('error', () => {
-        /*
-        Do nothing.
-        The handler is needed to avoid unhandled promise rejection
-        */
-      })
-
       availabilityQueue.on('end', () => {
         if (!objectDownloadQueue.length) {
-          fail('Failed to download the object from any availablable storage provider')
+          fail(`Failed to download object ${objectId} from any availablable storage provider`)
         }
       })
 
@@ -248,15 +324,15 @@ export class NetworkingService {
         if (availabilityQueue.length) {
           availabilityQueue.start()
         } else {
-          fail('Failed to download the object from any availablable storage provider')
+          fail(`Failed to download object ${objectId} from any availablable storage provider`)
         }
       })
 
-      objectDownloadQueue.on('success', (response: StorageNodeDownloadResponse) => {
+      objectDownloadQueue.on('success', ([endpoint, response]: [string, StorageNodeDownloadResponse]) => {
         availabilityQueue.removeAllListeners().end()
         objectDownloadQueue.removeAllListeners().end()
         response.data.on('close', finish).on('error', finish).on('end', finish)
-        sourceFound(response)
+        sourceFound(endpoint, response)
       })
     })
   }
@@ -265,34 +341,29 @@ export class NetworkingService {
     const {
       objectData: { objectId, size },
     } = downloadData
-
     if (this.stateCache.getPendingDownload(objectId)) {
       // Already downloading
       return null
     }
-
-    let resolveDownload: (response: StorageNodeDownloadResponse) => void, rejectDownload: (err: Error) => void
-    const downloadPromise = new Promise<StorageNodeDownloadResponse>((resolve, reject) => {
-      resolveDownload = resolve
-      rejectDownload = reject
+    const pendingDownload = this.stateCache.addPendingDownload(new PendingDownload(objectId, size))
+    return new Promise<StorageNodeDownloadResponse>((resolve, reject) => {
+      const onSourceFound = resolve
+      const onError = reject
+      // Queue the download
+      this.downloadQueue.push(() => this.downloadJob(pendingDownload, downloadData, onSourceFound, onError))
     })
-
-    // Queue the download
-    const pendingDownload = this.stateCache.newPendingDownload(objectId, size, downloadPromise)
-    this.downloadQueue.push(() => this.downloadJob(pendingDownload, downloadData, resolveDownload, rejectDownload))
-
-    return downloadPromise
   }
 
   async fetchSupportedDataObjects(): Promise<Map<string, DataObjectData>> {
-    const data =
-      this.config.buckets === 'all'
-        ? await this.queryNodeApi.getDistributionBucketsWithObjectsByWorkerId(this.config.workerId)
-        : await this.queryNodeApi.getDistributionBucketsWithObjectsByIds(this.config.buckets.map((id) => id.toString()))
+    const data = this.config.buckets
+      ? await this.queryNodeApi.getDistributionBucketsWithObjectsByIds(this.config.buckets.map((id) => id.toString()))
+      : typeof this.config.workerId === 'number'
+      ? await this.queryNodeApi.getDistributionBucketsWithObjectsByWorkerId(this.config.workerId)
+      : []
     const objectsData = new Map<string, DataObjectData>()
     data.forEach((bucket) => {
-      bucket.bagAssignments.forEach((a) => {
-        a.storageBag.objects.forEach((object) => {
+      bucket.bags.forEach((bag) => {
+        bag.objects.forEach((object) => {
           const { ipfsHash, id, size } = object
           objectsData.set(id, { contentHash: ipfsHash, objectId: id, size: parseInt(size) })
         })
@@ -308,7 +379,7 @@ export class NetworkingService {
       const endpoints = this.filterStorageNodeEndpoints(
         activeStorageOperators.map(({ id, operatorMetadata }) => ({
           bucketId: id,
-          endpoint: this.getApiEndpoint(operatorMetadata!.nodeEndpoint!),
+          endpoint: operatorMetadata?.nodeEndpoint ? this.getApiEndpoint(operatorMetadata.nodeEndpoint) : '',
         }))
       )
       this.logger.verbose('Checking nearby storage nodes...', { validEndpointsCount: endpoints.length })
@@ -327,9 +398,8 @@ export class NetworkingService {
     const start = Date.now()
     this.logger.debug(`Sending storage node response-time check request to: ${endpoint}`, { endpoint })
     try {
-      const api = new StorageNodeApi(endpoint, this.logging)
-      const reqConfig: AxiosRequestConfig = { headers: { connection: 'close' } }
-      await api.stateApi.stateApiGetVersion(reqConfig)
+      const api = new StorageNodeApi(endpoint, this.logging, this.config)
+      await api.getVersion()
       const responseTime = Date.now() - start
       this.logger.debug(`${endpoint} check request response time: ${responseTime}`, { endpoint, responseTime })
       this.stateCache.setStorageNodeEndpointResponseTime(endpoint, responseTime)

distributor-node/src/services/networking/PendingDownload.ts

@@ -0,0 +1,84 @@
+export enum PendingDownloadStatusType {
+  Waiting = 'Waiting',
+  LookingForSource = 'LookingForSource',
+  Downloading = 'Downloading',
+}
+
+export type PendingDownloadStatusWaiting = {
+  type: PendingDownloadStatusType.Waiting
+}
+
+export type PendingDownloadStatusLookingForSource = {
+  type: PendingDownloadStatusType.LookingForSource
+}
+
+export type PendingDownloadStatusDownloading = {
+  type: PendingDownloadStatusType.Downloading
+  source: string
+  contentType?: string
+}
+
+export type PendingDownloadStatus =
+  | PendingDownloadStatusWaiting
+  | PendingDownloadStatusLookingForSource
+  | PendingDownloadStatusDownloading
+
+export const STATUS_ORDER = [
+  PendingDownloadStatusType.Waiting,
+  PendingDownloadStatusType.LookingForSource,
+  PendingDownloadStatusType.Downloading,
+] as const
+
+export class PendingDownload {
+  private objectId: string
+  private objectSize: number
+  private status: PendingDownloadStatus = { type: PendingDownloadStatusType.Waiting }
+  private statusHandlers: Map<PendingDownloadStatusType, (() => void)[]> = new Map()
+  private cleanupHandlers: (() => void)[] = []
+
+  constructor(objectId: string, objectSize: number) {
+    this.objectId = objectId
+    this.objectSize = objectSize
+  }
+
+  setStatus(status: PendingDownloadStatus): void {
+    this.status = status
+    const handlers = this.statusHandlers.get(status.type) || []
+    handlers.forEach((handler) => handler())
+  }
+
+  getStatus(): PendingDownloadStatus {
+    return this.status
+  }
+
+  getObjectId(): string {
+    return this.objectId
+  }
+
+  getObjectSize(): number {
+    return this.objectSize
+  }
+
+  private registerStatusHandler(statusType: PendingDownloadStatusType, handler: () => void) {
+    const currentHandlers = this.statusHandlers.get(statusType) || []
+    this.statusHandlers.set(statusType, [...currentHandlers, handler])
+  }
+
+  private registerCleanupHandler(handler: () => void) {
+    this.cleanupHandlers.push(handler)
+  }
+
+  untilStatus<T extends PendingDownloadStatusType>(statusType: T): Promise<void> {
+    return new Promise((resolve, reject) => {
+      if (STATUS_ORDER.indexOf(this.status.type) >= STATUS_ORDER.indexOf(statusType)) {
+        return resolve()
+      }
+      this.registerStatusHandler(statusType, () => resolve())
+      this.registerCleanupHandler(() => reject(new Error(`Could not download object ${this.objectId} from any source`)))
+    })
+  }
+
+  cleanup(): void {
+    this.cleanupHandlers.forEach((handler) => handler())
+  }
+}

distributor-node/src/services/networking/distributor-node/generated/.openapi-generator-ignore

@@ -1,27 +0,0 @@
-# OpenAPI Generator Ignore
-# Generated by openapi-generator https://github.com/openapitools/openapi-generator
-
-# Use this file to prevent files from being overwritten by the generator.
-# The patterns follow closely to .gitignore or .dockerignore.
-
-# As an example, the C# client generator defines ApiClient.cs.
-# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
-#ApiClient.cs
-
-# You can match any string of characters against a directory, file or extension with a single asterisk (*):
-#foo/*/qux
-# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
-
-# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
-#foo/**/qux
-# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
-
-# You can also negate patterns with an exclamation (!).
-# For example, you can ignore all files in a docs folder with the file extension .md:
-#docs/*.md
-# Then explicitly reverse the ignore rule for a single file:
-#!docs/README.md
-
-git_push.sh
-.npmignore
-.gitignore

distributor-node/src/services/networking/distributor-node/generated/.openapi-generator/FILES

@@ -1,5 +0,0 @@
-api.ts
-base.ts
-common.ts
-configuration.ts
-index.ts

distributor-node/src/services/networking/distributor-node/generated/.openapi-generator/VERSION

@@ -1 +0,0 @@
-5.2.0

distributor-node/src/services/networking/distributor-node/generated/api.ts

@@ -1,394 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-/**
- * Distributor node API
- * Distributor node API
- *
- * The version of the OpenAPI document: 0.1.0
- * Contact: info@joystream.org
- *
- * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
- * https://openapi-generator.tech
- * Do not edit the class manually.
- */
-
-
-import { Configuration } from './configuration';
-import globalAxios, { AxiosPromise, AxiosInstance } from 'axios';
-// Some imports not used depending on template conditions
-// @ts-ignore
-import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from './common';
-// @ts-ignore
-import { BASE_PATH, COLLECTION_FORMATS, RequestArgs, BaseAPI, RequiredError } from './base';
-
-/**
- * @type BucketsResponse
- * @export
- */
-export type BucketsResponse = BucketsResponseOneOf | BucketsResponseOneOf1;
-
-/**
- * 
- * @export
- * @interface BucketsResponseOneOf
- */
-export interface BucketsResponseOneOf {
-    /**
-     * 
-     * @type {Array<number>}
-     * @memberof BucketsResponseOneOf
-     */
-    bucketIds: Array<number>;
-}
-/**
- * 
- * @export
- * @interface BucketsResponseOneOf1
- */
-export interface BucketsResponseOneOf1 {
-    /**
-     * 
-     * @type {number}
-     * @memberof BucketsResponseOneOf1
-     */
-    allByWorkerId: number;
-}
-/**
- * 
- * @export
- * @interface ErrorResponse
- */
-export interface ErrorResponse {
-    /**
-     * 
-     * @type {string}
-     * @memberof ErrorResponse
-     */
-    type?: string;
-    /**
-     * 
-     * @type {string}
-     * @memberof ErrorResponse
-     */
-    message: string;
-}
-/**
- * 
- * @export
- * @interface StatusResponse
- */
-export interface StatusResponse {
-    /**
-     * 
-     * @type {string}
-     * @memberof StatusResponse
-     */
-    id: string;
-    /**
-     * 
-     * @type {number}
-     * @memberof StatusResponse
-     */
-    objectsInCache: number;
-    /**
-     * 
-     * @type {number}
-     * @memberof StatusResponse
-     */
-    storageLimit: number;
-    /**
-     * 
-     * @type {number}
-     * @memberof StatusResponse
-     */
-    storageUsed: number;
-    /**
-     * 
-     * @type {number}
-     * @memberof StatusResponse
-     */
-    uptime: number;
-    /**
-     * 
-     * @type {number}
-     * @memberof StatusResponse
-     */
-    downloadsInProgress: number;
-}
-
-/**
- * PublicApi - axios parameter creator
- * @export
- */
-export const PublicApiAxiosParamCreator = function (configuration?: Configuration) {
-    return {
-        /**
-         * Returns a media file.
-         * @param {string} objectId Data Object ID
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        publicAsset: async (objectId: string, options: any = {}): Promise<RequestArgs> => {
-            // verify required parameter 'objectId' is not null or undefined
-            assertParamExists('publicAsset', 'objectId', objectId)
-            const localVarPath = `/asset/{objectId}`
-                .replace(`{${"objectId"}}`, encodeURIComponent(String(objectId)));
-            // use dummy base URL string because the URL constructor only accepts absolute URLs.
-            const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
-            let baseOptions;
-            if (configuration) {
-                baseOptions = configuration.baseOptions;
-            }
-
-            const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options};
-            const localVarHeaderParameter = {} as any;
-            const localVarQueryParameter = {} as any;
-
-
-    
-            setSearchParams(localVarUrlObj, localVarQueryParameter, options.query);
-            let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
-            localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
-
-            return {
-                url: toPathString(localVarUrlObj),
-                options: localVarRequestOptions,
-            };
-        },
-        /**
-         * Returns asset response headers (cache status, content type and/or length, accepted ranges etc.)
-         * @param {string} objectId Data Object ID
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        publicAssetHead: async (objectId: string, options: any = {}): Promise<RequestArgs> => {
-            // verify required parameter 'objectId' is not null or undefined
-            assertParamExists('publicAssetHead', 'objectId', objectId)
-            const localVarPath = `/asset/{objectId}`
-                .replace(`{${"objectId"}}`, encodeURIComponent(String(objectId)));
-            // use dummy base URL string because the URL constructor only accepts absolute URLs.
-            const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
-            let baseOptions;
-            if (configuration) {
-                baseOptions = configuration.baseOptions;
-            }
-
-            const localVarRequestOptions = { method: 'HEAD', ...baseOptions, ...options};
-            const localVarHeaderParameter = {} as any;
-            const localVarQueryParameter = {} as any;
-
-
-    
-            setSearchParams(localVarUrlObj, localVarQueryParameter, options.query);
-            let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
-            localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
-
-            return {
-                url: toPathString(localVarUrlObj),
-                options: localVarRequestOptions,
-            };
-        },
-        /**
-         * Returns list of distributed buckets
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        publicBuckets: async (options: any = {}): Promise<RequestArgs> => {
-            const localVarPath = `/buckets`;
-            // use dummy base URL string because the URL constructor only accepts absolute URLs.
-            const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
-            let baseOptions;
-            if (configuration) {
-                baseOptions = configuration.baseOptions;
-            }
-
-            const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options};
-            const localVarHeaderParameter = {} as any;
-            const localVarQueryParameter = {} as any;
-
-
-    
-            setSearchParams(localVarUrlObj, localVarQueryParameter, options.query);
-            let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
-            localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
-
-            return {
-                url: toPathString(localVarUrlObj),
-                options: localVarRequestOptions,
-            };
-        },
-        /**
-         * Returns json object describing current node status.
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        publicStatus: async (options: any = {}): Promise<RequestArgs> => {
-            const localVarPath = `/status`;
-            // use dummy base URL string because the URL constructor only accepts absolute URLs.
-            const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
-            let baseOptions;
-            if (configuration) {
-                baseOptions = configuration.baseOptions;
-            }
-
-            const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options};
-            const localVarHeaderParameter = {} as any;
-            const localVarQueryParameter = {} as any;
-
-
-    
-            setSearchParams(localVarUrlObj, localVarQueryParameter, options.query);
-            let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
-            localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
-
-            return {
-                url: toPathString(localVarUrlObj),
-                options: localVarRequestOptions,
-            };
-        },
-    }
-};
-
-/**
- * PublicApi - functional programming interface
- * @export
- */
-export const PublicApiFp = function(configuration?: Configuration) {
-    const localVarAxiosParamCreator = PublicApiAxiosParamCreator(configuration)
-    return {
-        /**
-         * Returns a media file.
-         * @param {string} objectId Data Object ID
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        async publicAsset(objectId: string, options?: any): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<any>> {
-            const localVarAxiosArgs = await localVarAxiosParamCreator.publicAsset(objectId, options);
-            return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
-        },
-        /**
-         * Returns asset response headers (cache status, content type and/or length, accepted ranges etc.)
-         * @param {string} objectId Data Object ID
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        async publicAssetHead(objectId: string, options?: any): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<void>> {
-            const localVarAxiosArgs = await localVarAxiosParamCreator.publicAssetHead(objectId, options);
-            return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
-        },
-        /**
-         * Returns list of distributed buckets
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        async publicBuckets(options?: any): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<BucketsResponse>> {
-            const localVarAxiosArgs = await localVarAxiosParamCreator.publicBuckets(options);
-            return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
-        },
-        /**
-         * Returns json object describing current node status.
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        async publicStatus(options?: any): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<StatusResponse>> {
-            const localVarAxiosArgs = await localVarAxiosParamCreator.publicStatus(options);
-            return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
-        },
-    }
-};
-
-/**
- * PublicApi - factory interface
- * @export
- */
-export const PublicApiFactory = function (configuration?: Configuration, basePath?: string, axios?: AxiosInstance) {
-    const localVarFp = PublicApiFp(configuration)
-    return {
-        /**
-         * Returns a media file.
-         * @param {string} objectId Data Object ID
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        publicAsset(objectId: string, options?: any): AxiosPromise<any> {
-            return localVarFp.publicAsset(objectId, options).then((request) => request(axios, basePath));
-        },
-        /**
-         * Returns asset response headers (cache status, content type and/or length, accepted ranges etc.)
-         * @param {string} objectId Data Object ID
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        publicAssetHead(objectId: string, options?: any): AxiosPromise<void> {
-            return localVarFp.publicAssetHead(objectId, options).then((request) => request(axios, basePath));
-        },
-        /**
-         * Returns list of distributed buckets
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        publicBuckets(options?: any): AxiosPromise<BucketsResponse> {
-            return localVarFp.publicBuckets(options).then((request) => request(axios, basePath));
-        },
-        /**
-         * Returns json object describing current node status.
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        publicStatus(options?: any): AxiosPromise<StatusResponse> {
-            return localVarFp.publicStatus(options).then((request) => request(axios, basePath));
-        },
-    };
-};
-
-/**
- * PublicApi - object-oriented interface
- * @export
- * @class PublicApi
- * @extends {BaseAPI}
- */
-export class PublicApi extends BaseAPI {
-    /**
-     * Returns a media file.
-     * @param {string} objectId Data Object ID
-     * @param {*} [options] Override http request option.
-     * @throws {RequiredError}
-     * @memberof PublicApi
-     */
-    public publicAsset(objectId: string, options?: any) {
-        return PublicApiFp(this.configuration).publicAsset(objectId, options).then((request) => request(this.axios, this.basePath));
-    }
-
-    /**
-     * Returns asset response headers (cache status, content type and/or length, accepted ranges etc.)
-     * @param {string} objectId Data Object ID
-     * @param {*} [options] Override http request option.
-     * @throws {RequiredError}
-     * @memberof PublicApi
-     */
-    public publicAssetHead(objectId: string, options?: any) {
-        return PublicApiFp(this.configuration).publicAssetHead(objectId, options).then((request) => request(this.axios, this.basePath));
-    }
-
-    /**
-     * Returns list of distributed buckets
-     * @param {*} [options] Override http request option.
-     * @throws {RequiredError}
-     * @memberof PublicApi
-     */
-    public publicBuckets(options?: any) {
-        return PublicApiFp(this.configuration).publicBuckets(options).then((request) => request(this.axios, this.basePath));
-    }
-
-    /**
-     * Returns json object describing current node status.
-     * @param {*} [options] Override http request option.
-     * @throws {RequiredError}
-     * @memberof PublicApi
-     */
-    public publicStatus(options?: any) {
-        return PublicApiFp(this.configuration).publicStatus(options).then((request) => request(this.axios, this.basePath));
-    }
-}
-
-

distributor-node/src/services/networking/distributor-node/generated/base.ts

@@ -1,71 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-/**
- * Distributor node API
- * Distributor node API
- *
- * The version of the OpenAPI document: 0.1.0
- * Contact: info@joystream.org
- *
- * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
- * https://openapi-generator.tech
- * Do not edit the class manually.
- */
-
-
-import { Configuration } from "./configuration";
-// Some imports not used depending on template conditions
-// @ts-ignore
-import globalAxios, { AxiosPromise, AxiosInstance } from 'axios';
-
-export const BASE_PATH = "http://localhost:3334/api/v1".replace(/\/+$/, "");
-
-/**
- *
- * @export
- */
-export const COLLECTION_FORMATS = {
-    csv: ",",
-    ssv: " ",
-    tsv: "\t",
-    pipes: "|",
-};
-
-/**
- *
- * @export
- * @interface RequestArgs
- */
-export interface RequestArgs {
-    url: string;
-    options: any;
-}
-
-/**
- *
- * @export
- * @class BaseAPI
- */
-export class BaseAPI {
-    protected configuration: Configuration | undefined;
-
-    constructor(configuration?: Configuration, protected basePath: string = BASE_PATH, protected axios: AxiosInstance = globalAxios) {
-        if (configuration) {
-            this.configuration = configuration;
-            this.basePath = configuration.basePath || this.basePath;
-        }
-    }
-};
-
-/**
- *
- * @export
- * @class RequiredError
- * @extends {Error}
- */
-export class RequiredError extends Error {
-    name: "RequiredError" = "RequiredError";
-    constructor(public field: string, msg?: string) {
-        super(msg);
-    }
-}

distributor-node/src/services/networking/distributor-node/generated/common.ts

@@ -1,138 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-/**
- * Distributor node API
- * Distributor node API
- *
- * The version of the OpenAPI document: 0.1.0
- * Contact: info@joystream.org
- *
- * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
- * https://openapi-generator.tech
- * Do not edit the class manually.
- */
-
-
-import { Configuration } from "./configuration";
-import { RequiredError,　RequestArgs } from "./base";
-import { AxiosInstance } from 'axios';
-
-/**
- *
- * @export
- */
-export const DUMMY_BASE_URL = 'https://example.com'
-
-/**
- *
- * @throws {RequiredError}
- * @export
- */
-export const assertParamExists = function (functionName: string, paramName: string, paramValue: unknown) {
-    if (paramValue === null || paramValue === undefined) {
-        throw new RequiredError(paramName, `Required parameter ${paramName} was null or undefined when calling ${functionName}.`);
-    }
-}
-
-/**
- *
- * @export
- */
-export const setApiKeyToObject = async function (object: any, keyParamName: string, configuration?: Configuration) {
-    if (configuration && configuration.apiKey) {
-        const localVarApiKeyValue = typeof configuration.apiKey === 'function'
-            ? await configuration.apiKey(keyParamName)
-            : await configuration.apiKey;
-        object[keyParamName] = localVarApiKeyValue;
-    }
-}
-
-/**
- *
- * @export
- */
-export const setBasicAuthToObject = function (object: any, configuration?: Configuration) {
-    if (configuration && (configuration.username || configuration.password)) {
-        object["auth"] = { username: configuration.username, password: configuration.password };
-    }
-}
-
-/**
- *
- * @export
- */
-export const setBearerAuthToObject = async function (object: any, configuration?: Configuration) {
-    if (configuration && configuration.accessToken) {
-        const accessToken = typeof configuration.accessToken === 'function'
-            ? await configuration.accessToken()
-            : await configuration.accessToken;
-        object["Authorization"] = "Bearer " + accessToken;
-    }
-}
-
-/**
- *
- * @export
- */
-export const setOAuthToObject = async function (object: any, name: string, scopes: string[], configuration?: Configuration) {
-    if (configuration && configuration.accessToken) {
-        const localVarAccessTokenValue = typeof configuration.accessToken === 'function'
-            ? await configuration.accessToken(name, scopes)
-            : await configuration.accessToken;
-        object["Authorization"] = "Bearer " + localVarAccessTokenValue;
-    }
-}
-
-/**
- *
- * @export
- */
-export const setSearchParams = function (url: URL, ...objects: any[]) {
-    const searchParams = new URLSearchParams(url.search);
-    for (const object of objects) {
-        for (const key in object) {
-            if (Array.isArray(object[key])) {
-                searchParams.delete(key);
-                for (const item of object[key]) {
-                    searchParams.append(key, item);
-                }
-            } else {
-                searchParams.set(key, object[key]);
-            }
-        }
-    }
-    url.search = searchParams.toString();
-}
-
-/**
- *
- * @export
- */
-export const serializeDataIfNeeded = function (value: any, requestOptions: any, configuration?: Configuration) {
-    const nonString = typeof value !== 'string';
-    const needsSerialization = nonString && configuration && configuration.isJsonMime
-        ? configuration.isJsonMime(requestOptions.headers['Content-Type'])
-        : nonString;
-    return needsSerialization
-        ? JSON.stringify(value !== undefined ? value : {})
-        : (value || "");
-}
-
-/**
- *
- * @export
- */
-export const toPathString = function (url: URL) {
-    return url.pathname + url.search + url.hash
-}
-
-/**
- *
- * @export
- */
-export const createRequestFunction = function (axiosArgs: RequestArgs, globalAxios: AxiosInstance, BASE_PATH: string, configuration?: Configuration) {
-    return (axios: AxiosInstance = globalAxios, basePath: string = BASE_PATH) => {
-        const axiosRequestArgs = {...axiosArgs.options, url: (configuration?.basePath || basePath) + axiosArgs.url};
-        return axios.request(axiosRequestArgs);
-    };
-}

distributor-node/src/services/networking/distributor-node/generated/configuration.ts

@@ -1,101 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-/**
- * Distributor node API
- * Distributor node API
- *
- * The version of the OpenAPI document: 0.1.0
- * Contact: info@joystream.org
- *
- * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
- * https://openapi-generator.tech
- * Do not edit the class manually.
- */
-
-
-export interface ConfigurationParameters {
-    apiKey?: string | Promise<string> | ((name: string) => string) | ((name: string) => Promise<string>);
-    username?: string;
-    password?: string;
-    accessToken?: string | Promise<string> | ((name?: string, scopes?: string[]) => string) | ((name?: string, scopes?: string[]) => Promise<string>);
-    basePath?: string;
-    baseOptions?: any;
-    formDataCtor?: new () => any;
-}
-
-export class Configuration {
-    /**
-     * parameter for apiKey security
-     * @param name security name
-     * @memberof Configuration
-     */
-    apiKey?: string | Promise<string> | ((name: string) => string) | ((name: string) => Promise<string>);
-    /**
-     * parameter for basic security
-     *
-     * @type {string}
-     * @memberof Configuration
-     */
-    username?: string;
-    /**
-     * parameter for basic security
-     *
-     * @type {string}
-     * @memberof Configuration
-     */
-    password?: string;
-    /**
-     * parameter for oauth2 security
-     * @param name security name
-     * @param scopes oauth2 scope
-     * @memberof Configuration
-     */
-    accessToken?: string | Promise<string> | ((name?: string, scopes?: string[]) => string) | ((name?: string, scopes?: string[]) => Promise<string>);
-    /**
-     * override base path
-     *
-     * @type {string}
-     * @memberof Configuration
-     */
-    basePath?: string;
-    /**
-     * base options for axios calls
-     *
-     * @type {any}
-     * @memberof Configuration
-     */
-    baseOptions?: any;
-    /**
-     * The FormData constructor that will be used to create multipart form data
-     * requests. You can inject this here so that execution environments that
-     * do not support the FormData class can still run the generated client.
-     *
-     * @type {new () => FormData}
-     */
-    formDataCtor?: new () => any;
-
-    constructor(param: ConfigurationParameters = {}) {
-        this.apiKey = param.apiKey;
-        this.username = param.username;
-        this.password = param.password;
-        this.accessToken = param.accessToken;
-        this.basePath = param.basePath;
-        this.baseOptions = param.baseOptions;
-        this.formDataCtor = param.formDataCtor;
-    }
-
-    /**
-     * Check if the given MIME is a JSON MIME.
-     * JSON MIME examples:
-     *   application/json
-     *   application/json; charset=UTF8
-     *   APPLICATION/JSON
-     *   application/vnd.company+json
-     * @param mime - MIME (Multipurpose Internet Mail Extensions)
-     * @return True if the given MIME is JSON, false otherwise.
-     */
-    public isJsonMime(mime: string): boolean {
-        const jsonMime: RegExp = new RegExp('^(application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(;.*)?$', 'i');
-        return mime !== null && (jsonMime.test(mime) || mime.toLowerCase() === 'application/json-patch+json');
-    }
-}

distributor-node/src/services/networking/distributor-node/generated/index.ts

@@ -1,18 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-/**
- * Distributor node API
- * Distributor node API
- *
- * The version of the OpenAPI document: 0.1.0
- * Contact: info@joystream.org
- *
- * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
- * https://openapi-generator.tech
- * Do not edit the class manually.
- */
-
-
-export * from "./api";
-export * from "./configuration";
-

distributor-node/src/services/networking/query-node/api.ts

@@ -22,6 +22,23 @@ import {
 } from './generated/queries'
 import { Maybe } from './generated/schema'
 
+const MAX_RESULTS_PER_QUERY = 1000
+
+type PaginationQueryVariables = {
+  limit: number
+  lastCursor?: Maybe<string>
+}
+
+type PaginationQueryResult<T = unknown> = {
+  edges: { node: T }[]
+  pageInfo: {
+    hasNextPage: boolean
+    endCursor?: Maybe<string>
+  }
+}
+
+type CustomVariables<T> = Omit<T, keyof PaginationQueryVariables>
+
 export class QueryNodeApi {
   private apolloClient: ApolloClient<NormalizedCacheObject>
   private logger: Logger
@@ -68,6 +85,35 @@ export class QueryNodeApi {
     return (await this.apolloClient.query<QueryT, VariablesT>({ query, variables })).data[resultKey]
   }
 
+  protected async multipleEntitiesWithPagination<
+    NodeT,
+    QueryT extends { [k: string]: PaginationQueryResult<NodeT> },
+    CustomVariablesT extends Record<string, unknown>
+  >(
+    query: DocumentNode,
+    variables: CustomVariablesT,
+    resultKey: keyof QueryT,
+    itemsPerPage = MAX_RESULTS_PER_QUERY
+  ): Promise<NodeT[]> {
+    let hasNextPage = true
+    let results: NodeT[] = []
+    let lastCursor: string | undefined
+    while (hasNextPage) {
+      const paginationVariables = { limit: itemsPerPage, lastCursor }
+      const queryVariables = { ...variables, ...paginationVariables }
+      const page = (
+        await this.apolloClient.query<QueryT, PaginationQueryVariables & CustomVariablesT>({
+          query,
+          variables: queryVariables,
+        })
+      ).data[resultKey]
+      results = results.concat(page.edges.map((e) => e.node))
+      hasNextPage = page.pageInfo.hasNextPage
+      lastCursor = page.pageInfo.endCursor || undefined
+    }
+    return results
+  }
+
   public getDataObjectDetails(objectId: string): Promise<DataObjectDetailsFragment | null> {
     return this.uniqueEntityQuery<GetDataObjectDetailsQuery, GetDataObjectDetailsQueryVariables>(
       GetDataObjectDetails,
@@ -93,9 +139,10 @@ export class QueryNodeApi {
   }
 
   public getActiveStorageBucketOperatorsData(): Promise<StorageBucketOperatorFieldsFragment[]> {
-    return this.multipleEntitiesQuery<
+    return this.multipleEntitiesWithPagination<
+      StorageBucketOperatorFieldsFragment,
       GetActiveStorageBucketOperatorsDataQuery,
-      GetActiveStorageBucketOperatorsDataQueryVariables
-    >(GetActiveStorageBucketOperatorsData, {}, 'storageBuckets')
+      CustomVariables<GetActiveStorageBucketOperatorsDataQueryVariables>
+    >(GetActiveStorageBucketOperatorsData, {}, 'storageBucketsConnection')
   }
 }

distributor-node/src/services/networking/query-node/codegen.yml

@@ -16,8 +16,8 @@ generates:
   src/services/networking/query-node/generated/schema.ts:
     hooks:
       afterOneFileWrite:
-        - prettier --write
-        - eslint --fix
+        - yarn prettier --write
+        - yarn eslint --fix
     plugins:
       - typescript
   src/services/networking/query-node/generated/queries.ts:
@@ -26,8 +26,8 @@ generates:
       typesPath: ./schema
     hooks:
       afterOneFileWrite:
-        - prettier --write
-        - eslint --fix
+        - yarn prettier --write
+        - yarn eslint --fix
     plugins:
       - typescript-operations
       - typescript-document-nodes

distributor-node/src/services/networking/query-node/generated/queries.ts

@@ -1,73 +1,130 @@
-import * as Types from './schema';
+import * as Types from './schema'
 
-import gql from 'graphql-tag';
-export type DataObjectDetailsFragment = { id: string, size: any, ipfsHash: string, isAccepted: boolean, storageBag: { storageAssignments: Array<{ storageBucket: { id: string, operatorMetadata?: Types.Maybe<{ nodeEndpoint?: Types.Maybe<string> }>, operatorStatus: { __typename: 'StorageBucketOperatorStatusMissing' } | { __typename: 'StorageBucketOperatorStatusInvited' } | { __typename: 'StorageBucketOperatorStatusActive' } } }>, distirbutionAssignments: Array<{ distributionBucket: { id: string, operators: Array<{ workerId: number, status: Types.DistributionBucketOperatorStatus }> } }> } };
+import gql from 'graphql-tag'
+export type DistributionBucketOperatorDetailsFragment = {
+  workerId: number
+  status: Types.DistributionBucketOperatorStatus
+}
 
-export type GetDataObjectDetailsQueryVariables = Types.Exact<{
-  id: Types.Scalars['ID'];
-}>;
+export type DistributionBucketDetailsFragment = {
+  id: string
+  operators: Array<DistributionBucketOperatorDetailsFragment>
+}
 
+export type StorageBucketDetailsFragment = {
+  id: string
+  operatorMetadata?: Types.Maybe<{ nodeEndpoint?: Types.Maybe<string> }>
+  operatorStatus:
+    | { __typename: 'StorageBucketOperatorStatusMissing' }
+    | { __typename: 'StorageBucketOperatorStatusInvited' }
+    | { __typename: 'StorageBucketOperatorStatusActive' }
+}
 
-export type GetDataObjectDetailsQuery = { storageDataObjectByUniqueInput?: Types.Maybe<DataObjectDetailsFragment> };
+export type DataObjectDetailsFragment = {
+  id: string
+  size: any
+  ipfsHash: string
+  isAccepted: boolean
+  storageBag: {
+    storageBuckets: Array<StorageBucketDetailsFragment>
+    distributionBuckets: Array<DistributionBucketDetailsFragment>
+  }
+}
 
-export type DistirubtionBucketWithObjectsFragment = { id: string, bagAssignments: Array<{ storageBag: { objects: Array<{ id: string, size: any, ipfsHash: string }> } }> };
+export type GetDataObjectDetailsQueryVariables = Types.Exact<{
+  id: Types.Scalars['ID']
+}>
 
-export type GetDistributionBucketsWithObjectsByIdsQueryVariables = Types.Exact<{
-  ids?: Types.Maybe<Array<Types.Scalars['ID']> | Types.Scalars['ID']>;
-}>;
+export type GetDataObjectDetailsQuery = { storageDataObjectByUniqueInput?: Types.Maybe<DataObjectDetailsFragment> }
 
+export type DistirubtionBucketWithObjectsFragment = {
+  id: string
+  bags: Array<{ objects: Array<{ id: string; size: any; ipfsHash: string }> }>
+}
 
-export type GetDistributionBucketsWithObjectsByIdsQuery = { distributionBuckets: Array<DistirubtionBucketWithObjectsFragment> };
+export type GetDistributionBucketsWithObjectsByIdsQueryVariables = Types.Exact<{
+  ids?: Types.Maybe<Array<Types.Scalars['ID']> | Types.Scalars['ID']>
+}>
 
-export type GetDistributionBucketsWithObjectsByWorkerIdQueryVariables = Types.Exact<{
-  workerId: Types.Scalars['Int'];
-}>;
+export type GetDistributionBucketsWithObjectsByIdsQuery = {
+  distributionBuckets: Array<DistirubtionBucketWithObjectsFragment>
+}
 
+export type GetDistributionBucketsWithObjectsByWorkerIdQueryVariables = Types.Exact<{
+  workerId: Types.Scalars['Int']
+}>
 
-export type GetDistributionBucketsWithObjectsByWorkerIdQuery = { distributionBuckets: Array<DistirubtionBucketWithObjectsFragment> };
+export type GetDistributionBucketsWithObjectsByWorkerIdQuery = {
+  distributionBuckets: Array<DistirubtionBucketWithObjectsFragment>
+}
 
-export type StorageBucketOperatorFieldsFragment = { id: string, operatorMetadata?: Types.Maybe<{ nodeEndpoint?: Types.Maybe<string> }> };
+export type StorageBucketOperatorFieldsFragment = {
+  id: string
+  operatorMetadata?: Types.Maybe<{ nodeEndpoint?: Types.Maybe<string> }>
+}
 
-export type GetActiveStorageBucketOperatorsDataQueryVariables = Types.Exact<{ [key: string]: never; }>;
+export type StorageBucketsConnectionFieldsFragment = {
+  edges: Array<{ node: StorageBucketOperatorFieldsFragment }>
+  pageInfo: { hasNextPage: boolean; endCursor?: Types.Maybe<string> }
+}
 
+export type GetActiveStorageBucketOperatorsDataQueryVariables = Types.Exact<{
+  limit: Types.Scalars['Int']
+  lastCursor?: Types.Maybe<Types.Scalars['String']>
+}>
 
-export type GetActiveStorageBucketOperatorsDataQuery = { storageBuckets: Array<StorageBucketOperatorFieldsFragment> };
+export type GetActiveStorageBucketOperatorsDataQuery = {
+  storageBucketsConnection: StorageBucketsConnectionFieldsFragment
+}
 
+export const StorageBucketDetails = gql`
+  fragment StorageBucketDetails on StorageBucket {
+    id
+    operatorMetadata {
+      nodeEndpoint
+    }
+    operatorStatus {
+      __typename
+    }
+  }
+`
+export const DistributionBucketOperatorDetails = gql`
+  fragment DistributionBucketOperatorDetails on DistributionBucketOperator {
+    workerId
+    status
+  }
+`
+export const DistributionBucketDetails = gql`
+  fragment DistributionBucketDetails on DistributionBucket {
+    id
+    operators {
+      ...DistributionBucketOperatorDetails
+    }
+  }
+  ${DistributionBucketOperatorDetails}
+`
 export const DataObjectDetails = gql`
-    fragment DataObjectDetails on StorageDataObject {
-  id
-  size
-  ipfsHash
-  isAccepted
-  storageBag {
-    storageAssignments {
-      storageBucket {
-        id
-        operatorMetadata {
-          nodeEndpoint
-        }
-        operatorStatus {
-          __typename
-        }
+  fragment DataObjectDetails on StorageDataObject {
+    id
+    size
+    ipfsHash
+    isAccepted
+    storageBag {
+      storageBuckets {
+        ...StorageBucketDetails
       }
-    }
-    distirbutionAssignments {
-      distributionBucket {
-        id
-        operators {
-          workerId
-          status
-        }
+      distributionBuckets {
+        ...DistributionBucketDetails
       }
     }
   }
-}
-    `;
+  ${StorageBucketDetails}
+  ${DistributionBucketDetails}
+`
 export const DistirubtionBucketWithObjects = gql`
-    fragment DistirubtionBucketWithObjects on DistributionBucket {
-  id
-  bagAssignments {
-    storageBag {
+  fragment DistirubtionBucketWithObjects on DistributionBucket {
+    id
+    bags {
       objects {
         id
         size
@@ -75,41 +132,62 @@ export const DistirubtionBucketWithObjects = gql`
       }
     }
   }
-}
-    `;
+`
 export const StorageBucketOperatorFields = gql`
-    fragment StorageBucketOperatorFields on StorageBucket {
-  id
-  operatorMetadata {
-    nodeEndpoint
+  fragment StorageBucketOperatorFields on StorageBucket {
+    id
+    operatorMetadata {
+      nodeEndpoint
+    }
   }
-}
-    `;
+`
+export const StorageBucketsConnectionFields = gql`
+  fragment StorageBucketsConnectionFields on StorageBucketConnection {
+    edges {
+      node {
+        ...StorageBucketOperatorFields
+      }
+    }
+    pageInfo {
+      hasNextPage
+      endCursor
+    }
+  }
+  ${StorageBucketOperatorFields}
+`
 export const GetDataObjectDetails = gql`
-    query getDataObjectDetails($id: ID!) {
-  storageDataObjectByUniqueInput(where: {id: $id}) {
-    ...DataObjectDetails
+  query getDataObjectDetails($id: ID!) {
+    storageDataObjectByUniqueInput(where: { id: $id }) {
+      ...DataObjectDetails
+    }
   }
-}
-    ${DataObjectDetails}`;
+  ${DataObjectDetails}
+`
 export const GetDistributionBucketsWithObjectsByIds = gql`
-    query getDistributionBucketsWithObjectsByIds($ids: [ID!]) {
-  distributionBuckets(where: {id_in: $ids}) {
-    ...DistirubtionBucketWithObjects
+  query getDistributionBucketsWithObjectsByIds($ids: [ID!]) {
+    distributionBuckets(where: { id_in: $ids }) {
+      ...DistirubtionBucketWithObjects
+    }
   }
-}
-    ${DistirubtionBucketWithObjects}`;
+  ${DistirubtionBucketWithObjects}
+`
 export const GetDistributionBucketsWithObjectsByWorkerId = gql`
-    query getDistributionBucketsWithObjectsByWorkerId($workerId: Int!) {
-  distributionBuckets(where: {operators_some: {workerId_eq: $workerId, status_eq: ACTIVE}}) {
-    ...DistirubtionBucketWithObjects
+  query getDistributionBucketsWithObjectsByWorkerId($workerId: Int!) {
+    distributionBuckets(where: { operators_some: { workerId_eq: $workerId, status_eq: ACTIVE } }) {
+      ...DistirubtionBucketWithObjects
+    }
   }
-}
-    ${DistirubtionBucketWithObjects}`;
+  ${DistirubtionBucketWithObjects}
+`
 export const GetActiveStorageBucketOperatorsData = gql`
-    query getActiveStorageBucketOperatorsData {
-  storageBuckets(where: {operatorStatus_json: {isTypeOf_eq: "StorageBucketOperatorStatusActive"}, operatorMetadata: {nodeEndpoint_contains: "http"}}, limit: 9999) {
-    ...StorageBucketOperatorFields
+  query getActiveStorageBucketOperatorsData($limit: Int!, $lastCursor: String) {
+    storageBucketsConnection(
+      first: $limit
+      after: $lastCursor
+      where: { operatorStatus_json: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } }
+    ) {
+      ...StorageBucketsConnectionFields
+    }
   }
-}
-    ${StorageBucketOperatorFields}`;
+  ${StorageBucketsConnectionFields}
+`

distributor-node/src/services/networking/query-node/queries/queries.graphql

@@ -1,28 +1,36 @@
+fragment DistributionBucketOperatorDetails on DistributionBucketOperator {
+  workerId
+  status
+}
+
+fragment DistributionBucketDetails on DistributionBucket {
+  id
+  operators {
+    ...DistributionBucketOperatorDetails
+  }
+}
+
+fragment StorageBucketDetails on StorageBucket {
+  id
+  operatorMetadata {
+    nodeEndpoint
+  }
+  operatorStatus {
+    __typename
+  }
+}
+
 fragment DataObjectDetails on StorageDataObject {
   id
   size
   ipfsHash
   isAccepted
   storageBag {
-    storageAssignments {
-      storageBucket {
-        id
-        operatorMetadata {
-          nodeEndpoint
-        }
-        operatorStatus {
-          __typename
-        }
-      }
+    storageBuckets {
+      ...StorageBucketDetails
     }
-    distirbutionAssignments {
-      distributionBucket {
-        id
-        operators {
-          workerId
-          status
-        }
-      }
+    distributionBuckets {
+      ...DistributionBucketDetails
     }
   }
 }
@@ -35,13 +43,11 @@ query getDataObjectDetails($id: ID!) {
 
 fragment DistirubtionBucketWithObjects on DistributionBucket {
   id
-  bagAssignments {
-    storageBag {
-      objects {
-        id
-        size
-        ipfsHash
-      }
+  bags {
+    objects {
+      id
+      size
+      ipfsHash
     }
   }
 }
@@ -65,14 +71,24 @@ fragment StorageBucketOperatorFields on StorageBucket {
   }
 }
 
-query getActiveStorageBucketOperatorsData {
-  storageBuckets(
-    where: {
-      operatorStatus_json: { isTypeOf_eq: "StorageBucketOperatorStatusActive" }
-      operatorMetadata: { nodeEndpoint_contains: "http" }
+fragment StorageBucketsConnectionFields on StorageBucketConnection {
+  edges {
+    node {
+      ...StorageBucketOperatorFields
     }
-    limit: 9999
+  }
+  pageInfo {
+    hasNextPage
+    endCursor
+  }
+}
+
+query getActiveStorageBucketOperatorsData($limit: Int!, $lastCursor: String) {
+  storageBucketsConnection(
+    first: $limit
+    after: $lastCursor
+    where: { operatorStatus_json: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } }
   ) {
-    ...StorageBucketOperatorFields
+    ...StorageBucketsConnectionFields
   }
 }