Merge pull request #2611 from shamil-gadelshin/storage_v2_runtime_auth_fixes

Storage v2 auth schema fixes.

runtime-modules/storage/src/lib.rs

@@ -319,6 +319,9 @@ pub trait Trait: frame_system::Trait + balances::Trait + membership::Trait {
     /// Max number of pending invitations per distribution bucket.
     type MaxNumberOfPendingInvitationsPerDistributionBucket: Get<u64>;
 
+    /// Max data object size in bytes.
+    type MaxDataObjectSize: Get<u64>;
+
     /// Demand the storage working group leader authorization.
     /// TODO: Refactor after merging with the Olympia release.
     fn ensure_storage_working_group_leader_origin(origin: Self::Origin) -> DispatchResult;
@@ -467,6 +470,9 @@ pub struct DataObject<Balance> {
 
     /// Object size in bytes.
     pub size: u64,
+
+    /// Content identifier presented as IPFS hash.
+    pub ipfs_content_id: Vec<u8>,
 }
 
 /// Type alias for the BagRecord.
@@ -652,9 +658,6 @@ pub type UploadParameters<T> = UploadParametersRecord<
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
 #[derive(Encode, Decode, Default, Clone, PartialEq, Eq, Debug)]
 pub struct UploadParametersRecord<MemberId, ChannelId, AccountId, Balance> {
-    /// Public key used authentication in upload to liaison.
-    pub authentication_key: Vec<u8>,
-
     /// Static or dynamic bag to upload data.
     pub bag_id: BagIdType<MemberId, ChannelId>,
 
@@ -1427,6 +1430,9 @@ decl_error! {
 
         /// Distribution family bound to a bag creation policy.
         DistributionFamilyBoundToBagCreationPolicy,
+
+        /// Max data object size exceeded.
+        MaxDataObjectSizeExceeded,
     }
 }
 
@@ -1477,6 +1483,9 @@ decl_module! {
         const MaxNumberOfPendingInvitationsPerDistributionBucket: u64 =
             T::MaxNumberOfPendingInvitationsPerDistributionBucket::get();
 
+        /// Exports const - max data object size in bytes.
+        const MaxDataObjectSize: u64 = T::MaxDataObjectSize::get();
+
         // ===== Storage Lead actions =====
 
         /// Delete storage bucket. Must be empty. Storage operator must be missing.
@@ -2900,6 +2909,7 @@ impl<T: Trait> Module<T> {
             accepted: false,
             deletion_prize,
             size: obj.size,
+            ipfs_content_id: obj.ipfs_content_id,
         });
 
         let mut next_data_object_id = Self::next_data_object_id();
@@ -3184,6 +3194,15 @@ impl<T: Trait> Module<T> {
             Error::<T>::NoObjectsOnUpload
         );
 
+        // Check data objects' max size.
+        ensure!(
+            params
+                .object_creation_list
+                .iter()
+                .all(|obj| obj.size <= T::MaxDataObjectSize::get()),
+            Error::<T>::MaxDataObjectSizeExceeded
+        );
+
         let bag = Self::ensure_bag_exists(&params.bag_id)?;
 
         let new_objects_number: u64 = params.object_creation_list.len().saturated_into();

runtime-modules/storage/src/tests/mocks.rs

@@ -65,6 +65,7 @@ parameter_types! {
     pub const DefaultChannelDynamicBagNumberOfStorageBuckets: u64 = 4;
     pub const DistributionBucketsPerBagValueConstraint: crate::DistributionBucketsPerBagValueConstraint =
         crate::StorageBucketsPerBagValueConstraint {min: 3, max_min_diff: 7};
+    pub const MaxDataObjectSize: u64 = 400;
 }
 
 pub const STORAGE_WG_LEADER_ACCOUNT_ID: u64 = 100001;
@@ -101,6 +102,7 @@ impl crate::Trait for Test {
     type DistributionBucketsPerBagValueConstraint = DistributionBucketsPerBagValueConstraint;
     type MaxNumberOfPendingInvitationsPerDistributionBucket =
         MaxNumberOfPendingInvitationsPerDistributionBucket;
+    type MaxDataObjectSize = MaxDataObjectSize;
 
     fn ensure_storage_working_group_leader_origin(origin: Self::Origin) -> DispatchResult {
         let account_id = ensure_signed(origin)?;

runtime-modules/storage/src/tests/mod.rs

@@ -24,7 +24,7 @@ use crate::{
 use mocks::{
     build_test_externalities, Balances, DataObjectDeletionPrize,
     DefaultChannelDynamicBagNumberOfStorageBuckets, DefaultMemberDynamicBagNumberOfStorageBuckets,
-    InitialStorageBucketsNumberForDynamicBag, MaxDistributionBucketFamilyNumber,
+    InitialStorageBucketsNumberForDynamicBag, MaxDataObjectSize, MaxDistributionBucketFamilyNumber,
     MaxDistributionBucketNumberPerFamily, MaxNumberOfDataObjectsPerBag, MaxRandomIterationNumber,
     Storage, Test, ANOTHER_DISTRIBUTION_PROVIDER_ID, ANOTHER_STORAGE_PROVIDER_ID,
     DEFAULT_DISTRIBUTION_PROVIDER_ACCOUNT_ID, DEFAULT_DISTRIBUTION_PROVIDER_ID,
@@ -360,7 +360,6 @@ fn update_storage_buckets_for_bags_succeeded_with_voucher_usage() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -424,7 +423,6 @@ fn update_storage_buckets_for_bags_fails_with_exceeding_the_voucher_objects_numb
 
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -470,7 +468,6 @@ fn update_storage_buckets_for_bags_fails_with_exceeding_the_voucher_objects_tota
 
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -671,7 +668,6 @@ fn upload_succeeded() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -695,6 +691,9 @@ fn upload_succeeded() {
             Storage::data_object_by_id(&bag_id, &data_object_id),
             DataObject {
                 size: upload_params.object_creation_list[0].size,
+                ipfs_content_id: upload_params.object_creation_list[0]
+                    .ipfs_content_id
+                    .clone(),
                 deletion_prize: DataObjectDeletionPrize::get(),
                 accepted: false,
             }
@@ -717,6 +716,28 @@ fn upload_succeeded() {
     });
 }
 
+#[test]
+fn upload_failed_with_exceeding_the_data_object_max_size() {
+    build_test_externalities().execute_with(|| {
+        let initial_balance = 1000;
+        increase_account_balance(&DEFAULT_MEMBER_ACCOUNT_ID, initial_balance);
+
+        let mut data_object_list = create_single_data_object();
+        data_object_list[0].size = MaxDataObjectSize::get() + 1;
+
+        let upload_params = UploadParameters::<Test> {
+            bag_id: BagId::<Test>::Static(StaticBagId::Council),
+            deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
+            object_creation_list: data_object_list,
+            expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
+        };
+
+        UploadFixture::default()
+            .with_params(upload_params.clone())
+            .call_and_assert(Err(Error::<Test>::MaxDataObjectSizeExceeded.into()));
+    });
+}
+
 #[test]
 fn upload_succeeded_with_data_size_fee() {
     build_test_externalities().execute_with(|| {
@@ -732,7 +753,6 @@ fn upload_succeeded_with_data_size_fee() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -768,7 +788,6 @@ fn upload_succeeded_with_active_storage_bucket_having_voucher() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id,
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -803,7 +822,6 @@ fn upload_fails_with_active_storage_bucket_with_voucher_object_number_limit_exce
 
         let upload_params = UploadParameters::<Test> {
             bag_id,
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -838,7 +856,6 @@ fn upload_fails_with_active_storage_bucket_with_voucher_object_size_limit_exceed
 
         let upload_params = UploadParameters::<Test> {
             bag_id,
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -864,7 +881,6 @@ fn upload_succeeded_with_dynamic_bag() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Dynamic(dynamic_bag_id.clone()),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -888,6 +904,9 @@ fn upload_succeeded_with_dynamic_bag() {
             Storage::data_object_by_id(&bag_id, &data_object_id),
             DataObject {
                 size: upload_params.object_creation_list[0].size,
+                ipfs_content_id: upload_params.object_creation_list[0]
+                    .ipfs_content_id
+                    .clone(),
                 deletion_prize: DataObjectDeletionPrize::get(),
                 accepted: false,
             }
@@ -902,7 +921,6 @@ fn upload_fails_with_non_existent_dynamic_bag() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Dynamic(dynamic_bag_id.clone()),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -921,7 +939,6 @@ fn upload_succeeded_with_non_empty_bag() {
 
         let upload_params1 = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_data_object_candidates(1, 2),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -933,7 +950,6 @@ fn upload_succeeded_with_non_empty_bag() {
 
         let upload_params2 = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_data_object_candidates(3, 2),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -964,7 +980,6 @@ fn upload_fails_with_zero_object_size() {
     build_test_externalities().execute_with(|| {
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: vec![DataObjectCreationParameters {
                 ipfs_content_id: vec![1],
@@ -984,7 +999,6 @@ fn upload_fails_with_empty_object_cid() {
     build_test_externalities().execute_with(|| {
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: vec![DataObjectCreationParameters {
                 ipfs_content_id: Vec::new(),
@@ -1007,7 +1021,6 @@ fn upload_fails_with_max_data_object_size_exceeded() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_data_object_candidates(1, invalid_object_number),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1024,7 +1037,6 @@ fn upload_fails_with_insufficient_balance_for_deletion_prize() {
     build_test_externalities().execute_with(|| {
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1043,7 +1055,6 @@ fn upload_fails_with_insufficient_balance_for_data_size_fee() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1078,7 +1089,6 @@ fn upload_fails_with_data_size_fee_changed() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1105,7 +1115,6 @@ fn upload_failed_with_blocked_uploading() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1140,7 +1149,6 @@ fn upload_failed_with_blacklisted_data_object() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: BagId::<Test>::Static(StaticBagId::Council),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list,
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1283,7 +1291,6 @@ fn accept_pending_data_objects_succeeded() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1348,7 +1355,6 @@ fn accept_pending_data_objects_fails_with_unrelated_storage_bucket() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1443,7 +1449,6 @@ fn accept_pending_data_objects_succeeded_with_dynamic_bag() {
         let bag_id = BagId::<Test>::Dynamic(dynamic_bag_id.clone());
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1773,7 +1778,6 @@ fn move_data_objects_succeeded() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: src_bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1873,7 +1877,6 @@ fn move_data_objects_succeeded_having_voucher() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: src_bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -1948,7 +1951,6 @@ fn move_data_objects_fails_with_exceeding_voucher_object_number_limit() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: src_bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -2007,7 +2009,6 @@ fn move_data_objects_fails_with_exceeding_voucher_objects_size_limit() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: src_bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -2086,7 +2087,6 @@ fn delete_data_objects_succeeded() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -2186,7 +2186,6 @@ fn delete_data_objects_fails_with_invalid_treasury_balance() {
         let council_bag_id = BagId::<Test>::Static(StaticBagId::Council);
         let upload_params = UploadParameters::<Test> {
             bag_id: council_bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: create_single_data_object(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -2231,7 +2230,6 @@ fn delete_data_objects_succeeded_with_voucher_usage() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -2657,7 +2655,6 @@ fn delete_storage_bucket_fails_with_non_empty_bucket() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),
@@ -2844,7 +2841,6 @@ fn storage_bucket_voucher_changed_event_fired() {
 
         let upload_params = UploadParameters::<Test> {
             bag_id: bag_id.clone(),
-            authentication_key: Vec::new(),
             deletion_prize_source_account_id: DEFAULT_MEMBER_ACCOUNT_ID,
             object_creation_list: object_creation_list.clone(),
             expected_data_size_fee: Storage::data_object_per_mega_byte_fee(),

runtime/src/lib.rs

@@ -694,6 +694,7 @@ parameter_types! {
     pub const DefaultChannelDynamicBagNumberOfStorageBuckets: u64 = 4; //TODO: adjust value
     pub const DistributionBucketsPerBagValueConstraint: storage::DistributionBucketsPerBagValueConstraint =
         storage::DistributionBucketsPerBagValueConstraint {min: 3, max_min_diff: 7}; //TODO: adjust value
+    pub const MaxDataObjectSize: u64 = 10 * 1024 * 1024 * 1024; // 10 GB
 }
 
 impl storage::Trait for Runtime {
@@ -721,6 +722,7 @@ impl storage::Trait for Runtime {
     type DistributionBucketOperatorId = DistributionBucketOperatorId;
     type MaxNumberOfPendingInvitationsPerDistributionBucket =
         MaxNumberOfPendingInvitationsPerDistributionBucket;
+    type MaxDataObjectSize = MaxDataObjectSize;
 
     fn ensure_storage_working_group_leader_origin(origin: Self::Origin) -> DispatchResult {
         StorageWorkingGroup::ensure_origin_is_active_leader(origin)

storage-node-v2/package.json

@@ -1,6 +1,6 @@
 {
   "name": "storage-node-v2",
-  "description": "Jostream storage subsystem.",
+  "description": "Joystream storage subsystem.",
   "version": "0.1.0",
   "author": "Joystream contributors",
   "bin": {

storage-node-v2/src/api-spec/openapi.yaml

@@ -174,6 +174,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ErrorResponse'
+        401:
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
 
 components:
   securitySchemes:

storage-node-v2/src/commands/server.ts

@@ -45,7 +45,10 @@ export default class Server extends ApiCommandBase {
     try {
       const port = flags.port
       const workerId = flags.worker ?? 0
-      const app = await createApp(api, account, workerId, flags.uploads)
+      const maxFileSize = await api.consts.storage.maxDataObjectSize.toNumber()
+      logger.debug(`Max file size runtime parameter: ${maxFileSize}`)
+
+      const app = await createApp(api, account, workerId, flags.uploads, maxFileSize)
       logger.info(`Listening on http://localhost:${port}`)
       app.listen(port)
     } catch (err) {

storage-node-v2/src/services/webApi/app.ts

@@ -17,13 +17,15 @@ import { httpLogger, errorLogger } from '../../services/logger'
  * @param account - KeyringPair instance
  * @param workerId - storage provider ID (worker ID)
  * @param uploadsDir - directory for the file uploading
+ * @param maxFileSize - max allowed file size
  * @returns Express promise.
  */
 export async function createApp(
   api: ApiPromise,
   account: KeyringPair,
   workerId: number,
-  uploadsDir: string
+  uploadsDir: string,
+  maxFileSize: number
 ): Promise<Express> {
   const spec = path.join(__dirname, './../../api-spec/openapi.yaml')
 
@@ -52,7 +54,16 @@ export async function createApp(
         basePath: path.join(__dirname, './controllers'),
         resolver: OpenApiValidator.resolvers.modulePathResolver,
       },
-      fileUploader: { dest: uploadsDir },
+      fileUploader: {
+        dest: uploadsDir,
+        // Busboy library settings
+        limits: {
+          // For multipart forms, the max number of file fields (Default: Infinity)
+          files: 1,
+          // For multipart forms, the max file size (in bytes) (Default: Infinity)
+          fileSize: maxFileSize,
+        },
+      },
       validateSecurity: {
         handlers: {
           UploadAuth: validateUpload(api, account),
@@ -65,12 +76,17 @@ export async function createApp(
 
   /* eslint-disable @typescript-eslint/no-unused-vars */
   app.use((err: Error, req: express.Request, res: express.Response, next: express.NextFunction) => {
-    // Request validation error handler.
+    // Express error handling recommendation:
+    // https://expressjs.com/en/guide/error-handling.html
+    if (res.headersSent) {
+      return next(err)
+    }
+
+    // Request error handler.
     if (err instanceof HttpError) {
-      res.status(err.status).json({
-        type: 'request_validation',
+      res.status(err.status || 500).json({
+        type: 'request_exception',
         message: err.message,
-        errors: err.errors,
       })
     } else {
       res.status(500).json({

storage-node-v2/src/services/webApi/controllers/publicApi.ts

@@ -11,6 +11,7 @@ import { hashFile } from '../../../services/helpers/hashing'
 import { createNonce, getTokenExpirationTime } from '../../../services/helpers/tokenNonceKeeper'
 import { getFileInfo } from '../../../services/helpers/fileInfo'
 import { parseBagId } from '../../helpers/bagTypes'
+import { BagId } from '@joystream/types/storage'
 import logger from '../../../services/logger'
 import { KeyringPair } from '@polkadot/keyring/types'
 import { ApiPromise } from '@polkadot/api'
@@ -19,6 +20,7 @@ import fs from 'fs'
 import path from 'path'
 import send from 'send'
 import { CLIError } from '@oclif/errors'
+import { hexToString } from '@polkadot/util'
 const fsPromises = fs.promises
 
 /**
@@ -107,10 +109,13 @@ export async function uploadFile(req: express.Request, res: express.Response): P
     const fileObj = getFileObject(req)
     cleanupFileName = fileObj.path
 
-    verifyFileSize(fileObj.size)
+    const api = getApi(res)
     await verifyFileMimeType(fileObj.path)
 
     const hash = await hashFile(fileObj.path)
+    const bagId = parseBagId(api, uploadRequest.bagId)
+
+    const accepted = await verifyDataObjectInfo(api, bagId, uploadRequest.dataObjectId, fileObj.size, hash)
 
     // Prepare new file name
     const newPath = fileObj.path.replace(fileObj.filename, hash)
@@ -119,11 +124,16 @@ export async function uploadFile(req: express.Request, res: express.Response): P
     await fsPromises.rename(fileObj.path, newPath)
     cleanupFileName = newPath
 
-    const api = getApi(res)
-    const bagId = parseBagId(api, uploadRequest.bagId)
-    await acceptPendingDataObjects(api, bagId, getAccount(res), getWorkerId(res), uploadRequest.storageBucketId, [
-      uploadRequest.dataObjectId,
-    ])
+    const workerId = getWorkerId(res)
+    if (!accepted) {
+      await acceptPendingDataObjects(api, bagId, getAccount(res), workerId, uploadRequest.storageBucketId, [
+        uploadRequest.dataObjectId,
+      ])
+    } else {
+      logger.warn(
+        `Received already accepted data object. DataObjectId = ${uploadRequest.dataObjectId} WorkerId = ${workerId}`
+      )
+    }
     res.status(201).json({
       id: hash,
     })
@@ -294,17 +304,41 @@ async function validateTokenRequest(api: ApiPromise, tokenRequest: UploadTokenRe
 }
 
 /**
- * Validates file size. It throws an error when file size exceeds the limit
+ * Validates the runtime info for the data object. It verifies contentID,
+ * file size, and 'accepted' status.
  *
- * @param fileSize - runtime API promise
- * @returns void promise.
+ * @param api - runtime API promise
+ * @param bagId - bag ID
+ * @param dataObjectId - data object ID to validate in runtime
+ * @param fileSize - file size to validate
+ * @param hash - file multihash
+ * @returns promise with the 'data object accepted' flag.
  */
-function verifyFileSize(fileSize: number) {
-  const MAX_FILE_SIZE = 1000000 // TODO: Get this const from the runtime
+async function verifyDataObjectInfo(
+  api: ApiPromise,
+  bagId: BagId,
+  dataObjectId: number,
+  fileSize: number,
+  hash: string
+): Promise<boolean> {
+  const dataObject = await api.query.storage.dataObjectsById(bagId, dataObjectId)
+
+  // Cannot get 'size' as a regular property.
+  const dataObjectSize = dataObject.getField('size')
+
+  if (dataObjectSize?.toNumber() !== fileSize) {
+    throw new WebApiError(`File size doesn't match the data object's size for data object ID = ${dataObjectId}`, 400)
+  }
 
-  if (fileSize > MAX_FILE_SIZE) {
-    throw new WebApiError('Max file size exceeded.', 400)
+  const runtimeHash = hexToString(dataObject.ipfsContentId.toString())
+  if (runtimeHash !== hash) {
+    throw new WebApiError(
+      `File multihash doesn't match the data object's ipfsContentId for data object ID = ${dataObjectId}`,
+      400
+    )
   }
+
+  return dataObject.accepted.valueOf()
 }
 
 /**

types/augment-codec/augment-api-consts.ts

@@ -193,6 +193,10 @@ declare module '@polkadot/api/types/consts' {
        * Exports const - "Distribution buckets per bag" value constraint.
        **/
       distributionBucketsPerBagValueConstraint: StorageBucketsPerBagValueConstraint & AugmentedConst<ApiType>;
+      /**
+       * Exports const - max data object size in bytes.
+       **/
+      maxDataObjectSize: u64 & AugmentedConst<ApiType>;
       /**
        * Exports const - max allowed distribution bucket family number.
        **/

types/augment-codec/augment-api-errors.ts

@@ -2315,6 +2315,10 @@ declare module '@polkadot/api/types/errors' {
        * Invalid operation with invites: storage provider was already invited.
        **/
       InvitedStorageProvider: AugmentedError<ApiType>;
+      /**
+       * Max data object size exceeded.
+       **/
+      MaxDataObjectSizeExceeded: AugmentedError<ApiType>;
       /**
        * Max distribution bucket family number limit exceeded.
        **/

types/augment-codec/augment-api-tx.ts

@@ -1365,7 +1365,7 @@ declare module '@polkadot/api/types/submittable' {
       /**
        * Upload new data objects. Development mode.
        **/
-      sudoUploadDataObjects: AugmentedSubmittable<(params: UploadParameters | { authenticationKey?: any; bagId?: any; objectCreationList?: any; deletionPrizeSourceAccountId?: any; expectedDataSizeFee?: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>, [UploadParameters]>;
+      sudoUploadDataObjects: AugmentedSubmittable<(params: UploadParameters | { bagId?: any; objectCreationList?: any; deletionPrizeSourceAccountId?: any; expectedDataSizeFee?: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>, [UploadParameters]>;
       /**
        * Add and remove hashes to the current blacklist.
        **/

types/augment/all/defs.json

@@ -563,7 +563,6 @@
         }
     },
     "UploadParameters": {
-        "authenticationKey": "Bytes",
         "bagId": "BagId",
         "objectCreationList": "Vec<DataObjectCreationParameters>",
         "deletionPrizeSourceAccountId": "GenericAccountId",
@@ -583,7 +582,8 @@
     "DataObject": {
         "accepted": "bool",
         "deletion_prize": "u128",
-        "size": "u64"
+        "size": "u64",
+        "ipfsContentId": "Bytes"
     },
     "DistributionBucketId": "u64",
     "DistributionBucketFamilyId": "u64",

types/augment/all/types.ts

@@ -366,6 +366,7 @@ export interface DAOId extends u64 {}
 export interface DataObject extends Struct {
   readonly accepted: bool;
   readonly deletion_prize: u128;
+  readonly ipfsContentId: Bytes;
 }
 
 /** @name DataObjectCreationParameters */
@@ -1292,7 +1293,6 @@ export interface UpdatePropertyValuesOperation extends Null {}
 
 /** @name UploadParameters */
 export interface UploadParameters extends Struct {
-  readonly authenticationKey: Bytes;
   readonly bagId: BagId;
   readonly objectCreationList: Vec<DataObjectCreationParameters>;
   readonly deletionPrizeSourceAccountId: GenericAccountId;

types/augment/augment-api-consts.ts

@@ -193,6 +193,10 @@ declare module '@polkadot/api/types/consts' {
        * Exports const - "Distribution buckets per bag" value constraint.
        **/
       distributionBucketsPerBagValueConstraint: StorageBucketsPerBagValueConstraint & AugmentedConst<ApiType>;
+      /**
+       * Exports const - max data object size in bytes.
+       **/
+      maxDataObjectSize: u64 & AugmentedConst<ApiType>;
       /**
        * Exports const - max allowed distribution bucket family number.
        **/

types/augment/augment-api-errors.ts

@@ -2315,6 +2315,10 @@ declare module '@polkadot/api/types/errors' {
        * Invalid operation with invites: storage provider was already invited.
        **/
       InvitedStorageProvider: AugmentedError<ApiType>;
+      /**
+       * Max data object size exceeded.
+       **/
+      MaxDataObjectSizeExceeded: AugmentedError<ApiType>;
       /**
        * Max distribution bucket family number limit exceeded.
        **/

types/augment/augment-api-tx.ts

@@ -1365,7 +1365,7 @@ declare module '@polkadot/api/types/submittable' {
       /**
        * Upload new data objects. Development mode.
        **/
-      sudoUploadDataObjects: AugmentedSubmittable<(params: UploadParameters | { authenticationKey?: any; bagId?: any; objectCreationList?: any; deletionPrizeSourceAccountId?: any; expectedDataSizeFee?: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>, [UploadParameters]>;
+      sudoUploadDataObjects: AugmentedSubmittable<(params: UploadParameters | { bagId?: any; objectCreationList?: any; deletionPrizeSourceAccountId?: any; expectedDataSizeFee?: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>, [UploadParameters]>;
       /**
        * Add and remove hashes to the current blacklist.
        **/

types/src/storage.ts

@@ -23,6 +23,7 @@ export type IDataObject = {
   accepted: bool
   deletion_prize: BalanceOf
   size: u64
+  ipfsContentId: Bytes
 }
 
 export class DataObject
@@ -30,6 +31,7 @@ export class DataObject
     accepted: bool,
     deletion_prize: BalanceOf,
     size: u64,
+    ipfsContentId: Bytes,
   })
   implements IDataObject {}
 
@@ -164,7 +166,6 @@ export class DataObjectCreationParameters
   implements IDataObjectCreationParameters {}
 
 export type IUploadParameters = {
-  authenticationKey: Bytes
   bagId: BagId
   objectCreationList: Vec<DataObjectCreationParameters>
   deletionPrizeSourceAccountId: AccountId
@@ -173,7 +174,6 @@ export type IUploadParameters = {
 
 export class UploadParameters
   extends JoyStructDecorated({
-    authenticationKey: Bytes,
     bagId: BagId,
     objectCreationList: Vec.with(DataObjectCreationParameters),
     deletionPrizeSourceAccountId: AccountId,