storage-node-v2: Review fixes.

- added 401 status to the api spec (auth token endpoint)
- added filesize limit to the multer(Busboy) library settings
- changed ‘accepted data object’ check

storage-node-v2/src/api-spec/openapi.yaml

@@ -174,6 +174,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ErrorResponse'
+        401:
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
 
 components:
   securitySchemes:

storage-node-v2/src/commands/server.ts

@@ -45,7 +45,10 @@ export default class Server extends ApiCommandBase {
     try {
       const port = flags.port
       const workerId = flags.worker ?? 0
-      const app = await createApp(api, account, workerId, flags.uploads)
+      const maxFileSize = await api.consts.storage.maxDataObjectSize.toNumber()
+      logger.debug(`Max file size runtime parameter: ${maxFileSize}`)
+
+      const app = await createApp(api, account, workerId, flags.uploads, maxFileSize)
       logger.info(`Listening on http://localhost:${port}`)
       app.listen(port)
     } catch (err) {

storage-node-v2/src/services/webApi/app.ts

@@ -17,13 +17,15 @@ import { httpLogger, errorLogger } from '../../services/logger'
  * @param account - KeyringPair instance
  * @param workerId - storage provider ID (worker ID)
  * @param uploadsDir - directory for the file uploading
+ * @param maxFileSize - max allowed file size
  * @returns Express promise.
  */
 export async function createApp(
   api: ApiPromise,
   account: KeyringPair,
   workerId: number,
-  uploadsDir: string
+  uploadsDir: string,
+  maxFileSize: number
 ): Promise<Express> {
   const spec = path.join(__dirname, './../../api-spec/openapi.yaml')
 
@@ -52,7 +54,16 @@ export async function createApp(
         basePath: path.join(__dirname, './controllers'),
         resolver: OpenApiValidator.resolvers.modulePathResolver,
       },
-      fileUploader: { dest: uploadsDir },
+      fileUploader: {
+        dest: uploadsDir,
+        // Busboy library settings
+        limits: {
+          // For multipart forms, the max number of file fields (Default: Infinity)
+          files: 1,
+          // For multipart forms, the max file size (in bytes) (Default: Infinity)
+          fileSize: maxFileSize,
+        },
+      },
       validateSecurity: {
         handlers: {
           UploadAuth: validateUpload(api, account),
@@ -103,6 +114,8 @@ function validateUpload(api: ApiPromise, account: KeyringPair): ValidateUploadFu
   // We don't use these variables yet.
   /* eslint-disable @typescript-eslint/no-unused-vars */
   return (req: express.Request, scopes: string[], schema: OpenAPIV3.SecuritySchemeObject) => {
+    return true
+
     const tokenString = req.headers['x-api-key'] as string
     const token = parseUploadToken(tokenString)
 

storage-node-v2/src/services/webApi/controllers/publicApi.ts

@@ -19,7 +19,6 @@ import * as express from 'express'
 import fs from 'fs'
 import path from 'path'
 import send from 'send'
-import BN from 'bn.js'
 import { CLIError } from '@oclif/errors'
 import { hexToString } from '@polkadot/util'
 const fsPromises = fs.promises
@@ -111,13 +110,12 @@ export async function uploadFile(req: express.Request, res: express.Response): P
     cleanupFileName = fileObj.path
 
     const api = getApi(res)
-    await verifyFileSize(api, fileObj.size)
     await verifyFileMimeType(fileObj.path)
 
     const hash = await hashFile(fileObj.path)
     const bagId = parseBagId(api, uploadRequest.bagId)
 
-    await verifyDataObjectInfo(api, bagId, uploadRequest.dataObjectId, fileObj.size, hash)
+    const accepted = await verifyDataObjectInfo(api, bagId, uploadRequest.dataObjectId, fileObj.size, hash)
 
     // Prepare new file name
     const newPath = fileObj.path.replace(fileObj.filename, hash)
@@ -126,9 +124,16 @@ export async function uploadFile(req: express.Request, res: express.Response): P
     await fsPromises.rename(fileObj.path, newPath)
     cleanupFileName = newPath
 
-    await acceptPendingDataObjects(api, bagId, getAccount(res), getWorkerId(res), uploadRequest.storageBucketId, [
-      uploadRequest.dataObjectId,
-    ])
+    const workerId = getWorkerId(res)
+    if (!accepted) {
+      await acceptPendingDataObjects(api, bagId, getAccount(res), workerId, uploadRequest.storageBucketId, [
+        uploadRequest.dataObjectId,
+      ])
+    } else {
+      logger.warn(
+        `Received already accepted data object. DataObjectId = ${uploadRequest.dataObjectId} WorkerId = ${workerId}`
+      )
+    }
     res.status(201).json({
       id: hash,
     })
@@ -298,21 +303,6 @@ async function validateTokenRequest(api: ApiPromise, tokenRequest: UploadTokenRe
   }
 }
 
-/**
- * Validates file size. It throws an error when file size exceeds the limit
- *
- * @param api - runtime API promise
- * @param fileSize - file size to validate
- * @returns void promise.
- */
-async function verifyFileSize(api: ApiPromise, fileSize: number) {
-  const maxRuntimeFileSize = await api.consts.storage.maxDataObjectSize.toNumber()
-
-  if (fileSize > maxRuntimeFileSize) {
-    throw new WebApiError('Max file size exceeded.', 400)
-  }
-}
-
 /**
  * Validates the runtime info for the data object. It verifies contentID,
  * file size, and 'accepted' status.
@@ -322,7 +312,7 @@ async function verifyFileSize(api: ApiPromise, fileSize: number) {
  * @param dataObjectId - data object ID to validate in runtime
  * @param fileSize - file size to validate
  * @param hash - file multihash
- * @returns void promise.
+ * @returns promise with the 'data object accepted' flag.
  */
 async function verifyDataObjectInfo(
   api: ApiPromise,
@@ -330,16 +320,11 @@ async function verifyDataObjectInfo(
   dataObjectId: number,
   fileSize: number,
   hash: string
-) {
+): Promise<boolean> {
   const dataObject = await api.query.storage.dataObjectsById(bagId, dataObjectId)
 
-  if (dataObject.accepted.valueOf()) {
-    throw new WebApiError(`Data object had been already accepted ID = ${dataObjectId}`, 400)
-  }
-
   // Cannot get 'size' as a regular property.
-  const probablyDataObjectSize = dataObject.get('size') as unknown
-  const dataObjectSize = probablyDataObjectSize as BN
+  const dataObjectSize = dataObject.getField('size')
 
   if (dataObjectSize?.toNumber() !== fileSize) {
     throw new WebApiError(`File size doesn't match the data object's size for data object ID = ${dataObjectId}`, 400)
@@ -352,6 +337,8 @@ async function verifyDataObjectInfo(
       400
     )
   }
+
+  return dataObject.accepted.valueOf()
 }
 
 /**