storage-node-v2: Start adding token authorization.

storage-node-v2/package.json

@@ -6,22 +6,20 @@
   "bin": {
     "storage-node": "./bin/run"
   },
-  "bugs": "https://github.com/shamil-gadelshin/storage-node-v2/issues",
+  "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {
     "@joystream/types": "^0.17.0",
     "@oclif/command": "^1",
     "@oclif/config": "^1",
     "@oclif/plugin-help": "^3",
     "@polkadot/api": "4.2.1",
-    "@types/express": "^4.17.11",
+    "@types/express": "4.17.1",
     "@types/multer": "^1.4.5",
-    "@types/swagger-ui-express": "^4.1.2",
     "blake3": "^2.1.4",
-    "express": "^4.17.1",
+    "express": "4.17.1",
     "express-openapi-validator": "^4.12.4",
     "multihashes": "^4.0.2",
     "openapi-editor": "^0.3.0",
-    "swagger-ui-express": "^4.1.6",
     "tslib": "^1"
   },
   "devDependencies": {
@@ -30,6 +28,7 @@
     "@types/chai": "^4",
     "@types/mocha": "^5",
     "@types/node": "^10",
+    "@types/swagger-ui-express": "^4.1.2",
     "@typescript-eslint/eslint-plugin": "3.8.0",
     "@typescript-eslint/parser": "3.8.0",
     "chai": "^4",
@@ -41,6 +40,7 @@
     "nyc": "^14",
     "prettier": "^2.3.0",
     "sinon": "^11.1.1",
+    "swagger-ui-express": "^4.1.6",
     "ts-node": "^8.8.2",
     "typescript": "^3.3"
   },
@@ -53,9 +53,10 @@
     "/npm-shrinkwrap.json",
     "/oclif.manifest.json"
   ],
-  "homepage": "https://github.com/shamil-gadelshin/storage-node-v2",
+  "homepage": "https://github.com/Joystream/joystream",
   "keywords": [
-    "oclif"
+    "joystream",
+    "storage-node"
   ],
   "license": "GPL-3.0-only",
   "main": "lib/index.js",

storage-node-v2/src/api-spec/openapi.yaml

@@ -10,24 +10,27 @@ info:
   version: 0.1.0
 externalDocs:
   description: Storage node API
-  url: joystream.org
+  url: https://github.com/Joystream/joystream/issues/2224
 servers:
   - url: http://localhost:3333/api/v1/
 
 tags:
-  - name: storage
-    description: Storage node APIs
+  - name: public
+    description: Public storage node API
 
 paths:
   /upload:
     post:
+      security:
+        - UploadAuth: []
       description: Upload data
       operationId: publicApi.upload
+      tags:
+        - public
       requestBody:
         content:
           multipart/form-data:
             schema:
-              # $ref: '#/components/schemas/NewPhoto'
               type: object
               required:
                 - dataObjectId
@@ -38,21 +41,21 @@ paths:
                   description: Data file
                   type: string
                   format: binary
-                dataObjectId: #TODO: convert to JSON
+                dataObjectId:
                   description: Data object runtime ID
                   type: string
                   pattern: '^\d+$' #integer
-                storageBucketId: #TODO: convert to JSON
-                  description: Data object runtime ID
+                storageBucketId:
+                  description: Storage bucket ID
                   type: string
                   pattern: '^\d+$' #integer
-                workerId: #TODO: convert to JSON
-                  description: Data object runtime ID
+                workerId: 
+                  description: Storage provider worker ID
                   type: string
                   pattern: '^\d+$' #integer
         required: true
       responses:
-        200:
+        201:
           description: Created
           content:
             application/json:
@@ -61,3 +64,43 @@ paths:
                 properties:
                   success:
                     type: boolean
+        401:
+          description: Unauthorized
+  /authToken:
+    post:
+      description: Get auth token from a server.
+      operationId: publicApi.authToken
+      tags:
+        - public
+      requestBody:
+        description: Token request parameters,
+        content:
+          application/json:    
+            schema:
+              $ref: "#/components/schemas/TokenRequest"
+      responses:
+        201:
+          description: Created
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  token:
+                    type: string
+components:
+  securitySchemes:
+    UploadAuth:
+      type: apiKey
+      in: header
+      name: x-api-key
+  schemas:
+    TokenRequest:
+      type: object
+      required:
+        - dataObjectId
+      properties:
+        dataObjectId:
+          type: integer
+          format: int64    
+

storage-node-v2/src/commands/server.ts

@@ -1,16 +1,14 @@
-import { Command, flags } from '@oclif/command'
-import { createServer } from '../services/webApi/server'
+import { flags } from '@oclif/command'
+import { createApp } from '../services/webApi/app'
+import ApiCommandBase from '../command-base/ApiCommandBase'
 
 // TODO: fix command not found error (error handling)
 // TODO: custom IP address?
-// TODO: parameters for --dev or key file
 
-export default class Server extends Command {
+export default class Server extends ApiCommandBase {
   static description = 'Starts the storage node server.'
 
   static flags = {
-    help: flags.help({ char: 'h' }),
-    dev: flags.boolean({ char: 'd', description: 'Use development mode' }),
     uploads: flags.string({
       char: 'u',
       required: true,
@@ -21,6 +19,7 @@ export default class Server extends Command {
       required: true,
       description: 'Server port.',
     }),
+    ...ApiCommandBase.keyflags,
   }
 
   static args = [{ name: 'file' }]
@@ -28,13 +27,23 @@ export default class Server extends Command {
   async run(): Promise<void> {
     const { flags } = this.parse(Server)
 
+    if (flags.dev) {
+      await this.ensureDevelopmentChain()
+    }
+
+    const account = this.getAccount(flags)
+
     try {
       const port = flags.port
-      const server = await createServer(flags.dev, flags.uploads)
+      const app = await createApp(account, flags.uploads)
       console.info(`Listening on http://localhost:${port}`)
-      server.listen(port)
+      app.listen(port)
     } catch (err) {
       console.error(`Error: ${err}`)
     }
   }
+
+  // Override exiting.
+  /* eslint-disable @typescript-eslint/no-empty-function */
+  async finally(): Promise<void> {}
 }

storage-node-v2/src/services/auth.ts

@@ -0,0 +1,21 @@
+import { KeyringPair } from '@polkadot/keyring/types'
+import { stringToU8a, u8aToHex } from '@polkadot/util'
+import { signatureVerify } from '@polkadot/util-crypto'
+
+export interface TokenRequest {
+  dataObjectId: number
+}
+
+export function verifyTokenSignature(tokenRequest: TokenRequest, signature: string, account: KeyringPair): boolean{
+  const message = JSON.stringify(tokenRequest)
+  const { isValid } = signatureVerify(message, signature, account.address)
+  
+  return isValid
+}
+
+export function signToken(tokenRequest: TokenRequest, account: KeyringPair): string{
+  const message = stringToU8a(JSON.stringify(tokenRequest))
+  const signature = account.sign(message)
+
+  return u8aToHex(signature)
+}

storage-node-v2/src/services/webApi/app.ts

@@ -0,0 +1,75 @@
+import express from 'express'
+import path from 'path'
+import cors from 'cors'
+import { Express, NextFunction } from 'express-serve-static-core'
+import * as OpenApiValidator from 'express-openapi-validator'
+import { OpenAPIV3 } from 'express-openapi-validator/dist/framework/types'
+import { KeyringPair } from '@polkadot/keyring/types'
+import { TokenRequest, verifyTokenSignature } from '../auth'
+
+// TODO: custom errors (including validation errors)
+// TODO: custom authorization errors
+
+export async function createApp(
+  account: KeyringPair,
+  uploadsDir: string
+): Promise<Express> {
+  const spec = path.join(__dirname, './../../api-spec/openapi.yaml')
+
+  const app = express()
+
+  app.use(cors())
+  app.use(express.json())
+
+  // TODO: check path
+  app.use('/files', express.static(uploadsDir))
+
+  app.use(
+    // Set parameters for each request.
+    (req: express.Request, res: express.Response, next: NextFunction) => {
+      res.locals.storageProviderAccount = account
+      next()
+    },
+    OpenApiValidator.middleware({
+      apiSpec: spec,
+      validateApiSpec: true,
+      validateResponses: true,
+      validateRequests: true,
+      operationHandlers: {
+        basePath: path.join(__dirname, './controllers'),
+        resolver: OpenApiValidator.resolvers.modulePathResolver,
+      },
+      fileUploader: { dest: uploadsDir },
+      validateSecurity: {
+        handlers: {
+          UploadAuth: validateUpload(account),
+        },
+      },
+    })
+  )
+
+  return app
+}
+
+type ValidateUploadFunction = (
+  req: express.Request,
+  scopes: string[],
+  schema: OpenAPIV3.SecuritySchemeObject
+) => boolean | Promise<boolean>
+
+function validateUpload(account: KeyringPair): ValidateUploadFunction {
+  // We don't use these variables yet.
+  /* eslint-disable @typescript-eslint/no-unused-vars */
+  return (
+    req: express.Request,
+    scopes: string[],
+    schema: OpenAPIV3.SecuritySchemeObject
+  ) => {
+    const tokenSignature = req.headers['x-api-key'] as string
+
+//TODO: token construction
+    const sourceTokenRequest: TokenRequest = { dataObjectId: parseInt(req.body.dataObjectId) }
+
+    return verifyTokenSignature(sourceTokenRequest, tokenSignature, account)
+  }
+}

storage-node-v2/src/services/webApi/controllers/publicApi.ts

@@ -1,7 +1,8 @@
 import * as express from 'express'
 import { acceptPendingDataObjects } from '../../runtime/extrinsics'
-import { getAlicePair } from '../../runtime/api'
+import { TokenRequest, signToken } from '../../auth'
 import { hashFile } from '../../../services/hashing'
+import { KeyringPair } from '@polkadot/keyring/types'
 import fs from 'fs'
 const fsPromises = fs.promises
 
@@ -21,25 +22,24 @@ export async function upload(
 ): Promise<void> {
   const uploadRequest: UploadRequest = req.body
 
+  console.log(uploadRequest)
+
   try {
     const fileObj = getFileObject(req)
-    console.log(fileObj)
 
     const hash = await hashFile(fileObj.path)
     const newPath = fileObj.path.replace(fileObj.filename, hash)
-    console.log(hash)
 
     // Overwrites existing file.
     await fsPromises.rename(fileObj.path, newPath)
 
-    // TODO: account
     await acceptPendingDataObjects(
-      getAlicePair(),
+      getAccount(res),
       uploadRequest.workerId,
       uploadRequest.storageBucketId,
       [uploadRequest.dataObjectId]
     )
-    res.status(200).json({
+    res.status(201).json({
       file: 'received',
     })
   } catch (err) {
@@ -49,6 +49,19 @@ export async function upload(
   }
 }
 
+export async function authToken(
+  req: express.Request,
+  res: express.Response
+): Promise<void> {
+  const account = getAccount(res)
+  const tokenRequest = getTokenRequest(req)
+  const signature = signToken(tokenRequest, account)
+
+  res.status(201).json({
+    token: signature,
+  })
+}
+
 function getFileObject(req: express.Request): Express.Multer.File {
   if (req.file) {
     return req.file
@@ -61,3 +74,20 @@ function getFileObject(req: express.Request): Express.Multer.File {
 
   throw new Error('No file uploaded')
 }
+
+function getAccount(res: express.Response): KeyringPair {
+  if (res.locals.storageProviderAccount) {
+    return res.locals.storageProviderAccount
+  }
+
+  throw new Error('No Joystream account loaded.')
+}
+
+function getTokenRequest(req: express.Request): TokenRequest {
+  const tokenRequest = req.body as TokenRequest
+  if (tokenRequest) {
+    return tokenRequest
+  }
+
+  throw new Error('No token request provided.')
+}

storage-node-v2/src/services/webApi/server.ts

@@ -1,39 +0,0 @@
-import express from 'express'
-import path from 'path'
-import cors from 'cors'
-import { Express } from 'express-serve-static-core'
-import * as OpenApiValidator from 'express-openapi-validator'
-
-// TODO: custom errors (including validation errors)
-
-export async function createServer(
-  devMode: boolean,
-  uploadsDir: string
-): Promise<Express> {
-  const server = express()
-  server.use(cors())
-  const spec = path.join(__dirname, './../../api-spec/openapi.yaml')
-
-  if (devMode) {
-    server.use('/spec', express.static(spec))
-  }
-
-  // TODO: check path
-  server.use('/files', express.static(uploadsDir))
-
-  server.use(
-    OpenApiValidator.middleware({
-      apiSpec: spec,
-      validateApiSpec: true,
-      validateResponses: true,
-      validateRequests: true,
-      operationHandlers: {
-        basePath: path.join(__dirname, './controllers'),
-        resolver: OpenApiValidator.resolvers.modulePathResolver,
-      },
-      fileUploader: { dest: uploadsDir },
-    })
-  )
-
-  return server
-}

yarn.lock

@@ -5009,15 +5009,6 @@
     "@types/qs" "*"
     "@types/range-parser" "*"
 
-"@types/express-serve-static-core@^4.17.18":
-  version "4.17.21"
-  resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.21.tgz#a427278e106bca77b83ad85221eae709a3414d42"
-  integrity sha512-gwCiEZqW6f7EoR8TTEfalyEhb1zA5jQJnRngr97+3pzMaO1RKoI1w2bw07TK72renMUVWcWS5mLI6rk1NqN0nA==
-  dependencies:
-    "@types/node" "*"
-    "@types/qs" "*"
-    "@types/range-parser" "*"
-
 "@types/express@*", "@types/express@^4.17.2":
   version "4.17.8"
   resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.8.tgz#3df4293293317e61c60137d273a2e96cd8d5f27a"
@@ -5028,6 +5019,15 @@
     "@types/qs" "*"
     "@types/serve-static" "*"
 
+"@types/express@4.17.1":
+  version "4.17.1"
+  resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.1.tgz#4cf7849ae3b47125a567dfee18bfca4254b88c5c"
+  integrity sha512-VfH/XCP0QbQk5B5puLqTLEeFgR8lfCJHZJKkInZ9mkYd+u8byX0kztXEQxEk4wZXJs8HI+7km2ALXjn4YKcX9w==
+  dependencies:
+    "@types/body-parser" "*"
+    "@types/express-serve-static-core" "*"
+    "@types/serve-static" "*"
+
 "@types/express@4.17.7":
   version "4.17.7"
   resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.7.tgz#42045be6475636d9801369cd4418ef65cdb0dd59"
@@ -5048,16 +5048,6 @@
     "@types/qs" "*"
     "@types/serve-static" "*"
 
-"@types/express@^4.17.11":
-  version "4.17.12"
-  resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.12.tgz#4bc1bf3cd0cfe6d3f6f2853648b40db7d54de350"
-  integrity sha512-pTYas6FrP15B1Oa0bkN5tQMNqOcVXa9j4FTFtO8DWI9kppKib+6NJtfTOOLcwxuuYvcX2+dVG6et1SxW/Kc17Q==
-  dependencies:
-    "@types/body-parser" "*"
-    "@types/express-serve-static-core" "^4.17.18"
-    "@types/qs" "*"
-    "@types/serve-static" "*"
-
 "@types/faker@^4.1.8":
   version "4.1.12"
   resolved "https://registry.yarnpkg.com/@types/faker/-/faker-4.1.12.tgz#065d37343677df1aa757c622650bd14666c42602"
@@ -13111,7 +13101,7 @@ express-openapi@^4.6.1:
     openapi-framework "0.24.5"
     openapi-types "1.3.5"
 
-express@^4.0.0, express@^4.16.4, express@^4.17.0, express@^4.17.1:
+express@4.17.1, express@^4.0.0, express@^4.16.4, express@^4.17.0, express@^4.17.1:
   version "4.17.1"
   resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134"
   integrity sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==