Procházet zdrojové kódy

storage-node: Fix review comments.

- change file logging transport
- introduce more log parameters
- update express winston error logger parameters (log level)
- update readme
Shamil Gadelshin před 3 roky
rodič
revize
2f527e86e1

+ 36 - 27
storage-node/README.md

@@ -705,49 +705,58 @@ USAGE
   $ storage-node server
 
 OPTIONS
-  -d, --uploads=uploads                              (required) Data uploading directory (absolute path).
+  -d, --uploads=uploads                                       (required) Data uploading directory (absolute path).
 
-  -e, --elasticSearchEndpoint=elasticSearchEndpoint  Elasticsearch endpoint (e.g.: http://some.com:8081).
-                                                     Log level could be set using the ELASTIC_LOG_LEVEL enviroment
-                                                     variable.
-                                                     Supported values: warn, error, debug, info. Default:debug
+  -e, --elasticSearchEndpoint=elasticSearchEndpoint           Elasticsearch endpoint (e.g.: http://some.com:8081).
+                                                              Log level could be set using the ELASTIC_LOG_LEVEL
+                                                              enviroment variable.
+                                                              Supported values: warn, error, debug, info. Default:debug
 
-  -h, --help                                         show CLI help
+  -h, --help                                                  show CLI help
 
-  -i, --syncInterval=syncInterval                    [default: 1] Interval between synchronizations (in minutes)
+  -i, --syncInterval=syncInterval                             [default: 1] Interval between synchronizations (in
+                                                              minutes)
 
-  -k, --keyFile=keyFile                              Key file for the account. Mandatory in non-dev environment.
+  -k, --keyFile=keyFile                                       Key file for the account. Mandatory in non-dev
+                                                              environment.
 
-  -l, --logFileName=logFileName                      Absolute path to the rolling log file. Creates up to 3 files with
-                                                     50MB each
+  -l, --logFilePath=logFilePath                               Absolute path to the rolling log files.
 
-  -m, --dev                                          Use development mode
+  -m, --dev                                                   Use development mode
 
-  -o, --port=port                                    (required) Server port.
+  -n, --logMaxFileNumber=logMaxFileNumber                     [default: 7] Maximum rolling log files number.
 
-  -p, --password=password                            Key file password (optional). Could be overriden by ACCOUNT_PWD
-                                                     environment variable.
+  -o, --port=port                                             (required) Server port.
 
-  -q, --queryNodeEndpoint=queryNodeEndpoint          (required) [default: http://localhost:8081/graphql] Query node
-                                                     endpoint (e.g.: http://some.com:8081/graphql)
+  -p, --password=password                                     Key file password (optional). Could be overriden by
+                                                              ACCOUNT_PWD environment variable.
 
-  -r, --syncWorkersNumber=syncWorkersNumber          [default: 20] Sync workers number (max async operations in
-                                                     progress).
+  -q, --queryNodeEndpoint=queryNodeEndpoint                   (required) [default: http://localhost:8081/graphql] Query
+                                                              node endpoint (e.g.: http://some.com:8081/graphql)
 
-  -s, --sync                                         Enable data synchronization.
+  -r, --syncWorkersNumber=syncWorkersNumber                   [default: 20] Sync workers number (max async operations in
+                                                              progress).
 
-  -t, --syncWorkersTimeout=syncWorkersTimeout        [default: 30] Asset downloading timeout for the syncronization (in
-                                                     minutes).
+  -s, --sync                                                  Enable data synchronization.
 
-  -u, --apiUrl=apiUrl                                [default: ws://localhost:9944] Runtime API URL. Mandatory in
-                                                     non-dev environment.
+  -t, --syncWorkersTimeout=syncWorkersTimeout                 [default: 30] Asset downloading timeout for the
+                                                              syncronization (in minutes).
 
-  -w, --worker=worker                                (required) Storage provider worker ID
+  -u, --apiUrl=apiUrl                                         [default: ws://localhost:9944] Runtime API URL. Mandatory
+                                                              in non-dev environment.
 
-  -y, --accountUri=accountUri                        Account URI (optional). Has a priority over the keyFile and
-                                                     password flags. Could be overriden by ACCOUNT_URI environment
-                                                     variable.
+  -w, --worker=worker                                         (required) Storage provider worker ID
+
+  -x, --logMaxFileSize=logMaxFileSize                         [default: 50000000] Maximum rolling log files size in
+                                                              bytes.
+
+  -y, --accountUri=accountUri                                 Account URI (optional). Has a priority over the keyFile
+                                                              and password flags. Could be overriden by ACCOUNT_URI
+                                                              environment variable.
+
+  -z, --logFileChangeFrequency=(yearly|monthly|daily|hourly)  [default: daily] Log files update frequency.
 ```
 
 _See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/v2.0.0/src/commands/server.ts)_
+
 <!-- commandsstop -->

+ 2 - 1
storage-node/package.json

@@ -55,7 +55,8 @@
     "url-join": "^4.0.1",
     "uuid": "^8.3.2",
     "winston": "^3.3.3",
-    "winston-elasticsearch": "^0.15.8"
+    "winston-elasticsearch": "^0.15.8",
+    "winston-daily-rotate-file": "^4.5.5"
   },
   "devDependencies": {
     "@graphql-codegen/cli": "^1.21.4",

+ 31 - 5
storage-node/src/commands/server.ts

@@ -1,7 +1,7 @@
 import { flags } from '@oclif/command'
 import { createApp } from '../services/webApi/app'
 import ApiCommandBase from '../command-base/ApiCommandBase'
-import logger, { initNewLogger } from '../services/logger'
+import logger, { initNewLogger, DatePatternByFrequency, Frequency } from '../services/logger'
 import { loadDataObjectIdCache } from '../services/caching/localDataObjects'
 import { ApiPromise } from '@polkadot/api'
 import { performSync, TempDirName } from '../services/sync/synchronizer'
@@ -75,10 +75,29 @@ export default class Server extends ApiCommandBase {
 Log level could be set using the ELASTIC_LOG_LEVEL enviroment variable.
 Supported values: warn, error, debug, info. Default:debug`,
     }),
-    logFileName: flags.string({
+    logFilePath: flags.string({
       char: 'l',
       required: false,
-      description: `Absolute path to the rolling log file. Creates up to 3 files with 50MB each`,
+      description: `Absolute path to the rolling log files.`,
+    }),
+    logMaxFileNumber: flags.integer({
+      char: 'n',
+      required: false,
+      default: 7,
+      description: `Maximum rolling log files number.`,
+    }),
+    logMaxFileSize: flags.integer({
+      char: 'x',
+      required: false,
+      default: 50000000,
+      description: `Maximum rolling log files size in bytes.`,
+    }),
+    logFileChangeFrequency: flags.enum({
+      char: 'z',
+      description: `Log files update frequency.`,
+      options: Object.keys(DatePatternByFrequency),
+      default: 'daily',
+      required: false,
     }),
     ...ApiCommandBase.flags,
   }
@@ -94,8 +113,15 @@ Supported values: warn, error, debug, info. Default:debug`,
       await loadDataObjectIdCache(flags.uploads, TempDirName)
     }
 
-    if (!_.isEmpty(flags.elasticSearchEndpoint) || !_.isEmpty(flags.logFileName)) {
-      initNewLogger({ logSource, elasticSearchEndpoint: flags.elasticSearchEndpoint, filename: flags.logFileName })
+    if (!_.isEmpty(flags.elasticSearchEndpoint) || !_.isEmpty(flags.logFilePath)) {
+      initNewLogger({
+        elasticSearchlogSource: logSource,
+        elasticSearchEndpoint: flags.elasticSearchEndpoint,
+        filePath: flags.logFilePath,
+        maxFileNumber: flags.logMaxFileNumber,
+        maxFileSize: flags.logMaxFileSize,
+        fileFrequency: flags.logFileChangeFrequency as Frequency, // type checked in the flags.enum
+      })
     }
 
     logger.info(`Query node endpoint set: ${flags.queryNodeEndpoint}`)

+ 79 - 22
storage-node/src/services/logger.ts

@@ -3,6 +3,9 @@ import ecsformat from '@elastic/ecs-winston-format'
 import expressWinston from 'express-winston'
 import { Handler, ErrorRequestHandler } from 'express'
 import { ElasticsearchTransport } from 'winston-elasticsearch'
+import 'winston-daily-rotate-file'
+import path from 'path'
+
 /**
  * Possible log levels.
  */
@@ -82,16 +85,27 @@ const proxy = new Proxy(InnerLogger, {
 export default proxy
 
 /**
- * Creates Express-Winston logger options.
+ * Creates Express-Winston default logger options.
  *
  */
-export function createExpressLoggerOptions(): expressWinston.LoggerOptions {
+export function createExpressDefaultLoggerOptions(): expressWinston.LoggerOptions {
   return {
     winstonInstance: proxy,
     level: 'http',
   }
 }
 
+/**
+ * Creates Express-Winston error logger options.
+ *
+ */
+export function createExpressErrorLoggerOptions(): expressWinston.LoggerOptions {
+  return {
+    winstonInstance: proxy,
+    level: 'error',
+  }
+}
+
 /**
  * Creates Express-Winston error logger.
  *
@@ -116,16 +130,11 @@ export function httpLogger(options: expressWinston.LoggerOptions): Handler {
 
 /**
  * Creates Winston logger with ElasticSearch and File transports.
- * @param logSource - source tag for log entries.
- * @param elasticSearchEndpoint - elastic search engine endpoint.
+ * @param customOptions - logger options
  * @returns Winston logger
  *
  */
-function createCustomLogger(customOptions: {
-  logSource: string
-  elasticSearchEndpoint?: string
-  filename?: string
-}): winston.Logger {
+function createCustomLogger(customOptions: LogConfig): winston.Logger {
   const loggerOptions = createDefaultLoggerOptions()
 
   // Transports
@@ -135,10 +144,17 @@ function createCustomLogger(customOptions: {
   }
 
   if (customOptions.elasticSearchEndpoint) {
-    transports.push(createElasticTransport(customOptions.logSource, customOptions.elasticSearchEndpoint))
+    transports.push(createElasticTransport(customOptions.elasticSearchlogSource, customOptions.elasticSearchEndpoint))
   }
-  if (customOptions.filename) {
-    transports.push(createFileTransport(customOptions.filename))
+  if (customOptions.filePath) {
+    transports.push(
+      createFileTransport(
+        customOptions.filePath,
+        customOptions.fileFrequency,
+        customOptions.maxFileNumber,
+        customOptions.maxFileSize
+      )
+    )
   }
 
   // Logger
@@ -157,11 +173,9 @@ function createCustomLogger(customOptions: {
 /**
  * Updates the default system logger with elastic search capabilities.
  *
- * @param logSource - source tag for log entries.
- * @param elasticSearchEndpoint - elastic search engine endpoint.
- * @param filename - absolute path to the log file.
+ * @param customOptions - logger options
  */
-export function initNewLogger(options: { logSource: string; elasticSearchEndpoint?: string; filename?: string }): void {
+export function initNewLogger(options: LogConfig): void {
   InnerLogger = createCustomLogger(options)
 }
 
@@ -196,16 +210,59 @@ function createElasticTransport(logSource: string, elasticSearchEndpoint: string
 /**
  * Creates winston logger file transport.
  *
- * @param fileName - log file name.
+ * @param fileName - log file path
+ * @param fileFrequency - file frequence (daily,montly, etc.)
+ * @param maxFiles - maximum number of the log files
+ * @param maxSize - maximum log file size
  * @returns winston file transport
  */
-function createFileTransport(filename: string): winston.transport {
+function createFileTransport(
+  filepath: string,
+  fileFrequency: Frequency,
+  maxFiles: number,
+  maxSize: number
+): winston.transport {
   const options = {
-    filename,
-    maxsize: 50000000, // 50 MB
-    maxFiles: 3,
+    filename: path.join(filepath, 'colossus-%DATE%.log'),
+    datePattern: DatePatternByFrequency[fileFrequency || 'daily'],
+    maxSize,
+    maxFiles,
     level: 'debug',
     format: ecsformat(),
   }
-  return new winston.transports.File(options)
+
+  return new winston.transports.DailyRotateFile(options)
+}
+
+export const DatePatternByFrequency = {
+  yearly: 'YYYY',
+  monthly: 'YYYY-MM',
+  daily: 'YYYY-MM-DD',
+  hourly: 'YYYY-MM-DD-HH',
+}
+
+/** File frequency for  */
+export type Frequency = keyof typeof DatePatternByFrequency
+
+/**
+ * Configuration for the ElasticSearch and File loggers
+ */
+export type LogConfig = {
+  /** Path to log files */
+  filePath?: string
+
+  /** Maximum log file size */
+  maxFileSize: number
+
+  /** Maximum number of the log files */
+  maxFileNumber: number
+
+  /** Log files update frequency (yearly, monthly, daily, hourly) */
+  fileFrequency: Frequency
+
+  /** Source tag for log entries. */
+  elasticSearchlogSource: string
+
+  /** Elastic search engine endpoint */
+  elasticSearchEndpoint?: string
 }

+ 9 - 3
storage-node/src/services/webApi/app.ts

@@ -9,7 +9,12 @@ import { ApiPromise } from '@polkadot/api'
 import { RequestData, verifyTokenSignature, parseUploadToken, UploadToken } from '../helpers/auth'
 import { checkRemoveNonce } from '../caching/tokenNonceKeeper'
 import { AppConfig } from './controllers/common'
-import { createExpressLoggerOptions, httpLogger, errorLogger } from '../../services/logger'
+import {
+  createExpressErrorLoggerOptions,
+  createExpressDefaultLoggerOptions,
+  httpLogger,
+  errorLogger,
+} from '../../services/logger'
 
 /**
  * Creates Express web application. Uses the OAS spec file for the API.
@@ -20,7 +25,7 @@ import { createExpressLoggerOptions, httpLogger, errorLogger } from '../../servi
 export async function createApp(config: AppConfig): Promise<Express> {
   const spec = path.join(__dirname, './../../api-spec/openapi.yaml')
   const app = express()
-  const expressLoggerOptions = createExpressLoggerOptions()
+  const expressLoggerOptions = createExpressDefaultLoggerOptions()
 
   app.use(cors())
   app.use(express.json())
@@ -58,7 +63,8 @@ export async function createApp(config: AppConfig): Promise<Express> {
   ) // Required signature.
 
   // Error logger
-  app.use(errorLogger(expressLoggerOptions))
+  const errorLoggerOptions = createExpressErrorLoggerOptions()
+  app.use(errorLogger(errorLoggerOptions))
 
   /* eslint-disable @typescript-eslint/no-unused-vars */
   app.use((err: Error, req: express.Request, res: express.Response, next: express.NextFunction) => {

+ 23 - 1
yarn.lock

@@ -14411,6 +14411,13 @@ file-selector@^0.2.2:
   dependencies:
     tslib "^2.0.3"
 
+file-stream-rotator@^0.5.7:
+  version "0.5.7"
+  resolved "https://registry.yarnpkg.com/file-stream-rotator/-/file-stream-rotator-0.5.7.tgz#868a2e5966f7640a17dd86eda0e4467c089f6286"
+  integrity sha512-VYb3HZ/GiAGUCrfeakO8Mp54YGswNUHvL7P09WQcXAJNSj3iQ5QraYSp3cIn1MUyw6uzfgN/EFOarCNa4JvUHQ==
+  dependencies:
+    moment "^2.11.2"
+
 file-system-cache@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/file-system-cache/-/file-system-cache-1.0.5.tgz#84259b36a2bbb8d3d6eb1021d3132ffe64cfff4f"
@@ -21448,7 +21455,7 @@ module-lookup-amd@^6.1.0:
     requirejs "^2.3.5"
     requirejs-config-file "^3.1.1"
 
-moment@^2.10.2, moment@^2.22.1, moment@^2.24.0:
+moment@^2.10.2, moment@^2.11.2, moment@^2.22.1, moment@^2.24.0:
   version "2.29.1"
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
   integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
@@ -22426,6 +22433,11 @@ object-filter-sequence@^1.0.0:
   resolved "https://registry.yarnpkg.com/object-filter-sequence/-/object-filter-sequence-1.0.0.tgz#10bb05402fff100082b80d7e83991b10db411692"
   integrity sha512-CsubGNxhIEChNY4cXYuA6KXafztzHqzLLZ/y3Kasf3A+sa3lL9thq3z+7o0pZqzEinjXT6lXDPAfVWI59dUyzQ==
 
+object-hash@^2.0.1:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.2.0.tgz#5ad518581eefc443bd763472b8ff2e9c2c0d54a5"
+  integrity sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==
+
 object-identity-map@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/object-identity-map/-/object-identity-map-1.0.2.tgz#2b4213a4285ca3a8cd2e696782c9964f887524e7"
@@ -30909,6 +30921,16 @@ windows-release@^3.1.0:
   dependencies:
     execa "^1.0.0"
 
+winston-daily-rotate-file@^4.5.5:
+  version "4.5.5"
+  resolved "https://registry.yarnpkg.com/winston-daily-rotate-file/-/winston-daily-rotate-file-4.5.5.tgz#cfa3a89f4eb0e4126917592b375759b772bcd972"
+  integrity sha512-ds0WahIjiDhKCiMXmY799pDBW+58ByqIBtUcsqr4oDoXrAI3Zn+hbgFdUxzMfqA93OG0mPLYVMiotqTgE/WeWQ==
+  dependencies:
+    file-stream-rotator "^0.5.7"
+    object-hash "^2.0.1"
+    triple-beam "^1.3.0"
+    winston-transport "^4.4.0"
+
 winston-elasticsearch@^0.15.8:
   version "0.15.9"
   resolved "https://registry.yarnpkg.com/winston-elasticsearch/-/winston-elasticsearch-0.15.9.tgz#a9490614f6b92d5e1977df927c77c29b3a66f247"