ConfigParser - fix for integer/array env values, allow explicit turning off logs via env

distributor-node/src/services/logging/LoggingService.ts

@@ -59,40 +59,44 @@ export class LoggingService {
   public static withAppConfig(config: ReadonlyConfig): LoggingService {
     const transports: winston.LoggerOptions['transports'] = []
 
-    const esTransport = config.endpoints.elasticSearch
-      ? new ElasticsearchTransport({
-          level: config.log?.elastic || 'warn',
-          format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
-          flushInterval: 5000,
-          source: config.id,
-          clientOpts: {
-            node: {
-              url: new URL(config.endpoints.elasticSearch),
-            },
+    let esTransport: ElasticsearchTransport | undefined
+    if (config.log?.elastic && config.log.elastic !== 'off') {
+      if (!config.endpoints.elasticSearch) {
+        throw new Error('config.endpoints.elasticSearch must be provided when elasticSeach logging is enabled!')
+      }
+      esTransport = new ElasticsearchTransport({
+        level: config.log.elastic,
+        format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
+        flushInterval: 5000,
+        source: config.id,
+        clientOpts: {
+          node: {
+            url: new URL(config.endpoints.elasticSearch),
           },
-        })
-      : undefined
-    if (esTransport) {
+        },
+      })
       transports.push(esTransport)
     }
 
     const fileTransport =
-      config.log?.file &&
-      new winston.transports.File({
-        filename: `${config.directories.logs}/logs.json`,
-        level: config.log.file,
-        format: winston.format.combine(pauseFormat({ id: 'file' }), escFormat()),
-      })
+      config.log?.file && config.log.file !== 'off'
+        ? new winston.transports.File({
+            filename: `${config.directories.logs}/logs.json`,
+            level: config.log.file,
+            format: winston.format.combine(pauseFormat({ id: 'file' }), escFormat()),
+          })
+        : undefined
     if (fileTransport) {
       transports.push(fileTransport)
     }
 
     const consoleTransport =
-      config.log?.console &&
-      new winston.transports.Console({
-        level: config.log.console,
-        format: winston.format.combine(pauseFormat({ id: 'cli' }), cliFormat),
-      })
+      config.log?.console && config.log.console !== 'off'
+        ? new winston.transports.Console({
+            level: config.log.console,
+            format: winston.format.combine(pauseFormat({ id: 'cli' }), cliFormat),
+          })
+        : undefined
     if (consoleTransport) {
       transports.push(consoleTransport)
     }

distributor-node/src/services/parsers/ConfigParserService.ts

@@ -4,7 +4,8 @@ import fs from 'fs'
 import path from 'path'
 import YAML from 'yaml'
 import _ from 'lodash'
-import { bytesizeUnits } from '../validation/schemas/configSchema'
+import configSchema, { bytesizeUnits } from '../validation/schemas/configSchema'
+import { JSONSchema4 } from 'json-schema'
 
 const MIN_CACHE_SIZE = 20 * Math.pow(1024, 3)
 
@@ -26,22 +27,53 @@ export class ConfigParserService {
     return intValue * Math.pow(1024, bytesizeUnits.indexOf(unit))
   }
 
-  private mergeEnvConfigTo(config: any) {
+  private schemaTypeOf(schema: JSONSchema4, path: string[]): JSONSchema4['type'] {
+    if (path.length === 0) {
+      return undefined
+    }
+    if (schema.properties && schema.properties[path[0]]) {
+      const item = schema.properties[path[0]]
+      if (item.type === 'object') {
+        return this.schemaTypeOf(item, path.slice(1))
+      } else {
+        return item.type
+      }
+    }
+  }
+
+  private mergeEnvConfigWith(config: Record<string, unknown>) {
     Object.entries(process.env)
       .filter(([k]) => k.startsWith('JOYSTREAM_DISTRIBUTOR__'))
-      .map(([k, v]) => {
-        console.log(k, v)
+      .forEach(([k, v]) => {
         const path = k
           .replace('JOYSTREAM_DISTRIBUTOR__', '')
           .split('__')
           .map((k) => _.camelCase(k))
-          .join('.')
-        _.set(config, path, v)
+
+        const valueType = this.schemaTypeOf(configSchema, path)
+        if (valueType === undefined) {
+          // Invalid key - skip
+        } else if (valueType === 'integer') {
+          _.set(config, path, parseInt(v || ''))
+        } else if (valueType === 'number') {
+          _.set(config, path, parseFloat(v || ''))
+        } else if (valueType === 'boolean') {
+          _.set(config, path, !!v)
+        } else if (valueType === 'array') {
+          try {
+            const parsed = JSON.parse(v || 'undefined')
+            _.set(config, path, parsed)
+          } catch (e) {
+            throw new Error(`Env value ${k} is not a valid JSON array`)
+          }
+        } else {
+          _.set(config, path, v)
+        }
       })
   }
 
   public loadConfing(configPath: string): Config {
-    let inputConfig = {}
+    let inputConfig: Record<string, unknown> = {}
     // Try to load config from file if exists
     if (fs.existsSync(configPath)) {
       const fileContent = fs.readFileSync(configPath).toString()
@@ -55,7 +87,7 @@ export class ConfigParserService {
     }
 
     // Override config with env variables
-    this.mergeEnvConfigTo(inputConfig)
+    this.mergeEnvConfigWith(inputConfig)
 
     // Validate the config
     const configJson = this.validator.validate('Config', inputConfig)

distributor-node/src/services/validation/schemas/configSchema.ts

@@ -30,9 +30,9 @@ export const configSchema: JSONSchema4 = {
       type: 'object',
       additionalProperties: false,
       properties: {
-        file: { type: 'string', enum: Object.keys(winston.config.npm.levels) },
-        console: { type: 'string', enum: Object.keys(winston.config.npm.levels) },
-        elastic: { type: 'string', enum: Object.keys(winston.config.npm.levels) },
+        file: { type: 'string', enum: [...Object.keys(winston.config.npm.levels), 'off'] },
+        console: { type: 'string', enum: [...Object.keys(winston.config.npm.levels), 'off'] },
+        elastic: { type: 'string', enum: [...Object.keys(winston.config.npm.levels), 'off'] },
       },
     },
     storageLimit: { type: 'string', pattern: bytesizeRegex.source },

distributor-node/src/types/generated/ConfigJson.d.ts

@@ -18,9 +18,9 @@ export interface ConfigJson {
     logs: string
   }
   log?: {
-    file?: 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly'
-    console?: 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly'
-    elastic?: 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly'
+    file?: 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly' | 'off'
+    console?: 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly' | 'off'
+    elastic?: 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly' | 'off'
   }
   storageLimit: string
   port: number

docker-compose.yml

@@ -64,10 +64,23 @@ services:
       - /cache
     ports:
       - 127.0.0.1:3334:3334
+    # Node configuration can be overriden via env, for exampe:
     # environment:
-      # Configuration overrides can be defined here, for example:
-      # JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE: http://graphql-server-mnt:4002/graphql
-      # JOYSTREAM_DISTRIBUTOR__LOGS__ELASTIC: null
+    #   JOYSTREAM_DISTRIBUTOR__ID: node-id
+    #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE: qn-endpoint
+    #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__SUBSTRATE_NODE: sn-endpoint
+    #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__ELASTIC_SEARCH: es-endpoint
+    #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__DATA: data-dir
+    #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__CACHE: cache-dir
+    #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__LOGS: logs-dir
+    #   JOYSTREAM_DISTRIBUTOR__LOG__CONSOLE: "off"
+    #   JOYSTREAM_DISTRIBUTOR__LOG__FILE: "off"
+    #   JOYSTREAM_DISTRIBUTOR__LOG__ELASTIC: "off"
+    #   JOYSTREAM_DISTRIBUTOR__STORAGE_LIMIT: 50G
+    #   JOYSTREAM_DISTRIBUTOR__PORT: 1234
+    #   JOYSTREAM_DISTRIBUTOR__KEYS: "[\"//Bob\"]"
+    #   JOYSTREAM_DISTRIBUTOR__BUCKETS: "[1,2]"
+    #   JOYSTREAM_DISTRIBUTOR__WORKER_ID: 0
     command: ["start"]
 
   db: