Allow distributor-node config through ENV + config improvements

distributor-node.Dockerfile

@@ -4,7 +4,6 @@ WORKDIR /joystream
 COPY ./types types
 COPY ./metadata-protobuf metadata-protobuf
 COPY ./distributor-node distributor-node
-COPY ./distributor-node/config/docker/config.docker.yml config.yml
 COPY ./yarn.lock yarn.lock
 COPY ./package.json package.json
 
@@ -21,5 +20,7 @@ RUN \
   yarn --frozen-lockfile --production &&\
   yarn cache clean
 
+ENV CONFIG_PATH ./distributor-node/config/docker/config.docker.yml
+
 ENTRYPOINT ["yarn", "joystream-distributor"]
 CMD ["start"]

distributor-node/config.yml

@@ -2,7 +2,7 @@ id: test-node
 endpoints:
   queryNode: http://localhost:8081/graphql
   substrateNode: ws://localhost:9944
-  elasticSearch: http://localhost:9200
+  # elasticSearch: http://localhost:9200
 directories:
   data: ./local/data
   cache: ./local/cache
@@ -10,7 +10,7 @@ directories:
 log:
   file: debug
   console: info
-  elastic: info
+  # elastic: info
 storageLimit: 100G
 port: 3334
 keys: [//Alice]

distributor-node/config/docker/config.docker.yml

@@ -1,13 +1,15 @@
-id: test-distributor-node
+id: distributor-node-docker
 endpoints:
   queryNode: http://graphql-server-mnt:4002/graphql
   substrateNode: ws://joystream-node:9944
+  # elasticSearch: http://elasticsearch:9200
 directories:
   data: /data
   cache: /cache
   logs: /logs
 log:
   console: info
+  # elastic: info
 storageLimit: 100G
 port: 3334
 keys: [//Alice]

distributor-node/src/app/index.ts

@@ -31,7 +31,11 @@ export class App {
     Object.entries(this.config.directories).forEach(([name, path]) => {
       const dirInfo = `${name} directory (${path})`
       if (!fs.existsSync(path)) {
-        throw new Error(`${dirInfo} doesn't exists!`)
+        try {
+          fs.mkdirSync(path, { recursive: true })
+        } catch (e) {
+          throw new Error(`${dirInfo} doesn't exist and cannot be created!`)
+        }
       }
       try {
         fs.accessSync(path, fs.constants.R_OK)

distributor-node/src/services/logging/LoggingService.ts

@@ -59,19 +59,19 @@ export class LoggingService {
   public static withAppConfig(config: ReadonlyConfig): LoggingService {
     const transports: winston.LoggerOptions['transports'] = []
 
-    const esTransport =
-      config.log?.elastic &&
-      new ElasticsearchTransport({
-        level: config.log?.elastic || 'warn',
-        format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
-        flushInterval: 5000,
-        source: config.id,
-        clientOpts: {
-          node: {
-            url: new URL(config.endpoints.elasticSearch),
+    const esTransport = config.endpoints.elasticSearch
+      ? new ElasticsearchTransport({
+          level: config.log?.elastic || 'warn',
+          format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
+          flushInterval: 5000,
+          source: config.id,
+          clientOpts: {
+            node: {
+              url: new URL(config.endpoints.elasticSearch),
+            },
           },
-        },
-      })
+        })
+      : undefined
     if (esTransport) {
       transports.push(esTransport)
     }

distributor-node/src/services/parsers/ConfigParserService.ts

@@ -26,19 +26,41 @@ export class ConfigParserService {
     return intValue * Math.pow(1024, bytesizeUnits.indexOf(unit))
   }
 
+  private mergeEnvConfigTo(config: any) {
+    Object.entries(process.env)
+      .filter(([k]) => k.startsWith('JOYSTREAM_DISTRIBUTOR__'))
+      .map(([k, v]) => {
+        console.log(k, v)
+        const path = k
+          .replace('JOYSTREAM_DISTRIBUTOR__', '')
+          .split('__')
+          .map((k) => _.camelCase(k))
+          .join('.')
+        _.set(config, path, v)
+      })
+  }
+
   public loadConfing(configPath: string): Config {
-    const fileContent = fs.readFileSync(configPath).toString()
-    let inputConfig: unknown
-    if (path.extname(configPath) === '.json') {
-      inputConfig = JSON.parse(fileContent)
-    } else if (path.extname(configPath) === '.yml' || path.extname(configPath) === '.yaml') {
-      inputConfig = YAML.parse(fileContent)
-    } else {
-      throw new Error('Unrecognized config format (use .yml or .json)')
+    let inputConfig = {}
+    // Try to load config from file if exists
+    if (fs.existsSync(configPath)) {
+      const fileContent = fs.readFileSync(configPath).toString()
+      if (path.extname(configPath) === '.json') {
+        inputConfig = JSON.parse(fileContent)
+      } else if (path.extname(configPath) === '.yml' || path.extname(configPath) === '.yaml') {
+        inputConfig = YAML.parse(fileContent)
+      } else {
+        throw new Error('Unrecognized config format (use .yml or .json)')
+      }
     }
 
+    // Override config with env variables
+    this.mergeEnvConfigTo(inputConfig)
+
+    // Validate the config
     const configJson = this.validator.validate('Config', inputConfig)
 
+    // Normalize values
     const directories = this.resolveConfigDirectoryPaths(configJson.directories, configPath)
     const storageLimit = this.parseBytesize(configJson.storageLimit)
 

distributor-node/src/types/generated/ConfigJson.d.ts

@@ -10,7 +10,7 @@ export interface ConfigJson {
   endpoints: {
     queryNode: string
     substrateNode: string
-    elasticSearch: string
+    elasticSearch?: string
   }
   directories: {
     data: string

docker-compose.yml

@@ -49,9 +49,12 @@ services:
     volumes:
       - /data
       - /cache
-      - /logs
     ports:
       - 127.0.0.1:3334:3334
+    # environment:
+      # Configuration overrides can be defined here, for example:
+      # JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE: http://graphql-server-mnt:4002/graphql
+      # JOYSTREAM_DISTRIBUTOR__LOGS__ELASTIC: null
     command: ["start"]
 
   db: