pulumi query-node refactor

devops/kubernetes/query-node/Pulumi.yaml

@@ -25,3 +25,12 @@ template:
     appsImage:
       description: The joystream image to use for running GraphQL servers
       default: joystream/apps:latest
+    dbPassword:
+      description: database password for indexer and processor databases  
+      required: true
+    blockHeight:
+      descroption: Block height to start indexing at
+      default: 0
+    joystreamWsEndpoint:
+      description: Joystream-node websocket endpoint used by indexer
+      required: true

devops/kubernetes/query-node/README.md

@@ -38,6 +38,8 @@ After cloning this repo, from this working directory, run these commands:
 
    ```bash
    $ pulumi config set-all --plaintext aws:region=us-east-1 --plaintext aws:profile=joystream-user \
+    --plaintext dbPassword=password --plaintext blockHeight=0 \
+    --plaintext joystreamWsEndpoint=ws://endpoint.somewhere.net:9944 \
     --plaintext isMinikube=true --plaintext skipProcessor=false
    ```
 
@@ -66,10 +68,6 @@ After cloning this repo, from this working directory, run these commands:
 
    If not using minikube, just specify the `appsImage` config.
 
-1. Create a `.env` file in this directory (`cp ../../../.env ./.env`) and set the database and other variables in it
-
-   Make sure to set `GRAPHQL_SERVER_PORT=4001`
-
 1. Stand up the Kubernetes cluster:
 
    Running `pulumi up -y` will deploy the EKS cluster. Note, provisioning a

devops/kubernetes/query-node/configMap.ts

@@ -2,7 +2,7 @@ import * as pulumi from '@pulumi/pulumi'
 import * as k8s from '@pulumi/kubernetes'
 import * as fs from 'fs'
 
-export class configMapFromFile extends pulumi.ComponentResource {
+export class ConfigMapFromFile extends pulumi.ComponentResource {
   public readonly configName?: pulumi.Output<string>
 
   constructor(name: string, args: ConfigMapArgs, opts: pulumi.ComponentResourceOptions = {}) {

devops/kubernetes/query-node/index.ts

@@ -2,14 +2,12 @@ import * as awsx from '@pulumi/awsx'
 import * as eks from '@pulumi/eks'
 import * as docker from '@pulumi/docker'
 import * as pulumi from '@pulumi/pulumi'
-import { configMapFromFile } from './configMap'
+import { ConfigMapFromFile } from './configMap'
 import * as k8s from '@pulumi/kubernetes'
 import { IndexerServiceDeployment } from './indexerDeployment'
 import { ProcessorServiceDeployment } from './processorDeployment'
 import { CaddyServiceDeployment } from 'pulumi-common'
 
-require('dotenv').config()
-
 const config = new pulumi.Config()
 const awsConfig = new pulumi.Config('aws')
 const isMinikube = config.getBoolean('isMinikube')
@@ -66,7 +64,7 @@ if (isMinikube) {
   joystreamAppsImage = repo.buildAndPushImage({
     context: './docker_dummy',
     dockerfile: './docker_dummy/Dockerfile',
-    args: { SOURCE_IMAGE: appsImage! },
+    args: { SOURCE_IMAGE: appsImage },
   })
 }
 
@@ -80,7 +78,7 @@ const ns = new k8s.core.v1.Namespace(name, {}, resourceOptions)
 // Export the Namespace name
 export const namespaceName = ns.metadata.name
 
-const defsConfig = new configMapFromFile(
+const defsConfig = new ConfigMapFromFile(
   'defs-config',
   {
     filePath: '../../../types/augment/all/defs.json',

devops/kubernetes/query-node/indexerDeployment.ts

@@ -2,6 +2,15 @@ import * as k8s from '@pulumi/kubernetes'
 import * as pulumi from '@pulumi/pulumi'
 import { PostgresServiceDeployment } from 'pulumi-common'
 
+const config = new pulumi.Config()
+const DB_PASS = config.require('dbPassword')
+const BLOCK_HEIGHT = config.require('blockHeight') || '0'
+const WS_PROVIDER_ENDPOINT_URI = config.require('joystreamWsEndpoint')
+
+const DB_USERNAME = 'postgres'
+const INDEXER_DATABASE_NAME = 'indexer'
+const DB_PORT = '5432'
+
 /**
  * ServiceDeployment is an example abstraction that uses a class to fold together the common pattern of a
  * Kubernetes Deployment and its associated Service object.
@@ -16,7 +25,7 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
 
     // Name passed in the constructor will be the endpoint for accessing the service
     const serviceName = name
-    let appLabels = { appClass: 'indexer' }
+    const appLabels = { appClass: 'indexer' }
 
     const indexerDbName = 'indexer-db'
     const indexerDb = new PostgresServiceDeployment(
@@ -24,56 +33,16 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
       {
         namespaceName: args.namespaceName,
         env: [
-          { name: 'POSTGRES_USER', value: process.env.DB_USER! },
-          { name: 'POSTGRES_PASSWORD', value: process.env.DB_PASS! },
-          { name: 'POSTGRES_DB', value: process.env.INDEXER_DB_NAME! },
+          { name: 'POSTGRES_USER', value: DB_USERNAME },
+          { name: 'POSTGRES_PASSWORD', value: DB_PASS },
+          { name: 'POSTGRES_DB', value: INDEXER_DATABASE_NAME },
+          { name: 'PGPORT', value: DB_PORT },
         ],
         storage: args.storage,
       },
       { parent: this }
     )
 
-    const indexerMigrationJob = new k8s.batch.v1.Job(
-      'indexer-db-migration',
-      {
-        metadata: {
-          namespace: args.namespaceName,
-        },
-        spec: {
-          backoffLimit: 0,
-          template: {
-            spec: {
-              containers: [
-                {
-                  name: 'db-migration',
-                  image: args.joystreamAppsImage,
-                  imagePullPolicy: 'IfNotPresent',
-                  resources: { requests: { cpu: '100m', memory: '100Mi' } },
-                  env: [
-                    {
-                      name: 'WARTHOG_DB_HOST',
-                      value: indexerDbName,
-                    },
-                    {
-                      name: 'DB_HOST',
-                      value: indexerDbName,
-                    },
-                    { name: 'WARTHOG_DB_DATABASE', value: process.env.INDEXER_DB_NAME! },
-                    { name: 'DB_NAME', value: process.env.INDEXER_DB_NAME! },
-                    { name: 'DB_PASS', value: process.env.DB_PASS! },
-                  ],
-                  command: ['/bin/sh', '-c'],
-                  args: ['yarn workspace query-node-root db:prepare; yarn workspace query-node-root db:migrate'],
-                },
-              ],
-              restartPolicy: 'Never',
-            },
-          },
-        },
-      },
-      { parent: this, dependsOn: indexerDb.service }
-    )
-
     this.deployment = new k8s.apps.v1.Deployment(
       'indexer',
       {
@@ -100,17 +69,18 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
                   image: 'joystream/hydra-indexer:3.0.0',
                   env: [
                     { name: 'DB_HOST', value: indexerDbName },
-                    { name: 'DB_NAME', value: process.env.INDEXER_DB_NAME! },
-                    { name: 'DB_PASS', value: process.env.DB_PASS! },
-                    { name: 'DB_USER', value: process.env.DB_USER! },
-                    { name: 'DB_PORT', value: process.env.DB_PORT! },
+                    { name: 'DB_NAME', value: INDEXER_DATABASE_NAME },
+                    { name: 'DB_PASS', value: DB_PASS },
+                    { name: 'DB_USER', value: DB_USERNAME },
+                    { name: 'DB_PORT', value: DB_PORT },
                     { name: 'INDEXER_WORKERS', value: '5' },
+                    // Why localhost works and we didn't have to use the container name 'redis'
                     { name: 'REDIS_URI', value: 'redis://localhost:6379/0' },
                     { name: 'DEBUG', value: 'index-builder:*' },
-                    { name: 'WS_PROVIDER_ENDPOINT_URI', value: process.env.WS_PROVIDER_ENDPOINT_URI! },
+                    { name: 'WS_PROVIDER_ENDPOINT_URI', value: WS_PROVIDER_ENDPOINT_URI },
                     { name: 'TYPES_JSON', value: 'types.json' },
-                    { name: 'PGUSER', value: process.env.DB_USER! },
-                    { name: 'BLOCK_HEIGHT', value: process.env.BLOCK_HEIGHT! },
+                    { name: 'PGUSER', value: DB_USERNAME },
+                    { name: 'BLOCK_HEIGHT', value: BLOCK_HEIGHT },
                   ],
                   volumeMounts: [
                     {
@@ -126,17 +96,18 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
                   name: 'hydra-indexer-gateway',
                   image: 'joystream/hydra-indexer-gateway:3.0.0',
                   env: [
-                    { name: 'WARTHOG_STARTER_DB_DATABASE', value: process.env.INDEXER_DB_NAME! },
+                    { name: 'WARTHOG_STARTER_DB_DATABASE', value: INDEXER_DATABASE_NAME },
                     { name: 'WARTHOG_STARTER_DB_HOST', value: indexerDbName },
-                    { name: 'WARTHOG_STARTER_DB_PASSWORD', value: process.env.DB_PASS! },
-                    { name: 'WARTHOG_STARTER_DB_PORT', value: process.env.DB_PORT! },
-                    { name: 'WARTHOG_STARTER_DB_USERNAME', value: process.env.DB_USER! },
+                    { name: 'WARTHOG_STARTER_DB_PASSWORD', value: DB_PASS },
+                    { name: 'WARTHOG_STARTER_DB_PORT', value: DB_PORT },
+                    { name: 'WARTHOG_STARTER_DB_USERNAME', value: DB_USERNAME },
+                    // Why localhost works and we didn't have to use the container name 'redis'
                     { name: 'WARTHOG_STARTER_REDIS_URI', value: 'redis://localhost:6379/0' },
-                    { name: 'WARTHOG_APP_PORT', value: process.env.WARTHOG_APP_PORT! },
-                    { name: 'PORT', value: process.env.WARTHOG_APP_PORT! },
+                    { name: 'WARTHOG_APP_PORT', value: '4001' },
+                    { name: 'PORT', value: '4001' },
                     { name: 'DEBUG', value: '*' },
                   ],
-                  ports: [{ name: 'hydra-port', containerPort: Number(process.env.WARTHOG_APP_PORT!) }],
+                  ports: [{ name: 'hydra-port', containerPort: 4001 }],
                 },
               ],
               volumes: [
@@ -151,7 +122,7 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
           },
         },
       },
-      { parent: this, dependsOn: indexerMigrationJob }
+      { parent: this, dependsOn: indexerDb.service }
     )
 
     // Create a Service for the Indexer
@@ -183,5 +154,5 @@ export interface ServiceDeploymentArgs {
   joystreamAppsImage: pulumi.Output<string>
   defsConfig: pulumi.Output<string> | undefined
   env?: Environment[]
-  storage: Number
+  storage: number
 }

devops/kubernetes/query-node/processorDeployment.ts

@@ -2,6 +2,12 @@ import * as k8s from '@pulumi/kubernetes'
 import * as pulumi from '@pulumi/pulumi'
 import { PostgresServiceDeployment } from 'pulumi-common'
 
+const config = new pulumi.Config()
+const DB_PASS = config.require('dbPassword')
+const DB_USERNAME = 'postgres'
+const PROCESSOR_DATABASE_NAME = 'processor'
+const DB_PORT = '5432'
+
 /**
  * ServiceDeployment is an example abstraction that uses a class to fold together the common pattern of a
  * Kubernetes Deployment and its associated Service object.
@@ -24,9 +30,10 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
       {
         namespaceName: args.namespaceName,
         env: [
-          { name: 'POSTGRES_USER', value: process.env.DB_USER! },
-          { name: 'POSTGRES_PASSWORD', value: process.env.DB_PASS! },
-          { name: 'POSTGRES_DB', value: process.env.DB_NAME! },
+          { name: 'POSTGRES_USER', value: DB_USERNAME },
+          { name: 'POSTGRES_PASSWORD', value: DB_PASS },
+          { name: 'POSTGRES_DB', value: PROCESSOR_DATABASE_NAME },
+          { name: 'PGPORT', value: DB_PORT },
         ],
         storage: args.storage,
       },
@@ -58,12 +65,20 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
                       name: 'DB_HOST',
                       value: processorDbName,
                     },
-                    { name: 'WARTHOG_DB_DATABASE', value: process.env.DB_NAME! },
-                    { name: 'DB_NAME', value: process.env.DB_NAME! },
-                    { name: 'DB_PASS', value: process.env.DB_PASS! },
+                    { name: 'WARTHOG_DB_DATABASE', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'WARTHOG_DB_USERNAME', value: DB_USERNAME },
+                    { name: 'WARTHOG_DB_PASSWORD', value: DB_PASS },
+                    { name: 'WARTHOG_DB_PORT', value: DB_PORT },
+                    { name: 'DB_NAME', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'DB_PASS', value: DB_PASS },
+                    { name: 'DB_USER', value: DB_USERNAME },
+                    { name: 'DB_PORT', value: DB_PORT },
                   ],
                   command: ['/bin/sh', '-c'],
-                  args: ['yarn workspace query-node-root db:prepare; yarn workspace query-node-root db:migrate'],
+                  args: [
+                    // 'yarn workspace query-node config:dev;',
+                    'yarn workspace query-node-root db:prepare; yarn workspace query-node-root db:migrate',
+                  ],
                 },
               ],
               restartPolicy: 'Never',
@@ -98,15 +113,18 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
                   imagePullPolicy: 'IfNotPresent',
                   env: [
                     { name: 'DB_HOST', value: processorDbName },
-                    { name: 'DB_PASS', value: process.env.DB_PASS! },
-                    { name: 'DB_USER', value: process.env.DB_USER! },
-                    { name: 'DB_PORT', value: process.env.DB_PORT! },
-                    { name: 'DB_NAME', value: process.env.DB_NAME! },
-                    { name: 'GRAPHQL_SERVER_HOST', value: process.env.GRAPHQL_SERVER_HOST! },
-                    { name: 'GRAPHQL_SERVER_PORT', value: process.env.GRAPHQL_SERVER_PORT! },
-                    { name: 'WS_PROVIDER_ENDPOINT_URI', value: process.env.WS_PROVIDER_ENDPOINT_URI! },
+                    { name: 'DB_PASS', value: DB_PASS },
+                    { name: 'DB_USER', value: DB_USERNAME },
+                    { name: 'DB_PORT', value: DB_PORT },
+                    { name: 'DB_NAME', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'WARTHOG_DB_DATABASE', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'WARTHOG_DB_USERNAME', value: DB_USERNAME },
+                    { name: 'WARTHOG_DB_PASSWORD', value: DB_PASS },
+                    { name: 'WARTHOG_APP_PORT', value: '4002' },
+                    // Why do we need this anyway?
+                    { name: 'GRAPHQL_SERVER_HOST', value: 'graphql-server' },
                   ],
-                  ports: [{ name: 'graph-ql-port', containerPort: Number(process.env.GRAPHQL_SERVER_PORT!) }],
+                  ports: [{ name: 'graph-ql-port', containerPort: 4002 }],
                   args: ['workspace', 'query-node-root', 'query-node:start:prod'],
                 },
               ],
@@ -163,9 +181,19 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
                       value: indexerURL,
                     },
                     { name: 'TYPEORM_HOST', value: processorDbName },
-                    { name: 'TYPEORM_DATABASE', value: process.env.DB_NAME! },
+                    { name: 'TYPEORM_DATABASE', value: PROCESSOR_DATABASE_NAME },
                     { name: 'DEBUG', value: 'index-builder:*' },
                     { name: 'PROCESSOR_POLL_INTERVAL', value: '1000' },
+                    { name: 'DB_PASS', value: DB_PASS },
+                    { name: 'DB_USER', value: DB_USERNAME },
+                    { name: 'DB_PORT', value: DB_PORT },
+                    { name: 'WARTHOG_DB_DATABASE', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'WARTHOG_DB_USERNAME', value: DB_USERNAME },
+                    { name: 'WARTHOG_DB_PASSWORD', value: DB_PASS },
+                    { name: 'WARTHOG_DB_PORT', value: DB_PORT },
+                    // What do these have to do with processor and why are they required!
+                    { name: 'WARTHOG_APP_HOST', value: 'graphql-server' },
+                    { name: 'WARTHOG_APP_PORT', value: '4002' },
                   ],
                   volumeMounts: [
                     {
@@ -175,6 +203,8 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
                     },
                   ],
                   command: ['/bin/sh', '-c'],
+                  // Should we really use "-e ../.env" args could there be some env vars we have not
+                  // correctly overridden for this container?
                   args: ['cd query-node && yarn hydra-processor run -e ../.env'],
                 },
               ],
@@ -206,5 +236,5 @@ export interface ServiceDeploymentArgs {
   defsConfig: pulumi.Output<string> | undefined
   externalIndexerUrl: string | undefined
   env?: Environment[]
-  storage: Number
+  storage: number
 }