Merge pull request #2848 from mnaamani/query-node-deployment-refactor

Devops: query-node deployment refactor

devops/kubernetes/query-node/Pulumi.yaml

@@ -25,3 +25,12 @@ template:
     appsImage:
       description: The joystream image to use for running GraphQL servers
       default: joystream/apps:latest
+    dbPassword:
+      description: database password for indexer and processor databases  
+      required: true
+    blockHeight:
+      descroption: Block height to start indexing at
+      default: 0
+    joystreamWsEndpoint:
+      description: Joystream-node websocket endpoint used by indexer
+      required: true

devops/kubernetes/query-node/README.md

@@ -38,6 +38,8 @@ After cloning this repo, from this working directory, run these commands:
 
    ```bash
    $ pulumi config set-all --plaintext aws:region=us-east-1 --plaintext aws:profile=joystream-user \
+    --plaintext dbPassword=password --plaintext blockHeight=0 \
+    --plaintext joystreamWsEndpoint=ws://endpoint.somewhere.net:9944 \
     --plaintext isMinikube=true --plaintext skipProcessor=false
    ```
 
@@ -66,10 +68,6 @@ After cloning this repo, from this working directory, run these commands:
 
    If not using minikube, just specify the `appsImage` config.
 
-1. Create a `.env` file in this directory (`cp ../../../.env ./.env`) and set the database and other variables in it
-
-   Make sure to set `GRAPHQL_SERVER_PORT=4001`
-
 1. Stand up the Kubernetes cluster:
 
    Running `pulumi up -y` will deploy the EKS cluster. Note, provisioning a

devops/kubernetes/query-node/configMap.ts

@@ -2,7 +2,7 @@ import * as pulumi from '@pulumi/pulumi'
 import * as k8s from '@pulumi/kubernetes'
 import * as fs from 'fs'
 
-export class configMapFromFile extends pulumi.ComponentResource {
+export class ConfigMapFromFile extends pulumi.ComponentResource {
   public readonly configName?: pulumi.Output<string>
 
   constructor(name: string, args: ConfigMapArgs, opts: pulumi.ComponentResourceOptions = {}) {

devops/kubernetes/query-node/index.ts

@@ -2,14 +2,12 @@ import * as awsx from '@pulumi/awsx'
 import * as eks from '@pulumi/eks'
 import * as docker from '@pulumi/docker'
 import * as pulumi from '@pulumi/pulumi'
-import { configMapFromFile } from './configMap'
+import { ConfigMapFromFile } from './configMap'
 import * as k8s from '@pulumi/kubernetes'
 import { IndexerServiceDeployment } from './indexerDeployment'
 import { ProcessorServiceDeployment } from './processorDeployment'
 import { CaddyServiceDeployment } from 'pulumi-common'
 
-require('dotenv').config()
-
 const config = new pulumi.Config()
 const awsConfig = new pulumi.Config('aws')
 const isMinikube = config.getBoolean('isMinikube')
@@ -19,12 +17,11 @@ const skipProcessor = config.getBoolean('skipProcessor')
 const useLocalRepo = config.getBoolean('useLocalRepo')
 
 export let kubeconfig: pulumi.Output<any>
-export let joystreamAppsImage: pulumi.Output<string>
+export let joystreamAppsImage: pulumi.Output<string> = pulumi.interpolate`${appsImage}`
 let provider: k8s.Provider
 
 if (skipProcessor && externalIndexerUrl) {
-  pulumi.log.error('Need to deploy atleast one component, Indexer or Processor')
-  throw new Error(`Please check the config settings for skipProcessor and externalIndexerUrl`)
+  pulumi.log.info('No Indexer or Processor will be deployed only the cluster')
 }
 
 if (isMinikube) {
@@ -59,15 +56,19 @@ if (isMinikube) {
   // Export the cluster's kubeconfig.
   kubeconfig = cluster.kubeconfig
 
-  // Create a repository
-  const repo = new awsx.ecr.Repository('joystream/apps')
-
-  // Build an image from an existing local/docker hub image and push to ECR
-  joystreamAppsImage = repo.buildAndPushImage({
-    context: './docker_dummy',
-    dockerfile: './docker_dummy/Dockerfile',
-    args: { SOURCE_IMAGE: appsImage! },
-  })
+  // Only deploy ECR and push image if we need to deploy processor from
+  // local image build.
+  if (!skipProcessor && useLocalRepo) {
+    // Create a repository
+    const repo = new awsx.ecr.Repository('joystream/apps')
+
+    // Build an image from an existing local/docker hub image and push to ECR
+    joystreamAppsImage = repo.buildAndPushImage({
+      context: './docker_dummy',
+      dockerfile: './docker_dummy/Dockerfile',
+      args: { SOURCE_IMAGE: appsImage },
+    })
+  }
 }
 
 const resourceOptions = { provider: provider }
@@ -80,7 +81,7 @@ const ns = new k8s.core.v1.Namespace(name, {}, resourceOptions)
 // Export the Namespace name
 export const namespaceName = ns.metadata.name
 
-const defsConfig = new configMapFromFile(
+const defsConfig = new ConfigMapFromFile(
   'defs-config',
   {
     filePath: '../../../types/augment/all/defs.json',
@@ -109,11 +110,14 @@ const caddyEndpoints = [
   `/indexer* {
     uri strip_prefix /indexer
     reverse_proxy indexer:4000
-}`,
+  }`,
   `/server* {
     uri strip_prefix /server
     reverse_proxy graphql-server:8081
-}`,
+  }`,
+  `/@apollographql/* {
+    reverse_proxy graphql-server:8081
+  }`,
 ]
 
 const lbReady = config.get('isLoadBalancerReady') === 'true'

devops/kubernetes/query-node/indexerDeployment.ts

@@ -14,9 +14,18 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
   constructor(name: string, args: ServiceDeploymentArgs, opts?: pulumi.ComponentResourceOptions) {
     super('indexer:service:IndexerServiceDeployment', name, {}, opts)
 
+    const config = new pulumi.Config()
+    const DB_PASS = config.require('dbPassword')
+    const BLOCK_HEIGHT = config.require('blockHeight') || '0'
+    const WS_PROVIDER_ENDPOINT_URI = config.require('joystreamWsEndpoint')
+
+    const DB_USERNAME = 'postgres'
+    const INDEXER_DATABASE_NAME = 'indexer'
+    const DB_PORT = '5432'
+
     // Name passed in the constructor will be the endpoint for accessing the service
     const serviceName = name
-    let appLabels = { appClass: 'indexer' }
+    const appLabels = { appClass: 'indexer' }
 
     const indexerDbName = 'indexer-db'
     const indexerDb = new PostgresServiceDeployment(
@@ -24,56 +33,16 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
       {
         namespaceName: args.namespaceName,
         env: [
-          { name: 'POSTGRES_USER', value: process.env.DB_USER! },
-          { name: 'POSTGRES_PASSWORD', value: process.env.DB_PASS! },
-          { name: 'POSTGRES_DB', value: process.env.INDEXER_DB_NAME! },
+          { name: 'POSTGRES_USER', value: DB_USERNAME },
+          { name: 'POSTGRES_PASSWORD', value: DB_PASS },
+          { name: 'POSTGRES_DB', value: INDEXER_DATABASE_NAME },
+          { name: 'PGPORT', value: DB_PORT },
         ],
         storage: args.storage,
       },
       { parent: this }
     )
 
-    const indexerMigrationJob = new k8s.batch.v1.Job(
-      'indexer-db-migration',
-      {
-        metadata: {
-          namespace: args.namespaceName,
-        },
-        spec: {
-          backoffLimit: 0,
-          template: {
-            spec: {
-              containers: [
-                {
-                  name: 'db-migration',
-                  image: args.joystreamAppsImage,
-                  imagePullPolicy: 'IfNotPresent',
-                  resources: { requests: { cpu: '100m', memory: '100Mi' } },
-                  env: [
-                    {
-                      name: 'WARTHOG_DB_HOST',
-                      value: indexerDbName,
-                    },
-                    {
-                      name: 'DB_HOST',
-                      value: indexerDbName,
-                    },
-                    { name: 'WARTHOG_DB_DATABASE', value: process.env.INDEXER_DB_NAME! },
-                    { name: 'DB_NAME', value: process.env.INDEXER_DB_NAME! },
-                    { name: 'DB_PASS', value: process.env.DB_PASS! },
-                  ],
-                  command: ['/bin/sh', '-c'],
-                  args: ['yarn workspace query-node-root db:prepare; yarn workspace query-node-root db:migrate'],
-                },
-              ],
-              restartPolicy: 'Never',
-            },
-          },
-        },
-      },
-      { parent: this, dependsOn: indexerDb.service }
-    )
-
     this.deployment = new k8s.apps.v1.Deployment(
       'indexer',
       {
@@ -100,17 +69,18 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
                   image: 'joystream/hydra-indexer:3.0.0',
                   env: [
                     { name: 'DB_HOST', value: indexerDbName },
-                    { name: 'DB_NAME', value: process.env.INDEXER_DB_NAME! },
-                    { name: 'DB_PASS', value: process.env.DB_PASS! },
-                    { name: 'DB_USER', value: process.env.DB_USER! },
-                    { name: 'DB_PORT', value: process.env.DB_PORT! },
+                    { name: 'DB_NAME', value: INDEXER_DATABASE_NAME },
+                    { name: 'DB_PASS', value: DB_PASS },
+                    { name: 'DB_USER', value: DB_USERNAME },
+                    { name: 'DB_PORT', value: DB_PORT },
                     { name: 'INDEXER_WORKERS', value: '5' },
+                    // localhost for redis should work since it is in the same deployment
                     { name: 'REDIS_URI', value: 'redis://localhost:6379/0' },
                     { name: 'DEBUG', value: 'index-builder:*' },
-                    { name: 'WS_PROVIDER_ENDPOINT_URI', value: process.env.WS_PROVIDER_ENDPOINT_URI! },
+                    { name: 'WS_PROVIDER_ENDPOINT_URI', value: WS_PROVIDER_ENDPOINT_URI },
                     { name: 'TYPES_JSON', value: 'types.json' },
-                    { name: 'PGUSER', value: process.env.DB_USER! },
-                    { name: 'BLOCK_HEIGHT', value: process.env.BLOCK_HEIGHT! },
+                    { name: 'PGUSER', value: DB_USERNAME },
+                    { name: 'BLOCK_HEIGHT', value: BLOCK_HEIGHT },
                   ],
                   volumeMounts: [
                     {
@@ -126,17 +96,18 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
                   name: 'hydra-indexer-gateway',
                   image: 'joystream/hydra-indexer-gateway:3.0.0',
                   env: [
-                    { name: 'WARTHOG_STARTER_DB_DATABASE', value: process.env.INDEXER_DB_NAME! },
+                    { name: 'WARTHOG_STARTER_DB_DATABASE', value: INDEXER_DATABASE_NAME },
                     { name: 'WARTHOG_STARTER_DB_HOST', value: indexerDbName },
-                    { name: 'WARTHOG_STARTER_DB_PASSWORD', value: process.env.DB_PASS! },
-                    { name: 'WARTHOG_STARTER_DB_PORT', value: process.env.DB_PORT! },
-                    { name: 'WARTHOG_STARTER_DB_USERNAME', value: process.env.DB_USER! },
+                    { name: 'WARTHOG_STARTER_DB_PASSWORD', value: DB_PASS },
+                    { name: 'WARTHOG_STARTER_DB_PORT', value: DB_PORT },
+                    { name: 'WARTHOG_STARTER_DB_USERNAME', value: DB_USERNAME },
+                    // localhost for redis should work since it is in the same deployment
                     { name: 'WARTHOG_STARTER_REDIS_URI', value: 'redis://localhost:6379/0' },
-                    { name: 'WARTHOG_APP_PORT', value: process.env.WARTHOG_APP_PORT! },
-                    { name: 'PORT', value: process.env.WARTHOG_APP_PORT! },
+                    { name: 'WARTHOG_APP_PORT', value: '4001' },
+                    { name: 'PORT', value: '4001' },
                     { name: 'DEBUG', value: '*' },
                   ],
-                  ports: [{ name: 'hydra-port', containerPort: Number(process.env.WARTHOG_APP_PORT!) }],
+                  ports: [{ name: 'hydra-port', containerPort: 4001 }],
                 },
               ],
               volumes: [
@@ -151,7 +122,7 @@ export class IndexerServiceDeployment extends pulumi.ComponentResource {
           },
         },
       },
-      { parent: this, dependsOn: indexerMigrationJob }
+      { parent: this, dependsOn: indexerDb.service }
     )
 
     // Create a Service for the Indexer
@@ -183,5 +154,5 @@ export interface ServiceDeploymentArgs {
   joystreamAppsImage: pulumi.Output<string>
   defsConfig: pulumi.Output<string> | undefined
   env?: Environment[]
-  storage: Number
+  storage: number
 }

devops/kubernetes/query-node/processorDeployment.ts

@@ -15,6 +15,12 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
   constructor(name: string, args: ServiceDeploymentArgs, opts?: pulumi.ComponentResourceOptions) {
     super('processor:service:ProcessorServiceDeployment', name, {}, opts)
 
+    const config = new pulumi.Config()
+    const DB_PASS = config.require('dbPassword')
+    const DB_USERNAME = 'postgres'
+    const PROCESSOR_DATABASE_NAME = 'processor'
+    const DB_PORT = '5432'
+
     // Name passed in the constructor will be the endpoint for accessing the service
     this.endpoint = 'graphql-server'
 
@@ -24,9 +30,10 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
       {
         namespaceName: args.namespaceName,
         env: [
-          { name: 'POSTGRES_USER', value: process.env.DB_USER! },
-          { name: 'POSTGRES_PASSWORD', value: process.env.DB_PASS! },
-          { name: 'POSTGRES_DB', value: process.env.DB_NAME! },
+          { name: 'POSTGRES_USER', value: DB_USERNAME },
+          { name: 'POSTGRES_PASSWORD', value: DB_PASS },
+          { name: 'POSTGRES_DB', value: PROCESSOR_DATABASE_NAME },
+          { name: 'PGPORT', value: DB_PORT },
         ],
         storage: args.storage,
       },
@@ -58,12 +65,20 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
                       name: 'DB_HOST',
                       value: processorDbName,
                     },
-                    { name: 'WARTHOG_DB_DATABASE', value: process.env.DB_NAME! },
-                    { name: 'DB_NAME', value: process.env.DB_NAME! },
-                    { name: 'DB_PASS', value: process.env.DB_PASS! },
+                    { name: 'WARTHOG_DB_DATABASE', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'WARTHOG_DB_USERNAME', value: DB_USERNAME },
+                    { name: 'WARTHOG_DB_PASSWORD', value: DB_PASS },
+                    { name: 'WARTHOG_DB_PORT', value: DB_PORT },
+                    { name: 'DB_NAME', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'DB_PASS', value: DB_PASS },
+                    { name: 'DB_USER', value: DB_USERNAME },
+                    { name: 'DB_PORT', value: DB_PORT },
                   ],
                   command: ['/bin/sh', '-c'],
-                  args: ['yarn workspace query-node-root db:prepare; yarn workspace query-node-root db:migrate'],
+                  args: [
+                    // 'yarn workspace query-node config:dev;',
+                    'yarn workspace query-node-root db:prepare; yarn workspace query-node-root db:migrate',
+                  ],
                 },
               ],
               restartPolicy: 'Never',
@@ -98,15 +113,18 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
                   imagePullPolicy: 'IfNotPresent',
                   env: [
                     { name: 'DB_HOST', value: processorDbName },
-                    { name: 'DB_PASS', value: process.env.DB_PASS! },
-                    { name: 'DB_USER', value: process.env.DB_USER! },
-                    { name: 'DB_PORT', value: process.env.DB_PORT! },
-                    { name: 'DB_NAME', value: process.env.DB_NAME! },
-                    { name: 'GRAPHQL_SERVER_HOST', value: process.env.GRAPHQL_SERVER_HOST! },
-                    { name: 'GRAPHQL_SERVER_PORT', value: process.env.GRAPHQL_SERVER_PORT! },
-                    { name: 'WS_PROVIDER_ENDPOINT_URI', value: process.env.WS_PROVIDER_ENDPOINT_URI! },
+                    { name: 'DB_PASS', value: DB_PASS },
+                    { name: 'DB_USER', value: DB_USERNAME },
+                    { name: 'DB_PORT', value: DB_PORT },
+                    { name: 'DB_NAME', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'WARTHOG_DB_DATABASE', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'WARTHOG_DB_USERNAME', value: DB_USERNAME },
+                    { name: 'WARTHOG_DB_PASSWORD', value: DB_PASS },
+                    { name: 'WARTHOG_APP_PORT', value: '4002' },
+                    // Why do we need this anyway?
+                    { name: 'GRAPHQL_SERVER_HOST', value: 'graphql-server' },
                   ],
-                  ports: [{ name: 'graph-ql-port', containerPort: Number(process.env.GRAPHQL_SERVER_PORT!) }],
+                  ports: [{ name: 'graph-ql-port', containerPort: 4002 }],
                   args: ['workspace', 'query-node-root', 'query-node:start:prod'],
                 },
               ],
@@ -163,9 +181,19 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
                       value: indexerURL,
                     },
                     { name: 'TYPEORM_HOST', value: processorDbName },
-                    { name: 'TYPEORM_DATABASE', value: process.env.DB_NAME! },
+                    { name: 'TYPEORM_DATABASE', value: PROCESSOR_DATABASE_NAME },
                     { name: 'DEBUG', value: 'index-builder:*' },
                     { name: 'PROCESSOR_POLL_INTERVAL', value: '1000' },
+                    { name: 'DB_PASS', value: DB_PASS },
+                    { name: 'DB_USER', value: DB_USERNAME },
+                    { name: 'DB_PORT', value: DB_PORT },
+                    { name: 'WARTHOG_DB_DATABASE', value: PROCESSOR_DATABASE_NAME },
+                    { name: 'WARTHOG_DB_USERNAME', value: DB_USERNAME },
+                    { name: 'WARTHOG_DB_PASSWORD', value: DB_PASS },
+                    { name: 'WARTHOG_DB_PORT', value: DB_PORT },
+                    // These are note required but must be defined or processor will not startup
+                    { name: 'WARTHOG_APP_HOST', value: 'graphql-server' },
+                    { name: 'WARTHOG_APP_PORT', value: '4002' },
                   ],
                   volumeMounts: [
                     {
@@ -174,8 +202,7 @@ export class ProcessorServiceDeployment extends pulumi.ComponentResource {
                       subPath: 'fileData',
                     },
                   ],
-                  command: ['/bin/sh', '-c'],
-                  args: ['cd query-node && yarn hydra-processor run -e ../.env'],
+                  args: ['workspace', 'query-node-root', 'processor:start'],
                 },
               ],
               volumes: [
@@ -206,5 +233,5 @@ export interface ServiceDeploymentArgs {
   defsConfig: pulumi.Output<string> | undefined
   externalIndexerUrl: string | undefined
   env?: Environment[]
-  storage: Number
+  storage: number
 }