Add database and redis containers for query node k8s

devops/infrastructure/query-node/.gitignore

@@ -0,0 +1,5 @@
+/bin/
+/node_modules/
+kubeconfig.yml
+package-lock.json
+.env

devops/infrastructure/query-node/Pulumi.dev.yaml

@@ -0,0 +1,4 @@
+encryptionsalt: v1:6JCZJ+FPZ/o=:v1:axvmG3ZflvqylUwZ:8J9F+CmjCVz8Z8lY9LhB0PlMHSIk8w==
+config:
+  aws:profile: joystream-user
+  aws:region: us-east-1

devops/infrastructure/query-node/Pulumi.yaml

@@ -0,0 +1,12 @@
+name: query-node
+runtime: nodejs
+description: Kubernetes IaC for Query Node
+template:
+  config:
+    aws:profile:
+      default: joystream-user
+    aws:region:
+      default: us-east-1
+    isMinikube:
+      description: Whether you are deploying to minikube
+      default: false

devops/infrastructure/query-node/README.md

@@ -0,0 +1,90 @@
+# Amazon EKS Cluster: Hello World!
+
+This example deploys an EKS Kubernetes cluster with query node
+
+## Deploying the App
+
+To deploy your infrastructure, follow the below steps.
+
+### Prerequisites
+
+1. [Install Pulumi](https://www.pulumi.com/docs/get-started/install/)
+1. [Install Node.js](https://nodejs.org/en/download/)
+1. Install a package manager for Node.js, such as [npm](https://www.npmjs.com/get-npm) or [Yarn](https://yarnpkg.com/en/docs/install).
+1. [Configure AWS Credentials](https://www.pulumi.com/docs/intro/cloud-providers/aws/setup/)
+1. Optional (for debugging): [Install kubectl](https://kubernetes.io/docs/tasks/tools/)
+
+### Steps
+
+After cloning this repo, from this working directory, run these commands:
+
+1. Install the required Node.js packages:
+
+   This installs the dependent packages [needed](https://www.pulumi.com/docs/intro/concepts/how-pulumi-works/) for our Pulumi program.
+
+   ```bash
+   $ npm install
+   ```
+
+1. Create a new stack, which is an isolated deployment target for this example:
+
+   This will initialize the Pulumi program in TypeScript.
+
+   ```bash
+   $ pulumi stack init
+   ```
+
+1. Set the required AWS configuration variables in `Pulumi.<stack>.yaml`
+
+1. Create a `.env` file in this directory and set the database and other variables in it
+
+1. Stand up the EKS cluster:
+
+   Running `pulumi up -y` will deploy the EKS cluster. Note, provisioning a
+   new EKS cluster takes between 10-15 minutes.
+
+1. Access the Kubernetes Cluster using `kubectl`
+
+   To access your new Kubernetes cluster using `kubectl`, we need to set up the
+   `kubeconfig` file and download `kubectl`. We can leverage the Pulumi
+   stack output in the CLI, as Pulumi facilitates exporting these objects for us.
+
+   ```bash
+   $ pulumi stack output kubeconfig --show-secrets > kubeconfig
+   $ export KUBECONFIG=$PWD/kubeconfig
+   $ kubectl get nodes
+   ```
+
+   We can also use the stack output to query the cluster for our newly created Deployment:
+
+   ```bash
+   $ kubectl get deployment $(pulumi stack output deploymentName) --namespace=$(pulumi stack output namespaceName)
+   $ kubectl get service $(pulumi stack output serviceName) --namespace=$(pulumi stack output namespaceName)
+   ```
+
+   To get logs
+
+   ```bash
+   $ kubectl config set-context --current --namespace=$(pulumi stack output namespaceName)
+   $ kubectl get pods
+   $ kubectl logs <PODNAME> --all-containers
+   ```
+
+   To see complete pulumi stack output
+
+   ```bash
+   $ pulumi stack output
+   ```
+
+   To execute a command
+
+   ```bash
+   $ kubectl exec --stdin --tty <PODNAME> -c colossus -- /bin/bash
+   ```
+
+1. Once you've finished experimenting, tear down your stack's resources by destroying and removing it:
+
+   ```bash
+   $ pulumi destroy --yes
+   $ pulumi stack rm --yes
+   ```

devops/infrastructure/query-node/index.ts

@@ -0,0 +1,43 @@
+import * as awsx from '@pulumi/awsx'
+import * as eks from '@pulumi/eks'
+// import * as k8s from '@pulumi/kubernetes'
+import * as pulumi from '@pulumi/pulumi'
+import * as k8sjs from './k8sjs'
+
+require('dotenv').config()
+
+const awsConfig = new pulumi.Config('aws')
+
+// Create a VPC for our cluster.
+const vpc = new awsx.ec2.Vpc('vpc', { numberOfAvailabilityZones: 2 })
+
+// Create an EKS cluster with the default configuration.
+const cluster = new eks.Cluster('eksctl-my-cluster', {
+  vpcId: vpc.id,
+  subnetIds: vpc.publicSubnetIds,
+  instanceType: 't2.micro',
+  providerCredentialOpts: {
+    profileName: awsConfig.get('profile'),
+  },
+})
+
+// Export the cluster's kubeconfig.
+export const kubeconfig = cluster.kubeconfig
+
+// Create a repository
+const repo = new awsx.ecr.Repository('my-repo')
+
+const redisLeader = new k8sjs.ServiceDeployment('redis-leader', {
+  image: 'redis:6.0-alpine',
+  ports: [6379],
+})
+
+const db = new k8sjs.ServiceDeployment('postgres-db', {
+  image: 'postgres:12',
+  ports: [5432],
+  env: [
+    { name: 'POSTGRES_USER', value: process.env.DB_USER! },
+    { name: 'POSTGRES_PASSWORD', value: process.env.DB_PASS! },
+    { name: 'POSTGRES_DB', value: process.env.INDEXER_DB_NAME! },
+  ],
+})

devops/infrastructure/query-node/k8sjs.ts

@@ -0,0 +1,80 @@
+import * as k8s from '@pulumi/kubernetes'
+import * as k8stypes from '@pulumi/kubernetes/types/input'
+import * as pulumi from '@pulumi/pulumi'
+
+/**
+ * ServiceDeployment is an example abstraction that uses a class to fold together the common pattern of a
+ * Kubernetes Deployment and its associated Service object.
+ */
+export class ServiceDeployment extends pulumi.ComponentResource {
+  public readonly deployment: k8s.apps.v1.Deployment
+  public readonly service: k8s.core.v1.Service
+  public readonly ipAddress?: pulumi.Output<string>
+
+  constructor(name: string, args: ServiceDeploymentArgs, opts?: pulumi.ComponentResourceOptions) {
+    super('k8sjs:service:ServiceDeployment', name, {}, opts)
+
+    const labels = { app: name }
+    const currentEnv = args.env || []
+    currentEnv.push({ name: 'GET_HOSTS_FROM', value: 'dns' })
+
+    const container: k8stypes.core.v1.Container = {
+      name,
+      image: args.image,
+      resources: args.resources || { requests: { cpu: '100m', memory: '100Mi' } },
+      env: currentEnv,
+      ports: args.ports && args.ports.map((p) => ({ containerPort: p })),
+    }
+    this.deployment = new k8s.apps.v1.Deployment(
+      name,
+      {
+        spec: {
+          selector: { matchLabels: labels },
+          replicas: args.replicas || 1,
+          template: {
+            metadata: { labels: labels },
+            spec: { containers: [container] },
+          },
+        },
+      },
+      { parent: this }
+    )
+
+    this.service = new k8s.core.v1.Service(
+      name,
+      {
+        metadata: {
+          name: name,
+          labels: this.deployment.metadata.labels,
+        },
+        spec: {
+          ports: args.ports && args.ports.map((p) => ({ port: p, targetPort: p })),
+          selector: this.deployment.spec.template.metadata.labels,
+          // Minikube does not implement services of type `LoadBalancer`; require the user to specify if we're
+          // running on minikube, and if so, create only services of type ClusterIP.
+          type: args.allocateIpAddress ? (args.isMinikube ? 'ClusterIP' : 'LoadBalancer') : undefined,
+        },
+      },
+      { parent: this }
+    )
+
+    if (args.allocateIpAddress) {
+      this.ipAddress = args.isMinikube ? this.service.spec.clusterIP : this.service.status.loadBalancer.ingress[0].ip
+    }
+  }
+}
+
+interface EnvironmentType {
+  name: string
+  value: string
+}
+
+export interface ServiceDeploymentArgs {
+  image: string
+  resources?: k8stypes.core.v1.ResourceRequirements
+  replicas?: number
+  ports?: number[]
+  env?: EnvironmentType[]
+  allocateIpAddress?: boolean
+  isMinikube?: boolean
+}

devops/infrastructure/query-node/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "query-node",
+  "devDependencies": {
+    "@types/node": "^10.0.0"
+  },
+  "dependencies": {
+    "@pulumi/aws": "^4.0.0",
+    "@pulumi/awsx": "^0.30.0",
+    "@pulumi/eks": "^0.31.0",
+    "@pulumi/kubernetes": "^3.0.0",
+    "@pulumi/pulumi": "^3.0.0",
+    "dotenv": "^10.0.0"
+  }
+}

devops/infrastructure/query-node/tsconfig.json

@@ -0,0 +1,18 @@
+{
+    "compilerOptions": {
+        "strict": true,
+        "outDir": "bin",
+        "target": "es2016",
+        "module": "commonjs",
+        "moduleResolution": "node",
+        "sourceMap": true,
+        "experimentalDecorators": true,
+        "pretty": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitReturns": true,
+        "forceConsistentCasingInFileNames": true
+    },
+    "files": [
+        "index.ts"
+    ]
+}