Add validator node on separate deployment, add bootnodes

devops/infrastructure/node-network/index.ts

@@ -5,12 +5,15 @@ import * as k8s from '@pulumi/kubernetes'
 import { configMapFromFile } from './configMap'
 import { CaddyServiceDeployment } from './caddy'
 import { getSubkeyContainers, getValidatorContainers } from './utils'
+// const { exec } = require('child_process')
 
 const config = new pulumi.Config()
 const awsConfig = new pulumi.Config('aws')
 const isMinikube = config.getBoolean('isMinikube')
+
 export let kubeconfig: pulumi.Output<any>
 export let joystreamAppsImage: pulumi.Output<string>
+
 let provider: k8s.Provider
 
 if (isMinikube) {
@@ -41,7 +44,6 @@ const resourceOptions = { provider: provider }
 const name = 'node-network'
 
 // Create a Kubernetes Namespace
-// const ns = new k8s.core.v1.Namespace(name, {}, { provider: cluster.provider })
 const ns = new k8s.core.v1.Namespace(name, {}, resourceOptions)
 
 // Export the Namespace name
@@ -61,10 +63,10 @@ const jsonModifyConfig = new configMapFromFile(
 const chainDataPath = '/chain-data'
 const networkSuffix = config.get('networkSuffix') || '8129'
 const chainSpecPath = `${chainDataPath}/chainspec-raw.json`
-const numberOfValidators = config.getNumber('numberOfValidators') || 2
+const numberOfValidators = config.getNumber('numberOfValidators') || 1
 
 const subkeyContainers = getSubkeyContainers(numberOfValidators, chainDataPath)
-// const validatorContainers = getValidatorContainers(numberOfValidators, dataPath, builderPath, chainSpecPath)
+const validatorContainers = getValidatorContainers(numberOfValidators, chainDataPath, chainSpecPath)
 
 const pvc = new k8s.core.v1.PersistentVolumeClaim(
   `${name}-pvc`,
@@ -137,7 +139,15 @@ const chainDataPrepareJob = new k8s.batch.v1.Job(
               name: 'json-modify',
               image: 'python',
               command: ['python'],
-              args: ['/scripts/json_modify.py', '--path', `${chainDataPath}/chainspec.json`, '--prefix', networkSuffix],
+              args: [
+                '/scripts/json_modify.py',
+                '--path',
+                `${chainDataPath}`,
+                '--prefix',
+                networkSuffix,
+                '--validators',
+                `${numberOfValidators}`,
+              ],
               volumeMounts: [
                 {
                   mountPath: '/scripts/json_modify.py',
@@ -185,6 +195,68 @@ const chainDataPrepareJob = new k8s.batch.v1.Job(
   { ...resourceOptions }
 )
 
+// async function executeCommand(url: string): Promise<string> {
+//   return new Promise((resolve, reject) => {
+//     exec(url, (err: string, stdout: string, stderr: string) => {
+//       if (err) reject(err)
+//       resolve(stdout.replace(/\r?\n|\r/g, ''))
+//     })
+//   })
+// }
+
+// const res = executeCommand("kubectl get pods | grep 'caddy-proxy' | awk '{print $1}'")
+
+// export const result = res
+
+const validatorLabels = { app: 'validator-nodes' }
+
+const validatorNode = new k8s.apps.v1.Deployment(
+  `validator-node`,
+  {
+    metadata: {
+      namespace: namespaceName,
+      labels: validatorLabels,
+    },
+    spec: {
+      replicas: 1,
+      selector: { matchLabels: validatorLabels },
+      template: {
+        metadata: {
+          labels: validatorLabels,
+        },
+        spec: {
+          containers: [...validatorContainers],
+          volumes: [
+            {
+              name: 'config-data',
+              persistentVolumeClaim: {
+                claimName: `${name}-pvc`,
+              },
+            },
+          ],
+        },
+      },
+    },
+  },
+  { ...resourceOptions, dependsOn: chainDataPrepareJob }
+)
+
+const validatorService = new k8s.core.v1.Service(
+  `node-1`,
+  {
+    metadata: {
+      labels: validatorLabels,
+      namespace: namespaceName,
+      name: 'node-1',
+    },
+    spec: {
+      ports: [{ name: 'port-1', port: 30333 }],
+      selector: validatorLabels,
+    },
+  },
+  resourceOptions
+)
+
 const deployment = new k8s.apps.v1.Deployment(
   `rpc-node`,
   {
@@ -245,7 +317,7 @@ const deployment = new k8s.apps.v1.Deployment(
       },
     },
   },
-  { ...resourceOptions, dependsOn: chainDataPrepareJob }
+  { ...resourceOptions, dependsOn: validatorNode }
 )
 
 // Export the Deployment name

devops/infrastructure/node-network/json_modify.py

@@ -2,9 +2,10 @@
 import argparse
 import json
 
-def main(chain_spec_path, prefix):
-    print("Updating chain spec file")
-    # all_nodes = module.params["all_nodes"]
+def main(chain_path, prefix, number_of_validators):
+    chain_spec_path = f"{chain_path}/chainspec.json"
+    print(f"Updating chain spec file {chain_spec_path}")
+    number_of_validators = int(number_of_validators)
 
     with open(chain_spec_path) as f:
         data = json.load(f)
@@ -15,27 +16,29 @@ def main(chain_spec_path, prefix):
         "protocolId": f'{data["protocolId"]}{prefix}'
     }
 
-    # boot_node_list = data["bootNodes"]
-    # for key in all_nodes:
-    #     if "validators" in all_nodes[key]["group_names"]:
-    #         public_key = all_nodes[key]["subkey_output"]["stderr"]
-    #         boot_node_list.append(f"/ip4/{key}/tcp/30333/p2p/{public_key}")
+    boot_node_list = data["bootNodes"]
+    for i in range(1, number_of_validators + 1):
+        public_key = open(f"{chain_path}/publickey{i}").read().replace('\n', '')
+        boot_node = f"/dns4/node-{i}/tcp/30333/p2p/{public_key}"
+        boot_node_list.append(boot_node)
 
     telemetry_endpoints = data["telemetryEndpoints"]
     telemetry_endpoints.append([
         "/dns/telemetry.joystream.org/tcp/443/x-parity-wss/%2Fsubmit%2F", 0])
 
-    # response["bootNodes"] = boot_node_list
+    response["bootNodes"] = boot_node_list
     response["telemetryEndpoints"] = telemetry_endpoints
 
     data.update(response)
     with open(chain_spec_path, 'w') as outfile:
         json.dump(data, outfile, indent=4)
+    print("Chain spec file updated")
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Modify Chain Spec file')
-    parser.add_argument('--path', required=True, help="Path to chain spec file")
+    parser.add_argument('--path', required=True, help="Path to chain data")
     parser.add_argument('--prefix', required=True, help="Network prefix")
+    parser.add_argument('--validators', required=True, help="Number of Validators")
     args = parser.parse_args()
     print(args.path)
-    main(chain_spec_path=args.path, prefix=args.prefix)
+    main(chain_path=args.path, prefix=args.prefix, number_of_validators=args.validators)

devops/infrastructure/node-network/utils.ts

@@ -5,7 +5,7 @@ export const getSubkeyContainers = (validators: number, dataPath: string) => {
       name: `subkey-node-${i}`,
       image: 'parity/subkey:latest',
       command: ['/bin/sh', '-c'],
-      args: [`subkey generate-node-key >> ${dataPath}/privatekey${i} 2>> ${dataPath}/publickey${i}`],
+      args: [`subkey generate-node-key > ${dataPath}/privatekey${i} 2> ${dataPath}/publickey${i}`],
       volumeMounts: [
         {
           name: 'config-data',