devops: separate ansible and aws from kubernetes deployments

.github/workflows/create-ami.yml

@@ -1,3 +1,5 @@
+# Creates an AWS AMI (system image) with compiled joystream-node and subkey
+# 
 name: Create AWS AMI
 
 on:
@@ -8,7 +10,7 @@ jobs:
     name: Build the code and run setup
     runs-on: ubuntu-latest
     env:
-      STACK_NAME: joystream-github-action-${{ github.run_number }}
+      STACK_NAME: create-joystream-node-ami-ga-${{ github.run_number }}
       KEY_NAME: joystream-github-action-key
     steps:
       - name: Extract branch name
@@ -18,7 +20,7 @@ jobs:
 
       - name: Set AMI Name environment variable
         shell: bash
-        run: echo "ami_name=joystream-${{ steps.extract_branch.outputs.branch }}-${{ github.run_number }}" >> $GITHUB_ENV
+        run: echo "ami_name=joystream-node-${{ steps.extract_branch.outputs.branch }}-${{ github.run_number }}" >> $GITHUB_ENV
         id: ami_name
 
       - name: Checkout
@@ -36,7 +38,7 @@ jobs:
         id: deploy_stack
         with:
           name: ${{ env.STACK_NAME }}
-          template: devops/infrastructure/cloudformation/single-instance.yml
+          template: devops/aws/cloudformation/single-instance.yml
           no-fail-on-empty-changeset: '1'
           parameter-overrides: 'KeyName=${{ env.KEY_NAME }}'
 
@@ -46,8 +48,8 @@ jobs:
       - name: Run playbook
         uses: dawidd6/action-ansible-playbook@v2
         with:
-          playbook: github-action-playbook.yml
-          directory: devops/infrastructure
+          playbook: create-joystream-node-ami-playbook.yml
+          directory: devops/aws
           requirements: requirements.yml
           key: ${{ secrets.SSH_PRIVATE_KEY }}
           inventory: |

.github/workflows/joystream-node-docker.yml

@@ -71,7 +71,7 @@ jobs:
             platform_tag: 'arm'
             file: 'joystream-node-armv7.Dockerfile'
     env:
-      STACK_NAME: joystream-ga-docker-${{ github.run_number }}-${{ matrix.platform_tag }}
+      STACK_NAME: build-joystream-node-docker-ga-${{ github.run_number }}-${{ matrix.platform_tag }}
     steps:
       - name: Extract branch name
         shell: bash
@@ -120,7 +120,7 @@ jobs:
         id: deploy_stack
         with:
           name: ${{ env.STACK_NAME }}
-          template: devops/infrastructure/cloudformation/single-instance-docker.yml
+          template: devops/aws/cloudformation/single-instance-docker.yml
           no-fail-on-empty-changeset: '1'
           parameter-overrides: 'KeyName=${{ env.KEY_NAME }},EC2AMI=ami-00d1ab6b335f217cf,EC2InstanceType=t4g.xlarge'
         if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}
@@ -129,7 +129,7 @@ jobs:
         uses: dawidd6/action-ansible-playbook@v2
         with:
           playbook: build-arm64-playbook.yml
-          directory: devops/infrastructure
+          directory: devops/aws
           requirements: requirements.yml
           key: ${{ secrets.SSH_PRIVATE_KEY }}
           inventory: |

devops/infrastructure/README.md → devops/aws/README.md

@@ -30,6 +30,12 @@ Copy and edit the file `deploy-config.sample.cfg` and update parameters like AWS
 Run the `deploy-infra.sh` script to deploy the infrastructure
 
 ```
-cd devops/infrastructure
+cd devops/aws
 ./deploy-infra.sh your-deploy-config.cfg
 ```
+
+# To tear down a network
+
+```
+./destroy-infra.sh your-deploy-config.cfg
+```

devops/infrastructure/cloudformation/infrastructure.yml → devops/aws/cloudformation/infrastructure.yml

@@ -1,3 +1,9 @@
+# Deploy inftrastructure required to run a new joystream chain.
+# This is comprised of:
+#   - N validators
+#   - One RPC node
+#   - s3 bucket with a build of Pionner
+
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
@@ -116,6 +122,9 @@ Resources:
             # Update all packages
             apt-get update -y
 
+            # Prevent interactive prompts that would interrup the installation
+            export DEBIAN_FRONTEND=noninteractive
+
             # Install the updates except docker, to avoid interactive prompt which blocks the flow of the script
             apt-mark hold docker.io
             apt-get upgrade -y

devops/infrastructure/cloudformation/single-instance-docker.yml → devops/aws/cloudformation/single-instance-docker.yml

@@ -1,3 +1,6 @@
+# Deploys and EC2 node with docker tools suitable for
+# building joystream node docker images
+
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
@@ -58,6 +61,9 @@ Resources:
             # Update all packages
             apt-get update -y
 
+            # Prevent interactive prompts that would interrup the installation
+            export DEBIAN_FRONTEND=noninteractive
+
             # Install the updates
             apt-get upgrade -y
 

devops/infrastructure/cloudformation/single-instance.yml → devops/aws/cloudformation/single-instance.yml

@@ -59,6 +59,9 @@ Resources:
             # Update all packages
             apt-get update -y
 
+            # Prevent interactive prompts that would interrup the installation
+            export DEBIAN_FRONTEND=noninteractive
+
             # Install the updates
             apt-get upgrade -y
 

devops/infrastructure/delete-stack.sh → devops/aws/destroy-infra.sh

@@ -6,7 +6,7 @@ source common.sh
 
 if [ -z "$1" ]; then
   echo "ERROR: Configuration file not passed"
-  echo "Please use ./delete-stack.sh PATH/TO/CONFIG to run this script"
+  echo "Please use ./destroy-infra.sh PATH/TO/CONFIG to run this script"
   exit 1
 else
   echo "Using $1 file for config"

devops/kubernetes/query-node/kubeconfig

@@ -0,0 +1,31 @@
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1UQXlNakV6TkRreU9Wb1hEVE14TVRBeU1ERXpORGt5T1Zvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBSjJvCm0xcmpMaEs5eEN1WVRqRjhBNnBNVDVxaTI0S21aUGVJZ0RjUWZmaFZaQU5vZGlFdmFNSVgwUE40Qzl5bGZVQ3YKRmc2amFsTlRLZ3IyV01Wd1JHVmV4aE5mTEdkdE81L21ZRVp1T0U0S3RwYmdFUjMyeEdDdUNuUFhHcUt1YUY3YQpST2Mvc1VMU0V6TVNHMUlwYUdDUlIweERja0JPVTdLWGg3eVBjL2xPNXpWTWdZWEVTUVVzeFUzajQ3WWRGZWVXCm9mTmhXcUN0cnkvMkJtVWhPWnB4YXBrUkRLa1QrZXpLV0tkbFl6ZVZKS3UvNWJnODgvb0sra1EvMjVkUmFadzYKdjJBN3MrZ09Xa2FPQ1NqR2NmNThTWWlBcTArR1dvUFI4dE1IOFNqSWdyZWVScjJFbU1tMWhFbDJZS3VTYkRDQwpIYkJza0NBMFh1T1l6eUZWMzA4Q0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZFK0pLY1FNYXAwbVBMS004d3AxZitjN3QxUjBNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBTFdQS2FScG5oVkZlRUg0NnpOb1ZhM3FMRkttbm50VUxjVnlROXNxT09DMzluWmlqdwo1aEwrRGRsSWJIK2dHU3hZY2RkWWx0Qm9UdDEyTG10OG84YWxETU5IWlEwTUdGNEx3bENjRDBUa3pTaFk0KzJnCmhKWGpKTnF6WTQxWTh4YzhVbThXY3poVWtTVVNaRDhia3hDd1htMjRYUW4rQktvNVh1Zm5SRTRYYVk2Y21zL1cKY2VjMzNGdDFwLzMzMjZDb3IwQjVpeC9iQmZuOGxRS2I4eU4zRm5yODc2NWpVWTVWa1FqdjYrc0ZaSWtkdkhwUApqNzgzbHJYOEJRRW5oamZKVFBCcVNPYUlWQVZRRCtNVVQ4OUppQTVNVzdaVVhwNVcrSnFyM2k2dkZwb2FXbG9uCk1kNVhFMFI0b0YvalFkcG16Y2xyMElTbmpjclBkQklQQ3kyQQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+    server: https://450F306AE798F0057C49C428A8BE608A.gr7.us-east-1.eks.amazonaws.com
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    namespace: query-node-ifgoohih
+    user: aws
+  name: aws
+current-context: aws
+kind: Config
+preferences: {}
+users:
+- name: aws
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1alpha1
+      args:
+      - eks
+      - get-token
+      - --cluster-name
+      - eksctl-query-node-eksCluster-7e1b8b4
+      command: aws
+      env:
+      - name: AWS_PROFILE
+        value: joystream-user
+      interactiveMode: IfAvailable
+      provideClusterInfo: false

setup.sh

@@ -3,6 +3,8 @@
 set -e
 
 if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    # Prevent interactive prompts that would interrup the installation
+    export DEBIAN_FRONTEND=noninteractive
     # code build tools
     sudo apt-get update
     sudo apt-get install -y coreutils clang llvm jq curl gcc xz-utils sudo pkg-config unzip libc6-dev make libssl-dev python