Merge pull request #2467 from ahhda/create-ami-github-action

DevOps - Github action for creating AMI's

.github/workflows/create-ami.yml

@@ -0,0 +1,63 @@
+name: Build code and create AMI
+
+on:
+  push:
+    branches:
+      - master
+      - olympia
+      - test_branch
+
+jobs:
+  build:
+    name: Build the code and run setup
+    runs-on: ubuntu-latest
+    env:
+      STACK_NAME: joystream-github-action-${{ github.run_number }}
+      KEY_NAME: joystream-github-action-key
+    steps:
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+
+    - name: Set AMI Name environment variable
+      shell: bash
+      run: echo "ami_name=joystream-${{ steps.extract_branch.outputs.branch }}-${{ github.run_number }}" >> $GITHUB_ENV
+      id: ami_name
+
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Deploy to AWS CloudFormation
+      uses: aws-actions/aws-cloudformation-github-deploy@v1
+      id: deploy_stack
+      with:
+        name: ${{ env.STACK_NAME }}
+        template: devops/infrastructure/single-instance.yml
+        no-fail-on-empty-changeset: "1"
+        parameter-overrides: "KeyName=${{ env.KEY_NAME }}"
+
+    - name: Install Ansible dependencies
+      run: pipx inject ansible-base boto3 botocore
+
+    - name: Run playbook
+      uses: dawidd6/action-ansible-playbook@v2
+      with:
+        playbook: github-action-playbook.yml
+        directory: devops/infrastructure
+        requirements: requirements.yml
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        inventory: |
+          [all]
+          ${{ steps.deploy_stack.outputs.PublicIp }}
+        options: |
+          --extra-vars "git_repo=https://github.com/${{ github.repository }} \
+                        branch_name=${{ steps.extract_branch.outputs.branch }} instance_id=${{ steps.deploy_stack.outputs.InstanceId }}
+                        stack_name=${{ env.STACK_NAME }} ami_name=${{ env.ami_name }}"

devops/infrastructure/github-action-playbook.yml

@@ -0,0 +1,40 @@
+---
+# Setup joystream code, build and Create AMI
+
+- name: Setup instance
+  hosts: all
+
+  tasks:
+    - block:
+      - name: Get code from git repo
+        include_role:
+          name: common
+          tasks_from: get-code-git
+
+      - name: Run setup and build
+        include_role:
+          name: common
+          tasks_from: run-setup-build
+
+      - name: Basic AMI Creation
+        amazon.aws.ec2_ami:
+          instance_id: "{{ instance_id }}"
+          wait: yes
+          name: "{{ ami_name }}"
+          launch_permissions:
+            group_names: ['all']
+          tags:
+            Name: "{{ ami_name }}"
+        register: ami_data
+        delegate_to: localhost
+
+      - name: Print AMI ID
+        debug:
+          msg: "AMI ID is: {{ ami_data.image_id }}"
+
+      always:
+      - name: Delete the stack
+        amazon.aws.cloudformation:
+          stack_name: "{{ stack_name }}"
+          state: "absent"
+        delegate_to: localhost

devops/infrastructure/requirements.yml

@@ -3,3 +3,4 @@ roles:
 - caddy_ansible.caddy_ansible
 collections:
 - community.aws
+- amazon.aws

devops/infrastructure/roles/common/tasks/run-setup-build.yml

@@ -13,3 +13,14 @@
   shell: . ~/.bash_profile && yarn cargo-build
   args:
     chdir: "{{ remote_code_path }}"
+  async: 3600
+  poll: 0
+  register: build_result
+
+- name: Check on build async task
+  async_status:
+    jid: "{{ build_result.ansible_job_id }}"
+  register: job_result
+  until: job_result.finished
+  retries: 36
+  delay: 100

devops/infrastructure/single-instance.yml

@@ -0,0 +1,106 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Parameters:
+  EC2InstanceType:
+    Type: String
+    Default: t2.xlarge
+  EC2AMI:
+    Type: String
+    Default: 'ami-09e67e426f25ce0d7'
+  DefaultAMI:
+    Type: String
+    Default: 'ami-09e67e426f25ce0d7'
+  KeyName:
+    Description: Name of an existing EC2 KeyPair to enable SSH access to the instance
+    Type: 'AWS::EC2::KeyPair::KeyName'
+    Default: 'joystream-key'
+    ConstraintDescription: must be the name of an existing EC2 KeyPair.
+
+Conditions:
+  HasAMIId: !Not [!Equals [!Ref EC2AMI, ""]]
+
+Resources:
+  SecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription:
+        !Sub 'Internal Security group for validator nodes ${AWS::StackName}'
+      SecurityGroupIngress:
+        - IpProtocol: tcp
+          FromPort: 22
+          ToPort: 22
+          CidrIp: 0.0.0.0/0
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_validator'
+
+  InstanceLaunchTemplate:
+    Type: AWS::EC2::LaunchTemplate
+    Metadata:
+      AWS::CloudFormation::Init:
+        config:
+          packages:
+            apt:
+              wget: []
+              unzip: []
+    Properties:
+      LaunchTemplateName: !Sub 'LaunchTemplate_${AWS::StackName}'
+      LaunchTemplateData:
+        ImageId: !If [HasAMIId, !Ref EC2AMI, !Ref DefaultAMI]
+        InstanceType: !Ref EC2InstanceType
+        KeyName: !Ref KeyName
+        SecurityGroupIds:
+          - !GetAtt SecurityGroup.GroupId
+        BlockDeviceMappings:
+          - DeviceName: /dev/sda1
+            Ebs:
+              VolumeSize: '30'
+        UserData:
+          Fn::Base64: !Sub |
+            #!/bin/bash -xe
+
+            # send script output to /tmp so we can debug boot failures
+            exec > /tmp/userdata.log 2>&1
+
+            # Update all packages
+            apt-get update -y
+
+            # Install the updates
+            apt-get upgrade -y
+
+            # Get latest cfn scripts and install them;
+            apt-get install -y python3-setuptools
+            mkdir -p /opt/aws/bin
+            wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz
+            python3 -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-py3-latest.tar.gz
+
+            /opt/aws/bin/cfn-signal -e $? -r "Instance Created" '${WaitHandle}'
+
+  Instance:
+    Type: AWS::EC2::Instance
+    Properties:
+      LaunchTemplate:
+        LaunchTemplateId: !Ref InstanceLaunchTemplate
+        Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_1'
+
+  WaitHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+
+  WaitCondition:
+    Type: AWS::CloudFormation::WaitCondition
+    Properties:
+      Handle: !Ref 'WaitHandle'
+      Timeout: '600'
+      Count: 1
+
+Outputs:
+  PublicIp:
+    Description: The DNS name for the created instance
+    Value:  !Sub "${Instance.PublicIp}"
+
+  InstanceId:
+    Description: The Instance ID
+    Value:  !Ref Instance