Update CICD in AWS

Author: vectornguyen76

.github/workflows/README.md

@@ -1,42 +1,65 @@
-# Workflows
+# CI/CD Deploy
 
-## Development - Build and Unittest
+## Workflows
 
-File: [development_pipeline.yml](development_pipeline.yml)
+### Development - Build and Unittest
 
-Event: On **Pull Request** → any branch into **develop**
+#### File: [development_pipeline.yml](development_pipeline.yml)
 
-Jobs:
+**Event:** On Pull Request → any branch into develop
 
-- Build
-- Unit Test
+**Jobs:**
 
-### Description:
+- Install dependencies (caches)
+- Run isort
+- Run black
+- Run flake8
+- Build images (caches)
+- Push images to Docker Hub
 
-## Staging - CI/CD Pipeline
+**Description:**
+This workflow is triggered on Pull Requests into the develop branch. It ensures a clean and standardized codebase by installing dependencies, checking code formatting with isort, black, and flake8, and finally building and pushing Docker images to Docker Hub.
 
-File: [staging_pipeline.yml](staging_pipeline.yml)
+### Staging - CI/CD Pipeline
 
-Event: On **Pull Request** → any branch into **staging**
+#### File: [staging_pipeline.yml](staging_pipeline.yml)
 
-Jobs:
+**Event:** On Pull Request → any branch into staging
 
-- Build
-- Unit Test
-- Deploy
+**Jobs:**
 
-### Description:
+- Install dependencies (caches)
+- Run isort
+- Run black
+- Run flake8
+- Build images (caches)
+- Push images to Docker Hub
+- Create infrastructure
+- Configure infrastructure
+- Deploy application using Docker Compose
+- Clean up following the concept of A/B deploy
 
-## Production - CI/CD Pipeline
+**Description:**
+This pipeline is designed for the staging environment and is triggered on Pull Requests into the staging branch. It includes steps to ensure code quality, build and push Docker images, create and configure necessary infrastructure, and deploy the application using Docker Compose. The cleanup process follows the A/B deployment concept.
 
-File: [production_pipeline.yml](production_pipeline.yml)
+### Production - CI/CD Pipeline
 
-Event: On **Pull Request** → any branch into **main**
+#### File: [production_pipeline.yml](production_pipeline.yml)
 
-Jobs:
+**Event:** On Pull Request → any branch into master
 
-- Build
-- Test
-- Deploy
+**Jobs:**
 
-### Description:
+- Install dependencies (caches)
+- Run isort
+- Run black
+- Run flake8
+- Build images (caches)
+- Push images to Docker Hub
+- Create infrastructure
+- Configure infrastructure
+- Deploy application using Docker Compose
+- Clean up following the concept of A/B deploy
+
+**Description:**
+The production pipeline is triggered on Pull Requests into the master branch, indicating changes are ready for deployment to the production environment. It follows a similar process to the staging pipeline but is specifically tailored for the production environment. The cleanup process adheres to the A/B deployment concept, ensuring a smooth transition between versions.

.github/workflows/ansible/deploy-app.yml renamed to .github/workflows/ansible/deploy_applications.yml

@@ -1,5 +1,5 @@
 ---
-- name: "configuration play."
+- name: "Deploy applications"
   hosts: web
   user: ubuntu
   become: true
@@ -9,11 +9,9 @@
     - ansible_host_key_checking: false
     - ansible_stdout_callback: yaml
 
-    - mlflow_image_name: vectornguyen76/mlflow
-    - mlflow_tag_name: latest
+    - api_image_image: vectornguyen76/flask_template_image
+    - api_image_tag: latest
 
-    - model_predictor_image_name: vectornguyen76/model_predictor
-    - model_predictor_tag_name: latest
   pre_tasks:
     - name: "wait 600 seconds for target connection to become reachable/usable."
       wait_for_connection:
@@ -34,6 +32,7 @@
           - python3-pip
           - virtualenv
           - python3-setuptools
+          - unzip
         state: latest
         update_cache: true
 

.github/workflows/ansible/inventory.txt renamed to .github/workflows/ansible/hosts

@@ -6,35 +6,23 @@
 
 - name: "Copy compressed app folder"
   copy:
-    src: "artifact-app.tar.gz"
-    dest: "/home/ubuntu/server/artifact-app.tar.gz"
+    src: "artifact.zip"
+    dest: "/home/ubuntu/server/artifact.zip"
 
 - name: "Extract app"
-  unarchive:
+  ansible.builtin.unarchive:
     remote_src: yes
-    src: "/home/ubuntu/server/artifact-app.tar.gz"
+    src: "/home/ubuntu/server/artifact.zip"
     dest: "/home/ubuntu/server"
 
-- name: Pull mlflow image
+- name: Pull api image
   community.docker.docker_image:
-    name: "{{ mlflow_image_name }}"
-    tag: "{{ mlflow_tag_name }}"
+    name: "{{ api_image }}"
+    tag: "{{ api_tag }}"
     source: pull
 
-- name: Pull model_predictor image
-  community.docker.docker_image:
-    name: "{{ model_predictor_image_name }}"
-    tag: "{{ model_predictor_tag_name }}"
-    source: pull
-
-- name: Run mlflow container
-  become: True
-  shell:
-    chdir: /home/ubuntu/server
-    cmd: "docker compose -f deployment/mlflow/docker-compose.yml up -d"
-
-- name: Run model_predictor container
+- name: Run docker compose
   become: True
   shell:
     chdir: /home/ubuntu/server
-    cmd: "bash deployment/deploy.sh deploy_run_predictor data/model_config/phase-1/prob-1/model-1.yml data/model_config/phase-1/prob-2/model-1.yml 5040"
+    cmd: "docker compose up -d"

.github/workflows/ansible/roles/deploy/tasks/main.yml

@@ -0,0 +1,7 @@
+## Create Stack
+
+aws cloudformation create-stack --stack-name server --template-body file://server.yml --parameters file://server-parameters.json --region us-east-1
+
+## Delete Stack
+
+aws cloudformation delete-stack --stack-name server --region us-east-1

.github/workflows/cloudformations/README.md

@@ -1,4 +1,8 @@
 [
+  {
+    "ParameterKey": "EnvironmentName",
+    "ParameterValue": "Search-Engine"
+  },
   {
     "ParameterKey": "VpcCIDR",
     "ParameterValue": "10.0.0.0/16"
@@ -12,7 +16,7 @@
     "ParameterValue": "t3.medium"
   },
   {
-    "ParameterKey": "KeyPair",
+    "ParameterKey": "KeyPairName",
     "ParameterValue": "my-keypair"
   },
   {

.github/workflows/files/ec2-parameters.json renamed to .github/workflows/cloudformations/server-parameters.json

@@ -1,6 +1,10 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: Creates EC2
+Description: Creates EC2 Server
 Parameters:
+  EnvironmentName:
+    Description: An environment name that will be prefixed to resource names
+    Type: String
+
   VpcCIDR:
     Description: "VPC range"
     Type: String
@@ -38,7 +42,7 @@ Resources:
       EnableDnsSupport: true
       Tags:
         - Key: Name
-          Value: "Main VPC"
+          Value: !Sub ${EnvironmentName} VPC
 
   InternetGateway:
     Type: AWS::EC2::InternetGateway
@@ -56,12 +60,12 @@ Resources:
   PublicSubnet:
     Type: AWS::EC2::Subnet
     Properties:
-      AvailabilityZone: "us-east-1a"
+      AvailabilityZone: !Select [0, !GetAZs ""]
       VpcId: !Ref VPC
       CidrBlock: !Ref PublicSubnetCIDR
       Tags:
         - Key: Name
-          Value: "Public Subnet AZ1"
+          Value: !Sub ${EnvironmentName} Public Subnet
 
   PublicRouteTable:
     Type: AWS::EC2::RouteTable
@@ -88,14 +92,18 @@ Resources:
   SecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
-      GroupName: mySecurityGroup
+      GroupName: !Sub ${EnvironmentName}-SecurityGroup
       GroupDescription: Allow http to client host
       VpcId: !Ref VPC
       SecurityGroupIngress:
         - IpProtocol: tcp
           FromPort: 80
           ToPort: 80
           CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 443
+          ToPort: 443
+          CidrIp: 0.0.0.0/0
         - IpProtocol: tcp
           FromPort: 22
           ToPort: 22
@@ -108,18 +116,11 @@ Resources:
           FromPort: 3000
           ToPort: 3000
           CidrIp: 0.0.0.0/0
-        - IpProtocol: tcp
-          FromPort: 5040
-          ToPort: 5040
-          CidrIp: 0.0.0.0/0
       SecurityGroupEgress:
         - IpProtocol: "-1"
           FromPort: -1
           ToPort: -1
           CidrIp: 0.0.0.0/0
-      Tags:
-        - Key: Name
-          Value: SecurityGroup
 
   # Server EC2 Instance
   ServerInstance:
@@ -135,27 +136,12 @@ Resources:
             - !Ref SecurityGroup
           SubnetId: !Ref PublicSubnet
       BlockDeviceMappings:
-        - DeviceName: "/dev/sdk"
+        - DeviceName: "/dev/sda1"
           Ebs:
-            VolumeSize: 20
-      # UserData:
-      #   Fn::Base64: !Sub |
-      #     #!/bin/bash
-      #     sudo apt-get update -y
-      #     sudo apt-get install ca-certificates curl gnupg
-      #     sudo install -m 0755 -d /etc/apt/keyrings
-      #     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-      #     sudo chmod a+r /etc/apt/keyrings/docker.gpg
-      #     echo \
-      #     "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-      #     "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
-      #     sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-      #     sudo apt-get update -y
-      #     apt-get install -y docker-ce
-      #     docker run -p 80:8080 tomcat:8.0
+            VolumeSize: 50
       Tags:
         - Key: Name
-          Value: ServerInstance
+          Value: !Sub ${EnvironmentName}-Instance
 
 Outputs:
   EC2InstanceConnection:

.github/workflows/files/ec2.yml renamed to .github/workflows/cloudformations/server.yml

@@ -1,7 +1,8 @@
 name: development
 on:
   pull_request:
-    branches: develop
+    branches:
+      - develop
 
   push:
     branches:
@@ -21,24 +22,26 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v2
-
-      - name: Cache Python dependencies
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+        uses: actions/checkout@v3
 
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: "3.9"
+          cache: "pip"
 
       - name: Install Python dependencies
         run: python -m pip install -r requirements.txt
 
+      - name: Run isort
+        run: isort --check-only --profile=black .
+
+      - name: Run black
+        run: black --check .
+
+      - name: Run flake8
+        run: flake8 --ignore=E501,W503,F401 .
+
       - name: Unit Tests and Integration Tests
         env:
           DATABASE_TEST_URL: postgresql://db_user:db_password@localhost/db_test