Merge pull request #38 from nyu-devops/su24-updates

Updates for Summer 2024 Semester

Author: rofrano

.devcontainer/devcontainer.json

@@ -8,10 +8,20 @@
 	"customizations": {
 		"vscode": {
 			"settings": {
+				"cSpell.words": [
+					"Rofrano",
+					"sqlalchemy",
+					"psycopg",
+					"pytest",
+					"tekton",
+					"creds",
+					"virtualenvs"
+				],				
 				"[python]": {
 					"editor.defaultFormatter": "ms-python.black-formatter",
 					"editor.formatOnSave": true
 				},
+				"git.mergeEditor": true,
 				"markdown-preview-github-styles.colorTheme": "light",
 				"makefile.extensionOutputFolder": "/tmp",
 				"python.testing.unittestEnabled": false,
@@ -36,36 +46,32 @@
 			"extensions": [
 				"ms-python.python",
 				"ms-python.vscode-pylance",
-				"VisualStudioExptTeam.vscodeintellicode",
 				"ms-python.pylint",
 				"ms-python.flake8",
 				"ms-python.black-formatter",
 				"ms-vscode.makefile-tools",
-				"cstrap.flask-snippets",
-				"wholroyd.jinja",
-				"ms-vscode.makefile-tools",
 				"yzhang.markdown-all-in-one",
+				"DavidAnson.vscode-markdownlint",
 				"bierner.github-markdown-preview",
 				"hnw.vscode-auto-open-markdown-preview",
-				"davidanson.vscode-markdownlint",
 				"bierner.markdown-preview-github-styles",
 				"tamasfe.even-better-toml",
 				"donjayamanne.githistory",
 				"GitHub.vscode-pull-request-github",
+				"github.vscode-github-actions",
 				"hbenl.vscode-test-explorer",
 				"LittleFoxTeam.vscode-python-test-adapter",
 				"njpwerner.autodocstring",
 				"wholroyd.jinja",
 				"redhat.vscode-yaml",
-				"rangav.vscode-thunder-client",
 				"redhat.fabric8-analytics",
-				"streetsidesoftware.code-spell-checker",
 				"ms-azuretools.vscode-docker",
 				"ms-kubernetes-tools.vscode-kubernetes-tools",
-				"github.vscode-github-actions",
+				"inercia.vscode-k3d",
+				"rangav.vscode-thunder-client",
 				"alexkrechik.cucumberautocomplete",
 				"Zignd.html-css-class-completion",
-				"streetsidesoftware.code-spell-checker",				
+				"streetsidesoftware.code-spell-checker",
 				"bbenoist.vagrant"
 			]		
 		}

.devcontainer/scripts/install-tools.sh

@@ -24,7 +24,7 @@ sudo sh -c 'echo "127.0.0.1 cluster-registry" >> /etc/hosts'
 echo "**********************************************************************"
 echo "Installing K9s..."
 echo "**********************************************************************"
-curl -L -o k9s.tar.gz "https://github.com/derailed/k9s/releases/download/v0.27.3/k9s_Linux_$ARCH.tar.gz"
+curl -L -o k9s.tar.gz "https://github.com/derailed/k9s/releases/download/v0.32.5/k9s_Linux_$ARCH.tar.gz"
 tar xvzf k9s.tar.gz
 sudo install -c -m 0755 k9s /usr/local/bin
 rm k9s.tar.gz
@@ -34,3 +34,9 @@ echo "Installing Skaffold..."
 echo "**********************************************************************"
 curl -Lo skaffold "https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-$ARCH"
 sudo install skaffold /usr/local/bin/
+
+echo "**********************************************************************"
+echo "Installing DevSpace..."
+echo "**********************************************************************"
+curl -Lo devspace "https://github.com/loft-sh/devspace/releases/latest/download/devspace-linux-$ARCH"
+sudo install -c -m 0755 devspace /usr/local/bin

Makefile

@@ -1,8 +1,8 @@
 # These can be overidden with env vars.
-REGISTRY ?= cluster-registry:32000
+REGISTRY ?= cluster-registry:5000
 NAMESPACE ?= nyu-devops
-IMAGE_NAME ?= lab-flask-bdd
-IMAGE_TAG ?= 1.0
+IMAGE_NAME ?= petshop
+IMAGE_TAG ?= 1.0.0
 IMAGE ?= $(REGISTRY)/$(NAMESPACE)/$(IMAGE_NAME):$(IMAGE_TAG)
 PLATFORM ?= "linux/amd64,linux/arm64"
 CLUSTER ?= nyu-devops
@@ -61,7 +61,7 @@ secret: ## Generate a secret hex key
 .PHONY: cluster
 cluster: ## Create a K3D Kubernetes cluster with load balancer and registry
 	$(info Creating Kubernetes cluster with a registry and 1 node...)
-	k3d cluster create nyu-devops --agents 1 --registry-create cluster-registry:0.0.0.0:32000 --port '8080:80@loadbalancer'
+	k3d cluster create nyu-devops --agents 1 --registry-create cluster-registry:0.0.0.0:5000 --port '8080:80@loadbalancer'
 
 .PHONY: cluster-rm
 cluster-rm: ## Remove a K3D Kubernetes cluster
@@ -71,15 +71,25 @@ cluster-rm: ## Remove a K3D Kubernetes cluster
 ##@ Deploy
 
 .PHONY: push
-image-push: ## Push to a Docker image registry
+push: ## Push to a Docker image registry
 	$(info Logging into IBM Cloud cluster $(CLUSTER)...)
 	docker push $(IMAGE)
 
+.PHONY: postgres
+postgres: ## Deploy the PostgreSQL service on local Kubernetes
+	$(info Deploying PostgreSQL service to Kubernetes...)
+	kubectl apply -f k8s/postgres
+
 .PHONY: deploy
-deploy: ## Deploy the service on local Kubernetes
+deploy: postgres ## Deploy the service on local Kubernetes
 	$(info Deploying service locally...)
 	kubectl apply -f k8s/
 
+.PHONY: undeploy
+undeploy: ## Delete the deployment on local Kubernetes
+	$(info Removing service on Kubernetes...)
+	kubectl delete -f k8s/
+
 ############################################################
 # COMMANDS FOR BUILDING THE IMAGE
 ############################################################

README.md

@@ -78,22 +78,64 @@ Pytest is configured to automatically include the flags `--pspec --cov=service -
 
 These tests require the service to be running because unlike the the TDD unit tests that test the code locally, these BDD integration tests are using Selenium to manipulate a web page on a running server.
 
-Run the tests using `behave`
+#### Run using two shells
 
 Start the server in a separate bash shell:
 
 ```sh
 honcho start
 ```
 
-Then start behave in your original bash shell:
+Then start `behave` in your original bash shell:
 
 ```sh
 behave
 ```
 
 You will see the results of the tests scroll down yur screen using the familiar red/green/refactor colors.
 
+#### Run using Kubernetes
+
+You can also use Kubernetes to host your application and test against it with BDD. The commands to do this are:
+
+```bash
+make cluster
+make build
+make push
+make deploy
+```
+
+What did these commands do?
+
+| Command | What does it do? |
+|---------|------------------|
+| make cluster | Creates a local Kubernetes cluster using `k3d` |
+| make build | Builds the Docker image |
+| make push | Pushes the image to the local Docker registry |
+| make deploy | Deploys the application using the image that was just built and pushed |
+
+Now you can just run `behave` against the application running in the local Kubernetes cluster
+
+```bash
+behave
+```
+
+### See what images are in the local registry
+
+You can use the `curl` command to query what images you have pushed to your local Docker registry. This will return `JSON` so you might want to use the silent flag `-s` and pipe it through `jq` like this:
+
+```bash
+curl -XGET http://localhost:5000/v2/_catalog -s | jq
+```
+
+That will return all of the image names without the tags.
+
+To get the tags use:
+
+```bash
+curl -XGET http://localhost:5000/v2/<image-name>/tags/list -s | jq
+```
+
 ## What's featured in the project?
 
 ```text
@@ -108,7 +150,7 @@ You will see the results of the tests scroll down yur screen using the familiar
 
 ## License
 
-Copyright (c) 2016, 2023, John J. Rofrano. All rights reserved.
+Copyright (c) 2016, 2024, John J. Rofrano. All rights reserved.
 
 Licensed under the Apache License. See [LICENSE](LICENSE)
 

features/environment.py

@@ -34,6 +34,7 @@ def get_chrome():
     """Creates a headless Chrome driver"""
     options = webdriver.ChromeOptions()
     options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
     options.add_argument("--headless")
     return webdriver.Chrome(options=options)
 
@@ -43,4 +44,4 @@ def get_firefox():
     options = webdriver.FirefoxOptions()
     options.add_argument("--headless")
     return webdriver.Firefox(options=options)    
-    
+    

features/steps/pets_steps.py

@@ -1,5 +1,5 @@
 ######################################################################
-# Copyright 2016, 2023 John J. Rofrano. All Rights Reserved.
+# Copyright 2016, 2024 John J. Rofrano. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,24 +23,29 @@
     https://selenium-python.readthedocs.io/waits.html
 """
 import requests
-from behave import given
+from compare3 import expect
+from behave import given  # pylint: disable=no-name-in-module
 
 # HTTP Return Codes
 HTTP_200_OK = 200
 HTTP_201_CREATED = 201
 HTTP_204_NO_CONTENT = 204
 
+WAIT_TIMEOUT = 60
+
+
 @given('the following pets')
 def step_impl(context):
     """ Delete all Pets and load new ones """
 
-    # List all of the pets and delete them one by one
+    # Get a list all of the pets
     rest_endpoint = f"{context.base_url}/pets"
-    context.resp = requests.get(rest_endpoint)
-    assert(context.resp.status_code == HTTP_200_OK)
+    context.resp = requests.get(rest_endpoint, timeout=WAIT_TIMEOUT)
+    expect(context.resp.status_code).equal_to(HTTP_200_OK)
+    # and delete them one by one
     for pet in context.resp.json():
-        context.resp = requests.delete(f"{rest_endpoint}/{pet['id']}")
-        assert(context.resp.status_code == HTTP_204_NO_CONTENT)
+        context.resp = requests.delete(f"{rest_endpoint}/{pet['id']}", timeout=WAIT_TIMEOUT)
+        expect(context.resp.status_code).equal_to(HTTP_204_NO_CONTENT)
 
     # load the database with new pets
     for row in context.table:
@@ -51,5 +56,5 @@ def step_impl(context):
             "gender": row['gender'],
             "birthday": row['birthday']
         }
-        context.resp = requests.post(rest_endpoint, json=payload)
-        assert(context.resp.status_code == HTTP_201_CREATED)
+        context.resp = requests.post(rest_endpoint, json=payload, timeout=WAIT_TIMEOUT)
+        expect(context.resp.status_code).equal_to(HTTP_201_CREATED)

features/steps/web_steps.py

@@ -1,5 +1,5 @@
 ######################################################################
-# Copyright 2016, 2023 John J. Rofrano. All Rights Reserved.
+# Copyright 2016, 2024 John J. Rofrano. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@
     https://selenium-python.readthedocs.io/waits.html
 """
 import logging
-from behave import when, then
+from behave import when, then  # pylint: disable=no-name-in-module
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import Select, WebDriverWait
 from selenium.webdriver.support import expected_conditions

k3d-config.yaml

@@ -1,6 +1,6 @@
 apiVersion: k3d.io/v1alpha3
 kind: Simple
-name: devops
+name: nyu-devops
 servers: 1
 agents: 1
 ports:
@@ -11,9 +11,9 @@ registries:
   create: 
     name: cluster-registry
     host: "0.0.0.0"
-    hostPort: "32000"
+    hostPort: "5000"
   config: |
     mirrors:
-      "cluster-registry:32000":
+      "cluster-registry":
         endpoint:
-          - http://cluster-registry:32000
+          - http://cluster-registry:5000

k8s/deployment.yaml

@@ -1,31 +1,29 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: lab-flask-bdd
+  name: petshop
   labels:
-    app: lab-flask-bdd
+    app: petshop
 spec:
-  replicas: 2
+  replicas: 1
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 0%
       maxUnavailable: 50%
   selector:
     matchLabels:
-      app: lab-flask-bdd
+      app: petshop
   template:
     metadata:
       labels:
-        app: lab-flask-bdd
+        app: petshop
     spec:
-      imagePullSecrets:
-      - name: all-icr-io
       restartPolicy: Always
       containers:
-      - name: lab-flask-bdd
-        # image: cluster-registry:32000/lab-flask-bdd:1.0
-        image: lab-flask-bdd
+      - name: petshop
+        image: cluster-registry:5000/nyu-devops/petshop:1.0.0
+        # image: petshop
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8080

k8s/ingress.yaml

@@ -2,7 +2,7 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
-  name: lab-flask-bdd
+  name: petshop
   annotations:
     nginx.ingress.kubernetes.io/rewrite-target: /
 spec:
@@ -13,6 +13,6 @@ spec:
         pathType: Prefix
         backend:
           service:
-            name: lab-flask-bdd
+            name: petshop
             port:
               number: 8080

k8s/pv.yaml renamed to k8s/postgres/pv.yaml

@@ -11,4 +11,3 @@ spec:
   persistentVolumeReclaimPolicy: Recycle
   hostPath:
     path: /data/pv0001
-  storageClassName: "default"

k8s/postgres/pvc.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi

k8s/secret.yaml renamed to k8s/postgres/secret.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  labels:
+    app: postgres
+spec:
+  type: ClusterIP
+  selector:
+    app: postgres
+  ports:
+    - port: 5432
+      targetPort: 5432