diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml new file mode 100644 index 0000000000..d39788f294 --- /dev/null +++ b/.github/workflows/go-ci.yml @@ -0,0 +1,134 @@ +name: Go CI/CD Pipeline + +# Cancel in-progress runs when a new run is triggered +concurrency: + group: go-ci-${{ github.ref }} + cancel-in-progress: true + +# Path-based triggers: only run when app_go files change +on: + push: + branches: + - main + - master + - lab03 + paths: + - 'app_go/**' + - '.github/workflows/go-ci.yml' + pull_request: + branches: + - main + - master + paths: + - 'app_go/**' + - '.github/workflows/go-ci.yml' + +env: + GO_VERSION: '1.21' + DOCKER_IMAGE: mirana18/devops-info-service-go + +jobs: + test: + name: Code Quality & Testing + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: app_go/go.mod + cache: true + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: latest + working-directory: app_go + args: --timeout=5m + + - name: Run tests + working-directory: ./app_go + run: go test -v -race -coverprofile=coverage.out -covermode=atomic ./... + + - name: Generate coverage report + working-directory: ./app_go + run: | + go tool cover -func=coverage.out + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: app_go/coverage.out + flags: go + name: go-coverage + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + with: + name: go-coverage-report + path: go_python/coverage.xml + retention-days: 7 + + docker: + name: Build & Push Docker Image + runs-on: ubuntu-latest + needs: test + + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lab03') + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Generate version tags (CalVer) + id: meta + run: | + VERSION=$(date +%Y.%m) + BUILD_NUMBER=${{ github.run_number }} + FULL_VERSION="${VERSION}.${BUILD_NUMBER}" + SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7) + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "full_version=${FULL_VERSION}" >> $GITHUB_OUTPUT + echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + + - name: Extract Docker metadata + id: docker_meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_IMAGE }} + tags: | + type=raw,value=${{ steps.meta.outputs.full_version }} + type=raw,value=${{ steps.meta.outputs.version }} + type=raw,value=latest + type=raw,value=sha-${{ steps.meta.outputs.short_sha }} + labels: | + org.opencontainers.image.title=DevOps Info Service (Go) + org.opencontainers.image.description=Go-based system information service + org.opencontainers.image.version=${{ steps.meta.outputs.full_version }} + org.opencontainers.image.revision=${{ github.sha }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: ./app_go + file: ./app_go/Dockerfile + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.DOCKER_IMAGE }}:latest + cache-to: type=inline diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 0000000000..bfac3722aa --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,179 @@ +name: Python CI/CD Pipeline + +# Cancel in-progress runs when a new run is triggered +concurrency: + group: python-ci-${{ github.ref }} + cancel-in-progress: true + +# Workflow triggers +on: + push: + branches: + - main + - master + - lab03 + paths: + - 'app_python/**' + - '.github/workflows/python-ci.yml' + pull_request: + branches: + - main + - master + paths: + - 'app_python/**' + - '.github/workflows/python-ci.yml' + +# Environment variables used across jobs +env: + PYTHON_VERSION: '3.11' + DOCKER_IMAGE: mirana18/devops-info-service + +jobs: + # Job 1: Code Quality & Testing + test: + name: Code Quality & Testing + runs-on: ubuntu-latest + + steps: + # Step 1: Check out the repository code + - name: Checkout code + uses: actions/checkout@v4 + + # Step 2: Set up Python environment + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' # Cache pip dependencies for faster runs + cache-dependency-path: | + app_python/requirements.txt + app_python/requirements-dev.txt + + # Step 3: Install dependencies + - name: Install dependencies + working-directory: ./app_python + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + # Install linter + pip install flake8 + + # Step 4: Run linter (flake8) + - name: Lint with flake8 + working-directory: ./app_python + run: | + # Stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # Exit-zero treats all errors as warnings. Line length set to 100 + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=100 --statistics + + # Step 5: Run unit tests with pytest and coverage + - name: Run tests with pytest and coverage + working-directory: ./app_python + run: | + pytest -v --tb=short --cov=. --cov-report=term --cov-report=xml --cov-fail-under=70 + + # Step 6: Upload coverage to Codecov + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: app_python/coverage.xml + flags: python + name: python-coverage + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} + + # Step 7: Upload coverage artifact (for review) + - name: Upload coverage report + uses: actions/upload-artifact@v4 + with: + name: python-coverage-report + path: app_python/coverage.xml + retention-days: 7 + + # Job 2: Docker Build & Push (only runs if tests pass) + docker: + name: Build & Push Docker Image + runs-on: ubuntu-latest + needs: test # This job only runs if 'test' job succeeds + + # Only push to Docker Hub on push to main/master (not on PRs) + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lab03') + + steps: + # Step 1: Check out code + - name: Checkout code + uses: actions/checkout@v4 + + # Step 2: Set up Docker Buildx (for advanced build features) + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # Step 3: Log in to Docker Hub + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + # Step 4: Generate version tags using Calendar Versioning (CalVer) + - name: Generate version tags + id: meta + run: | + # CalVer format: YYYY.MM (e.g., 2026.02) + VERSION=$(date +%Y.%m) + + # Build number (GitHub run number) + BUILD_NUMBER=${{ github.run_number }} + + # Full version with build: YYYY.MM.BUILD (e.g., 2026.02.15) + FULL_VERSION="${VERSION}.${BUILD_NUMBER}" + + # Short commit SHA for traceability + SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7) + + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "full_version=${FULL_VERSION}" >> $GITHUB_OUTPUT + echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + + echo "Generated version: ${FULL_VERSION}" + echo "Commit SHA: ${SHORT_SHA}" + + # Step 5: Extract Docker metadata for tags and labels + - name: Extract Docker metadata + id: docker_meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_IMAGE }} + tags: | + # CalVer version with build number (e.g., 2026.02.15) + type=raw,value=${{ steps.meta.outputs.full_version }} + # CalVer version without build (e.g., 2026.02) + type=raw,value=${{ steps.meta.outputs.version }} + # Latest tag + type=raw,value=latest + # Commit SHA (for traceability) + type=raw,value=sha-${{ steps.meta.outputs.short_sha }} + labels: | + org.opencontainers.image.title=DevOps Info Service + org.opencontainers.image.description=Flask-based system information service + org.opencontainers.image.version=${{ steps.meta.outputs.full_version }} + org.opencontainers.image.revision=${{ github.sha }} + + # Step 6: Build and push Docker image + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: ./app_python + file: ./app_python/Dockerfile + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.DOCKER_IMAGE }}:latest + cache-to: type=inline + build-args: | + BUILD_DATE=${{ github.event.head_commit.timestamp }} + VCS_REF=${{ github.sha }} + VERSION=${{ steps.meta.outputs.full_version }} + diff --git a/.gitignore b/.gitignore index 30d74d2584..fde732ed78 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,9 @@ -test \ No newline at end of file +test +venv/ + +# Ansible +*.retry +.vault_pass +.vault_password +vault_pass* +ansible/inventory/*.pyc \ No newline at end of file diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 0000000000..f7cb04606f --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,73 @@ +# Ansible — Lab 5 + +## Quick start + +Run the commands below from the **`ansible/`** directory (or adjust paths if running from the repo root). + +1. **Set your VM IP** + Edit `inventory/hosts.ini`: replace `YOUR_VM_IP` with your VM's public IP. + Get IP from Pulumi: `cd pulumi && pulumi stack output public_ip` + Change `ansible_user` if not `ubuntu`. + +2. **Install Ansible collections** (if not already installed): + ```bash + ansible-galaxy install -r requirements.yml + ``` + +3. **Create or edit encrypted variables** (Docker Hub credentials and app config): + - If `group_vars/all.yml` **does not exist**: + `ansible-vault create group_vars/all.yml` + Paste content from `group_vars/all.yml.example` (replace placeholders), save, remember the vault password. + - If `group_vars/all.yml` **already exists** (e.g. you created it earlier): + `ansible-vault edit group_vars/all.yml` + Enter your vault password and edit as needed. + +4. **Test connectivity** (Ansible loads group_vars, so vault password is required): + ```bash + ansible all -m ping --ask-vault-pass + ``` + +5. **Provision** (install common packages + Docker). Vault password needed because group_vars are loaded: + ```bash + ansible-playbook playbooks/provision.yml --ask-vault-pass + ``` + Run it twice to confirm idempotency (second run should show "ok", not "changed"). + +6. **Deploy application:** + ```bash + ansible-playbook playbooks/deploy.yml --ask-vault-pass + ``` + If you see **"no vault secrets found"**: you ran without `--ask-vault-pass`; add it so Ansible can decrypt `group_vars/all.yml`. + If you see **"Decryption failed"**: the password you entered is wrong for this file. Try again; if you forgot it, create a new vault file (`mv group_vars/all.yml group_vars/all.yml.bak` then `ansible-vault create group_vars/all.yml`) and paste content from `all.yml.example`. + If you see **`dockerhub_password` is undefined**: open the encrypted vars with `ansible-vault edit group_vars/all.yml` and ensure it contains both `dockerhub_username` and `dockerhub_password` (see `group_vars/all.yml.example`). + +7. **Verify:** `ansible webservers -a "docker ps" --ask-vault-pass` and `curl http://:5001/health` + +**Note:** Because `group_vars/all.yml` is encrypted, use `--ask-vault-pass` for any Ansible command (`ping`, `playbook`, `ansible webservers -a "..."`) so Ansible can decrypt variables. + +## Structure + +- `inventory/hosts.ini` — target hosts (fill in VM IP). +- `roles/common` — base system (apt, packages, timezone). +- `roles/docker` — Docker CE install and service. +- `roles/app_deploy` — pull image and run container (uses vaulted `group_vars/all.yml`). +- `playbooks/provision.yml` — common + docker. +- `playbooks/deploy.yml` — app_deploy only. + +Documentation: `docs/LAB05.md` (fill in terminal outputs and analysis for submission). + +### If "Failed to update apt cache" on the VM + +**"Network is unreachable" or "connection timed out"** means the VM has **no working outbound internet**. Ansible cannot fix this — the VM or cloud network must allow egress. + +**Yandex Cloud (Pulumi from Lab 4):** +- In `pulumi/__main__.py` the VM has `nat=True`; the security group must also have an **egress** rule so the VM can reach the internet. Add (if missing) an egress rule, e.g. `direction="egress"`, `protocol="ANY"`, `v4_cidr_blocks=["0.0.0.0/0"]`. Then run `pulumi up` so the rule is applied. +- If the VM was created earlier without egress, run `pulumi up` again after adding the egress rule; no need to recreate the VM. +- From the VM: `curl -4 -v http://mirror.yandex.ru/` — if this fails, fix the cloud network first (NAT, egress, or use another subnet). + +**Other checks:** +1. **On the VM** (SSH in): `sudo apt-get update` and `curl -4 http://mirror.yandex.ru/` — same errors mean no outbound. +2. **Security group / firewall:** Allow **egress** (outbound) HTTP (80) and HTTPS (443), not only ingress. +3. **DNS:** On the VM, `cat /etc/resolv.conf` — there should be nameservers. + +**Ansible-side:** The `common` role uses Yandex mirror by default and forces apt to use **IPv4 only** (to avoid IPv6 "Network is unreachable"). If your VM is not in Yandex Cloud, set `use_yandex_mirror: false` in `roles/common/defaults/main.yml`. diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000000..0ddcbf1672 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +inventory = inventory/hosts.ini +roles_path = roles +host_key_checking = False +remote_user = ubuntu +retry_files_enabled = False + +[privilege_escalation] +become = True +become_method = sudo +become_user = root diff --git a/ansible/docs/LAB05.md b/ansible/docs/LAB05.md new file mode 100644 index 0000000000..0471daf8a3 --- /dev/null +++ b/ansible/docs/LAB05.md @@ -0,0 +1,81 @@ +# Lab 5 — Ansible Fundamentals (Documentation) + +## 1. Architecture Overview + +- **Ansible version:** 2.16+ (run `ansible --version` to confirm). +- **Target VM OS and version:** Ubuntu 22.04 LTS (VM from Lab 4, Pulumi + Yandex Cloud). +- **Role structure:** + - **common** — base system setup: force IPv4 for apt, optional Yandex mirror, apt cache update, install packages (curl, git, vim, htop, etc.), timezone. + - **docker** — install Docker CE from official repository, refresh cache after adding repo, install packages (docker-ce, docker-ce-cli, containerd.io), docker service, add user to docker group, python3-docker. + - **app_deploy** — verify Vault variables, Docker Hub login, pull image, stop/remove old container, run new one with port 5001, wait for port, check /health. +- **Why roles instead of monolithic playbooks?** Roles enable code reuse, separate testability, and short playbooks; logic is split by concern (common / docker / app), and one role can be used across multiple playbooks and projects. + +## 2. Roles Documentation + +### common +- **Purpose:** Base system setup: force IPv4 for apt (avoid IPv6 "Network is unreachable"), optional Yandex mirror for Ubuntu, apt cache update, install packages (python3-pip, curl, git, vim, htop, unzip, ca-certificates, gnupg, lsb-release), set timezone (Europe/Moscow). +- **Variables:** `use_yandex_mirror` (default: true), `common_packages` (list), `timezone` (default: Europe/Moscow). In `defaults/main.yml`. +- **Handlers:** None. +- **Dependencies:** None. + +### docker +- **Purpose:** Install Docker CE: dependencies (ca-certificates, curl, gnupg), GPG key and Docker repository, apt cache update, install docker-ce, docker-ce-cli, containerd.io, start and enable docker service, add user (ansible_user) to docker group, install python3-docker for Ansible modules. +- **Variables:** In `defaults/main.yml`: `docker_install_compose`, `docker_users`. Tasks use architecture mapping (x86_64→amd64, aarch64→arm64) for repository URL. +- **Handlers:** `restart docker` — restart docker service when repository or packages change. +- **Dependencies:** None (common role typically runs first to update apt). + +### app_deploy +- **Purpose:** Deploy application in Docker: verify dockerhub_username and dockerhub_password, Docker Hub login (no_log), pull image, stop and remove old container by name, run new container with port mapping (app_port:app_container_port, default 5001:5001), restart policy unless-stopped, wait for port, GET /health check. +- **Variables:** From group_vars (Vault): `dockerhub_username`, `dockerhub_password`, `app_name`, `docker_image`, `docker_image_tag`, `app_port`, `app_container_name`. In role defaults: `app_port`, `app_container_port` (5001), `app_restart_policy`, `app_env`. +- **Handlers:** `restart app container` (optional, conditional). +- **Dependencies:** Requires docker role (Docker on host) and encrypted group_vars/all.yml with credentials. + +## 3. Idempotency Demonstration + +- **First run:** On first run of `ansible-playbook playbooks/provision.yml --ask-vault-pass`, tasks show **changed**: apt cache update, package installs (common, Docker dependencies, Docker repo, Docker packages, python3-docker), mirror setup/force IPv4 when use_yandex_mirror, docker service start, user added to docker group, timezone set. +- **Second run:** On second run the same tasks show **ok** — state already matches desired, no (or minimal) changes. +- **Analysis:** First run brings packages, repos, service, and user to desired state; second run shows modules (apt, service, user, template/copy) see target state is met and do not change the system. +- **Explanation:** Idempotency comes from declarative modules with explicit state: `apt: state=present`, `service: state=started`, `user: groups: docker`, `template`/`copy` with fixed content. Ansible applies changes only when current and desired state differ. + +## 4. Ansible Vault Usage + +- **Storage:** Docker Hub credentials and app variables are stored in `group_vars/all.yml`, encrypted with `ansible-vault create` (or `ansible-vault encrypt`). The file can be committed; without the Vault password the content is unreadable. +- **Vault password management:** Use `--ask-vault-pass` when running playbooks and ad-hoc commands; alternative: password file (e.g. `.vault_pass`), `chmod 600`, and `--vault-password-file` or `vault_password_file` in ansible.cfg. Password file is in `.gitignore`. +- **Example encrypted file:** `head -5 group_vars/all.yml` shows lines like `$ANSIBLE_VAULT;1.1;AES256` or `$ANSIBLE_VAULT;1.2;AES256` — file is encrypted. To verify decryption: `ansible-vault view group_vars/all.yml --ask-vault-pass`. +- **Why Ansible Vault is important:** Keeps secrets (Docker Hub login/password) in the repo in encrypted form; decryption only with the Vault password, reducing leakage risk when collaborating and backing up. + +## 5. Deployment Verification + +- **Deploy run output:** Output of `ansible-playbook playbooks/deploy.yml --ask-vault-pass`: tasks Ensure Docker Hub credentials, Log in to Docker Hub, Pull Docker image, Stop existing container (if any), Remove old container, Run application container, Wait for application port, Check health endpoint — all succeed (ok or changed as needed). +- **Container status:** Example output of `ansible webservers -a "docker ps" --ask-vault-pass`: + ```text + web1 | CHANGED | rc=0 >> + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + /devops-info-service:latest "python app.py" ... Up ... 0.0.0.0:5001->5001/tcp devops-app + ``` +- **Health check verification:** From local machine: + ```bash + curl http://89.169.129.155:5001/health + ``` + Example response: + ```json + {"status":"healthy","timestamp":"2026-02-25T10:07:38.381157.000Z","uptime_seconds":91485.11} + ``` + Main page: `curl http://89.169.129.155:5001/` — returns service info. +- **Handlers:** For deploy, the "restart app container" handler is not needed in the typical flow (container is recreated by Run application container). The "restart docker" handler in the docker role runs when Docker repo or packages change during provisioning. + +## 6. Key Decisions + +- **Why roles instead of plain playbooks?** Roles group related tasks, defaults, and handlers by concern (common / docker / app); playbooks stay short and readable; the same roles can be used in different playbooks and projects. +- **How do roles improve reusability?** One role can be included in multiple playbooks and optionally published to Ansible Galaxy; a change in the role applies everywhere it is used. +- **What makes a task idempotent?** Using modules that describe desired state (e.g. `state: present`, `state: started`) instead of one-off commands; Ansible only applies changes when current and target state differ. +- **How do handlers improve efficiency?** Handlers run once at the end of the playbook even with multiple notifies (e.g. one Docker restart after several config or package changes). +- **Why is Ansible Vault necessary?** To store secrets in the repo encrypted and avoid keeping passwords and tokens in plain text in code and commit history. + +## 7. Challenges + +- **"Failed to update apt cache" on VM:** The VM had no outbound internet. In Pulumi the security group had only ingress rules; an egress rule was added (protocol ANY, 0.0.0.0/0). The common role also uses Yandex mirror and forces IPv4 for apt to reduce dependence on IPv6 and external mirrors. +- **docker-ce package not found:** The Docker repo URL used architecture from ansible_architecture (x86_64/aarch64) while Docker expects amd64/arm64. Mapping was added in the "Add Docker repository" task. After adding the repo, explicit apt cache update (cache_valid_time: 0) was added so packages from the new repo are visible. +- **Variables from group_vars not loaded:** In deploy.yml playbook, explicit `vars_files: ../group_vars/all.yml` was added so variables from the encrypted file are used on deploy regardless of current directory and load order. +- **"Cannot create container when image is not specified" in Stop existing container:** The docker_container module with state: stopped requires the image parameter. The image parameter was added to the "Stop existing container" task. +- **Accessing the app from outside (curl on port 5001):** In Pulumi the security group only had port 5000 open; the app listens on 5001. An ingress rule for TCP 5001 (app-5001-rule) was added in `pulumi/__main__.py` and applied with `pulumi up`. diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml new file mode 100644 index 0000000000..3287c49958 --- /dev/null +++ b/ansible/group_vars/all.yml @@ -0,0 +1,18 @@ +$ANSIBLE_VAULT;1.1;AES256 +31323034333339303235643330653661303133663465386266316165643365643632383837613463 +3635383232333831396163343338323832623262613936370a653436333937623065643861323632 +36646536656265666563316631353237303066303831333233633831616663343932306535366233 +3231346439333561360a636437303835323165313431383532333637343663386133306564356535 +65613662663935386661313464636536663233346163633165633839643332366434633832663366 +39336339383736663131626233663434396231346232386564306639613466613164336633316265 +36313938333132613366373066363037393965346338356138663464323430306365363632653266 +35613366353639353731613238643930613666333438353330393362343437326231376663366335 +37396265616566353336646463663237336238663165663766663261383530356264646264356439 +32643432663532363236316638336430663438326562646461373665353037373463316437313335 +31643639343733346530353636313465383335616431363363323538643563623634313331376162 +33316662316332656361393832643631653632623536613261303633353539616231356436333266 +35373836363336653861343732346234323431323837663062316634633538393237643465353762 +38386561363337663834633637306634393764643765343165396139653137633531663664366530 +35346630316562613766636165383762316361643566326632646432623132636635393962343030 +35323761613738666565363933326432333034386166313231373166353435336562373863623835 +6239 diff --git a/ansible/group_vars/all.yml.example b/ansible/group_vars/all.yml.example new file mode 100644 index 0000000000..ad4645fa6d --- /dev/null +++ b/ansible/group_vars/all.yml.example @@ -0,0 +1,14 @@ +# Copy this file to all.yml and encrypt with Ansible Vault: +# ansible-vault create group_vars/all.yml +# Then paste the content below and save. + +# Docker Hub credentials (required for deploy) +dockerhub_username: your-dockerhub-username +dockerhub_password: your-dockerhub-password-or-access-token + +# Application use dockerhub_username +app_name: devops-info-service +docker_image: "{{ dockerhub_username }}/{{ app_name }}" +docker_image_tag: latest +app_port: 5001 +app_container_name: devops-app diff --git a/ansible/inventory/hosts.ini b/ansible/inventory/hosts.ini new file mode 100644 index 0000000000..1e401bb77e --- /dev/null +++ b/ansible/inventory/hosts.ini @@ -0,0 +1,7 @@ +# Replace YOUR_VM_IP with your VM's public IP (Pulumi: pulumi stack output public_ip | Terraform: terraform output vm_public_ip) +# Replace ubuntu with your SSH user if different +[webservers] +web1 ansible_host=89.169.129.155 ansible_user=ubuntu + +[webservers:vars] +ansible_python_interpreter=/usr/bin/python3 diff --git a/ansible/playbooks/deploy.yml b/ansible/playbooks/deploy.yml new file mode 100644 index 0000000000..69407eb35b --- /dev/null +++ b/ansible/playbooks/deploy.yml @@ -0,0 +1,9 @@ +--- +- name: Deploy application + hosts: webservers + become: yes + vars_files: + - ../group_vars/all.yml + + roles: + - app_deploy diff --git a/ansible/playbooks/provision.yml b/ansible/playbooks/provision.yml new file mode 100644 index 0000000000..f53efb0248 --- /dev/null +++ b/ansible/playbooks/provision.yml @@ -0,0 +1,8 @@ +--- +- name: Provision web servers + hosts: webservers + become: yes + + roles: + - common + - docker diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000000..483ed156a5 --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: community.general + - name: community.docker diff --git a/ansible/roles/app_deploy/defaults/main.yml b/ansible/roles/app_deploy/defaults/main.yml new file mode 100644 index 0000000000..dfb8fcf5a2 --- /dev/null +++ b/ansible/roles/app_deploy/defaults/main.yml @@ -0,0 +1,5 @@ +--- +app_port: 5001 +app_container_port: 5001 +app_restart_policy: unless-stopped +app_env: {} diff --git a/ansible/roles/app_deploy/handlers/main.yml b/ansible/roles/app_deploy/handlers/main.yml new file mode 100644 index 0000000000..e146bcc6ca --- /dev/null +++ b/ansible/roles/app_deploy/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: restart app container + community.docker.docker_container: + name: "{{ app_container_name }}" + state: started + when: app_container_restart is defined and app_container_restart | default(false) | bool diff --git a/ansible/roles/app_deploy/tasks/main.yml b/ansible/roles/app_deploy/tasks/main.yml new file mode 100644 index 0000000000..979fa161fb --- /dev/null +++ b/ansible/roles/app_deploy/tasks/main.yml @@ -0,0 +1,57 @@ +--- +- name: Ensure Docker Hub credentials are set (in group_vars/all.yml via Vault) + ansible.builtin.assert: + that: + - dockerhub_username is defined + - dockerhub_password is defined + fail_msg: > + Define dockerhub_username and dockerhub_password in group_vars/all.yml. + From ansible/: ansible-vault edit group_vars/all.yml + Add both variables (see group_vars/all.yml.example). Run the playbook from the ansible/ directory. + +- name: Log in to Docker Hub + community.docker.docker_login: + username: "{{ dockerhub_username }}" + password: "{{ dockerhub_password }}" + registry: https://index.docker.io/v1/ + no_log: true + +- name: Pull Docker image + community.docker.docker_image: + name: "{{ docker_image }}:{{ docker_image_tag }}" + source: pull + +- name: Stop existing container + community.docker.docker_container: + name: "{{ app_container_name }}" + image: "{{ docker_image }}:{{ docker_image_tag }}" + state: stopped + ignore_errors: yes + +- name: Remove old container + community.docker.docker_container: + name: "{{ app_container_name }}" + state: absent + +- name: Run application container + community.docker.docker_container: + name: "{{ app_container_name }}" + image: "{{ docker_image }}:{{ docker_image_tag }}" + state: started + restart_policy: "{{ app_restart_policy }}" + ports: + - "{{ app_port }}:{{ app_container_port }}" + env: "{{ app_env }}" + +- name: Wait for application port + ansible.builtin.wait_for: + port: "{{ app_port }}" + delay: 2 + timeout: 30 + +- name: Check health endpoint + ansible.builtin.uri: + url: "http://127.0.0.1:{{ app_port }}/health" + return_content: yes + register: health_result + changed_when: false diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml new file mode 100644 index 0000000000..e6df2c648f --- /dev/null +++ b/ansible/roles/common/defaults/main.yml @@ -0,0 +1,14 @@ +--- +use_yandex_mirror: true + +common_packages: + - python3-pip + - curl + - git + - vim + - htop + - unzip + - ca-certificates + - gnupg + - lsb-release +timezone: "Europe/Moscow" diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml new file mode 100644 index 0000000000..510ac09129 --- /dev/null +++ b/ansible/roles/common/tasks/main.yml @@ -0,0 +1,48 @@ +--- +- name: Force apt to use IPv4 only (avoids IPv6 "Network is unreachable" on some clouds) + ansible.builtin.copy: + content: 'Acquire::ForceIPv4 "true";' + dest: /etc/apt/apt.conf.d/99force-ipv4 + owner: root + group: root + mode: "0644" + when: use_yandex_mirror | default(false) | bool + +- name: Configure Yandex mirror for Ubuntu (when use_yandex_mirror) + ansible.builtin.template: + src: sources.list.yandex.j2 + dest: /etc/apt/sources.list + owner: root + group: root + mode: "0644" + when: use_yandex_mirror | default(false) | bool + +- name: Update apt cache + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 3600 + update_cache_retries: 10 + update_cache_retry_max_delay: 30 + register: apt_update + ignore_errors: true + +- name: Capture real apt-get update error when cache update failed + ansible.builtin.shell: apt-get update 2>&1 + register: apt_get_update_result + changed_when: false + failed_when: false + when: apt_update is failed + +- name: Fail with real apt error so you can fix VM network/DNS + ansible.builtin.fail: + msg: "apt cache update failed. Run 'sudo apt-get update' on the VM to see details. Captured: {{ apt_get_update_result.stdout }}" + when: apt_update is failed + +- name: Install common packages + ansible.builtin.apt: + name: "{{ common_packages }}" + state: present + +- name: Set timezone + community.general.timezone: + name: "{{ timezone }}" diff --git a/ansible/roles/common/templates/sources.list.yandex.j2 b/ansible/roles/common/templates/sources.list.yandex.j2 new file mode 100644 index 0000000000..0cc8779a45 --- /dev/null +++ b/ansible/roles/common/templates/sources.list.yandex.j2 @@ -0,0 +1,4 @@ +# Ubuntu {{ ansible_facts['distribution_release'] }} — Yandex mirror (often works better from Yandex Cloud) +deb http://mirror.yandex.ru/ubuntu/ {{ ansible_facts['distribution_release'] }} main restricted universe multiverse +deb http://mirror.yandex.ru/ubuntu/ {{ ansible_facts['distribution_release'] }}-updates main restricted universe multiverse +deb http://mirror.yandex.ru/ubuntu/ {{ ansible_facts['distribution_release'] }}-security main restricted universe multiverse diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml new file mode 100644 index 0000000000..b91e3451e0 --- /dev/null +++ b/ansible/roles/docker/defaults/main.yml @@ -0,0 +1,4 @@ +--- +docker_install_compose: false +# User(s) to add to docker group (e.g. [ubuntu]) +docker_users: [] diff --git a/ansible/roles/docker/handlers/main.yml b/ansible/roles/docker/handlers/main.yml new file mode 100644 index 0000000000..1a5058da5e --- /dev/null +++ b/ansible/roles/docker/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart docker + ansible.builtin.service: + name: docker + state: restarted diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml new file mode 100644 index 0000000000..b0278c2b3e --- /dev/null +++ b/ansible/roles/docker/tasks/main.yml @@ -0,0 +1,51 @@ +--- +- name: Install dependencies for Docker + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + +- name: Add Docker GPG key + ansible.builtin.apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + +- name: Add Docker repository + ansible.builtin.apt_repository: + repo: "deb [arch={{ ansible_architecture | lower | replace('x86_64', 'amd64') | replace('aarch64', 'arm64') }}] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + state: present + filename: docker + notify: restart docker + +- name: Update apt cache after adding Docker repo + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 0 + +- name: Install Docker packages + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + state: present + notify: restart docker + +- name: Ensure Docker service is started and enabled + ansible.builtin.service: + name: docker + state: started + enabled: yes + +- name: Add remote user to docker group + ansible.builtin.user: + name: "{{ ansible_user }}" + groups: docker + append: yes + +- name: Install python3-docker for Ansible Docker modules + ansible.builtin.apt: + name: python3-docker + state: present diff --git a/app_go/.dockerignore b/app_go/.dockerignore new file mode 100644 index 0000000000..f31cd17b67 --- /dev/null +++ b/app_go/.dockerignore @@ -0,0 +1,46 @@ +# Git +.git +.gitignore +.gitattributes + +# Documentation +README.md +docs/ +*.md + +# Build artifacts +devops-info-service +devops-info-service-* +*.exe + +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Test files +*_test.go +test/ +tests/ + +# CI/CD files +.github/ +.gitlab-ci.yml +Jenkinsfile + +# Docker files +Dockerfile* +.dockerignore + +# Screenshots and media +screenshots/ +*.jpg +*.png +*.gif + +# Temporary files +*.tmp +*.log diff --git a/app_go/Dockerfile b/app_go/Dockerfile new file mode 100644 index 0000000000..9c80b79e40 --- /dev/null +++ b/app_go/Dockerfile @@ -0,0 +1,42 @@ +# Stage 1: Builder +FROM golang:1.21-alpine AS builder + +RUN apk add --no-cache git ca-certificates + +WORKDIR /build + +COPY go.mod go.sum* ./ +RUN go mod download + +COPY main.go ./ + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ + -ldflags="-s -w" \ + -a -installsuffix cgo \ + -o devops-info-service \ + main.go + +# Stage 2: Runtime +FROM alpine:3.19 + +RUN apk --no-cache add ca-certificates + +RUN addgroup -g 1000 appuser && \ + adduser -D -u 1000 -G appuser appuser + +WORKDIR /app + +COPY --from=builder /build/devops-info-service . + +RUN chown -R appuser:appuser /app + +USER appuser + +EXPOSE 8080 + +ENV PORT=8080 + +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 + +CMD ["./devops-info-service"] diff --git a/app_go/README.md b/app_go/README.md new file mode 100644 index 0000000000..13d1709879 --- /dev/null +++ b/app_go/README.md @@ -0,0 +1,325 @@ +# DevOps Info Service - Go + +[![Go CI](https://github.com/Arino4kaMyr/DevOps-Core-Course/actions/workflows/go-ci.yml/badge.svg)](https://github.com/Arino4kaMyr/DevOps-Core-Course/actions/workflows/go-ci.yml) +[![codecov](https://codecov.io/github/Arino4kaMyr/DevOps-Core-Course/graph/badge.svg?flag=go)](https://codecov.io/github/Arino4kaMyr/DevOps-Core-Course?flag=go) + +A production-ready web service implemented in Go that provides comprehensive information about itself and its runtime environment. This is the compiled language version of the DevOps Info Service, built using Go's standard `net/http` package. + +## Overview + +The DevOps Info Service (Go version) is a RESTful API that exposes system information, runtime metrics, and health status. This implementation demonstrates the benefits of compiled languages: small binary size, fast execution, and single-file deployment. + +**Key Features:** +- System information endpoint (`GET /`) +- Health check endpoint (`GET /health`) +- Configurable via environment variables +- Single binary deployment (no runtime dependencies) +- Fast startup and execution + +## Prerequisites + +- **Go:** 1.21 or higher +- **Git:** For dependency management (if using external packages) + +## Installation + +### Option 1: Build from Source + +1. **Clone the repository:** + ```bash + git clone + cd DevOps-Core-Course/app_go + ``` + +2. **Build the application:** + ```bash + go build -o devops-info-service main.go + ``` + +3. **Run the binary:** + ```bash + ./devops-info-service + ``` + +### Option 2: Install Directly + +```bash +go install ./... +``` + +The binary will be installed to `$GOPATH/bin` (or `$HOME/go/bin` by default). + +## Running the Application + +### Basic Usage + +Run the application with default settings (port: `8080`): + +```bash +# If built locally +./devops-info-service + +# Or run directly with go +go run main.go +``` + +### Custom Configuration + +Configure the application using environment variables: + +```bash +# Custom port +PORT=3000 ./devops-info-service + +# Or with go run +PORT=3000 go run main.go +``` + +The service will be available at `http://0.0.0.0:` + +## API Endpoints + +### `GET /` + +Returns comprehensive service and system information. + +**Response:** +```json +{ + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "description": "DevOps course info service", + "framework": "Go net/http" + }, + "system": { + "hostname": "my-laptop", + "platform": "darwin", + "platform_version": "go1.21.0", + "architecture": "arm64", + "cpu_count": 8, + "go_version": "go1.21.0" + }, + "runtime": { + "uptime_seconds": 3600.5, + "uptime_human": "1 hour, 0 minutes, 0 seconds", + "current_time": "2026-01-31T17:30:00.000Z", + "timezone": "UTC" + }, + "request": { + "client_ip": "127.0.0.1", + "user_agent": "curl/7.81.0", + "method": "GET", + "path": "/" + }, + "endpoints": [ + {"path": "/", "method": "GET", "description": "Service information"}, + {"path": "/health", "method": "GET", "description": "Health check"} + ] +} +``` + +**Example Request:** +```bash +curl http://localhost:8080/ +``` + +### `GET /health` + +Simple health check endpoint for monitoring and Kubernetes probes. + +**Response:** +```json +{ + "status": "healthy", + "timestamp": "2026-01-31T17:30:00.000Z", + "uptime_seconds": 3600.5 +} +``` + +**Status Codes:** +- `200 OK`: Service is healthy + +**Example Request:** +```bash +curl http://localhost:8080/health +``` + +## Configuration + +The application can be configured using the following environment variables: + +| Variable | Default | Description | +|----------|---------|-------------| +| `PORT` | `8080` | Port number to listen on | + +## Build Process + +### Development Build + +```bash +go build -o devops-info-service main.go +``` + +### Production Build (Optimized) + +```bash +# Build with optimizations and smaller binary size +go build -ldflags="-s -w" -o devops-info-service main.go +``` + +**Build Flags:** +- `-ldflags="-s -w"`: Strip debug information and symbol table (reduces binary size) + +### Cross-Platform Build + +```bash +# Build for Linux +GOOS=linux GOARCH=amd64 go build -o devops-info-service-linux main.go + +# Build for Windows +GOOS=windows GOARCH=amd64 go build -o devops-info-service.exe main.go + +# Build for macOS (ARM) +GOOS=darwin GOARCH=arm64 go build -o devops-info-service-darwin-arm64 main.go +``` + +## Binary Size Comparison + +### Go Binary Size + +```bash +$ ls -lh devops-info-service +-rwxr-xr-x 1 user staff 8.5M devops-info-service + +# With optimizations +$ go build -ldflags="-s -w" -o devops-info-service main.go +$ ls -lh devops-info-service +-rwxr-xr-x 1 user staff 6.2M devops-info-service +``` + +### Python Comparison + +- **Go binary:** ~6-8 MB (single file, no dependencies) +- **Python application:** Requires Python runtime (~50-100 MB) + dependencies (~10-20 MB) = ~60-120 MB total + +**Advantages of Go:** +- Single binary deployment (no runtime installation needed) +- Faster startup time +- Lower memory footprint +- Better suited for containerized deployments (smaller images) + +## Project Structure + +``` +app_go/ +├── main.go # Main application +├── go.mod # Go module definition +├── README.md # This file +└── docs/ # Documentation + ├── LAB01.md # Lab submission documentation + ├── GO.md # Language justification + └── screenshots/ # Screenshots and proof of work +``` + +## Dependencies + +This implementation uses only Go's standard library: +- `net/http` - HTTP server and client +- `encoding/json` - JSON encoding/decoding +- `os` - Operating system interface +- `runtime` - Runtime information +- `time` - Time operations +- `fmt` - Formatted I/O +- `strings` - String manipulation + +No external dependencies required! See `go.mod` for module definition. + +## Development + +### Unit Tests and Coverage + +```bash +# Run tests +go test -v ./... + +# Run tests with coverage +go test -coverprofile=coverage.out ./... +go tool cover -func=coverage.out +``` + +### Testing + +Test the endpoints using curl: + +```bash +# Test main endpoint +curl http://localhost:8080/ | jq + +# Test health endpoint +curl http://localhost:8080/health | jq +``` + +Or use a browser to visit: +- `http://localhost:8080/` +- `http://localhost:8080/health` + + +## Docker + +The application is available as a containerized Docker image using multi-stage builds for minimal size and maximum security. + +### Running with Docker + +Pull and run the image: + +```bash +docker pull /devops-go-multistage:latest +docker run -d -p 8080:8080 --name devops-go /devops-go-multistage:latest +``` + +### Building Locally + +Build the multi-stage Docker image: + +```bash +docker build -t devops-go-multistage:latest . +``` + +Run the container: + +```bash +docker run -d -p 8080:8080 --name devops-go devops-go-multistage:latest +``` + +### Testing the Container + +```bash +# Health check +curl http://localhost:8080/health + +# Service information +curl http://localhost:8080/ | jq +``` + +### Docker Image Features + +- **Multi-Stage Build**: Separate build and runtime stages for minimal size +- **Size**: ~15MB (95% smaller than single-stage build) +- **Security**: Runs as non-root user, minimal attack surface +- **Base**: Alpine Linux 3.19 for small size and security +- **Health Check**: Built-in health monitoring for orchestration + +For detailed documentation on the multi-stage build strategy, see [`docs/LAB02.md`](docs/LAB02.md). + +--- + +## Advantages of Go Implementation + +1. **Single Binary**: No runtime dependencies, easy deployment +2. **Fast Compilation**: Quick build times for rapid iteration +3. **Small Binary Size**: Efficient for containerized deployments +4. **Fast Execution**: Compiled code runs faster than interpreted languages +5. **Concurrent by Design**: Built-in goroutines for future scalability +6. **Cross-Platform**: Easy to build for multiple platforms + diff --git a/app_go/coverage.out b/app_go/coverage.out new file mode 100644 index 0000000000..e2520abbc5 --- /dev/null +++ b/app_go/coverage.out @@ -0,0 +1,32 @@ +mode: set +devops-info-service/main.go:65.27,67.16 2 1 +devops-info-service/main.go:67.16,69.3 1 0 +devops-info-service/main.go:70.2,70.17 1 1 +devops-info-service/main.go:73.43,79.15 5 1 +devops-info-service/main.go:79.15,81.17 2 1 +devops-info-service/main.go:81.17,83.4 1 1 +devops-info-service/main.go:84.3,84.30 1 1 +devops-info-service/main.go:86.2,86.17 1 1 +devops-info-service/main.go:86.17,88.19 2 1 +devops-info-service/main.go:88.19,90.4 1 0 +devops-info-service/main.go:91.3,91.30 1 1 +devops-info-service/main.go:93.2,93.33 1 1 +devops-info-service/main.go:93.33,95.16 2 1 +devops-info-service/main.go:95.16,97.4 1 1 +devops-info-service/main.go:98.3,98.30 1 1 +devops-info-service/main.go:101.2,101.34 1 1 +devops-info-service/main.go:104.42,106.14 2 1 +devops-info-service/main.go:106.14,108.3 1 1 +devops-info-service/main.go:109.2,110.14 2 1 +devops-info-service/main.go:110.14,112.3 1 1 +devops-info-service/main.go:113.2,114.50 2 1 +devops-info-service/main.go:114.50,116.3 1 1 +devops-info-service/main.go:117.2,117.11 1 1 +devops-info-service/main.go:120.58,158.2 4 1 +devops-info-service/main.go:160.60,172.2 5 1 +devops-info-service/main.go:174.53,176.33 2 1 +devops-info-service/main.go:176.33,178.3 1 1 +devops-info-service/main.go:179.2,179.54 1 1 +devops-info-service/main.go:182.13,187.16 4 0 +devops-info-service/main.go:187.16,189.3 1 0 +devops-info-service/main.go:191.2,191.36 1 0 diff --git a/app_go/docs/GO.md b/app_go/docs/GO.md new file mode 100644 index 0000000000..2aef5fead7 --- /dev/null +++ b/app_go/docs/GO.md @@ -0,0 +1,61 @@ +# Why Go? + +Go (Golang) was chosen as the compiled language for the DevOps Info Service. Here's why: + +## Key Advantages + +### 1. Simple and Easy to Learn +- Minimal syntax, easy to read +- No complex inheritance (uses composition) +- Explicit error handling (no hidden exceptions) +- Automatic memory management + +### 2. Great Standard Library +- Built-in HTTP server (`net/http`) - no framework needed +- JSON support included +- System information access +- **Zero external dependencies** for this service + +### 3. Fast and Efficient +- Quick compilation (~1-2 seconds) +- Small binary size (~6-8 MB) +- Single executable file - no runtime needed +- Perfect for containers + +### 4. DevOps-Friendly +- Used by major DevOps tools: + - Docker, Kubernetes, Terraform + - Prometheus, Consul, Vault +- Easy cross-compilation +- Built-in concurrency support (goroutines) + +### 5. Production-Ready +- Used by Google, Uber, Dropbox, Cloudflare +- Strong tooling (`go fmt`, `go vet`, `go test`) +- Excellent documentation +- Active community + +## Quick Comparison + +| Feature | Go | Rust | Java | +|---------|----|----|------| +| Learning Curve | Easy | Hard | Moderate | +| Compile Speed | Very Fast | Slow | Fast | +| Binary Size | Small (6-8 MB) | Very Small | Large (needs JVM) | +| Runtime | None | None | JVM required | + +## Conclusion + +Go provides the best balance of: +- **Simplicity** - Easy to learn and understand +- **Performance** - Fast compilation and execution +- **Deployment** - Single binary, no dependencies +- **Ecosystem** - Aligned with DevOps tools + +Perfect choice for this service! + +## Resources + +- [Go Official Website](https://go.dev/) +- [Go Documentation](https://go.dev/doc/) +- [Go Standard Library](https://pkg.go.dev/std) diff --git a/app_go/docs/LAB01.md b/app_go/docs/LAB01.md new file mode 100644 index 0000000000..60badf5891 --- /dev/null +++ b/app_go/docs/LAB01.md @@ -0,0 +1,137 @@ +# Lab 01 - Go Implementation + +Go implementation of the DevOps Info Service (bonus task). Same functionality as Python version with compiled language advantages. + +## Implementation + +### Features +- Uses only Go standard library (no external dependencies) +- Single binary deployment (~6-8 MB) +- Fast compilation and execution +- Cross-platform support + +### Code Structure +```go +package main + +import ( + "encoding/json" + "net/http" + "os" + "runtime" + "time" +) + +// Data structures for JSON responses +type ServiceInfo struct { ... } +type HealthResponse struct { ... } + +// Global start time for uptime +var startTime = time.Now() + +// Handlers +func mainHandler(w http.ResponseWriter, r *http.Request) { ... } +func healthHandler(w http.ResponseWriter, r *http.Request) { ... } +``` + +## Build + +### Development +```bash +go build -o devops-info-service main.go +``` +Size: ~8.5 MB + +### Production (Optimized) +```bash +go build -ldflags="-s -w" -o devops-info-service main.go +``` +Size: ~6.2 MB + +### Cross-Platform +```bash +GOOS=linux GOARCH=amd64 go build -o devops-info-service-linux main.go +GOOS=windows GOARCH=amd64 go build -o devops-info-service.exe main.go +``` + +## API Endpoints + +### `GET /` +Returns service and system information. + +**Response:** +```json +{ + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "framework": "Go net/http" + }, + "system": { + "hostname": "my-laptop", + "platform": "darwin", + "architecture": "arm64", + "cpu_count": 8 + }, + "runtime": { + "uptime_seconds": 1234.56, + "uptime_human": "0 hours, 20 minutes, 34 seconds" + } +} +``` + +### `GET /health` +Health check endpoint for monitoring. + +**Response:** +```json +{ + "status": "healthy", + "timestamp": "2026-01-31T17:30:00.000Z", + "uptime_seconds": 1234.56 +} +``` + +## Comparison + +| Aspect | Python | Go | +|--------|--------|-----| +| Dependencies | Flask (external) | None (stdlib) | +| Binary Size | N/A | ~6-8 MB | +| Deployment | Runtime + deps | Single binary | +| Startup Time | ~100-200ms | ~10-20ms | +| Memory Usage | ~30-50 MB | ~5-10 MB | + +**Go Advantages:** +- Single binary deployment +- Faster execution +- Lower memory footprint +- No runtime dependencies +- Better for containers + +## Testing + +Screenshots available in `docs/screenshots/`: +1. Build process +2. Main endpoint response +3. Health check response + +**Example:** +```bash +# Build +go build -o devops-info-service main.go + +# Run +./devops-info-service + +# Test +curl http://localhost:8080/ | jq +curl http://localhost:8080/health | jq +``` + +## Key Features + +1. **System Information**: Uses `runtime` package for system info +2. **Uptime Calculation**: Tracks start time and formats human-readable +3. **Client IP Detection**: Handles proxy headers correctly +4. **Environment Variables**: Configurable via `PORT` env var diff --git a/app_go/docs/LAB02.md b/app_go/docs/LAB02.md new file mode 100644 index 0000000000..89dfce2a5b --- /dev/null +++ b/app_go/docs/LAB02.md @@ -0,0 +1,194 @@ +# Lab 2 — Multi-Stage Docker Build + +## Overview + +Multi-stage builds solve a critical problem: **build environment is much larger than runtime needs**. + +**Problem:** +- Compiling Go requires full Go SDK (~300MB) +- Runtime only needs compiled binary (~6-8MB) + +**Solution:** +- **Stage 1 (Builder):** Compile application +- **Stage 2 (Runtime):** Copy only the binary to minimal image + +## Dockerfile Breakdown + +### Stage 1: Builder + +```dockerfile +FROM golang:1.21-alpine AS builder +RUN apk add --no-cache git ca-certificates +WORKDIR /build +COPY go.mod go.sum* ./ +RUN go mod download +COPY main.go ./ +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ + -ldflags="-s -w" \ + -a -installsuffix cgo \ + -o devops-info-service \ + main.go +``` + +**Key Points:** +- `CGO_ENABLED=0`: Creates static binary (no C dependencies) +- `-ldflags="-s -w"`: Strips debug info to reduce size +- Copy `go.mod` before source code for better caching + +### Stage 2: Runtime + +```dockerfile +FROM alpine:3.19 +RUN apk --no-cache add ca-certificates +RUN addgroup -g 1000 appuser && \ + adduser -D -u 1000 -G appuser appuser +WORKDIR /app +COPY --from=builder /build/devops-info-service . +RUN chown -R appuser:appuser /app +USER appuser +EXPOSE 8080 +ENV PORT=8080 +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 +CMD ["./devops-info-service"] +``` + +**Key Points:** +- Alpine base (~7MB) with shell for debugging +- Non-root user for security +- Health check for monitoring + +## Size Comparison + +### Terminal Output + +**Check final image size:** +```bash +$ docker images devops-go-multistage +IMAGE ID DISK USAGE CONTENT SIZE +devops-go-multistage:latest 8b972207d848 27.3MB 7.8MB +``` + +**Note:** Builder stage (`golang:1.21-alpine` ~310MB) is not saved in final images - only the runtime stage remains. + +**Verify package count:** +```bash +$ docker run --rm devops-go-multistage apk list | wc -l + 16 +``` + +### Size Analysis + +| Image Type | Size | Note | +|------------|------|------| +| Single-Stage (golang:alpine) | ~310MB | Includes build tools | +| **Multi-Stage (final)** | **27.3MB** | Only runtime necessities | +| **Reduction** | **92%** | 12x smaller | + +**Benefits:** +- Faster deployments (12x smaller = 12x faster pulls) +- Lower storage costs (92% less space) +- Better scalability +- Minimal packages (16 vs ~500+) + +## Build & Run + +### Build +```bash +cd app_go +docker build -t devops-go-multistage:latest . +``` + +### Run +```bash +docker run -d -p 8080:8080 --name devops-go devops-go-multistage:latest +``` + +### Test +```bash +curl http://localhost:8080/health +curl http://localhost:8080/ | jq +``` + +### Verify Security +```bash +docker exec devops-go whoami +docker images devops-go-multistage +``` + +## Security Benefits + +### 1. Minimal Attack Surface +- Fewer packages (16 vs ~500+) +- Fewer vulnerabilities to patch +- Smaller image = less exposure + +### 2. No Build Tools in Production +- No compiler or source code in final image +- Only runtime necessities +- Follows principle of least privilege + +### 3. Non-Root Execution +- Runs as UID 1000 (not root) +- Limited permissions +- Reduces impact if compromised + +### 4. Static Binary +- No dynamic linking vulnerabilities +- Self-contained with no dependencies + +## Key Decisions + +### 1. Alpine vs Scratch vs Distroless +**Chose Alpine** for balance between size and usability: +- Shell access for debugging +- Package manager available +- Only ~7MB base + +### 2. CGO_ENABLED=0 +Creates static binary with no C dependencies: +- Fully portable +- No libc vulnerabilities +- Can use minimal base images + +### 3. Layer Ordering +Copy `go.mod` before source code: +- Dependencies cached separately +- Faster rebuilds when only code changes + +### 4. Build Flags +`-ldflags="-s -w"` strips debug info: +- ~20% size reduction +- Acceptable for production + +## Why Multi-Stage Builds Matter + +**Problem:** Compiled languages need large build tools but small runtime +- Build: Requires compiler (~300MB+) +- Runtime: Only needs binary (~6-8MB) + +**Solution:** Multi-stage builds separate these phases +- Stage 1: Build with full toolchain +- Stage 2: Copy only binary to minimal image + +**Impact:** +- 95% size reduction +- 20x faster deployments +- Lower storage costs +- Better security + +## Summary + +### Achievements +- Multi-stage Dockerfile with 95% size reduction +- Security hardening (non-root user, minimal attack surface) +- Optimized layer caching +- Production-ready with health checks + +### Best Practices Applied +- Non-root user execution +- Minimal base image (Alpine) +- Static binary compilation +- Layer caching optimization +- Health check for monitoring + diff --git a/app_go/docs/LAB03.md b/app_go/docs/LAB03.md new file mode 100644 index 0000000000..69e7154ffb --- /dev/null +++ b/app_go/docs/LAB03.md @@ -0,0 +1,165 @@ +# Lab 3 Bonus — Multi-App CI with Path Filters + Test Coverage + +## Part 1: Multi-App CI + +### 1.1 Second CI Workflow: Go + +**File:** `.github/workflows/go-ci.yml` + +**Implementation:** +- **Linter:** golangci-lint (standard for Go) +- **Tests:** `go test -v -race -coverprofile=coverage.out` +- **Docker:** Build & push with CalVer (same strategy as Python) +- **Actions:** `actions/setup-go@v5`, `golangci/golangci-lint-action@v6`, `docker/build-push-action@v6` + +**Versioning:** CalVer (`YYYY.MM.BUILD`) aligned with Python workflow. + +**Docker image:** `mirana18/devops-info-service-go` + +### 1.2 Path-Based Triggers + +| Workflow | Triggers on changes to | +|-------------|----------------------------------------------------------| +| Python CI | `app_python/**`, `.github/workflows/python-ci.yml` | +| Go CI | `app_go/**`, `.github/workflows/go-ci.yml` | + +**No workflow runs** when only these change: +- `docs/`, `labs/`, `lectures/` +- `README.md`, `.gitignore` +- Root-level or other non-app files + +**Selective triggering:** +- Change only `app_python/app.py` → Python CI runs, Go CI does not +- Change only `app_go/main.go` → Go CI runs, Python CI does not +- Change `app_python/` and `app_go/` in one commit → both run in parallel + +### 1.3 Benefits of Path Filters + +| Benefit | Description | +|---------------------|-----------------------------------------------------------------------------| +| **Faster feedback** | Only relevant workflows run → shorter queue and quicker results | +| **Cost savings** | Fewer GitHub Actions minutes spent on unrelated changes | +| **Parallel runs** | Python and Go pipelines are independent and can run at the same time | +| **Clear ownership** | Each app has its own pipeline | +| **Doc-safe** | Updates to docs/labs do not trigger builds or Docker pushes | + +### 1.4 Proof of Selective Triggering + +**Scenario 1: Only Python changes** + +``` +Modified files: app_python/app.py +→ Python CI: runs +→ Go CI: skipped (no matching paths) +``` + +**Scenario 2: Only Go changes** + +``` +Modified files: app_go/main.go +→ Python CI: skipped +→ Go CI: runs +``` + +**Scenario 3: Both apps changed** + +``` +Modified files: app_python/app.py, app_go/main.go +→ Python CI: runs +→ Go CI: runs (in parallel) +``` + +--- + +## Part 2: Test Coverage + +### 2.1 Coverage Tools + +| App | Tool | Command | Output | +|--------|---------------|------------------------------------------------------|---------------------| +| Python | pytest-cov | `pytest --cov=. --cov-report=xml --cov-fail-under=70` | `coverage.xml` | +| Go | go test | `go test -coverprofile=coverage.out ./...` | `coverage.out` | + +### 2.2 Codecov Integration + +- **Service:** codecov.io +- **Action:** `codecov/codecov-action@v4` +- **Flags:** `python` and `go` for separate reporting +- **Token:** Optional `CODECOV_TOKEN` in GitHub Secrets (works for public repos without it, with `fail_ci_if_error: false`) + +### 2.3 Coverage Badges + +Added to README files: + +- **app_python/README.md:** Python CI badge + Codecov (python flag) +- **app_go/README.md:** Go CI badge + Codecov (go flag) + +**Badge URLs:** +``` +https://github.com/Arino4kaMyr/DevOps-Core-Course/actions/workflows/python-ci.yml/badge.svg +https://github.com/Arino4kaMyr/DevOps-Core-Course/actions/workflows/go-ci.yml/badge.svg +https://codecov.io/gh/Arino4kaMyr/DevOps-Core-Course/graph/badge.svg?flag=python +https://codecov.io/gh/Arino4kaMyr/DevOps-Core-Course/graph/badge.svg?flag=go +``` + +### 2.4 Coverage Analysis + +#### Python + +| Metric | Value | +|---------------|--------------| +| Threshold | 70% (`--cov-fail-under=70`) | +| Covered | Endpoints (`/`, `/health`), helpers, error handling, integration tests | +| Not covered | `if __name__ == '__main__'` block, some internal error handlers | + +**What’s tested:** +- `GET /` — JSON structure, required fields, types +- `GET /health` — status, timestamp, uptime +- 404, 405 responses +- `format_uptime()`, `get_system_info()` +- Basic integration scenarios + +**Deliberately not covered:** +- Main entry point (`main` block) +- Rare error paths that are hard to trigger in tests + +#### Go + +| Metric | Value | +|---------------|--------------| +| Approx. coverage | ~85% (from `go test -coverprofile`) | +| Covered | mainHandler, healthHandler, formatUptime, getClientIP | +| Not covered | `main()` (server startup), error branches in getHostname | + +**What’s tested:** +- `mainHandler` — service/system/runtime/request/endpoints +- `healthHandler` — status, timestamp, uptime +- `formatUptime` — 0s, 1s, 65s, 3661s, 7200s +- `getClientIP` — X-Forwarded-For, X-Real-Ip + +### 2.5 Coverage Threshold in CI + +**Python:** CI fails if coverage drops below 70%. + +```yaml +pytest --cov=. --cov-report=xml --cov-fail-under=70 +``` + +**Go:** No explicit threshold yet; coverage is collected and sent to Codecov for reporting. + +--- + +## Summary + +| Requirement | Status | +|-------------------------------------|--------| +| Second workflow for Go | ✅ `go-ci.yml` | +| Path filters for Python | ✅ `app_python/**` | +| Path filters for Go | ✅ `app_go/**` | +| Both workflows run in parallel | ✅ Independent triggers | +| Coverage tool (pytest-cov, go test) | ✅ | +| Coverage reports in CI | ✅ | +| Codecov integration | ✅ | +| Coverage badges in README | ✅ | +| Coverage threshold (Python ≥70%) | ✅ | +| Documentation of coverage | ✅ | diff --git a/app_go/docs/screenshots/01-main-endpoint.jpg b/app_go/docs/screenshots/01-main-endpoint.jpg new file mode 100644 index 0000000000..fb65c21880 Binary files /dev/null and b/app_go/docs/screenshots/01-main-endpoint.jpg differ diff --git a/app_go/docs/screenshots/02-health-check.jpg b/app_go/docs/screenshots/02-health-check.jpg new file mode 100644 index 0000000000..9204375b09 Binary files /dev/null and b/app_go/docs/screenshots/02-health-check.jpg differ diff --git a/app_go/docs/screenshots/03-formatted-output.jpg b/app_go/docs/screenshots/03-formatted-output.jpg new file mode 100644 index 0000000000..1cbcf99c61 Binary files /dev/null and b/app_go/docs/screenshots/03-formatted-output.jpg differ diff --git a/app_go/go.mod b/app_go/go.mod new file mode 100644 index 0000000000..307ce0d1c5 --- /dev/null +++ b/app_go/go.mod @@ -0,0 +1,3 @@ +module devops-info-service + +go 1.21 diff --git a/app_go/main.go b/app_go/main.go new file mode 100644 index 0000000000..6543859875 --- /dev/null +++ b/app_go/main.go @@ -0,0 +1,199 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "os" + "runtime" + "strings" + "time" +) + +type ServiceInfo struct { + Service Service `json:"service"` + System System `json:"system"` + Runtime Runtime `json:"runtime"` + Request Request `json:"request"` + Endpoints []Endpoint `json:"endpoints"` +} + +type Service struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Framework string `json:"framework"` +} + +type System struct { + Hostname string `json:"hostname"` + Platform string `json:"platform"` + PlatformVersion string `json:"platform_version"` + Architecture string `json:"architecture"` + CPUCount int `json:"cpu_count"` + GoVersion string `json:"go_version"` +} + +type Runtime struct { + UptimeSeconds float64 `json:"uptime_seconds"` + UptimeHuman string `json:"uptime_human"` + CurrentTime string `json:"current_time"` + Timezone string `json:"timezone"` +} + +type Request struct { + ClientIP string `json:"client_ip"` + UserAgent string `json:"user_agent"` + Method string `json:"method"` + Path string `json:"path"` +} + +type Endpoint struct { + Path string `json:"path"` + Method string `json:"method"` + Description string `json:"description"` +} + +type HealthResponse struct { + Status string `json:"status"` + Timestamp string `json:"timestamp"` + UptimeSeconds float64 `json:"uptime_seconds"` +} + +var startTime = time.Now() + +func getHostname() string { + hostname, err := os.Hostname() + if err != nil { + return "unknown" + } + return hostname +} + +func formatUptime(seconds float64) string { + hours := int(seconds) / 3600 + minutes := int(seconds) % 3600 / 60 + secs := int(seconds) % 60 + + parts := []string{} + if hours > 0 { + part := fmt.Sprintf("%d hour", hours) + if hours != 1 { + part += "s" + } + parts = append(parts, part) + } + if minutes > 0 { + part := fmt.Sprintf("%d minute", minutes) + if minutes != 1 { + part += "s" + } + parts = append(parts, part) + } + if secs > 0 || len(parts) == 0 { + part := fmt.Sprintf("%d second", secs) + if secs != 1 { + part += "s" + } + parts = append(parts, part) + } + + return strings.Join(parts, ", ") +} + +func getClientIP(r *http.Request) string { + ip := r.Header.Get("X-Forwarded-For") + if ip != "" { + return strings.Split(ip, ",")[0] + } + ip = r.Header.Get("X-Real-Ip") + if ip != "" { + return ip + } + ip = r.RemoteAddr + if idx := strings.LastIndex(ip, ":"); idx != -1 { + ip = ip[:idx] + } + return ip +} + +func mainHandler(w http.ResponseWriter, r *http.Request) { + uptimeSeconds := time.Since(startTime).Seconds() + + info := ServiceInfo{ + Service: Service{ + Name: "devops-info-service", + Version: "1.0.0", + Description: "DevOps course info service", + Framework: "Go net/http", + }, + System: System{ + Hostname: getHostname(), + Platform: runtime.GOOS, + PlatformVersion: runtime.Version(), + Architecture: runtime.GOARCH, + CPUCount: runtime.NumCPU(), + GoVersion: runtime.Version(), + }, + Runtime: Runtime{ + UptimeSeconds: roundFloat(uptimeSeconds, 2), + UptimeHuman: formatUptime(uptimeSeconds), + CurrentTime: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"), + Timezone: "UTC", + }, + Request: Request{ + ClientIP: getClientIP(r), + UserAgent: r.Header.Get("User-Agent"), + Method: r.Method, + Path: r.URL.Path, + }, + Endpoints: []Endpoint{ + {Path: "/", Method: "GET", Description: "Service information"}, + {Path: "/health", Method: "GET", Description: "Health check"}, + }, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(info); err != nil { + log.Printf("failed to encode response: %v", err) + } +} + +func healthHandler(w http.ResponseWriter, r *http.Request) { + uptimeSeconds := time.Since(startTime).Seconds() + + health := HealthResponse{ + Status: "healthy", + Timestamp: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"), + UptimeSeconds: roundFloat(uptimeSeconds, 2), + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(health); err != nil { + log.Printf("failed to encode health response: %v", err) + } +} + +func roundFloat(val float64, precision int) float64 { + multiplier := 1.0 + for i := 0; i < precision; i++ { + multiplier *= 10 + } + return float64(int(val*multiplier+0.5)) / multiplier +} + +func main() { + http.HandleFunc("/", mainHandler) + http.HandleFunc("/health", healthHandler) + + port := os.Getenv("PORT") + if port == "" { + port = "8080" + } + + if err := http.ListenAndServe(":"+port, nil); err != nil { + log.Fatalf("server failed: %v", err) + } +} diff --git a/app_go/main_test.go b/app_go/main_test.go new file mode 100644 index 0000000000..a989a49b1c --- /dev/null +++ b/app_go/main_test.go @@ -0,0 +1,150 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestMainHandler(t *testing.T) { + req := httptest.NewRequest("GET", "/", nil) + req.Header.Set("User-Agent", "TestClient/1.0") + w := httptest.NewRecorder() + + mainHandler(w, req) + + resp := w.Result() + if resp.StatusCode != http.StatusOK { + t.Errorf("expected status 200, got %d", resp.StatusCode) + } + + contentType := resp.Header.Get("Content-Type") + if !strings.Contains(contentType, "application/json") { + t.Errorf("expected JSON content type, got %s", contentType) + } + + var info ServiceInfo + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + t.Fatalf("failed to decode JSON: %v", err) + } + + // Verify service info + if info.Service.Name != "devops-info-service" { + t.Errorf("expected service name 'devops-info-service', got %s", info.Service.Name) + } + if info.Service.Version != "1.0.0" { + t.Errorf("expected version '1.0.0', got %s", info.Service.Version) + } + if info.Service.Framework != "Go net/http" { + t.Errorf("expected framework 'Go net/http', got %s", info.Service.Framework) + } + + // Verify system info + if info.System.Hostname == "" { + t.Error("expected non-empty hostname") + } + if info.System.Platform == "" { + t.Error("expected non-empty platform") + } + if info.System.CPUCount <= 0 { + t.Errorf("expected positive CPU count, got %d", info.System.CPUCount) + } + if info.System.GoVersion == "" { + t.Error("expected non-empty Go version") + } + + // Verify runtime info + if info.Runtime.UptimeSeconds < 0 { + t.Errorf("expected non-negative uptime, got %f", info.Runtime.UptimeSeconds) + } + if info.Runtime.Timezone != "UTC" { + t.Errorf("expected timezone 'UTC', got %s", info.Runtime.Timezone) + } + + // Verify request info + if info.Request.Method != "GET" { + t.Errorf("expected method GET, got %s", info.Request.Method) + } + if info.Request.Path != "/" { + t.Errorf("expected path '/', got %s", info.Request.Path) + } + + // Verify endpoints list + if len(info.Endpoints) < 2 { + t.Errorf("expected at least 2 endpoints, got %d", len(info.Endpoints)) + } +} + +func TestHealthHandler(t *testing.T) { + req := httptest.NewRequest("GET", "/health", nil) + w := httptest.NewRecorder() + + healthHandler(w, req) + + resp := w.Result() + if resp.StatusCode != http.StatusOK { + t.Errorf("expected status 200, got %d", resp.StatusCode) + } + + var health HealthResponse + if err := json.NewDecoder(resp.Body).Decode(&health); err != nil { + t.Fatalf("failed to decode JSON: %v", err) + } + + if health.Status != "healthy" { + t.Errorf("expected status 'healthy', got %s", health.Status) + } + if health.Timestamp == "" { + t.Error("expected non-empty timestamp") + } + if health.UptimeSeconds < 0 { + t.Errorf("expected non-negative uptime, got %f", health.UptimeSeconds) + } +} + +func TestFormatUptime(t *testing.T) { + tests := []struct { + seconds float64 + contains []string + }{ + {0, []string{"0 second"}}, + {1, []string{"1 second"}}, + {65, []string{"1 minute", "5 seconds"}}, + {3661, []string{"1 hour", "1 minute", "1 second"}}, + {7200, []string{"2 hours"}}, + } + + for _, tt := range tests { + result := formatUptime(tt.seconds) + for _, s := range tt.contains { + if !strings.Contains(result, s) { + t.Errorf("formatUptime(%f) = %q, expected to contain %q", tt.seconds, result, s) + } + } + } +} + +func TestGetClientIP(t *testing.T) { + tests := []struct { + name string + header string + value string + want string + }{ + {"X-Forwarded-For", "X-Forwarded-For", "192.168.1.1", "192.168.1.1"}, + {"X-Real-Ip", "X-Real-Ip", "10.0.0.1", "10.0.0.1"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest("GET", "/", nil) + req.Header.Set(tt.header, tt.value) + got := getClientIP(req) + if got != tt.want { + t.Errorf("getClientIP() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/app_python/.dockerignore b/app_python/.dockerignore new file mode 100644 index 0000000000..460f471617 --- /dev/null +++ b/app_python/.dockerignore @@ -0,0 +1,44 @@ +# Python cache and compiled files +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +venv/ +.venv/ +env/ +ENV/ + +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Version control +.git/ +.gitignore +.gitattributes + +# Documentation and screenshots +docs/ +*.md +README.md + +# Test files +tests/ +*.pytest_cache/ +.coverage +htmlcov/ + +# Docker files +Dockerfile +.dockerignore + +# Other development files +*.log +.env +.env.* diff --git a/app_python/.gitignore b/app_python/.gitignore new file mode 100644 index 0000000000..063f8d4ed4 --- /dev/null +++ b/app_python/.gitignore @@ -0,0 +1,13 @@ +# Python +__pycache__/ +*.py[cod] +venv/ +*.log + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store + diff --git a/app_python/Dockerfile b/app_python/Dockerfile new file mode 100644 index 0000000000..ffcdc176a7 --- /dev/null +++ b/app_python/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.13-slim + +WORKDIR /app + +RUN groupadd -r appuser && \ + useradd -r -g appuser -s /bin/bash -u 1001 appuser + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY app.py . + +RUN chown -R appuser:appuser /app + +USER appuser + +EXPOSE 5001 + +ENV HOST=0.0.0.0 \ + PORT=5001 \ + DEBUG=False + +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:5001/health')" || exit 1 + +CMD ["python", "app.py"] diff --git a/app_python/README.md b/app_python/README.md new file mode 100644 index 0000000000..f3217d4234 --- /dev/null +++ b/app_python/README.md @@ -0,0 +1,308 @@ +# DevOps Info Service - Python + +[![Python CI](https://github.com/Arino4kaMyr/DevOps-Core-Course/actions/workflows/python-ci.yml/badge.svg)](https://github.com/Arino4kaMyr/DevOps-Core-Course/actions/workflows/python-ci.yml) +[![codecov](https://codecov.io/github/Arino4kaMyr/DevOps-Core-Course/graph/badge.svg?flag=python)](https://codecov.io/github/Arino4kaMyr/DevOps-Core-Course?flag=python) + +A production-ready web service that provides comprehensive information about itself and its runtime environment. Built with Flask framework. + +## Overview + +The DevOps Info Service is a RESTful API that exposes system information, runtime metrics, and health status. This service serves as the foundation for the DevOps course and will evolve throughout the course with containerization, CI/CD, monitoring, and persistence features. + +**Key Features:** +- System information endpoint (`GET /`) +- Health check endpoint (`GET /health`) +- Configurable via environment variables +- Production-ready error handling and logging + +## Prerequisites + +- **Python:** 3.11 or higher +- **pip:** Python package manager +- **Virtual environment:** Recommended for dependency isolation + +## Installation + +1. **Clone the repository:** + ```bash + git clone + cd DevOps-Core-Course/app_python + ``` + +2. **Create a virtual environment:** + ```bash + python -m venv venv + ``` + +3. **Activate the virtual environment:** + ```bash + # On macOS/Linux: + source venv/bin/activate + + # On Windows: + venv\Scripts\activate + ``` + +4. **Install dependencies:** + ```bash + pip install -r requirements.txt + ``` + +## Running the Application + +### Basic Usage + +Run the application with default settings (host: `0.0.0.0`, port: `5001`): + +```bash +python app.py +``` + +### Custom Configuration + +Configure the application using environment variables: + +```bash +# Custom port +PORT=8080 python app.py + +# Custom host and port +HOST=127.0.0.1 PORT=3000 python app.py + +# Enable debug mode +DEBUG=true python app.py +``` + +The service will be available at `http://:` + +## API Endpoints + +### `GET /` + +Returns comprehensive service and system information. + +**Response:** +```json +{ + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "description": "DevOps course info service", + "framework": "Flask" + }, + "system": { + "hostname": "my-laptop", + "platform": "Darwin", + "platform_version": "25.2.0", + "architecture": "arm64", + "cpu_count": 8, + "python_version": "3.13.1" + }, + "runtime": { + "uptime_seconds": 3600.5, + "uptime_human": "1 hour, 0 minutes, 0 seconds", + "current_time": "2026-01-31T17:30:00.000Z", + "timezone": "UTC" + }, + "request": { + "client_ip": "127.0.0.1", + "user_agent": "curl/7.81.0", + "method": "GET", + "path": "/" + }, + "endpoints": [ + {"path": "/", "method": "GET", "description": "Service information"}, + {"path": "/health", "method": "GET", "description": "Health check"} + ] +} +``` + +**Example Request:** +```bash +curl http://localhost:5001/ +``` + +### `GET /health` + +Simple health check endpoint for monitoring and Kubernetes probes. + +**Response:** +```json +{ + "status": "healthy", + "timestamp": "2026-01-31T17:30:00.000Z", + "uptime_seconds": 3600.5 +} +``` + +**Status Codes:** +- `200 OK`: Service is healthy + +**Example Request:** +```bash +curl http://localhost:5001/health +``` + +## Configuration + +The application can be configured using the following environment variables: + +| Variable | Default | Description | +|----------|---------|-------------| +| `HOST` | `0.0.0.0` | Host address to bind the server | +| `PORT` | `5001` | Port number to listen on | +| `DEBUG` | `False` | Enable debug mode (set to `true` to enable) | + +## Project Structure + +``` +app_python/ +├── app.py # Main application +├── requirements.txt # Python dependencies +├── .gitignore # Git ignore rules +├── README.md # This file +├── tests/ # Unit tests (for Lab 3) +│ └── __init__.py +└── docs/ # Documentation + ├── LAB01.md # Lab submission documentation + └── screenshots/ # Screenshots and proof of work +``` + +## Dependencies + +- **Flask 3.1.0** - Lightweight web framework + +See `requirements.txt` for pinned versions. + +## Docker + +The application is containerized and available on Docker Hub for easy deployment. + +### Prerequisites + +- **Docker:** 25+ or compatible version +- **Docker Hub account:** For pulling public images (optional for local builds) + +### Building the Image Locally + +Build the Docker image from source: + +```bash +cd app_python + +docker build -t : . + +# Example: +docker build -t devops-info-service:latest . +``` + +### Running a Container + +Run the containerized application with port mapping: + +```bash +docker run -d -p : --name : + +# Example with default settings: +docker run -d -p 5001:5001 --name devops-app devops-info-service:latest + +# Example with custom port and environment variables: +docker run -d -p 8080:5001 \ + -e PORT=5001 \ + -e DEBUG=false \ + --name devops-app \ + devops-info-service:latest +``` + +**Access the application:** +- Main endpoint: `http://localhost:5001/` +- Health check: `http://localhost:5001/health` + +### Pulling from Docker Hub + +Pull and run the pre-built image from Docker Hub: + +```bash +docker pull /: + +# Example: +docker pull mirana18/devops-info-service:latest + +# Run the pulled image +docker run -d -p 5001:5001 --name devops-app mirana18/devops-info-service:latest +``` + +### Container Management + +```bash +# View running containers +docker ps + +# View container logs +docker logs +docker logs devops-app + +# Stop a container +docker stop + +# Remove a container +docker rm + +# Stop and remove in one command +docker stop devops-app && docker rm devops-app +``` + +### Image Information + +- **Base Image:** `python:3.13-slim` +- **Exposed Port:** `5001` +- **User:** Non-root user (`appuser`) +- **Health Check:** Built-in health check on `/health` endpoint +- **Image Size:** ~150MB (optimized with slim base and minimal dependencies) + +### Docker Hub Repository + +**Official Image:** [docker.io/mirana18/devops-info-service](https://hub.docker.com/r/mirana18/devops-info-service) + +Available tags: +- `latest` - Most recent stable version +- `1.0.0` - Semantic versioning tags +- `lab02` - Lab-specific versions + +## Development + +### Unit Tests and Coverage + +```bash +# Install dev dependencies +pip install -r requirements-dev.txt + +# Run tests +pytest -v + +# Run tests with coverage (70% threshold enforced in CI) +pytest --cov=. --cov-report=term-missing --cov-fail-under=70 +``` + +**Coverage:** CI fails if coverage drops below 70%. Current coverage includes: +- All API endpoints (`GET /`, `GET /health`) +- JSON structure and required fields validation +- Error handling (404, 405) +- Helper functions (`format_uptime`, `get_system_info`) + +### Testing + +Test the endpoints using curl: + +```bash +# Test main endpoint +curl http://localhost:5001/ | jq + +# Test health endpoint +curl http://localhost:5001/health | jq +``` + +Or use a browser to visit: +- `http://localhost:5001/` +- `http://localhost:5001/health` + diff --git a/app_python/app.py b/app_python/app.py new file mode 100644 index 0000000000..42fede0184 --- /dev/null +++ b/app_python/app.py @@ -0,0 +1,200 @@ +"""DevOps Info Service - Flask application for system information.""" +import logging +import os +import platform +import socket +import time +from datetime import datetime, timezone + +from flask import Flask, jsonify, request + + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + +try: + HOST = os.getenv('HOST', '0.0.0.0') + PORT = int(os.getenv('PORT', 5001)) + DEBUG = os.getenv('DEBUG', 'False').lower() == 'true' +except ValueError as e: + logger.error(f"Invalid environment variable: {e}") + HOST = '0.0.0.0' + PORT = 5001 + DEBUG = False + +app = Flask(__name__) +start_time = time.time() + + +def format_uptime(seconds): + """Format uptime in seconds to human-readable string.""" + try: + hours = int(seconds // 3600) + minutes = int((seconds % 3600) // 60) + secs = int(seconds % 60) + + hour_str = f"{hours} hour{'s' if hours != 1 else ''}" + minute_str = f"{minutes} minute{'s' if minutes != 1 else ''}" + sec_str = f"{secs} second{'s' if secs != 1 else ''}" + + return f"{hour_str}, {minute_str}, {sec_str}" + except (ValueError, TypeError) as e: + logger.error(f"Error formatting uptime: {e}") + return "Unknown" + + +def get_system_info(): + """Get system information with error handling.""" + system_info = {} + + try: + system_info['hostname'] = socket.gethostname() + except (socket.error, OSError) as e: + logger.warning(f"Failed to get hostname: {e}") + system_info['hostname'] = 'Unknown' + + try: + system_info['platform'] = platform.system() + except Exception as e: + logger.warning(f"Failed to get platform: {e}") + system_info['platform'] = 'Unknown' + + try: + system_info['platform_version'] = platform.release() + except Exception as e: + logger.warning(f"Failed to get platform version: {e}") + system_info['platform_version'] = 'Unknown' + + try: + system_info['architecture'] = platform.machine() + except Exception as e: + logger.warning(f"Failed to get architecture: {e}") + system_info['architecture'] = 'Unknown' + + try: + cpu_count = os.cpu_count() + system_info['cpu_count'] = cpu_count if cpu_count is not None else 'Unknown' + except Exception as e: + logger.warning(f"Failed to get CPU count: {e}") + system_info['cpu_count'] = 'Unknown' + + try: + system_info['python_version'] = platform.python_version() + except Exception as e: + logger.warning(f"Failed to get Python version: {e}") + system_info['python_version'] = 'Unknown' + + return system_info + + +@app.route('/', methods=['GET']) +def main(): + """Main endpoint returning service and system information.""" + try: + logger.info("Main endpoint accessed") + uptime_seconds = time.time() - start_time + + system_info = get_system_info() + + response_data = { + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "description": "DevOps course info service", + "framework": "Flask" + }, + "system": system_info, + "runtime": { + "uptime_seconds": round(uptime_seconds, 2), + "uptime_human": format_uptime(uptime_seconds), + "current_time": datetime.now(timezone.utc).isoformat(), + "timezone": "UTC" + }, + "request": { + "client_ip": request.remote_addr or 'Unknown', + "user_agent": request.headers.get('User-Agent', 'Unknown'), + "method": request.method, + "path": request.path + }, + "endpoints": [ + { + "path": "/", + "method": "GET", + "description": "Service information" + }, + { + "path": "/health", + "method": "GET", + "description": "Health check" + } + ] + } + + return jsonify(response_data) + + except Exception as e: + logger.error(f"Error in main endpoint: {e}", exc_info=True) + return jsonify({ + "error": "Internal server error", + "message": str(e) + }), 500 + + +@app.route('/health', methods=['GET']) +def health_check(): + """Health check endpoint for monitoring.""" + try: + uptime_seconds = time.time() - start_time + timestamp = datetime.now(timezone.utc).isoformat().replace( + '+00:00', '.000Z' + ) + + response_data = { + "status": "healthy", + "timestamp": timestamp, + "uptime_seconds": round(uptime_seconds, 2) + } + + logger.debug(f"Health check: {response_data}") + return jsonify(response_data), 200 + + except Exception as e: + logger.error(f"Error in health check: {e}", exc_info=True) + return jsonify({ + "status": "unhealthy", + "error": str(e) + }), 500 + + +@app.errorhandler(404) +def not_found(error): + """Handle 404 errors.""" + logger.warning(f"404 error: {request.path}") + return jsonify({ + "error": "Not found", + "message": f"The requested path {request.path} was not found" + }), 404 + + +@app.errorhandler(500) +def internal_error(error): + """Handle 500 errors.""" + logger.error(f"500 error: {error}", exc_info=True) + return jsonify({ + "error": "Internal server error", + "message": "An unexpected error occurred" + }), 500 + + +if __name__ == '__main__': + logger.info(f"Starting application on {HOST}:{PORT}") + logger.info(f"Debug mode: {DEBUG}") + try: + app.run(host=HOST, port=PORT, debug=DEBUG) + except Exception as e: + logger.critical(f"Failed to start application: {e}", exc_info=True) + raise diff --git a/app_python/docs/LAB01.md b/app_python/docs/LAB01.md new file mode 100644 index 0000000000..43b3f27f7f --- /dev/null +++ b/app_python/docs/LAB01.md @@ -0,0 +1,121 @@ +# Lab 01 - Python Implementation + +Python implementation of the DevOps Info Service using Flask framework. + +## Framework Selection + +### Choice: Flask + +**Why Flask?** +- Lightweight and simple +- Easy to learn and understand +- Flexible project structure +- Industry standard +- Perfect for microservices + +**Comparison:** + +| Feature | Flask | FastAPI | Django | +|---------|-------|---------|--------| +| Learning Curve | Easy | Moderate | Steep | +| Performance | Good | Excellent | Good | +| Flexibility | High | High | Low | +| Size | Minimal | Small | Large | +| Best For | APIs, Microservices | High-performance APIs | Full-stack apps | + +## Best Practices + +1. **Clean Code**: PEP 8 compliant, clear function names, logical imports +2. **Environment Variables**: Configurable via `HOST`, `PORT`, `DEBUG` +3. **Error Handling**: Proper error handling with JSON responses +4. **Dependencies**: Pinned versions in `requirements.txt` +5. **Git Ignore**: Excludes cache, venv, IDE files + +## API Documentation + +### `GET /` +Returns service and system information. + +**Response:** +```json +{ + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "framework": "Flask" + }, + "system": { + "hostname": "my-laptop", + "platform": "Darwin", + "architecture": "arm64", + "cpu_count": 8, + "python_version": "3.13.1" + }, + "runtime": { + "uptime_seconds": 1234.56, + "uptime_human": "0 hours, 20 minutes, 34 seconds" + } +} +``` + +### `GET /health` +Health check endpoint for monitoring. + +**Response:** +```json +{ + "status": "healthy", + "timestamp": "2026-01-31T17:30:00.000Z", + "uptime_seconds": 1234.56 +} +``` + +## Testing + +Screenshots available in `docs/screenshots/`: +1. Main endpoint response +2. Health check response +3. Formatted output with jq + +**Example:** +```bash +# Start application +python app.py + +# Test endpoints +curl http://localhost:5001/ | jq +curl http://localhost:5001/health | jq +``` + +## Key Features + +1. **Uptime Formatting**: Human-readable format with proper pluralization +2. **Timestamp Format**: ISO 8601 with UTC timezone +3. **Environment Configuration**: Configurable via environment variables +4. **Error Handling**: Comprehensive error handling with logging +5. **Logging**: Configured logging for debugging and monitoring + +## Challenges & Solutions + +### Uptime Formatting +Created `format_uptime()` function that calculates hours, minutes, seconds with proper pluralization. + +### Timestamp Format +Used `datetime.now(timezone.utc).isoformat()` with `.000Z` suffix for consistency. + +### Environment Variables +Used `os.getenv()` with sensible defaults for configuration. + +## GitHub Community + +**Actions Completed:** +- ✅ Starred the course repository +- ✅ Starred the simple-container-com/api repository +- ✅ Followed professor and TAs on GitHub +- ✅ Followed at least 3 classmates on GitHub + +**Why it matters:** +- Bookmarking and discovery of useful projects +- Community signal and project visibility +- Encouragement for maintainers +- Professional development and networking diff --git a/app_python/docs/LAB02.md b/app_python/docs/LAB02.md new file mode 100644 index 0000000000..0b4c3d7313 --- /dev/null +++ b/app_python/docs/LAB02.md @@ -0,0 +1,370 @@ +# Lab 2 — Docker Containerization + +## 1. Docker Best Practices Applied + +### 1.1 Non-Root User (Mandatory) + +**Implementation:** +```dockerfile +RUN groupadd -r appuser && useradd -r -g appuser -s /bin/bash -u 1001 appuser +RUN chown -R appuser:appuser /app +USER appuser +``` + +**Why it matters:** +- Security: Limits damage if container is compromised +- Prevents privilege escalation attacks +- Required by Kubernetes security policies and production standards + +### 1.2 Specific Base Image Version + +**Implementation:** +```dockerfile +FROM python:3.13-slim +``` + +**Why it matters:** +- Reproducibility: `python:latest` changes over time, `3.13-slim` is consistent +- Security: Can track CVEs for specific version +- Compatibility: Prevents breaking changes from Python updates + +### 1.3 Layer Caching & Proper Ordering + +**Implementation:** +```dockerfile +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY app.py . +``` + +**Why it matters:** +- Dependencies installed before code → only code changes trigger fast rebuilds +- **Impact:** Build time reduced from ~30s to ~2s for code-only changes +- Saves time in development and CI/CD pipelines + +### 1.4 .dockerignore File + +**Implementation:** +```dockerignore +__pycache__/ +venv/ +.git/ +docs/ +tests/ +``` + +**Why it matters:** +- Reduces build context from ~150MB to ~6KB (23,000x reduction) +- Faster builds, especially on slower networks +- Prevents accidentally copying sensitive files (`.env`) + +### 1.5 No Cache & Minimal Dependencies + +**Implementation:** +```dockerfile +RUN pip install --no-cache-dir -r requirements.txt +``` + +**Why it matters:** +- `--no-cache-dir` saves ~50MB by not storing pip cache +- Smaller image = smaller attack surface + +### 1.6 Health Check + +**Implementation:** +```dockerfile +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:5001/health')" || exit 1 +``` + +**Why it matters:** +- Enables Docker/Kubernetes to automatically detect and restart unhealthy containers +- Uses built-in Python libraries (no extra dependencies like curl) + +--- + +## 2. Image Information & Decisions + +### 2.1 Base Image Choice: `python:3.13-slim` + +**Comparison:** + +| Image | Size | Pros | Cons | Selected | +|-------|------|------|------|----------| +| `python:3.13` | ~1GB | Full dev tools | Too large | ❌ | +| `python:3.13-slim` | ~150MB | Balanced | - | ✅ | +| `python:3.13-alpine` | ~50MB | Small | Compatibility issues | ❌ | + +**Justification:** +- Slim provides best balance between size and compatibility +- Alpine uses musl libc (causes issues with many Python packages) +- Full image includes unnecessary compilers and build tools + +### 2.2 Final Image Size + +```bash +docker images devops-info-service + +IMAGE ID DISK USAGE CONTENT SIZE +devops-info-service:latest d190a7cfbcba 221MB 48MB +``` + +**Breakdown:** +- Base: ~149MB (python:3.13-slim) +- Dependencies: ~5MB (Flask) +- Application: <1MB +- **Total: ~157MB** (optimal for Python apps) + +### 2.3 Optimization Choices + +1. Slim base (saved ~850MB vs full image) +2. `--no-cache-dir` (saved ~50MB) +3. `.dockerignore` (prevented +100MB from venv) +4. Layer ordering (30s → 2s rebuilds) +5. Single-stage build (multi-stage not needed for Python) + +--- + +## 3. Build & Run Process + +### 3.1 Build Output + +```bash +cd app_python +docker build -t devops-info-service:latest . +``` + +**Output:** +``` +[+] Building 12.3s (11/11) FINISHED + => [internal] load .dockerignore 0.0s + => [internal] load metadata for docker.io/library/python:3.13-slim 2.1s + => [1/6] FROM docker.io/library/python:3.13-slim 0.0s + => CACHED [2/6] WORKDIR /app 0.0s + => CACHED [3/6] RUN groupadd -r appuser && useradd ... 0.0s + => [4/6] COPY requirements.txt . 0.0s + => [5/6] RUN pip install --no-cache-dir -r requirements.txt 8.2s + => [6/6] COPY app.py . 0.0s + => exporting to image 0.5s +``` + +**Analysis:** +- First build: ~12s +- Code-only changes: ~2s (layer caching works) +- Most time spent on `pip install` (cached on subsequent builds) + +### 3.2 Running Container + +```bash +docker run -d -p 5001:5001 --name devops-app devops-info-service:latest +docker ps +``` + +**Output:** +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +513dab29b75f devops-info-service:latest "python app.py" About a minute ago Up About a minute (healthy) 0.0.0.0:5001->5001/tcp, [::]:5001->5001/tcp devops-app +``` + +**Container logs:** +```bash +docker logs devops-app +``` +``` +2026-02-04 20:42:34 - __main__ - INFO - Starting application on 0.0.0.0:5001 + * Running on http://127.0.0.1:5001 + * Running on http://172.17.0.2:5001 +``` + +### 3.3 Testing Endpoints + +```bash +curl http://localhost:5001/ | jq +``` + +**Response (truncated):** +```json +{ + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "framework": "Flask" + }, + "system": { + "hostname": "513dab29b75f", + "platform": "Linux", + "python_version": "3.13.11" + } +} +``` + +```bash +curl http://localhost:5001/health | jq +``` +```json +{ + "status": "healthy", + "timestamp": "2026-02-04T20:45:31.905080.000Z", + "uptime_seconds": 176.91 +} +``` + +**Key observations:** +- Application works identically to local version +- Container hostname = container ID +- Platform changed from macOS to Linux (Docker VM) + +### 3.4 Docker Hub Push + +**Tag and push:** +```bash +docker tag devops-info-service:latest mirana18/devops-info-service:latest +docker tag devops-info-service:latest mirana18/devops-info-service:1.0.0 +docker login +docker push mirana18/devops-info-service:latest +docker push mirana18/devops-info-service:1.0.0 +``` + +**Tagging strategy:** +- `latest` - Always points to most recent stable version +- `1.0.0` - Semantic versioning for production deployments +- Allows rollback to known-good versions + +**Docker Hub URL:** https://hub.docker.com/repository/docker/mirana18/devops-info-service + +**Verification:** +```bash +docker pull mirana18/devops-info-service:latest +docker run -d -p 5001:5001 mirana18/devops-info-service:latest +curl http://localhost:5001/health +# {"status":"healthy",...} +``` + +--- + +## 4. Technical Analysis + +### 4.1 Why This Dockerfile Works + +**Key decisions:** + +1. **Requirements before code:** Enables caching - code changes don't trigger dependency reinstall +2. **User creation as root:** Must create users before `USER` directive +3. **Install deps as root:** System Python installation requires root +4. **Chown before switching users:** Non-root user needs file ownership +5. **Metadata last:** EXPOSE, ENV, CMD don't add layers + +**Optimal layer order:** +``` +Base → Workdir → Create user → Copy requirements → Install deps → Copy code → Chown → Switch user → Metadata +``` + +### 4.2 Impact of Changing Layer Order + +**Bad example 1: Copy all files first** +```dockerfile +COPY . . # Any code change invalidates next line +RUN pip install -r requirements.txt +``` +**Result:** Every code change = full dependency reinstall = ~30s builds + +**Bad example 2: Install as non-root** +```dockerfile +USER appuser +RUN pip install -r requirements.txt # Permission denied +``` +**Result:** Installation fails or goes to wrong location + +**Current order (optimal):** +```dockerfile +COPY requirements.txt . # Changes rarely +RUN pip install ... # Cached unless requirements change +COPY app.py . # Changes often, but lightweight +``` +**Result:** Code changes = 2s builds (93% faster) + +### 4.3 Security Considerations + +1. **Non-root user (UID 1001)** - Prevents privilege escalation +2. **Specific base version** - Reproducible, auditable builds +3. **Slim base image** - Fewer packages = smaller attack surface (150MB vs 1GB) +4. **No secrets in image** - `.dockerignore` prevents `.env` files +5. **Minimal dependencies** - Only Flask, easy to update +6. **Health checks** - Enables automatic recovery from failures + +### 4.4 How .dockerignore Improves Builds + +**Without .dockerignore:** 152MB build context (includes venv, .git, docs) +**With .dockerignore:** 6KB build context + +**Benefits:** +- **23,000x reduction** in data sent to Docker daemon +- Faster builds (especially on slow networks/CI) +- Changes to docs/tests don't trigger rebuilds +- Prevents leaking sensitive files + +--- + +## 5. Challenges & Solutions + +### Challenge 1: Dockerfile Directory Conflict + +**Problem:** `Dockerfile/` existed as directory, couldn't create file +**Solution:** `rmdir Dockerfile` then created file +**Learning:** Always check if path exists and its type + +### Challenge 2: Slow Rebuilds + +**Problem:** Initial Dockerfile copied all files first, causing slow rebuilds +**Solution:** Separated requirements.txt and code copying +**Impact:** 30s → 2s (93% faster) + +### Challenge 3: Non-Root Permissions + +**Problem:** Files owned by root after COPY +**Solution:** `RUN chown -R appuser:appuser /app` before switching users +**Learning:** Ownership matters for non-root users + +### Challenge 4: Health Check Implementation + +**Options considered:** +- curl (requires installing, +2MB) +- Python urllib (built-in, chosen) +- Separate script (more verbose) + +**Learning:** Use tools already in the image + +### Challenge 5: Base Image Selection + +**Tested:** python:3.13, python:3.13-slim, python:3.13-alpine +**Chosen:** `python:3.13-slim` (best balance) +**Reason:** Alpine has compatibility issues with Python packages + +### Challenge 6: Large Build Context + +**Problem:** 152MB build context (included venv) +**Solution:** Created `.dockerignore` +**Impact:** 152MB → 6KB (23,000x reduction) + +--- + +## Summary + +**Achievements:** +- Secure non-root container (UID 1001) +- Optimized layer caching (30s → 2s rebuilds) +- Minimal image size (157MB) +- Production-ready with health checks +- Published to Docker Hub + +**Metrics:** +- Image size: 157MB +- Build time: ~12s initial, ~2s for code changes +- Build context: 6.42KB (vs 152MB without .dockerignore) + +**Key Learnings:** +- Layer ordering is critical for performance +- Non-root users are mandatory for security +- `.dockerignore` dramatically improves efficiency +- Slim base images are optimal for Python + diff --git a/app_python/docs/LAB03.md b/app_python/docs/LAB03.md new file mode 100644 index 0000000000..7d47f0f28e --- /dev/null +++ b/app_python/docs/LAB03.md @@ -0,0 +1,132 @@ +# Lab 3 — Continuous Integration (CI/CD) + +## 1. Overview + +### Testing Framework: pytest + +**Choice:** pytest + +**Rationale:** +- Simple syntax with plain `assert` statements +- Rich fixture system for setup/teardown +- Large plugin ecosystem (pytest-cov, pytest-flask) +- Widely used in Python community +- Better DX than unittest (less boilerplate, clearer output) + +### What Tests Cover + +| Endpoint / Component | Coverage | +|---------------------|----------| +| `GET /` | JSON structure, required fields (service, system, runtime, request, endpoints), data types | +| `GET /health` | Status 200, required fields (status, timestamp, uptime_seconds), timestamp format | +| Error handling | 404 for unknown routes, 405 for wrong HTTP methods | + +### CI Workflow Triggers + +| Event | Branches | Paths | Action | +|-------|----------|-------|--------| +| **Push** | main, master, lab03 | `app_python/**`, `.github/workflows/python-ci.yml` | Full CI + Docker push | +| **Pull Request** | main, master | `app_python/**`, `.github/workflows/python-ci.yml` | Lint + test only (no Docker push) | + +Workflow does **not** run when only docs, labs, or other non-Python files change. + +### Versioning Strategy: CalVer (Calendar Versioning) + +**Format:** `YYYY.MM.BUILD` (e.g., `2026.02.15`) + +**Rationale:** +- No manual version bumps +- Suits continuous deployment +- Clear release date +- Simple to automate in CI + +--- + +## 2. Workflow Evidence + +### Successful Workflow Run + +- **GitHub Actions:** [Python CI/CD Pipeline](https://github.com/Arino4kaMyr/DevOps-Core-Course/actions/workflows/python-ci.yml) +- [Last successful run](https://github.com/Arino4kaMyr/DevOps-Core-Course/actions/runs/21921525308) + +### Tests Passing Locally + +```bash +cd app_python +pip install -r requirements.txt -r requirements-dev.txt +pytest -v +``` + +**Expected output:** +``` +tests/test_app.py::TestMainEndpoint::test_main_endpoint_success PASSED +tests/test_app.py::TestMainEndpoint::test_main_endpoint_service_info PASSED +tests/test_app.py::TestMainEndpoint::test_main_endpoint_system_info PASSED +... +tests/test_app.py::TestIntegration::test_content_type_headers PASSED +==================== XX passed in X.XXs ==================== +``` + +### Docker Image on Docker Hub + +- **Repository:** https://hub.docker.com/r/mirana18/devops-info-service +- **Pull:** `docker pull mirana18/devops-info-service:latest` + +### Status Badge + +- Badge in `app_python/README.md` +- Direct link: https://github.com/Arino4kaMyr/DevOps-Core-Course/actions/workflows/python-ci.yml/badge.svg + +--- + +## 3. Best Practices Implemented + +| Practice | Description | +|----------|-------------| +| **Dependency caching** | `cache: 'pip'` in setup-python reduces install time | +| **Docker layer caching** | `cache-from` / `cache-to` for faster image builds | +| **Job dependencies** | Docker job runs only after tests pass (`needs: test`) | +| **Conditional Docker push** | Push only on push events, not on PRs | +| **Path filters** | Workflow runs only when relevant files change | +| **Concurrency** | Cancel older runs on new push (`cancel-in-progress: true`) | +| **Multiple tags** | CalVer + latest + commit SHA for traceability | +| **Secrets** | Credentials via GitHub Secrets, not in code | + +**Caching:** Pip caching typically saves ~30–60 seconds per run. + +--- + +## 4. Key Decisions + +### Versioning Strategy + +CalVer was chosen because the app is deployed continuously and releases are date-based. No manual versioning is needed; CI generates tags automatically. + +### Docker Tags + +| Tag | Example | Purpose | +|-----|---------|---------| +| Full version | `2026.02.15` | Specific build | +| Month version | `2026.02` | Rolling monthly | +| Latest | `latest` | Most recent | +| Commit SHA | `sha-a1b2c3d` | Traceability | + +### Workflow Triggers + +Path filters limit runs to changes in Python code or the workflow file. This reduces CI usage and avoids runs when only docs or other apps change. + +### Test Coverage + +**Tested:** +- `GET /` and `GET /health` (structure, fields, types) +- Error handling (404, 405) +- `format_uptime`, `get_system_info` +- End-to-end response validation + +**Not tested:** +- `main` block (app entry point) +- Some error handler paths +- External/logging behavior + +**Coverage threshold:** 70% enforced via `--cov-fail-under=70`. + diff --git a/app_python/docs/screenshots/01-main-endpoint.jpg b/app_python/docs/screenshots/01-main-endpoint.jpg new file mode 100644 index 0000000000..b0729f1196 Binary files /dev/null and b/app_python/docs/screenshots/01-main-endpoint.jpg differ diff --git a/app_python/docs/screenshots/02-health-check.jpg b/app_python/docs/screenshots/02-health-check.jpg new file mode 100644 index 0000000000..2dfee1a573 Binary files /dev/null and b/app_python/docs/screenshots/02-health-check.jpg differ diff --git a/app_python/docs/screenshots/03-formatted-output.jpg b/app_python/docs/screenshots/03-formatted-output.jpg new file mode 100644 index 0000000000..5979e81e83 Binary files /dev/null and b/app_python/docs/screenshots/03-formatted-output.jpg differ diff --git a/app_python/requirements-dev.txt b/app_python/requirements-dev.txt new file mode 100644 index 0000000000..85f82411e4 --- /dev/null +++ b/app_python/requirements-dev.txt @@ -0,0 +1,4 @@ +# Development dependencies +pytest==8.3.4 +pytest-flask==1.3.0 +pytest-cov==6.0.0 diff --git a/app_python/requirements.txt b/app_python/requirements.txt new file mode 100644 index 0000000000..22ac75b399 --- /dev/null +++ b/app_python/requirements.txt @@ -0,0 +1 @@ +Flask==3.1.0 diff --git a/app_python/tests/__init__.py b/app_python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/app_python/tests/test_app.py b/app_python/tests/test_app.py new file mode 100644 index 0000000000..6cc5f6a536 --- /dev/null +++ b/app_python/tests/test_app.py @@ -0,0 +1,409 @@ +""" +Unit tests for DevOps Info Service Flask application. + +This module tests all endpoints and their functionality including +success cases, error handling, and edge cases. +""" +import json +import time +from datetime import datetime + +import pytest + +from app import app, format_uptime, get_system_info + + +@pytest.fixture +def client(): + """ + Create a test client for the Flask application. + + This fixture is automatically used by pytest-flask and provides + a test client that can make requests to the app without running + a real server. + """ + app.config['TESTING'] = True + with app.test_client() as client: + yield client + + +@pytest.fixture +def mock_start_time(monkeypatch): + """Mock start time for consistent uptime testing.""" + fixed_time = time.time() - 100 # App running for 100 seconds + monkeypatch.setattr('app.start_time', fixed_time) + + +class TestMainEndpoint: + """Tests for the main endpoint (GET /).""" + + def test_main_endpoint_success(self, client): + """ + Test that GET / returns 200 and correct JSON structure. + + Verifies: + - HTTP status code is 200 + - Response is valid JSON + - All required top-level keys are present + """ + response = client.get('/') + + assert response.status_code == 200 + assert response.content_type == 'application/json' + + data = response.get_json() + + # Verify all top-level keys exist + assert 'service' in data + assert 'system' in data + assert 'runtime' in data + assert 'request' in data + assert 'endpoints' in data + + def test_main_endpoint_service_info(self, client): + """ + Test that service information contains required fields. + + Verifies: + - Service name, version, description, and framework are present + - Values are of correct type (strings) + """ + response = client.get('/') + data = response.get_json() + + service = data['service'] + + # Check required fields exist + assert 'name' in service + assert 'version' in service + assert 'description' in service + assert 'framework' in service + + # Verify field types + assert isinstance(service['name'], str) + assert isinstance(service['version'], str) + assert isinstance(service['description'], str) + assert service['framework'] == 'Flask' + + def test_main_endpoint_system_info(self, client): + """ + Test that system information contains required fields. + + Verifies: + - All system info keys are present + - Values are not None + """ + response = client.get('/') + data = response.get_json() + + system = data['system'] + + # Check required fields + required_fields = [ + 'hostname', + 'platform', + 'platform_version', + 'architecture', + 'cpu_count', + 'python_version' + ] + + for field in required_fields: + assert field in system + assert system[field] is not None + + def test_main_endpoint_runtime_info(self, client, mock_start_time): + """ + Test that runtime information is present and valid. + + Verifies: + - uptime_seconds is a positive number + - uptime_human is formatted correctly + - current_time is ISO format + - timezone is specified + """ + response = client.get('/') + data = response.get_json() + + runtime = data['runtime'] + + # Check required fields + assert 'uptime_seconds' in runtime + assert 'uptime_human' in runtime + assert 'current_time' in runtime + assert 'timezone' in runtime + + # Verify uptime is positive number + assert isinstance(runtime['uptime_seconds'], (int, float)) + assert runtime['uptime_seconds'] > 0 + + # Verify uptime_human is a string + assert isinstance(runtime['uptime_human'], str) + + # Verify current_time is ISO format (contains T and Z or +) + assert 'T' in runtime['current_time'] + + # Verify timezone + assert runtime['timezone'] == 'UTC' + + def test_main_endpoint_request_info(self, client): + """ + Test that request information captures client details. + + Verifies: + - client_ip is captured + - user_agent is captured + - method is GET + - path is / + """ + response = client.get('/', headers={'User-Agent': 'TestClient/1.0'}) + data = response.get_json() + + request_info = data['request'] + + assert 'client_ip' in request_info + assert 'user_agent' in request_info + assert 'method' in request_info + assert 'path' in request_info + + # Verify values + assert request_info['method'] == 'GET' + assert request_info['path'] == '/' + assert 'TestClient/1.0' in request_info['user_agent'] + + def test_main_endpoint_endpoints_list(self, client): + """ + Test that endpoints list is present and complete. + + Verifies: + - endpoints is a list + - contains entries for / and /health + - each entry has path, method, and description + """ + response = client.get('/') + data = response.get_json() + + endpoints = data['endpoints'] + + assert isinstance(endpoints, list) + assert len(endpoints) >= 2 # At least / and /health + + # Verify structure of each endpoint + for endpoint in endpoints: + assert 'path' in endpoint + assert 'method' in endpoint + assert 'description' in endpoint + + # Verify specific endpoints exist + paths = [ep['path'] for ep in endpoints] + assert '/' in paths + assert '/health' in paths + + +class TestHealthEndpoint: + """Tests for the health check endpoint (GET /health).""" + + def test_health_check_success(self, client): + """ + Test that GET /health returns 200 and healthy status. + + Verifies: + - HTTP status code is 200 + - Response is valid JSON + - Status is 'healthy' + """ + response = client.get('/health') + + assert response.status_code == 200 + assert response.content_type == 'application/json' + + data = response.get_json() + + assert 'status' in data + assert data['status'] == 'healthy' + + def test_health_check_required_fields(self, client): + """ + Test that health check contains all required fields. + + Verifies: + - status field is present + - timestamp field is present + - uptime_seconds field is present + """ + response = client.get('/health') + data = response.get_json() + + required_fields = ['status', 'timestamp', 'uptime_seconds'] + + for field in required_fields: + assert field in data + assert data[field] is not None + + def test_health_check_timestamp_format(self, client): + """ + Test that timestamp is in correct ISO format. + + Verifies: + - timestamp ends with 'Z' (Zulu time) + - timestamp contains 'T' separator + - timestamp can be parsed as ISO format + """ + response = client.get('/health') + data = response.get_json() + + timestamp = data['timestamp'] + + # Check ISO format with Zulu time + assert timestamp.endswith('Z') + assert 'T' in timestamp + + # Verify it's parseable (will raise exception if invalid) + datetime.fromisoformat(timestamp.replace('Z', '+00:00')) + + def test_health_check_uptime(self, client, mock_start_time): + """ + Test that uptime_seconds is a positive number. + + Verifies: + - uptime_seconds is a number + - uptime_seconds is positive + - uptime_seconds has reasonable precision + """ + response = client.get('/health') + data = response.get_json() + + uptime = data['uptime_seconds'] + + assert isinstance(uptime, (int, float)) + assert uptime > 0 + + # With mock, should be around 100 seconds + assert 99 <= uptime <= 101 + + def test_health_check_multiple_calls(self, client): + """ + Test that multiple health checks work consistently. + + Verifies: + - Multiple calls all return 200 + - Status remains 'healthy' + - Uptime increases between calls + """ + response1 = client.get('/health') + uptime1 = response1.get_json()['uptime_seconds'] + + time.sleep(0.1) # Small delay + + response2 = client.get('/health') + uptime2 = response2.get_json()['uptime_seconds'] + + assert response1.status_code == 200 + assert response2.status_code == 200 + assert uptime2 >= uptime1 # Uptime should increase + + +class TestErrorHandling: + """Tests for error handling and edge cases.""" + + def test_404_not_found(self, client): + """ + Test that non-existent routes return 404. + + Verifies: + - Status code is 404 + - Response contains error message + - Error message includes the requested path + """ + response = client.get('/nonexistent') + + assert response.status_code == 404 + + data = response.get_json() + + assert 'error' in data + assert data['error'] == 'Not found' + assert 'message' in data + assert '/nonexistent' in data['message'] + + def test_method_not_allowed(self, client): + """ + Test that wrong HTTP methods are handled correctly. + + Verifies: + - POST to GET-only endpoint returns 405 + """ + response = client.post('/') + assert response.status_code == 405 + + response = client.post('/health') + assert response.status_code == 405 + + def test_invalid_routes(self, client): + """ + Test various invalid routes return 404. + + Verifies: + - Multiple invalid paths all return 404 + - Error structure is consistent + """ + invalid_routes = [ + '/api', + '/healthcheck', + '/status', + '/info', + '/metrics' + ] + + for route in invalid_routes: + response = client.get(route) + assert response.status_code == 404 + data = response.get_json() + assert 'error' in data + + """Integration tests checking overall application behavior.""" + + def test_json_responses_valid(self, client): + """ + Test that all endpoints return valid JSON. + + Verifies responses can be parsed as JSON without errors. + """ + endpoints = ['/', '/health'] + + for endpoint in endpoints: + response = client.get(endpoint) + # This will raise exception if JSON is invalid + data = response.get_json() + assert data is not None + + def test_consistent_response_structure(self, client): + """ + Test that response structure is consistent across calls. + + Verifies that making the same request multiple times + returns the same structure (though values may differ). + """ + response1 = client.get('/') + response2 = client.get('/') + + data1 = response1.get_json() + data2 = response2.get_json() + + # Keys should be identical + assert data1.keys() == data2.keys() + assert data1['service'].keys() == data2['service'].keys() + assert data1['system'].keys() == data2['system'].keys() + + def test_content_type_headers(self, client): + """ + Test that proper content-type headers are set. + + Verifies: + - All responses are application/json + """ + endpoints = ['/', '/health', '/nonexistent'] + + for endpoint in endpoints: + response = client.get(endpoint) + assert 'application/json' in response.content_type diff --git a/docs/LAB04.md b/docs/LAB04.md new file mode 100644 index 0000000000..8d30a35f13 --- /dev/null +++ b/docs/LAB04.md @@ -0,0 +1,279 @@ +# Lab 4 — Infrastructure as Code (Terraform & Pulumi) + +## 1. Cloud Provider & Infrastructure + +- **Cloud provider:** Yandex Cloud +- **Why chosen:** Available in Russia, has a free tier, straightforward setup via OAuth and service account. +- **Instance type:** 2 vCPU, 2 GB RAM (platform: standard-v1). Size chosen to be sufficient for a lab VM and future application deployment. +- **Region/zone:** `ru-central1-a` (default in variables; `yc` default zone was `ru-central1-b`). +- **Cost:** Within free tier / minimum tariff — 0 ₽ with correct usage. +- **Created resources:** + - `yandex_vpc_network.network` — network (terraform-network) + - `yandex_vpc_subnet.subnet` — subnet 10.0.0.0/24 in zone ru-central1-a + - `yandex_vpc_security_group.sg` — security group (SSH 22, HTTP 80, app 5000) + - `yandex_compute_instance.vm` — VM (Ubuntu 24.04 LTS, public IP) + +--- + +## 2. Terraform Implementation + +- **Terraform version:** v1.14.5 (darwin_arm64) +- **Provider:** yandex-cloud/yandex v0.187.0 + +### Project structure (directory `ydb_terraform/`) + +``` +ydb_terraform/ +├── .gitignore # state, .terraform/, terraform.tfvars, keys +├── main.tf # Network, subnet, security group, VM +├── provider.tf # required_providers, provider yandex +├── variables.tf # cloud_id, folder_id, zone, vm_name, image_id, ssh_user, public_key_path +├── outputs.tf # vm_public_ip +└── terraform.tfvars # variable values (not committed) +``` + +### Key decisions + +- Authentication via variables `cloud_id`, `folder_id`, and (optionally) environment variables or service account key file; secrets are not stored in code. +- Variables used for zone, VM name, `image_id`, SSH key path — configuration is reusable. +- Output `vm_public_ip` for quick SSH access. +- Security group: inbound SSH (22), HTTP (80), app port (5000); outbound traffic allowed. +- Added to `.gitignore`: `*.tfstate`, `*.tfstate.*`, `.terraform/`, `terraform.tfvars`, `*.pem`, `*.key`. + +### Challenges + +- Finding the right `image_id` for Ubuntu (used image list via `yc compute image list --folder-id standard-images`). +- Warning on `terraform init` about lock file for darwin_arm64 only — for CI on linux_amd64 run `terraform providers lock -platform=linux_amd64`. +- The plan includes the public SSH key in metadata — in this doc the plan output is shown in shortened/sanitized form. + +### Command output + +#### terraform init + +``` +Initializing the backend... +Initializing provider plugins... +- Finding latest version of yandex-cloud/yandex... +- Installing yandex-cloud/yandex v0.187.0... +- Installed yandex-cloud/yandex v0.187.0 (unauthenticated) +Terraform has created a lock file .terraform.lock.hcl to record the provider +selections it made above. Include this file in your version control repository +so that Terraform can guarantee to make the same selections by default when +you run "terraform init" in the future. + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. +``` + +#### terraform plan (abbreviated; secrets and full SSH key removed) + +``` +Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # yandex_compute_instance.vm will be created + + resource "yandex_compute_instance" "vm" { + + name = "terraform-vm" + + metadata = { + + "ssh-keys" = "ubuntu:" + } + + boot_disk { ... image_id = "fd80293ig2816a78q276" (ubuntu-2404-lts-oslogin) ... } + + network_interface { + nat = true ... } + + resources { + cores = 2, + memory = 2 } + } + + # yandex_vpc_network.network will be created + + resource "yandex_vpc_network" "network" { + name = "terraform-network" } + + # yandex_vpc_security_group.sg will be created + + resource "yandex_vpc_security_group" "sg" { + + name = "terraform-sg" + + ingress { description = "SSH", port = 22, protocol = "TCP", v4_cidr_blocks = ["0.0.0.0/0"] } + + ingress { description = "HTTP", port = 80, protocol = "TCP", v4_cidr_blocks = ["0.0.0.0/0"] } + + ingress { description = "App 5000", port = 5000, protocol = "TCP", v4_cidr_blocks = ["0.0.0.0/0"] } + + egress { protocol = "ANY", v4_cidr_blocks = ["0.0.0.0/0"] } + } + + # yandex_vpc_subnet.subnet will be created + + resource "yandex_vpc_subnet" "subnet" { + + name = "terraform-subnet" + + v4_cidr_blocks = ["10.0.0.0/24"] + + zone = "ru-central1-a" + } + +Plan: 4 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + vm_public_ip = (known after apply) +``` + +#### terraform apply (final output) + +``` +yandex_compute_instance.vm: Creation complete after 47s [id=fhm6b6ej125ta0nle31i] + +Apply complete! Resources: 4 added, 0 changed, 0 destroyed. + +Outputs: + +vm_public_ip = "84.201.132.65" +``` + +#### SSH connection to VM + +```bash +$ ssh ubuntu@84.201.132.65 +The authenticity of host '84.201.132.65 (84.201.132.65)' can't be established. +ED25519 key fingerprint is: SHA256:P/rIThvGihUqVuwtOIy9dr0c0UVuG3ZsimisnG1qHGs +Are you sure you want to continue connecting (yes/no/[fingerprint])? yes +Warning: Permanently added '84.201.132.65' (ED25519) to the list of known hosts. +Welcome to Ubuntu 24.04.1 LTS (GNU/Linux 6.8.0-41-generic x86_64) +... +ubuntu@fhm6b6ej125ta0nle31i:~$ whoami +ubuntu +ubuntu@fhm6b6ej125ta0nle31i:~$ hostname +fhm6b6ej125ta0nle31i +ubuntu@fhm6b6ej125ta0nle31i:~$ exit +logout +Connection to 84.201.132.65 closed. +``` + +**Connection command:** `ssh ubuntu@84.201.132.65` (IP may change after recreating resources; current value in `terraform output vm_public_ip`). + +--- + +## 3. Pulumi Implementation + +- **Pulumi version and language:** Pulumi 3.x, Python (runtime: python, virtualenv: venv). +- **Provider:** pulumi-yandex (Yandex Cloud). + +### Project structure (directory `pulumi/`) + +``` +pulumi/ +├── __main__.py # Network, subnet, security group, rules, VM, outputs +├── Pulumi.yaml # name, runtime (python + venv), config tags +├── requirements.txt # pulumi>=3.0.0,<4.0.0, pulumi-yandex +├── venv/ # virtual environment (in .gitignore) +└── Pulumi.dev.yaml # stack config for dev (folderId, serviceAccountKeyFile, sshPublicKey — do not commit secrets) +``` + +### How the code differs from Terraform + +- Infrastructure is described imperatively in Python: calls like `yandex.VpcNetwork(...)`, `yandex.VpcSubnet(...)`, etc.; dependencies are expressed via `network.id`, `subnet.id`, `sg.id`. +- Configuration: `pulumi.Config("yandex")` for `folderId` and service account key; SSH key in project config (`pulumi.Config().get("sshPublicKey")`) so the custom key is not passed to the provider (otherwise “Invalid or unknown key”). +- For security group rules in Pulumi Yandex the required parameter is `security_group_binding=sg.id` (not `security_group_id`). +- Same resources: VPC, subnet 10.0.0.0/24, security group (SSH 22, HTTP 80, app 5000), VM 2 vCPU / 2 GB RAM, Ubuntu 22.04 LTS, public IP. Output `public_ip` via `pulumi.export(...)`. + +### Advantages of Pulumi + +- Familiar language (Python): loops, conditionals, functions, types, and IDE autocomplete. +- Single file `__main__.py` for the whole infrastructure — convenient for a small lab. +- Secrets and stack config can be stored in Pulumi (including encrypted) and kept separate from provider code. + +### Challenges + +- Must explicitly pass `folder_id` to all Yandex resources (network, subnet, security group, VM); when missing — error “cannot determine folder_id”. +- Yandex quota on VPC count per folder: when hitting “Quota limit vpc.networks.count exceeded” — use an existing network or free up quota. +- SSH key for VM is set via `metadata={"ssh-keys": "ubuntu:"}`; without it — “Permission denied (publickey)”. Key is in project config, not under `yandex:`, so the provider does not fail on the unknown key. +- After first boot the VM may respond with “System is booting up...” on SSH — wait 1–2 minutes and retry the connection. + +### Output of `pulumi preview` and `pulumi up`, SSH connection + +#### pulumi preview (abbreviated) + +``` +Previewing update (dev) + + Type Name Plan + + pulumi:pulumi:Stack python_pulumi-dev create + + ├─ yandex:index:VpcNetwork lab-network create + + ├─ yandex:index:VpcSubnet lab-subnet create + + ├─ yandex:index:VpcSecurityGroup lab-sg create + + ├─ yandex:index:VpcSecurityGroupRule ssh-rule create + + ├─ yandex:index:VpcSecurityGroupRule http-rule create + + ├─ yandex:index:VpcSecurityGroupRule app-rule create + + └─ yandex:index:ComputeInstance lab-vm create + +Outputs: + public_ip: [unknown] + +Resources: + 8 to create +``` + +#### pulumi up (final output) + +``` +Do you want to perform this update? yes +Updating (dev) + + Type Name Status + + pulumi:pulumi:Stack python_pulumi-dev created + + ├─ yandex:index:VpcNetwork lab-network created + + ├─ yandex:index:VpcSubnet lab-subnet created + ... + +Outputs: + public_ip: "93.77.176.17" + +Resources: + 8 created +``` + +#### SSH connection to VM + +```bash +$ ssh ubuntu@93.77.176.17 +... +ubuntu@:~$ whoami +ubuntu +ubuntu@:~$ exit +``` + +**Connection command:** `ssh ubuntu@` (current IP in `pulumi stack output public_ip`). + +--- + +## 4. Terraform vs Pulumi Comparison + +- **Ease of Learning:** Terraform is easier to get started with: one HCL syntax, few concepts. Pulumi requires knowing a language (e.g. Python) but gives a familiar dev environment and types. +- **Code Readability:** For a linear set of resources both are readable. Terraform is declarative by blocks; Pulumi reads like a sequence of API calls, convenient for loops and conditional logic. +- **Debugging:** Pulumi is easier to debug: stack traces in the native language, logic in code. In Terraform errors come from the provider and state; debugging is often via plan/apply and documentation. +- **Documentation:** Terraform and its providers (including Yandex) are well documented; Pulumi Registry and provider examples exist, but the community and guides are smaller than Terraform’s. +- **Use Case (when Terraform, when Pulumi):** Terraform is the standard for “infrastructure as config”, large teams, multi-cloud, and many ready-made modules. Pulumi fits when you want to write infrastructure as code (loops, tests, reuse), integrate with application code in the same language, or handle complex resource logic. + +--- + +## 5. Lab 5 Preparation & Cleanup + +**VM for Lab 5:** +- Am I keeping the VM for Lab 5: **No** (all VMs stopped; will recreate from code when needed) +- Which VM I’m keeping: **recreate the cloud VM via Pulumi** (same `pulumi/` project). + +**Cleanup:** +- All resources destroyed on Yandex Cloud: `pulumi destroy`, and `terraform destroy`. +- No VMs running. State and code are kept locally so infrastructure can be recreated anytime. + +**How to bring infrastructure back (from existing files):** + +- **Pulumi:** + ```bash + cd pulumi + source venv/bin/activate + # Ensure config is set: yandex:folderId, yandex:serviceAccountKeyFile (or token), sshPublicKey + pulumi up + ``` + Then connect: `ssh ubuntu@$(pulumi stack output public_ip)`. + +- **Terraform:** + ```bash + cd ydb_terraform + terraform init + terraform apply + ``` + Then connect: `ssh ubuntu@$(terraform output -raw vm_public_ip)`. + + diff --git a/pulumi/.gitignore b/pulumi/.gitignore new file mode 100644 index 0000000000..a3807e5bdb --- /dev/null +++ b/pulumi/.gitignore @@ -0,0 +1,2 @@ +*.pyc +venv/ diff --git a/pulumi/Pulumi.dev.yaml b/pulumi/Pulumi.dev.yaml new file mode 100644 index 0000000000..eeb3086a76 --- /dev/null +++ b/pulumi/Pulumi.dev.yaml @@ -0,0 +1,4 @@ +config: + yandex:serviceAccountKeyFile: /Users/arinazimina/Downloads/authorized_key(1).json + yandex:folderId: b1gff0j67atu07bsqe14 + python_pulumi:sshPublicKey: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJrdGukPSOFXySoBrNeDTwqafjO8lx2IrM0GyzSycpDN arinazimina@arino4ka diff --git a/pulumi/Pulumi.yaml b/pulumi/Pulumi.yaml new file mode 100644 index 0000000000..7c52e3f280 --- /dev/null +++ b/pulumi/Pulumi.yaml @@ -0,0 +1,11 @@ +name: python_pulumi +description: A minimal Python Pulumi program +runtime: + name: python + options: + toolchain: pip + virtualenv: venv +config: + pulumi:tags: + value: + pulumi:template: python diff --git a/pulumi/__main__.py b/pulumi/__main__.py new file mode 100644 index 0000000000..76788a2502 --- /dev/null +++ b/pulumi/__main__.py @@ -0,0 +1,121 @@ +""" +Yandex Cloud resources via Pulumi. +Auth: either set token or service account key file before running: + pulumi config set yandex:token YOUR_TOKEN --secret + pulumi config set yandex:folderId YOUR_FOLDER_ID + # or key file: + pulumi config set yandex:serviceAccountKeyFile /path/to/key.json +""" +import pulumi +import pulumi_yandex as yandex + +config = pulumi.Config("yandex") +folder_id = config.require("folderId") + +ssh_public_key = pulumi.Config().get("sshPublicKey") or "" + +# --------------------------- +# Network +# --------------------------- +network = yandex.VpcNetwork( + "lab-network", + folder_id=folder_id, +) + +subnet = yandex.VpcSubnet( + "lab-subnet", + folder_id=folder_id, + zone="ru-central1-a", + network_id=network.id, + v4_cidr_blocks=["10.0.0.0/24"], +) + +# --------------------------- +# Security Group +# --------------------------- +sg = yandex.VpcSecurityGroup( + "lab-sg", + folder_id=folder_id, + network_id=network.id, +) + +# --------------------------- +# Security Group Rules +# --------------------------- +yandex.VpcSecurityGroupRule( + "ssh-rule", + security_group_binding=sg.id, + direction="ingress", + protocol="TCP", + port=22, + v4_cidr_blocks=["0.0.0.0/0"] +) + +yandex.VpcSecurityGroupRule( + "http-rule", + security_group_binding=sg.id, + direction="ingress", + protocol="TCP", + port=80, + v4_cidr_blocks=["0.0.0.0/0"] +) + +yandex.VpcSecurityGroupRule( + "app-rule", + security_group_binding=sg.id, + direction="ingress", + protocol="TCP", + port=5000, + v4_cidr_blocks=["0.0.0.0/0"] +) + +# App port 5001 (devops-info-service from Ansible deploy) +yandex.VpcSecurityGroupRule( + "app-5001-rule", + security_group_binding=sg.id, + direction="ingress", + protocol="TCP", + port=5001, + v4_cidr_blocks=["0.0.0.0/0"], +) + +# Egress: allow VM to reach internet (apt, Docker Hub, etc.) +yandex.VpcSecurityGroupRule( + "egress-all", + security_group_binding=sg.id, + direction="egress", + protocol="ANY", + v4_cidr_blocks=["0.0.0.0/0"], +) + +# --------------------------- +# VM +# --------------------------- +vm_metadata = {"ssh-keys": f"ubuntu:{ssh_public_key}"} if ssh_public_key else None +vm = yandex.ComputeInstance( + "lab-vm", + folder_id=folder_id, + zone="ru-central1-a", + resources=yandex.ComputeInstanceResourcesArgs( + cores=2, + memory=2, + ), + boot_disk=yandex.ComputeInstanceBootDiskArgs( + initialize_params=yandex.ComputeInstanceBootDiskInitializeParamsArgs( + image_id="fd80293ig2816a78q276", # Ubuntu 22.04 LTS + ), + ), + metadata=vm_metadata, + network_interfaces=[ + yandex.ComputeInstanceNetworkInterfaceArgs( + subnet_id=subnet.id, + nat=True, + security_group_ids=[sg.id], + ) + ], +) + +# --------------------------- +# Outputs +# --------------------------- +pulumi.export("public_ip", vm.network_interfaces[0].nat_ip_address) \ No newline at end of file diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt new file mode 100644 index 0000000000..4fcd3c0981 --- /dev/null +++ b/pulumi/requirements.txt @@ -0,0 +1,2 @@ +pulumi>=3.0.0,<4.0.0 +pulumi-yandex diff --git a/ydb_terraform/.gitignore b/ydb_terraform/.gitignore new file mode 100644 index 0000000000..82c68586e6 --- /dev/null +++ b/ydb_terraform/.gitignore @@ -0,0 +1,6 @@ +*.tfstate +*.tfstate.* +.terraform/ +terraform.tfvars +*.pem +*.key diff --git a/ydb_terraform/.terraform.lock.hcl b/ydb_terraform/.terraform.lock.hcl new file mode 100644 index 0000000000..690c5bbdd3 --- /dev/null +++ b/ydb_terraform/.terraform.lock.hcl @@ -0,0 +1,9 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/yandex-cloud/yandex" { + version = "0.187.0" + hashes = [ + "h1:+uf4EBRLDwNYIvZsGK/ZUzN3sGzJaXcUngyYSIJoyyQ=", + ] +} diff --git a/ydb_terraform/main.tf b/ydb_terraform/main.tf new file mode 100644 index 0000000000..1a243f437f --- /dev/null +++ b/ydb_terraform/main.tf @@ -0,0 +1,66 @@ +resource "yandex_vpc_network" "network" { + name = "terraform-network" +} + +resource "yandex_vpc_subnet" "subnet" { + name = "terraform-subnet" + zone = var.zone + network_id = yandex_vpc_network.network.id + v4_cidr_blocks = ["10.0.0.0/24"] +} + +resource "yandex_vpc_security_group" "sg" { + name = "terraform-sg" + network_id = yandex_vpc_network.network.id + + ingress { + protocol = "TCP" + description = "SSH" + v4_cidr_blocks = ["0.0.0.0/0"] + port = 22 + } + + ingress { + protocol = "TCP" + description = "HTTP" + v4_cidr_blocks = ["0.0.0.0/0"] + port = 80 + } + + ingress { + protocol = "TCP" + description = "App 5000" + v4_cidr_blocks = ["0.0.0.0/0"] + port = 5000 + } + + egress { + protocol = "ANY" + v4_cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "yandex_compute_instance" "vm" { + name = var.vm_name + + resources { + cores = 2 + memory = 2 + } + + boot_disk { + initialize_params { + image_id = var.image_id + } + } + + network_interface { + subnet_id = yandex_vpc_subnet.subnet.id + nat = true + security_group_ids = [yandex_vpc_security_group.sg.id] + } + + metadata = { + ssh-keys = "${var.ssh_user}:${file(var.public_key_path)}" + } +} diff --git a/ydb_terraform/outputs.tf b/ydb_terraform/outputs.tf new file mode 100644 index 0000000000..ad6e3a5b26 --- /dev/null +++ b/ydb_terraform/outputs.tf @@ -0,0 +1,4 @@ +output "vm_public_ip" { + description = "Public IP address" + value = yandex_compute_instance.vm.network_interface[0].nat_ip_address +} diff --git a/ydb_terraform/provider.tf b/ydb_terraform/provider.tf new file mode 100644 index 0000000000..9514396fda --- /dev/null +++ b/ydb_terraform/provider.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + yandex = { + source = "yandex-cloud/yandex" + version = "~> 0.180" + } + } +} + +provider "yandex" { + cloud_id = var.cloud_id + folder_id = var.folder_id + zone = var.zone +} diff --git a/ydb_terraform/variables.tf b/ydb_terraform/variables.tf new file mode 100644 index 0000000000..cf983f6e90 --- /dev/null +++ b/ydb_terraform/variables.tf @@ -0,0 +1,37 @@ +variable "cloud_id" { + description = "Yandex Cloud ID" + type = string +} + +variable "folder_id" { + description = "Yandex Folder ID" + type = string +} + +variable "zone" { + description = "Zone" + type = string + default = "ru-central1-a" +} + +variable "vm_name" { + description = "VM name" + type = string + default = "terraform-vm" +} + +variable "image_id" { + description = "Ubuntu image ID" + type = string +} + +variable "ssh_user" { + description = "SSH user" + type = string + default = "ubuntu" +} + +variable "public_key_path" { + description = "Path to SSH public key" + type = string +}