From 2bc1d1ed7c9405dd5a9b0ba9ebae78491a403cd5 Mon Sep 17 00:00:00 2001 From: GabrielPalmar Date: Fri, 11 Jul 2025 13:46:26 -0500 Subject: [PATCH 1/4] feat(valkey): Added Valkey support for caching layer --- Dockerfile | 5 ++++- README.md | 0 app/opensense.py | 56 +++++++++++++++++++++++++++++++++++++++++----- k8s/deployment.yml | 28 ++++++++++++++++++++--- requirements.in | 3 ++- requirements.txt | 20 ++++++++++------- 6 files changed, 94 insertions(+), 18 deletions(-) create mode 100644 README.md diff --git a/Dockerfile b/Dockerfile index 10ec59e..b529ce0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,10 @@ RUN pip install --no-cache-dir -r /app/requirements.txt --require-hashes && \ chown -R appuser:appgroup /app ENV FLASK_APP=app.main.py:app \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + REDIS_HOST=localhost \ + REDIS_PORT=6379 \ + CACHE_TTL=300 USER appuser diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/app/opensense.py b/app/opensense.py index c89cc56..6938fbb 100644 --- a/app/opensense.py +++ b/app/opensense.py @@ -1,10 +1,46 @@ '''Module to get entries from OpenSenseMap API and get the average temperature''' from datetime import datetime, timezone, timedelta +import os import requests +import redis +REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') +REDIS_PORT = int(os.environ.get('REDIS_PORT', 6379)) +REDIS_DB = int(os.environ.get('REDIS_DB', 0)) +CACHE_TTL = int(os.environ.get('CACHE_TTL', 300)) + +try: + redis_client = redis.StrictRedis( + host=REDIS_HOST, + port=REDIS_PORT, + db=REDIS_DB, + decode_responses=True, + socket_connect_timeout=240, + socket_timeout=240 + ) + + redis_client.ping() + REDIS_AVAILABLE = True + print("Connected to Redis successfully!") +except (redis.ConnectionError, redis.TimeoutError) as e: + REDIS_AVAILABLE = False + print("Could not connect to Redis.") def get_temperature(): '''Function to get the average temperature from OpenSenseMap API.''' + cache_key = "temperature_data" + + if REDIS_AVAILABLE: + try: + cached_data = redis_client.get(cache_key) + if cached_data: + print("Using cached data from Redis.") + return cached_data + except redis.RedisError as e: + print(f"Redis error: {e}. Proceeding without cache.") + + print("Fetching new data from OpenSenseMap API...") + # Ensuring that data is not older than 1 hour. subs_time = datetime.now(timezone.utc) - timedelta(hours=1) time_iso = subs_time.isoformat().replace("+00:00", "Z") @@ -35,8 +71,18 @@ def get_temperature(): total_sum = sum(temp_list) average = total_sum / len(temp_list) if temp_list else 0 - if average < 10: - return f'Average temperature: {average:.2f} °C (Warning: Too cold)\n' - if 10 < average <= 36: - return f'Average temperature: {average:.2f} °C (Good)\n' - return f'Average temperature: {average:.2f} °C (Warning: Too hot)\n' + if average <= 10: + result = f'Average temperature: {average:.2f} °C (Warning: Too cold)\n' + elif 10 < average <= 36: + result = f'Average temperature: {average:.2f} °C (Good)\n' + else: + result = f'Average temperature: {average:.2f} °C (Warning: Too hot)\n' + + if REDIS_AVAILABLE: + try: + redis_client.setex(cache_key, CACHE_TTL, result) + print("Data cached in Redis.") + except redis.RedisError as e: + print(f"Redis error while caching data: {e}") + + return result diff --git a/k8s/deployment.yml b/k8s/deployment.yml index 3e88205..b78c095 100644 --- a/k8s/deployment.yml +++ b/k8s/deployment.yml @@ -26,9 +26,6 @@ spec: image: ghcr.io/gabrielpalmar/hivebox:0.4.0@sha256:31dccc066ffd02ef65850ed8125fc2dadf0bd65958fb49bee0517e40afab2e1c ports: - containerPort: 5000 - env: - - name: FLASK_ENV - value: "production" securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -63,6 +60,31 @@ spec: volumeMounts: - name: tmp-volume mountPath: /tmp + - name: valkey + image: valkey/valkey:8-alpine3.22@sha256:0d27f0bca0249f61d060029a6aaf2e16b2c417d68d02a508e1dfb763fa2948b4 + ports: + - containerPort: 6379 + command: ["valkey-server"] + args: ["--save", "", "--appendonly", "no"] + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + resources: + limits: + memory: "256Mi" + cpu: "250m" + requests: + memory: "128Mi" + cpu: "100m" + volumeMounts: + - name: valkey-data + mountPath: /data volumes: - name: tmp-volume emptyDir: {} + - name: valkey-data + emptyDir: {} diff --git a/requirements.in b/requirements.in index 24b01e9..01b49b0 100644 --- a/requirements.in +++ b/requirements.in @@ -1,3 +1,4 @@ Flask==3.1.1 requests==2.32.4 -prometheus-client==0.22.1 \ No newline at end of file +prometheus-client==0.22.1 +redis==6.2.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index bf7c886..b74be37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,15 +2,15 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in +# pip-compile --generate-hashes --output-file=requirements.txt requirements.in # blinker==1.9.0 \ --hash=sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf \ --hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc # via flask -certifi==2025.4.26 \ - --hash=sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6 \ - --hash=sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3 +certifi==2025.7.9 \ + --hash=sha256:c1d2ec05395148ee10cf672ffc28cd37ea0ab0d99f9cc74c43e588cbd111b079 \ + --hash=sha256:d842783a14f8fdd646895ac26f719a061408834473cfc10203f6a575beb15d39 # via requests charset-normalizer==3.4.2 \ --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \ @@ -196,15 +196,19 @@ prometheus-client==0.22.1 \ --hash=sha256:190f1331e783cf21eb60bca559354e0a4d4378facecf78f5428c39b675d20d28 \ --hash=sha256:cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094 # via -r requirements.in +redis==6.2.0 \ + --hash=sha256:c8ddf316ee0aab65f04a11229e94a64b2618451dab7a67cb2f77eb799d872d5e \ + --hash=sha256:e821f129b75dde6cb99dd35e5c76e8c49512a5a0d8dfdc560b2fbd44b85ca977 + # via -r requirements.in requests==2.32.4 \ --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 # via -r requirements.in -urllib3==2.4.0 \ - --hash=sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466 \ - --hash=sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813 +urllib3==2.5.0 \ + --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ + --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc # via requests werkzeug==3.1.3 \ --hash=sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e \ --hash=sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746 - # via flask \ No newline at end of file + # via flask From ae2a7d1230955aa308e2af8e4a42c8d3f2617915 Mon Sep 17 00:00:00 2001 From: GabrielPalmar Date: Mon, 14 Jul 2025 11:43:00 -0500 Subject: [PATCH 2/4] fix(docs): Adding README file and resolving pylint issues. --- .github/workflows/pylint.yml | 1 + README.md | 30 ++++++++++++++++++++++++++++++ app/opensense.py | 11 ++++------- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 98ebc38..7af33ff 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -31,6 +31,7 @@ jobs: pip install requests pip install vcrpy pip install prometheus_client + pip install redis - name: Analysing the code with pylint run: | # Set PYTHONPATH so pylint can find the app module diff --git a/README.md b/README.md index e69de29..d037a33 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,30 @@ +# HiveBox Project + +## Introduction + +DevOps project that includes most used technologies in the industry. Focusing in the CI and CD tools and integrating them simultaneously to achieve topics as the following: + +- Software Production. +- Agile Planning. +- QA and Quality Gates. +- Code and Programming. +- Operating System. +- Docker Containers. +- Kubernetes and Cloud. +- Observability and Monitoring. +- Continuous Integration/Delivery/Deployment. +- Automation and Infrastructure as Code. + +For more information please refer to the Project webpage: [HiveBox](https://devopsroadmap.io/projects/hivebox/) + +## Technologies Used +- Python (Flask, Prometheus, Redis, Requests) +- Docker +- Kubernetes +- GitHub Actions +- SonarQube +- Terrascan + +## Development + +To adhere to industry standards, the Repository's Project section was used, leveraging the Kanban board to process tasks in an organized manner. \ No newline at end of file diff --git a/app/opensense.py b/app/opensense.py index 6938fbb..6a5960a 100644 --- a/app/opensense.py +++ b/app/opensense.py @@ -7,7 +7,7 @@ REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') REDIS_PORT = int(os.environ.get('REDIS_PORT', 6379)) REDIS_DB = int(os.environ.get('REDIS_DB', 0)) -CACHE_TTL = int(os.environ.get('CACHE_TTL', 300)) +CACHE_TTL = int(os.environ.get('CACHE_TTL', 300)) try: redis_client = redis.StrictRedis( @@ -29,7 +29,6 @@ def get_temperature(): '''Function to get the average temperature from OpenSenseMap API.''' cache_key = "temperature_data" - if REDIS_AVAILABLE: try: cached_data = redis_client.get(cache_key) @@ -42,10 +41,7 @@ def get_temperature(): print("Fetching new data from OpenSenseMap API...") # Ensuring that data is not older than 1 hour. - subs_time = datetime.now(timezone.utc) - timedelta(hours=1) - time_iso = subs_time.isoformat().replace("+00:00", "Z") - - api_endpoint = "https://api.opensensemap.org/boxes" + time_iso = datetime.now(timezone.utc) - timedelta(hours=1).isoformat().replace("+00:00", "Z") params = { "date": time_iso, @@ -53,7 +49,8 @@ def get_temperature(): } print('Getting data from OpenSenseMap API...') - response = requests.get(api_endpoint, params=params, timeout=240) + + response = requests.get("https://api.opensensemap.org/boxes", params=params, timeout=240) print('Data retrieved successfully!') res = [d.get('sensors') for d in response.json() if 'sensors' in d] From 104a19dbb0160f954219faff82c8aba66018af6b Mon Sep 17 00:00:00 2001 From: GabrielPalmar Date: Mon, 14 Jul 2025 12:04:40 -0500 Subject: [PATCH 3/4] chore(tests): Added test support for cached entries --- README.md | 15 ++++++++++----- app/opensense.py | 2 +- tests/test_modules.py | 18 +++++++++++++++++- version.txt | 2 +- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d037a33..c556a8a 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,16 @@ For more information please refer to the Project webpage: [HiveBox](https://devo ## Technologies Used - Python (Flask, Prometheus, Redis, Requests) - Docker -- Kubernetes -- GitHub Actions -- SonarQube -- Terrascan +- Kubernetes (Minikube, Kind) +- Dependabot +- GitHub Actions: + - SonarQube + - Terrascan + - Pylint + - Hadolint ## Development -To adhere to industry standards, the Repository's Project section was used, leveraging the Kanban board to process tasks in an organized manner. \ No newline at end of file +To adhere to industry standards, the Repository's Project section was used, leveraging the Kanban board to process tasks in an organized manner. + +[Kanban Board](https://github.com/users/GabrielPalmar/projects/1) \ No newline at end of file diff --git a/app/opensense.py b/app/opensense.py index 6a5960a..3cbab8c 100644 --- a/app/opensense.py +++ b/app/opensense.py @@ -41,7 +41,7 @@ def get_temperature(): print("Fetching new data from OpenSenseMap API...") # Ensuring that data is not older than 1 hour. - time_iso = datetime.now(timezone.utc) - timedelta(hours=1).isoformat().replace("+00:00", "Z") + time_iso = (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat().replace("+00:00", "Z") params = { "date": time_iso, diff --git a/tests/test_modules.py b/tests/test_modules.py index f915c06..98bf594 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -26,7 +26,6 @@ def test_metrics_endpoint(): response = client.get('/metrics') assert response.status_code == 200 -# Add a simple test for the opensense module def test_opensense_get_temperature(): """Test that opensense.get_temperature returns a string""" result = opensense.get_temperature() @@ -63,3 +62,20 @@ def test_opensense_get_temperature_too_hot(): with mock.patch('app.opensense.requests.get', return_value=mock_response(40)): result = opensense.get_temperature() assert 'Too hot' in result + +def test_opensense_cache_get(): + """Test that cached data is used if available""" + with mock.patch('app.opensense.redis_client.get', return_value="cached_result"), \ + mock.patch('app.opensense.requests.get') as mock_requests: + result = opensense.get_temperature() + assert result == "cached_result" + mock_requests.assert_not_called() + +def test_opensense_cache_setex(): + """Test that data is cached after fetching""" + with mock.patch('app.opensense.redis_client.get', return_value=None), \ + mock.patch('app.opensense.redis_client.setex') as mock_setex, \ + mock.patch('app.opensense.requests.get', return_value=mock_response(25)): + result = opensense.get_temperature() + assert 'Average temperature' in result + mock_setex.assert_called() diff --git a/version.txt b/version.txt index 44bb5d1..79a2734 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.4.1 \ No newline at end of file +0.5.0 \ No newline at end of file From 6ac22acbdb49331fe13e3fca8bee983dd4f98700 Mon Sep 17 00:00:00 2001 From: GabrielPalmar Date: Mon, 14 Jul 2025 12:13:36 -0500 Subject: [PATCH 4/4] fix(ci): Resolving cached entries test --- tests/test_modules.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 98bf594..9430757 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -65,7 +65,8 @@ def test_opensense_get_temperature_too_hot(): def test_opensense_cache_get(): """Test that cached data is used if available""" - with mock.patch('app.opensense.redis_client.get', return_value="cached_result"), \ + with mock.patch('app.opensense.REDIS_AVAILABLE', True), \ + mock.patch('app.opensense.redis_client.get', return_value="cached_result"), \ mock.patch('app.opensense.requests.get') as mock_requests: result = opensense.get_temperature() assert result == "cached_result" @@ -73,7 +74,8 @@ def test_opensense_cache_get(): def test_opensense_cache_setex(): """Test that data is cached after fetching""" - with mock.patch('app.opensense.redis_client.get', return_value=None), \ + with mock.patch('app.opensense.REDIS_AVAILABLE', True), \ + mock.patch('app.opensense.redis_client.get', return_value=None), \ mock.patch('app.opensense.redis_client.setex') as mock_setex, \ mock.patch('app.opensense.requests.get', return_value=mock_response(25)): result = opensense.get_temperature()