From 9498dbe61afe7d47aeb94b5e9aa29f835d57f962 Mon Sep 17 00:00:00 2001 From: AHarmlessPyro Date: Sat, 30 Dec 2023 17:16:22 -0600 Subject: [PATCH 1/6] Add custom request timeout --- Dockerfile | 3 +++ src/handler.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 728a1a9..da975c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,12 @@ ARG COG_REPO ARG COG_MODEL ARG COG_VERSION +ARG REQUEST_TIMEOUT=600 FROM r8.im/${COG_REPO}/${COG_MODEL}@sha256:${COG_VERSION} +ENV REQUEST_TIMEOUT=${REQUEST_TIMEOUT} + # Install necessary packages and Python 3.10 RUN apt-get update && apt-get upgrade -y && \ apt-get install -y --no-install-recommends software-properties-common curl git openssh-server && \ diff --git a/src/handler.py b/src/handler.py index 669f2e0..031f6c4 100644 --- a/src/handler.py +++ b/src/handler.py @@ -1,5 +1,6 @@ import time import subprocess +import os import runpod import requests @@ -46,7 +47,8 @@ def run_inference(inference_request): Run inference on a request. ''' response = cog_session.post(url=f'{LOCAL_URL}/predictions', - json=inference_request, timeout=600) + json=inference_request, + timeout=int(os.environ.get("REQUEST_TIMEOUT",600))) return response.json() From c9f5718f5abb12d9c50cadd9237cfd5f45a22b8a Mon Sep 17 00:00:00 2001 From: AHarmlessPyro Date: Sat, 30 Dec 2023 19:04:46 -0600 Subject: [PATCH 2/6] Move from arg to env --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index da975c6..9ddd2f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,10 @@ ARG COG_REPO ARG COG_MODEL ARG COG_VERSION -ARG REQUEST_TIMEOUT=600 FROM r8.im/${COG_REPO}/${COG_MODEL}@sha256:${COG_VERSION} -ENV REQUEST_TIMEOUT=${REQUEST_TIMEOUT} +ENV REQUEST_TIMEOUT=600 # Install necessary packages and Python 3.10 RUN apt-get update && apt-get upgrade -y && \ From ca88eecd962ac075b3536b0992d93f7097851b68 Mon Sep 17 00:00:00 2001 From: AHarmlessPyro Date: Sat, 30 Dec 2023 19:08:39 -0600 Subject: [PATCH 3/6] Move arg down --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9ddd2f7..f12c0b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ ARG COG_VERSION FROM r8.im/${COG_REPO}/${COG_MODEL}@sha256:${COG_VERSION} -ENV REQUEST_TIMEOUT=600 +ARG REQUEST_TIMEOUT=600 +ENV REQUEST_TIMEOUT=${REQUEST_TIMEOUT} # Install necessary packages and Python 3.10 RUN apt-get update && apt-get upgrade -y && \ From 02df0173d60757c3479b1d2dd5231cc6b554c004 Mon Sep 17 00:00:00 2001 From: AHarmlessPyro Date: Sat, 30 Dec 2023 19:44:10 -0600 Subject: [PATCH 4/6] List timeout details on build --- Dockerfile | 3 +-- src/handler.py | 31 ++++++++++++++++++++----------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index f12c0b1..9ddd2f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,7 @@ ARG COG_VERSION FROM r8.im/${COG_REPO}/${COG_MODEL}@sha256:${COG_VERSION} -ARG REQUEST_TIMEOUT=600 -ENV REQUEST_TIMEOUT=${REQUEST_TIMEOUT} +ENV REQUEST_TIMEOUT=600 # Install necessary packages and Python 3.10 RUN apt-get update && apt-get upgrade -y && \ diff --git a/src/handler.py b/src/handler.py index 031f6c4..898b5b7 100644 --- a/src/handler.py +++ b/src/handler.py @@ -6,11 +6,13 @@ import requests from requests.adapters import HTTPAdapter, Retry +TIMEOUT = int(os.environ.get("REQUEST_TIMEOUT", "600")) + LOCAL_URL = "http://127.0.0.1:5000" cog_session = requests.Session() retries = Retry(total=10, backoff_factor=0.1, status_forcelist=[502, 503, 504]) -cog_session.mount('http://', HTTPAdapter(max_retries=retries)) +cog_session.mount("http://", HTTPAdapter(max_retries=retries)) # ----------------------------- Start API Service ---------------------------- # @@ -22,9 +24,9 @@ # Automatic Functions # # ---------------------------------------------------------------------------- # def wait_for_service(url): - ''' + """ Check if the service is ready to receive requests. - ''' + """ while True: try: health = requests.get(url, timeout=120) @@ -43,12 +45,17 @@ def wait_for_service(url): def run_inference(inference_request): - ''' + """ Run inference on a request. - ''' - response = cog_session.post(url=f'{LOCAL_URL}/predictions', - json=inference_request, - timeout=int(os.environ.get("REQUEST_TIMEOUT",600))) + """ + response = cog_session.post( + url=f"{LOCAL_URL}/predictions", + json=inference_request, + timeout=int(os.environ.get("REQUEST_TIMEOUT", 600)), + ) + if response.status_code != 200: + print(response.status_code) + print(response.text) return response.json() @@ -56,9 +63,9 @@ def run_inference(inference_request): # RunPod Handler # # ---------------------------------------------------------------------------- # def handler(event): - ''' + """ This is the handler function that will be called by the serverless. - ''' + """ json = run_inference({"input": event["input"]}) @@ -66,8 +73,10 @@ def handler(event): if __name__ == "__main__": - wait_for_service(url=f'{LOCAL_URL}/health-check') + wait_for_service(url=f"{LOCAL_URL}/health-check") print("Cog API Service is ready. Starting RunPod serverless handler...") + if TIMEOUT != 600: + print(f"Using non-default timeout of {TIMEOUT}s") runpod.serverless.start({"handler": handler}) From 78b63d9e78e70a9a3dd28bba1043b7b7972ce24a Mon Sep 17 00:00:00 2001 From: AHarmlessPyro Date: Sat, 30 Dec 2023 20:13:11 -0600 Subject: [PATCH 5/6] more debug info --- src/handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/handler.py b/src/handler.py index 898b5b7..01fda8f 100644 --- a/src/handler.py +++ b/src/handler.py @@ -56,6 +56,7 @@ def run_inference(inference_request): if response.status_code != 200: print(response.status_code) print(response.text) + print(response.text) return response.json() @@ -76,7 +77,6 @@ def handler(event): wait_for_service(url=f"{LOCAL_URL}/health-check") print("Cog API Service is ready. Starting RunPod serverless handler...") - if TIMEOUT != 600: - print(f"Using non-default timeout of {TIMEOUT}s") + print(f"Using request timeout of {TIMEOUT}s") runpod.serverless.start({"handler": handler}) From 8d8965697fe833839e87dce651b54c4b27e65798 Mon Sep 17 00:00:00 2001 From: AHarmlessPyro Date: Sat, 30 Dec 2023 20:21:03 -0600 Subject: [PATCH 6/6] Change env arg name --- src/handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/handler.py b/src/handler.py index 01fda8f..4a73942 100644 --- a/src/handler.py +++ b/src/handler.py @@ -6,7 +6,7 @@ import requests from requests.adapters import HTTPAdapter, Retry -TIMEOUT = int(os.environ.get("REQUEST_TIMEOUT", "600")) +TIMEOUT = int(os.environ.get("_RUNPOD_REQUEST_TIMEOUT", "600")) LOCAL_URL = "http://127.0.0.1:5000" @@ -51,7 +51,7 @@ def run_inference(inference_request): response = cog_session.post( url=f"{LOCAL_URL}/predictions", json=inference_request, - timeout=int(os.environ.get("REQUEST_TIMEOUT", 600)), + timeout=TIMEOUT, ) if response.status_code != 200: print(response.status_code)