fix conflicts

Author: JATAYU000

openml/_api/clients/http.py

@@ -2,20 +2,29 @@
 
 import hashlib
 import json
+import logging
+import math
+import random
 import time
-from collections.abc import Callable
+import xml
+from collections.abc import Callable, Mapping
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
 from urllib.parse import urlencode, urljoin, urlparse
 
 import requests
+import xmltodict
 from requests import Response
 
 from openml.__version__ import __version__
-from openml.exceptions import OpenMLHashException
-
-if TYPE_CHECKING:
-    from openml._api.config import DelayMethod
+from openml._api.config import RetryPolicy
+from openml.exceptions import (
+    OpenMLHashException,
+    OpenMLNotAuthorizedError,
+    OpenMLServerError,
+    OpenMLServerException,
+    OpenMLServerNoResult,
+)
 
 
 class HTTPCache:
@@ -111,21 +120,202 @@ def __init__(  # noqa: PLR0913
         api_key: str,
         timeout: int,
         retries: int,
-        delay_method: DelayMethod,
-        delay_time: int,
+        retry_policy: RetryPolicy,
         cache: HTTPCache | None = None,
     ) -> None:
         self.server = server
         self.base_url = base_url
         self.api_key = api_key
         self.timeout = timeout
         self.retries = retries
-        self.delay_method = delay_method
-        self.delay_time = delay_time
+        self.retry_policy = retry_policy
         self.cache = cache
 
+        self.retry_func = (
+            self._human_delay if retry_policy == RetryPolicy.HUMAN else self._robot_delay
+        )
         self.headers: dict[str, str] = {"user-agent": f"openml-python/{__version__}"}
 
+    def _robot_delay(self, n: int) -> float:
+        wait = (1 / (1 + math.exp(-(n * 0.5 - 4)))) * 60
+        variation = random.gauss(0, wait / 10)
+        return max(1.0, wait + variation)
+
+    def _human_delay(self, n: int) -> float:
+        return max(1.0, n)
+
+    def _parse_exception_response(
+        self,
+        response: Response,
+    ) -> tuple[int | None, str]:
+        content_type = response.headers.get("Content-Type", "").lower()
+
+        if "json" in content_type:
+            server_exception = response.json()
+            server_error = server_exception["detail"]
+            code = server_error.get("code")
+            message = server_error.get("message")
+            additional_information = server_error.get("additional_information")
+        else:
+            server_exception = xmltodict.parse(response.text)
+            server_error = server_exception["oml:error"]
+            code = server_error.get("oml:code")
+            message = server_error.get("oml:message")
+            additional_information = server_error.get("oml:additional_information")
+
+        if code is not None:
+            code = int(code)
+
+        if message and additional_information:
+            full_message = f"{message} - {additional_information}"
+        elif message:
+            full_message = message
+        elif additional_information:
+            full_message = additional_information
+        else:
+            full_message = ""
+
+        return code, full_message
+
+    def _raise_code_specific_error(
+        self,
+        code: int,
+        message: str,
+        url: str,
+        files: Mapping[str, Any] | None,
+    ) -> None:
+        if code in [111, 372, 512, 500, 482, 542, 674]:
+            # 512 for runs, 372 for datasets, 500 for flows
+            # 482 for tasks, 542 for evaluations, 674 for setups
+            # 111 for dataset descriptions
+            raise OpenMLServerNoResult(code=code, message=message, url=url)
+
+        # 163: failure to validate flow XML (https://www.openml.org/api_docs#!/flow/post_flow)
+        if code in [163] and files is not None and "description" in files:
+            # file_elements['description'] is the XML file description of the flow
+            message = f"\n{files['description']}\n{message}"
+
+        if code in [
+            102,  # flow/exists post
+            137,  # dataset post
+            350,  # dataset/42 delete
+            310,  # flow/<something> post
+            320,  # flow/42 delete
+            400,  # run/42 delete
+            460,  # task/42 delete
+        ]:
+            raise OpenMLNotAuthorizedError(
+                message=(
+                    f"The API call {url} requires authentication via an API key.\nPlease configure "
+                    "OpenML-Python to use your API as described in this example:"
+                    "\nhttps://openml.github.io/openml-python/latest/examples/Basics/introduction_tutorial/#authentication"
+                )
+            )
+
+        # Propagate all server errors to the calling functions, except
+        # for 107 which represents a database connection error.
+        # These are typically caused by high server load,
+        # which means trying again might resolve the issue.
+        # DATABASE_CONNECTION_ERRCODE
+        if code != 107:
+            raise OpenMLServerException(code=code, message=message, url=url)
+
+    def _validate_response(
+        self,
+        method: str,
+        url: str,
+        files: Mapping[str, Any] | None,
+        response: Response,
+    ) -> Exception | None:
+        if (
+            "Content-Encoding" not in response.headers
+            or response.headers["Content-Encoding"] != "gzip"
+        ):
+            logging.warning(f"Received uncompressed content from OpenML for {url}.")
+
+        if response.status_code == 200:
+            return None
+
+        if response.status_code == requests.codes.URI_TOO_LONG:
+            raise OpenMLServerError(f"URI too long! ({url})")
+
+        retry_raise_e: Exception | None = None
+
+        try:
+            code, message = self._parse_exception_response(response)
+
+        except (requests.exceptions.JSONDecodeError, xml.parsers.expat.ExpatError) as e:
+            if method != "GET":
+                extra = f"Status code: {response.status_code}\n{response.text}"
+                raise OpenMLServerError(
+                    f"Unexpected server error when calling {url}. Please contact the "
+                    f"developers!\n{extra}"
+                ) from e
+
+            retry_raise_e = e
+
+        except Exception as e:
+            # If we failed to parse it out,
+            # then something has gone wrong in the body we have sent back
+            # from the server and there is little extra information we can capture.
+            raise OpenMLServerError(
+                f"Unexpected server error when calling {url}. Please contact the developers!\n"
+                f"Status code: {response.status_code}\n{response.text}",
+            ) from e
+
+        if code is not None:
+            self._raise_code_specific_error(
+                code=code,
+                message=message,
+                url=url,
+                files=files,
+            )
+
+        if retry_raise_e is None:
+            retry_raise_e = OpenMLServerException(code=code, message=message, url=url)
+
+        return retry_raise_e
+
+    def _request(  # noqa: PLR0913
+        self,
+        method: str,
+        url: str,
+        params: Mapping[str, Any],
+        headers: Mapping[str, str],
+        timeout: float | int,
+        files: Mapping[str, Any] | None,
+        **request_kwargs: Any,
+    ) -> tuple[Response | None, Exception | None]:
+        retry_raise_e: Exception | None = None
+        response: Response | None = None
+
+        try:
+            response = requests.request(
+                method=method,
+                url=url,
+                params=params,
+                headers=headers,
+                timeout=timeout,
+                files=files,
+                **request_kwargs,
+            )
+        except (
+            requests.exceptions.ChunkedEncodingError,
+            requests.exceptions.ConnectionError,
+            requests.exceptions.SSLError,
+        ) as e:
+            retry_raise_e = e
+
+        if response is not None:
+            retry_raise_e = self._validate_response(
+                method=method,
+                url=url,
+                files=files,
+                response=response,
+            )
+
+        return response, retry_raise_e
+
     def request(
         self,
         method: str,
@@ -137,6 +327,7 @@ def request(
         **request_kwargs: Any,
     ) -> Response:
         url = urljoin(self.server, urljoin(self.base_url, path))
+        retries = max(1, self.retries)
 
         # prepare params
         params = request_kwargs.pop("params", {}).copy()
@@ -148,6 +339,9 @@ def request(
         headers.update(self.headers)
 
         timeout = request_kwargs.pop("timeout", self.timeout)
+        files = request_kwargs.pop("files", None)
+
+        use_cache = False
 
         if use_cache and self.cache is not None:
             cache_key = self.cache.get_key(url, params)
@@ -158,14 +352,28 @@ def request(
             except Exception:
                 raise  # propagate unexpected cache errors
 
-        response = requests.request(
-            method=method,
-            url=url,
-            params=params,
-            headers=headers,
-            timeout=timeout,
-            **request_kwargs,
-        )
+        for retry_counter in range(1, retries + 1):
+            response, retry_raise_e = self._request(
+                method=method,
+                url=url,
+                params=params,
+                headers=headers,
+                timeout=timeout,
+                files=files,
+                **request_kwargs,
+            )
+
+            # executed successfully
+            if retry_raise_e is None:
+                break
+            # tries completed
+            if retry_counter >= retries:
+                raise retry_raise_e
+
+            delay = self.retry_func(retry_counter)
+            time.sleep(delay)
+
+        assert response is not None
 
         if md5_checksum is not None:
             self._verify_checksum(response, md5_checksum)

openml/_api/config.py

@@ -4,7 +4,7 @@
 from enum import Enum
 
 
-class DelayMethod(str, Enum):
+class RetryPolicy(str, Enum):
     HUMAN = "human"
     ROBOT = "robot"
 
@@ -26,8 +26,7 @@ class APISettings:
 @dataclass
 class ConnectionConfig:
     retries: int = 3
-    delay_method: DelayMethod = DelayMethod.HUMAN
-    delay_time: int = 1  # seconds
+    retry_policy: RetryPolicy = RetryPolicy.HUMAN
 
 
 @dataclass

openml/_api/resources/base/__init__.py

@@ -0,0 +1,13 @@
+from openml._api.resources.base.base import APIVersion, ResourceAPI, ResourceType
+from openml._api.resources.base.resources import DatasetsAPI, TasksAPI
+from openml._api.resources.base.versions import ResourceV1, ResourceV2
+
+__all__ = [
+    "APIVersion",
+    "DatasetsAPI",
+    "ResourceAPI",
+    "ResourceType",
+    "ResourceV1",
+    "ResourceV2",
+    "TasksAPI",
+]

openml/_api/resources/base/base.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from openml._api.clients import HTTPClient
+
+
+class APIVersion(str, Enum):
+    V1 = "v1"
+    V2 = "v2"
+
+
+class ResourceType(str, Enum):
+    DATASET = "dataset"
+    TASK = "task"
+    TASK_TYPE = "task_type"
+    EVALUATION_MEASURE = "evaluation_measure"
+    ESTIMATION_PROCEDURE = "estimation_procedure"
+    EVALUATION = "evaluation"
+    FLOW = "flow"
+    STUDY = "study"
+    RUN = "run"
+    SETUP = "setup"
+    USER = "user"
+
+
+class ResourceAPI(ABC):
+    api_version: APIVersion
+    resource_type: ResourceType
+
+    def __init__(self, http: HTTPClient):
+        self._http = http
+
+    def _get_not_implemented_message(self, method_name: str | None = None) -> str:
+        version = getattr(self.api_version, "name", "Unknown version")
+        resource = getattr(self.resource_type, "name", "Unknown resource")
+        method_info = f" Method: {method_name}" if method_name else ""
+        return (
+            f"{self.__class__.__name__}: {version} API does not support this "
+            f"functionality for resource: {resource}.{method_info}"
+        )
+
+    @abstractmethod
+    def delete(self, resource_id: int) -> bool: ...
+
+    @abstractmethod
+    def publish(self) -> None: ...