experimental model framework

Author: fkiraly

openml/__init__.py

@@ -48,6 +48,7 @@
     OpenMLSupervisedTask,
     OpenMLTask,
 )
+from openml._get import get
 
 
 def populate_cache(
@@ -120,4 +121,5 @@ def populate_cache(
     "utils",
     "_api_calls",
     "__version__",
+    "get",
 ]

openml/_get.py

@@ -0,0 +1,9 @@
+"""Global get dispatch utility."""
+
+# currently just a forward to models
+# to discuss and possibly
+# todo: add global get utility here
+# in general, e.g., datasets will not have same name as models etc
+from openml.models import get
+
+__all__ = ["get"]

openml/base/__init__.py

@@ -0,0 +1,6 @@
+"""Module of base classes."""
+
+from openml.base._base import OpenMLBase
+from openml.base._base_pkg import _BasePkg
+
+__all__ = ["_BasePkg", "OpenMLBase"]

openml/base.py openml/base/_base.pyopenml/base.py renamed to openml/base/_base.py

@@ -10,8 +10,7 @@
 
 import openml._api_calls
 import openml.config
-
-from .utils import _get_rest_api_type_alias, _tag_openml_base
+from openml.utils import _get_rest_api_type_alias, _tag_openml_base
 
 
 class OpenMLBase(ABC):

openml/base/_base_pkg.py

@@ -0,0 +1,120 @@
+"""Base Packager class."""
+
+import inspect
+from pathlib import Path
+import sys
+import textwrap
+
+from skbase.base import BaseObject
+from skbase.utils.dependencies import _check_estimator_deps
+
+
+class _BasePkg(BaseObject):
+
+    _tags = {
+        "python_dependencies": None,
+        "python_version": None,
+        # package register and manifest
+        "pkg_id": None,  # object id contained, "__multiple" if multiple
+        "pkg_obj": "reference",  # or "code"
+        "pkg_obj_type": None,  # openml API type
+        "pkg_compression": "zlib",  # compression
+    }
+
+    def __init__(self):
+        super().__init__()
+
+    def materialize(self):
+        try:
+            _check_estimator_deps(obj=self)
+        except ModuleNotFoundError as e:
+            # prettier message, so the reference is to the pkg_id
+            # currently, we cannot simply pass the object name to skbase
+            # in the error message, so this is a hack
+            # todo: fix this in scikit-base
+            msg = str(e)
+            if len(msg) > 11:
+                msg = msg[11:]
+            raise ModuleNotFoundError(msg) from e
+
+        return self._materialize()
+
+    def _materialize(self):
+        raise RuntimeError("abstract method")
+
+    def serialize(self):
+        cls_str = class_to_source(type(self))
+        compress_method = self.get_tag("pkg_compression")
+        if compress_method in [None, "None"]:
+            return cls_str
+
+        cls_str = cls_str.encode("utf-8")
+        exec(f"import {compress_method}")
+        compressed_str = eval(f"{compress_method}.compress(cls_str)")
+
+        return compressed_str
+
+
+def _has_source(obj) -> bool:
+    """
+    Return True if inspect.getsource(obj) should succeed.
+    """
+    module_name = getattr(obj, "__module__", None)
+    if not module_name or module_name not in sys.modules:
+        return False
+
+    module = sys.modules[module_name]
+    file = getattr(module, "__file__", None)
+    if not file:
+        return False
+
+    return Path(file).suffix == ".py"
+
+
+def class_to_source(cls) -> str:
+    """Return full source definition of python class as string.
+
+    Parameters
+    ----------
+    cls : class to serialize
+
+    Returns
+    -------
+    str : complete definition of cls, as str.
+        Imports are not contained or serialized.
+    """""
+
+    # Fast path: class has retrievable source
+    if _has_source(cls):
+        source = inspect.getsource(cls)
+        return textwrap.dedent(source)
+
+    # Fallback for dynamically created classes
+    lines = []
+
+    bases = [base.__name__ for base in cls.__bases__ if base is not object]
+    base_str = f"({', '.join(bases)})" if bases else ""
+    lines.append(f"class {cls.__name__}{base_str}:")
+
+    body_added = False
+
+    for name, value in cls.__dict__.items():
+        if name.startswith("__") and name.endswith("__"):
+            continue
+
+        if inspect.isfunction(value):
+            if _has_source(value):
+                method_src = inspect.getsource(value)
+                method_src = textwrap.indent(textwrap.dedent(method_src), "    ")
+                lines.append(method_src)
+            else:
+                lines.append(f"    def {name}(self): ...")
+            body_added = True
+        else:
+            lines.append(f"    {name} = {repr(value)}")
+            body_added = True
+
+    if not body_added:
+        lines.append("    pass")
+
+    return "\n".join(lines)

openml/models/__init__.py

@@ -0,0 +1,5 @@
+"""Module with packaging adapters."""
+
+from openml.models._get import get
+
+__all__ = ["get"]

openml/models/_get.py

@@ -0,0 +1,63 @@
+
+"""Model retrieval utility."""
+
+from functools import lru_cache
+
+
+def get(id: str):
+    """Retrieve model object with unique identifier.
+
+    Parameter
+    ---------
+    id : str
+        unique identifier of object to retrieve
+
+    Returns
+    -------
+    class
+        retrieved object
+
+    Raises
+    ------
+    ModuleNotFoundError
+        if dependencies of object to retrieve are not satisfied
+    """
+
+    id_lookup = _id_lookup()
+    obj = id_lookup.get(id)
+    if obj is None:
+        raise ValueError(
+            f"Error in openml.get, object with package id {id} "
+            "does not exist."
+        )
+    return obj().materialize()
+
+
+# todo: need to generalize this later to more types
+# currently intentionally retrieves only classifiers
+# todo: replace this, optionally, by database backend
+def _id_lookup(obj_type=None):
+    return _id_lookup_cached(obj_type=obj_type).copy()
+
+
+@lru_cache
+def _id_lookup_cached(obj_type=None):
+    all_objs = _all_objects(obj_type=obj_type)
+
+    # todo: generalize that pkg can contain more than one object
+    lookup_dict = {obj.get_class_tag("pkg_id"): obj for obj in all_objs}
+
+    return lookup_dict
+
+
+@lru_cache
+def _all_objects(obj_type=None):
+    from skbase.lookup import all_objects
+
+    from openml.models.apis._classifier import _ModelPkgClassifier
+
+    clses = all_objects(
+        object_types=_ModelPkgClassifier, package_name="openml", return_names=False
+    )
+
+    return clses

openml/models/apis/__init__.py

@@ -0,0 +1,5 @@
+"""Module with packaging adapters."""
+
+from openml.models.apis._classifier import _ModelPkgClassifier
+
+__all__ = ["_ModelPkgClassifier"]

openml/models/apis/_classifier.py

@@ -0,0 +1,24 @@
+"""Base package for sklearn classifiers."""
+
+from openml.models.base import _OpenmlModelPkg
+
+
+class _ModelPkgClassifier(_OpenmlModelPkg):
+
+    _tags = {
+        # tags specific to API type
+        "pkg_obj_type": "classifier",
+    }
+
+    def get_obj_tags(self):
+        """Return tags of the object as a dictionary."""
+        return {}  # this needs to be implemented
+
+    def get_obj_param_names(self):
+        """Return parameter names of the object as a list.
+
+        Returns
+        -------
+        list: names of object parameters
+        """
+        return list(self.materialize()().get_params().keys())

openml/models/base/__init__.py

@@ -0,0 +1,5 @@
+"""Module with packaging adapters."""
+
+from openml.models.base._base import _OpenmlModelPkg
+
+__all__ = ["_OpenmlModelPkg"]