diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d497b4a..9882b66 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,9 @@ {"image": "mcr.microsoft.com/devcontainers/base:ubuntu", "features": { + "ghcr.io/rocker-org/devcontainer-features/quarto-cli": + {"installChromium": true, "installTinyTex": true}, + "ghcr.io/rocker-org/devcontainer-features/apt-packages:1": + {"packages": "ca-certificates,fonts-liberation,libasound2,libatk-bridge2.0-0,libatk1.0-0,libc6,libcairo2,libcups2,libdbus-1-3,libexpat1,libfontconfig1,libgbm1,libgcc1,libglib2.0-0,libgtk-3-0,libnspr4,libnss3,libpango-1.0-0,libpangocairo-1.0-0,libstdc++6,libx11-6,libx11-xcb1,libxcb1,libxcomposite1,libxcursor1,libxdamage1,libxext6,libxfixes3,libxi6,libxrandr2,libxrender1,libxss1,libxtst6,lsb-release,wget,xdg-utils"}, "ghcr.io/rocker-org/devcontainer-features/miniforge:2": {} }, "postCreateCommand": "conda env create --file environment.yml" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..d321c1b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,42 @@ +on: + workflow_dispatch: + +name: Quarto Publish + +jobs: + build-deploy: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + permissions: + contents: write + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Set up Quarto + uses: quarto-dev/quarto-actions/setup@v2 + with: + tinytex: true + # version: "pre-release" + - name: Set up Python + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: test + auto-update-conda: true + python-version: "3.10" + channels: conda-forge + allow-softlinks: true + channel-priority: flexible + show-channel-urls: true + - name: Install dependencies + run: | + conda install --yes -c numpy scipy scikit-learn statsmodels pandas seaborn jupyter tabulate + - name: Render and Publish + run: | + git config --global user.email "mgraffg@ieee.org" + git config --global user.name "mgraffg" + cd quarto + quarto publish gh-pages CompStats.qmd --no-browser + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4ccf5c8..5c09734 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,4 @@ cython_debug/ #.idea/ .vscode/settings.json +quarto/CompStats_files/ \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 9260ee9..043cbb2 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -7,7 +7,8 @@ "ms-toolsai.jupyter", "ms-python.vscode-pylance", "ms-python.python", - "ms-python.pylint" + "ms-python.pylint", + "quarto.quarto" ], // List of extensions recommended by VS Code that should not be recommended for users of this workspace. "unwantedRecommendations": [ diff --git a/CompStats/__init__.py b/CompStats/__init__.py index 304ae65..cb88852 100644 --- a/CompStats/__init__.py +++ b/CompStats/__init__.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.1.11' +__version__ = '0.1.12' from CompStats.bootstrap import StatisticSamples from CompStats.measurements import CI, SE, difference_p_value from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference diff --git a/CompStats/interface.py b/CompStats/interface.py index a4262fb..4b0e580 100644 --- a/CompStats/interface.py +++ b/CompStats/interface.py @@ -90,6 +90,7 @@ class Perf(object): """ def __init__(self, y_true, *y_pred, + name:str=None, score_func=balanced_accuracy_score, error_func=None, num_samples: int=500, @@ -100,8 +101,13 @@ def __init__(self, y_true, *y_pred, self.score_func = score_func self.error_func = error_func algs = {} - for k, v in enumerate(y_pred): - algs[f'alg-{k+1}'] = np.asanyarray(v) + if name is not None: + if isinstance(name, str): + name = [name] + else: + name = [f'alg-{k+1}' for k, _ in enumerate(y_pred)] + for key, v in zip(name, y_pred): + algs[key] = np.asanyarray(v) algs.update(**kwargs) self.predictions = algs self.y_true = y_true @@ -186,6 +192,7 @@ def __call__(self, y_pred, name=None): k = 1 name = f'alg-{k}' self.best = None + self.statistic = None self.predictions[name] = np.asanyarray(y_pred) samples = self._statistic_samples calls = samples.calls @@ -296,14 +303,23 @@ def statistic(self): >>> perf.statistic {'alg-1': 1.0, 'forest': 0.9500891265597148} """ - + if hasattr(self, '_statistic') and self._statistic is not None: + return self._statistic + BiB = True if self.score_func is not None else False data = sorted([(k, self.statistic_func(self.y_true, v)) for k, v in self.predictions.items()], - key=lambda x: self.sorting_func(x[1]), - reverse=self.statistic_samples.BiB) + key=lambda x: self.sorting_func(x[1]), + reverse=BiB) if len(data) == 1: - return data[0][1] - return dict(data) + self._statistic = data[0][1] + else: + self._statistic = dict(data) + return self._statistic + + @statistic.setter + def statistic(self, value): + """statistic setter""" + self._statistic = value @property def se(self): @@ -509,7 +525,7 @@ def y_true(self, value): algs[c] = value[c].to_numpy() self.predictions.update(algs) return - self._y_true = value + self._y_true = np.asanyarray(value) @property def score_func(self): diff --git a/CompStats/tests/test_interface.py b/CompStats/tests/test_interface.py index f2d02c4..1ff7626 100644 --- a/CompStats/tests/test_interface.py +++ b/CompStats/tests/test_interface.py @@ -23,6 +23,13 @@ from CompStats.tests.test_performance import DATA +def test_Perf_name(): + """Test Perf name keyword""" + from CompStats.metrics import f1_score + score = f1_score([1, 0, 1], [1, 0, 0], name='algo') + assert 'algo' in score.predictions + + def test_Perf_plot_col_wrap(): """Test plot when 2 classes""" from CompStats.metrics import f1_score diff --git a/README.rst b/README.rst index 4ac5f92..82da3ba 100644 --- a/README.rst +++ b/README.rst @@ -70,6 +70,8 @@ Let us incorporate another predictions, now with Naive Bayes classifier, and His >>> nb = GaussianNB().fit(X_train, y_train) >>> score(nb.predict(X_val), name='Naive Bayes') +>>> hist = HistGradientBoostingClassifier().fit(X_train, y_train) +>>> score(hist.predict(X_val), name='Hist. Grad. Boost. Tree') Statistic with its standard error (se) statistic (se) diff --git a/environment.yml b/environment.yml index 5039519..3e1ea8c 100644 --- a/environment.yml +++ b/environment.yml @@ -9,6 +9,8 @@ dependencies: - pytest - tqdm - sphinx + - yaml + - jupyter - pip: - pandas - seaborn \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8108be5..6ffe615 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,8 @@ dependencies = [ 'numpy', 'scikit-learn>=1.3.0', 'pandas', - 'seaborn>=0.13.0' + 'seaborn>=0.13.0', + 'statsmodels' ] dynamic = ['version'] @@ -26,6 +27,9 @@ classifiers = [ [tool.setuptools.dynamic] version = {attr = 'CompStats.__version__'} +[tool.setuptools] +packages = ['CompStats', 'CompStats.tests'] + [project.urls] Homepage = "https://compstats.readthedocs.io" Repository = "https://github.com/INGEOTEC/CompStats" diff --git a/quarto/CompStats.qmd b/quarto/CompStats.qmd new file mode 100644 index 0000000..c9f37c7 --- /dev/null +++ b/quarto/CompStats.qmd @@ -0,0 +1,99 @@ +--- +title: "CompStats" +format: + dashboard: + logo: images/ingeotec.png + orientation: columns + nav-buttons: [github] + theme: cosmo +execute: + freeze: auto +--- + +# Introduction + +## Column + +::: {.card title='Introduction'} +Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals. +::: + +::: {.card title='Installing using conda'} + +`CompStats` can be install using the conda package manager with the following instruction. + +```{sh} +conda install --channel conda-forge CompStats +``` +::: + +::: {.card title='Installing using pip'} +A more general approach to installing `CompStats` is through the use of the command pip, as illustrated in the following instruction. + +```{sh} +pip install CompStats +``` +::: + +# scikit-learn Users + +## Column + +To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm. + +```{python} +#| echo: true + +from sklearn.svm import LinearSVC +from sklearn.naive_bayes import GaussianNB +from sklearn.ensemble import RandomForestClassifier +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split +from sklearn.base import clone +from CompStats.metrics import f1_score +``` + +The first step is to load the digits problem and split the dataset into training and validation sets. The second step is to estimate the parameters of a linear Support Vector Machine and predict the validation set's classes. The predictions are stored in the variable `hy`. + +```{python} +#| echo: true + +X, y = load_digits(return_X_y=True) +_ = train_test_split(X, y, test_size=0.3) +X_train, X_val, y_train, y_val = _ +m = LinearSVC().fit(X_train, y_train) +hy = m.predict(X_val) +``` + +Once the predictions are available, it is time to measure the algorithm's performance, as seen in the following code. It is essential to note that the API used in `sklearn.metrics` is followed; the difference is that the function returns an instance with different methods that can be used to estimate different performance statistics and compare algorithms. + +## Column + +```{python} +#| echo: true + +score = f1_score(y_val, hy, average='macro') +score +``` + +Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis. + +```{python} +#| echo: true + +ens = RandomForestClassifier().fit(X_train, y_train) +score(ens.predict(X_val), name='Random Forest') +``` + +Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below. + +```{python} +#| echo: true + +nb = GaussianNB().fit(X_train, y_train) +_ = score(nb.predict(X_val), name='Naive Bayes') +hist = HistGradientBoostingClassifier().fit(X_train, y_train) +_ = score(hist.predict(X_val), name='Hist. Grad. Boost. Tree') +score.plot() +``` \ No newline at end of file diff --git a/quarto/images/ingeotec.png b/quarto/images/ingeotec.png new file mode 100644 index 0000000..188d23d Binary files /dev/null and b/quarto/images/ingeotec.png differ