From 56dc0ee241fb2809039d42383d38527867b42d43 Mon Sep 17 00:00:00 2001 From: Dmitry Zharikov Date: Sat, 8 Oct 2022 14:10:25 +0300 Subject: [PATCH 1/8] [ADD]Precision --- hw/hw1/HW1.ipynb | 52 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/hw/hw1/HW1.ipynb b/hw/hw1/HW1.ipynb index 5a7c263..0912e0d 100644 --- a/hw/hw1/HW1.ipynb +++ b/hw/hw1/HW1.ipynb @@ -33,19 +33,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "6c928452-693b-4a9d-bc66-284765b6ec50", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\dimaz\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "from typing import NamedTuple, Union\n", "import tests\n", - "import torch" + "import torch\n", + "from decimal import Decimal\n", + "import numpy as np" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "id": "93c1466c-0e00-42ee-9e6d-f822da820a91", "metadata": {}, "outputs": [], @@ -84,7 +95,24 @@ " torch.isnan(tensor) | torch.isinf(tensor),\n", " torch.full_like(tensor, fill_value=nan),\n", " tensor,\n", - " )" + " )\n", + "\n", + "\n", + "def tests(cases, metric):\n", + " flg = False\n", + " for index, case in enumerate(cases):\n", + " print(f\"\\ncase: {index}\\n\")\n", + " for k in case['topk']:\n", + " result = metric(case[\"output\"], case[\"target\"], k)\n", + " if result == case[\"expected\"][k]:\n", + " print(k, 'is cool')\n", + " else:\n", + " print(k, 'ALERT!')\n", + " print(result)\n", + " flg = True\n", + " break\n", + " if flg:\n", + " break" ] }, { @@ -99,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "id": "679e4cb5-0fde-43e6-92ff-de003a18057a", "metadata": {}, "outputs": [], @@ -109,12 +137,14 @@ " # target_sorted_by_output ~ (users, items)\n", " target_sorted_by_output = prepare_target(output, target)\n", " # YOUR CODE HERE\n", - " return 0" + " topk = min(target.shape[1], topk)\n", + " result = np.mean(list(map(lambda x: x/topk, np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))\n", + " return result" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "id": "a600b149-385b-40ad-b97d-b016426a4e17", "metadata": {}, "outputs": [], @@ -253,7 +283,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10.7 64-bit (microsoft store)", "language": "python", "name": "python3" }, @@ -267,11 +297,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.0" + "version": "3.10.7" }, "vscode": { "interpreter": { - "hash": "afdf057ef1ef2906fc2cc2ffd617646692fe5d919d63b76727650bd7046d9edf" + "hash": "570dd33043ea43a5929913ff205be499021c2ab8ef3bcad34e41dcd3206e6bcb" } } }, From 84a8362b97cc34301383dabe6e88fa03a495d1d9 Mon Sep 17 00:00:00 2001 From: Dmitry Zharikov Date: Sat, 8 Oct 2022 14:40:23 +0300 Subject: [PATCH 2/8] [ADD] Recall --- hw/hw1/HW1.ipynb | 234 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 230 insertions(+), 4 deletions(-) diff --git a/hw/hw1/HW1.ipynb b/hw/hw1/HW1.ipynb index 0912e0d..607fadd 100644 --- a/hw/hw1/HW1.ipynb +++ b/hw/hw1/HW1.ipynb @@ -98,7 +98,7 @@ " )\n", "\n", "\n", - "def tests(cases, metric):\n", + "def my_tests(cases, metric):\n", " flg = False\n", " for index, case in enumerate(cases):\n", " print(f\"\\ncase: {index}\\n\")\n", @@ -164,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 65, "id": "d67da32b-2e3a-4e1d-a3df-1f32d117b95b", "metadata": {}, "outputs": [], @@ -174,15 +174,241 @@ " # target_sorted_by_output ~ (users, items)\n", " target_sorted_by_output = prepare_target(output, target)\n", " # YOUR CODE HERE\n", - " return 0" + " result = torch.div(torch.sum(target_sorted_by_output[:, :topk], dim=1), torch.sum(target_sorted_by_output, dim=1)).mean()\n", + " return result" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "b7aceada", + "metadata": {}, + "outputs": [], + "source": [ + "cases = [\n", + " {\n", + " \"output\": torch.Tensor([[0, 0, 0, 0]]),\n", + " \"target\": torch.Tensor([[0, 0, 0, 0]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 0.0,\n", + " 3: 0.0,\n", + " 10: 0.0,\n", + " 100: 0.0,\n", + " }\n", + " },\n", + " {\n", + " \"output\": torch.Tensor([[1, 1, 1, 1]]),\n", + " \"target\": torch.Tensor([[1, 1, 1, 1]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 1 / 4,\n", + " 3: 3 / 4,\n", + " 10: 4 / 4,\n", + " 100: 4 / 4,\n", + " }\n", + " },\n", + " {\n", + " \"output\": torch.Tensor([[0, 0, 0, 0]]),\n", + " \"target\": torch.Tensor([[1, 1, 1, 1]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 1 / 4,\n", + " 3: 3 / 4,\n", + " 10: 4 / 4,\n", + " 100: 4 / 4,\n", + " }\n", + " },\n", + " {\n", + " \"output\": torch.Tensor([[1, 1, 1, 1]]),\n", + " \"target\": torch.Tensor([[0, 0, 0, 0]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 0.0,\n", + " 3: 0.0,\n", + " 10: 0.0,\n", + " 100: 0.0,\n", + " }\n", + " },\n", + " {\n", + " \"output\": torch.Tensor([[0.5, 0.4, 0.3, 0.2]]),\n", + " \"target\": torch.Tensor([[1, 0, 1, 0]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 1 / 2,\n", + " 3: 2 / 2,\n", + " 10: 2 / 2,\n", + " 100: 2 / 2,\n", + " }\n", + " },\n", + " {\n", + " \"output\": torch.Tensor(\n", + " [\n", + " [9, 5, 3, 0, 7, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 8, 2, 0, 10],\n", + " [0, 0, 1, 5, 9, 3, 0, 0, 0, 0, 0, 4, 0, 0, 10, 7, 0, 2, 8, 6],\n", + " [0, 1, 4, 8, 6, 5, 3, 7, 10, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0],\n", + " [7, 8, 0, 0, 1, 0, 4, 0, 10, 0, 0, 6, 0, 0, 0, 9, 2, 3, 5, 0],\n", + " ]\n", + " ),\n", + " \"target\": torch.Tensor(\n", + " [\n", + " [1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0],\n", + " [1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0],\n", + " [0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0],\n", + " [0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0],\n", + " ]\n", + " ),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: (1 / 8) / 4,\n", + " 3: (2 / 8 + 1 / 8 + 1 / 10 + 1 / 9) / 4,\n", + " 10: (5 / 8 + 3 / 8 + 5 / 10 + 3 / 9) / 4,\n", + " 100: 1.0,\n", + " }\n", + " },\n", + " ]" ] }, { "cell_type": "code", "execution_count": null, - "id": "e77cb6cc-2ca7-483a-abe9-75cbaf92539c", + "id": "2d5f7614", "metadata": {}, "outputs": [], + "source": [ + "target_sorted_by_output = prepare_target(output, target)\n", + "# YOUR CODE HERE\n", + "topk = min(target.shape[1], topk)\n", + "relevant = \n", + "result = np.mean(list(map(lambda x: x/torch.sum(target_sorted_by_output, dim=1), np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "36e44a65", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0., 1., 1., 0., 0., 1., 1., 1., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0.,\n", + " 0., 1.],\n", + " [1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 1., 1., 0., 1.,\n", + " 0., 1.],\n", + " [0., 1., 0., 1., 0., 1., 0., 1., 1., 0., 0., 1., 0., 1., 0., 1., 0., 1.,\n", + " 1., 0.],\n", + " [0., 0., 1., 0., 1., 0., 0., 1., 0., 0., 1., 1., 1., 1., 1., 0., 0., 0.,\n", + " 1., 0.]])" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "topk = 3\n", + "case = cases[5]\n", + "output = case['output']\n", + "target = case['target']\n", + "target_sorted_by_output = prepare_target(output, target)\n", + "target_sorted_by_output" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "b7c37982", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(0.1465)" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "bfb58204", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([2., 1., 1., 1.])" + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.sum(target_sorted_by_output[:, :topk], dim=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "4e9719ef", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\dimaz\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\numpy\\core\\_methods.py:164: FutureWarning: The input object of type 'Tensor' is an array-like implementing one of the corresponding protocols (`__array__`, `__array_interface__` or `__array_struct__`); but not a sequence (or 0-D). In the future, this object will be coerced as if it was first converted using `np.array(obj)`. To retain the old behaviour, you have to either modify the type 'Tensor', or assign to an empty array created with `np.empty(correct_shape, dtype=object)`.\n", + " arr = asanyarray(a)\n", + "C:\\Users\\dimaz\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\numpy\\core\\_methods.py:164: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n", + " arr = asanyarray(a)\n" + ] + }, + { + "ename": "AttributeError", + "evalue": "'torch.dtype' object has no attribute 'type'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 13\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m np\u001b[39m.\u001b[39;49mmean(\u001b[39mlist\u001b[39;49m(\u001b[39mmap\u001b[39;49m(\u001b[39mlambda\u001b[39;49;00m x: x\u001b[39m/\u001b[39;49mtorch\u001b[39m.\u001b[39;49msum(target_sorted_by_output, dim\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m), np\u001b[39m.\u001b[39;49marray(torch\u001b[39m.\u001b[39;49msum(target_sorted_by_output[:, :topk], dim\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)))))\n", + "File \u001b[1;32m<__array_function__ internals>:180\u001b[0m, in \u001b[0;36mmean\u001b[1;34m(*args, **kwargs)\u001b[0m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\numpy\\core\\fromnumeric.py:3432\u001b[0m, in \u001b[0;36mmean\u001b[1;34m(a, axis, dtype, out, keepdims, where)\u001b[0m\n\u001b[0;32m 3429\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 3430\u001b[0m \u001b[39mreturn\u001b[39;00m mean(axis\u001b[39m=\u001b[39maxis, dtype\u001b[39m=\u001b[39mdtype, out\u001b[39m=\u001b[39mout, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m-> 3432\u001b[0m \u001b[39mreturn\u001b[39;00m _methods\u001b[39m.\u001b[39m_mean(a, axis\u001b[39m=\u001b[39maxis, dtype\u001b[39m=\u001b[39mdtype,\n\u001b[0;32m 3433\u001b[0m out\u001b[39m=\u001b[39mout, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\numpy\\core\\_methods.py:190\u001b[0m, in \u001b[0;36m_mean\u001b[1;34m(a, axis, dtype, out, keepdims, where)\u001b[0m\n\u001b[0;32m 188\u001b[0m ret \u001b[39m=\u001b[39m arr\u001b[39m.\u001b[39mdtype\u001b[39m.\u001b[39mtype(ret \u001b[39m/\u001b[39m rcount)\n\u001b[0;32m 189\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m--> 190\u001b[0m ret \u001b[39m=\u001b[39m ret\u001b[39m.\u001b[39;49mdtype\u001b[39m.\u001b[39;49mtype(ret \u001b[39m/\u001b[39m rcount)\n\u001b[0;32m 191\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 192\u001b[0m ret \u001b[39m=\u001b[39m ret \u001b[39m/\u001b[39m rcount\n", + "\u001b[1;31mAttributeError\u001b[0m: 'torch.dtype' object has no attribute 'type'" + ] + } + ], + "source": [ + "np.mean(list(map(lambda x: x/torch.sum(target_sorted_by_output, dim=1), np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "e77cb6cc-2ca7-483a-abe9-75cbaf92539c", + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'function' object has no attribute 'run_recall'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 16\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m tests\u001b[39m.\u001b[39;49mrun_recall(recall)\n", + "\u001b[1;31mAttributeError\u001b[0m: 'function' object has no attribute 'run_recall'" + ] + } + ], "source": [ "tests.run_recall(recall)" ] From 157aa73a8f5f940abdb7f527e110190486e3b66f Mon Sep 17 00:00:00 2001 From: Dmitry Zharikov Date: Sat, 8 Oct 2022 14:43:47 +0300 Subject: [PATCH 3/8] [ADD] Recall --- hw/hw1/HW1.ipynb | 263 +++++------------------------------------------ 1 file changed, 28 insertions(+), 235 deletions(-) diff --git a/hw/hw1/HW1.ipynb b/hw/hw1/HW1.ipynb index 607fadd..10ed935 100644 --- a/hw/hw1/HW1.ipynb +++ b/hw/hw1/HW1.ipynb @@ -56,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 2, "id": "93c1466c-0e00-42ee-9e6d-f822da820a91", "metadata": {}, "outputs": [], @@ -105,7 +105,7 @@ " for k in case['topk']:\n", " result = metric(case[\"output\"], case[\"target\"], k)\n", " if result == case[\"expected\"][k]:\n", - " print(k, 'is cool')\n", + " print(k, 'good')\n", " else:\n", " print(k, 'ALERT!')\n", " print(result)\n", @@ -127,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 3, "id": "679e4cb5-0fde-43e6-92ff-de003a18057a", "metadata": {}, "outputs": [], @@ -144,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 4, "id": "a600b149-385b-40ad-b97d-b016426a4e17", "metadata": {}, "outputs": [], @@ -164,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 5, "id": "d67da32b-2e3a-4e1d-a3df-1f32d117b95b", "metadata": {}, "outputs": [], @@ -175,240 +175,17 @@ " target_sorted_by_output = prepare_target(output, target)\n", " # YOUR CODE HERE\n", " result = torch.div(torch.sum(target_sorted_by_output[:, :topk], dim=1), torch.sum(target_sorted_by_output, dim=1)).mean()\n", + " if torch.isnan(result):\n", + " result = 0.0\n", " return result" ] }, { "cell_type": "code", - "execution_count": 43, - "id": "b7aceada", - "metadata": {}, - "outputs": [], - "source": [ - "cases = [\n", - " {\n", - " \"output\": torch.Tensor([[0, 0, 0, 0]]),\n", - " \"target\": torch.Tensor([[0, 0, 0, 0]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 0.0,\n", - " 3: 0.0,\n", - " 10: 0.0,\n", - " 100: 0.0,\n", - " }\n", - " },\n", - " {\n", - " \"output\": torch.Tensor([[1, 1, 1, 1]]),\n", - " \"target\": torch.Tensor([[1, 1, 1, 1]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 1 / 4,\n", - " 3: 3 / 4,\n", - " 10: 4 / 4,\n", - " 100: 4 / 4,\n", - " }\n", - " },\n", - " {\n", - " \"output\": torch.Tensor([[0, 0, 0, 0]]),\n", - " \"target\": torch.Tensor([[1, 1, 1, 1]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 1 / 4,\n", - " 3: 3 / 4,\n", - " 10: 4 / 4,\n", - " 100: 4 / 4,\n", - " }\n", - " },\n", - " {\n", - " \"output\": torch.Tensor([[1, 1, 1, 1]]),\n", - " \"target\": torch.Tensor([[0, 0, 0, 0]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 0.0,\n", - " 3: 0.0,\n", - " 10: 0.0,\n", - " 100: 0.0,\n", - " }\n", - " },\n", - " {\n", - " \"output\": torch.Tensor([[0.5, 0.4, 0.3, 0.2]]),\n", - " \"target\": torch.Tensor([[1, 0, 1, 0]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 1 / 2,\n", - " 3: 2 / 2,\n", - " 10: 2 / 2,\n", - " 100: 2 / 2,\n", - " }\n", - " },\n", - " {\n", - " \"output\": torch.Tensor(\n", - " [\n", - " [9, 5, 3, 0, 7, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 8, 2, 0, 10],\n", - " [0, 0, 1, 5, 9, 3, 0, 0, 0, 0, 0, 4, 0, 0, 10, 7, 0, 2, 8, 6],\n", - " [0, 1, 4, 8, 6, 5, 3, 7, 10, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0],\n", - " [7, 8, 0, 0, 1, 0, 4, 0, 10, 0, 0, 6, 0, 0, 0, 9, 2, 3, 5, 0],\n", - " ]\n", - " ),\n", - " \"target\": torch.Tensor(\n", - " [\n", - " [1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0],\n", - " [1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0],\n", - " [0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0],\n", - " [0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0],\n", - " ]\n", - " ),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: (1 / 8) / 4,\n", - " 3: (2 / 8 + 1 / 8 + 1 / 10 + 1 / 9) / 4,\n", - " 10: (5 / 8 + 3 / 8 + 5 / 10 + 3 / 9) / 4,\n", - " 100: 1.0,\n", - " }\n", - " },\n", - " ]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2d5f7614", - "metadata": {}, - "outputs": [], - "source": [ - "target_sorted_by_output = prepare_target(output, target)\n", - "# YOUR CODE HERE\n", - "topk = min(target.shape[1], topk)\n", - "relevant = \n", - "result = np.mean(list(map(lambda x: x/torch.sum(target_sorted_by_output, dim=1), np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "id": "36e44a65", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[0., 1., 1., 0., 0., 1., 1., 1., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0.,\n", - " 0., 1.],\n", - " [1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 1., 1., 0., 1.,\n", - " 0., 1.],\n", - " [0., 1., 0., 1., 0., 1., 0., 1., 1., 0., 0., 1., 0., 1., 0., 1., 0., 1.,\n", - " 1., 0.],\n", - " [0., 0., 1., 0., 1., 0., 0., 1., 0., 0., 1., 1., 1., 1., 1., 0., 0., 0.,\n", - " 1., 0.]])" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "topk = 3\n", - "case = cases[5]\n", - "output = case['output']\n", - "target = case['target']\n", - "target_sorted_by_output = prepare_target(output, target)\n", - "target_sorted_by_output" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "id": "b7c37982", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(0.1465)" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 60, - "id": "bfb58204", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([2., 1., 1., 1.])" - ] - }, - "execution_count": 60, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.sum(target_sorted_by_output[:, :topk], dim=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "id": "4e9719ef", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "C:\\Users\\dimaz\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\numpy\\core\\_methods.py:164: FutureWarning: The input object of type 'Tensor' is an array-like implementing one of the corresponding protocols (`__array__`, `__array_interface__` or `__array_struct__`); but not a sequence (or 0-D). In the future, this object will be coerced as if it was first converted using `np.array(obj)`. To retain the old behaviour, you have to either modify the type 'Tensor', or assign to an empty array created with `np.empty(correct_shape, dtype=object)`.\n", - " arr = asanyarray(a)\n", - "C:\\Users\\dimaz\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\numpy\\core\\_methods.py:164: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n", - " arr = asanyarray(a)\n" - ] - }, - { - "ename": "AttributeError", - "evalue": "'torch.dtype' object has no attribute 'type'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 13\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m np\u001b[39m.\u001b[39;49mmean(\u001b[39mlist\u001b[39;49m(\u001b[39mmap\u001b[39;49m(\u001b[39mlambda\u001b[39;49;00m x: x\u001b[39m/\u001b[39;49mtorch\u001b[39m.\u001b[39;49msum(target_sorted_by_output, dim\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m), np\u001b[39m.\u001b[39;49marray(torch\u001b[39m.\u001b[39;49msum(target_sorted_by_output[:, :topk], dim\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)))))\n", - "File \u001b[1;32m<__array_function__ internals>:180\u001b[0m, in \u001b[0;36mmean\u001b[1;34m(*args, **kwargs)\u001b[0m\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\numpy\\core\\fromnumeric.py:3432\u001b[0m, in \u001b[0;36mmean\u001b[1;34m(a, axis, dtype, out, keepdims, where)\u001b[0m\n\u001b[0;32m 3429\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 3430\u001b[0m \u001b[39mreturn\u001b[39;00m mean(axis\u001b[39m=\u001b[39maxis, dtype\u001b[39m=\u001b[39mdtype, out\u001b[39m=\u001b[39mout, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m-> 3432\u001b[0m \u001b[39mreturn\u001b[39;00m _methods\u001b[39m.\u001b[39m_mean(a, axis\u001b[39m=\u001b[39maxis, dtype\u001b[39m=\u001b[39mdtype,\n\u001b[0;32m 3433\u001b[0m out\u001b[39m=\u001b[39mout, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\numpy\\core\\_methods.py:190\u001b[0m, in \u001b[0;36m_mean\u001b[1;34m(a, axis, dtype, out, keepdims, where)\u001b[0m\n\u001b[0;32m 188\u001b[0m ret \u001b[39m=\u001b[39m arr\u001b[39m.\u001b[39mdtype\u001b[39m.\u001b[39mtype(ret \u001b[39m/\u001b[39m rcount)\n\u001b[0;32m 189\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m--> 190\u001b[0m ret \u001b[39m=\u001b[39m ret\u001b[39m.\u001b[39;49mdtype\u001b[39m.\u001b[39;49mtype(ret \u001b[39m/\u001b[39m rcount)\n\u001b[0;32m 191\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 192\u001b[0m ret \u001b[39m=\u001b[39m ret \u001b[39m/\u001b[39m rcount\n", - "\u001b[1;31mAttributeError\u001b[0m: 'torch.dtype' object has no attribute 'type'" - ] - } - ], - "source": [ - "np.mean(list(map(lambda x: x/torch.sum(target_sorted_by_output, dim=1), np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))" - ] - }, - { - "cell_type": "code", - "execution_count": 66, + "execution_count": 6, "id": "e77cb6cc-2ca7-483a-abe9-75cbaf92539c", "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "'function' object has no attribute 'run_recall'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 16\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m tests\u001b[39m.\u001b[39;49mrun_recall(recall)\n", - "\u001b[1;31mAttributeError\u001b[0m: 'function' object has no attribute 'run_recall'" - ] - } - ], + "outputs": [], "source": [ "tests.run_recall(recall)" ] @@ -430,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "1daf876a-4b0b-48af-a38d-57423e5ff46d", "metadata": {}, "outputs": [], @@ -445,10 +222,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "39353a96-0ce9-493d-a56c-29c85759bf7b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "AssertionError", + "evalue": "Inputs:{\n \"output\": [\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ]\n ],\n \"target\": [\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ]\n ],\n \"topk\": 1\n}\nThe (d)types do not match: != .", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mAssertionError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 13\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m tests\u001b[39m.\u001b[39;49mrun_map(mnap)\n\u001b[0;32m 2\u001b[0m tests\u001b[39m.\u001b[39mrun_mnap(mnap)\n", + "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:278\u001b[0m, in \u001b[0;36mrun_map\u001b[1;34m(func)\u001b[0m\n\u001b[0;32m 180\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrun_map\u001b[39m(func: Callable) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m 181\u001b[0m cases \u001b[39m=\u001b[39m [\n\u001b[0;32m 182\u001b[0m {\n\u001b[0;32m 183\u001b[0m \u001b[39m\"\u001b[39m\u001b[39moutput\u001b[39m\u001b[39m\"\u001b[39m: torch\u001b[39m.\u001b[39mTensor([[\u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m]]),\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 276\u001b[0m },\n\u001b[0;32m 277\u001b[0m ]\n\u001b[1;32m--> 278\u001b[0m \u001b[39mreturn\u001b[39;00m _run_tests(partial(func, normalized\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m), cases)\n", + "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:488\u001b[0m, in \u001b[0;36m_run_tests\u001b[1;34m(func, cases)\u001b[0m\n\u001b[0;32m 486\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m case\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m 487\u001b[0m actual \u001b[39m=\u001b[39m func(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mcase, topk\u001b[39m=\u001b[39mk)\n\u001b[1;32m--> 488\u001b[0m torch\u001b[39m.\u001b[39;49mtesting\u001b[39m.\u001b[39;49massert_close(\n\u001b[0;32m 489\u001b[0m actual,\n\u001b[0;32m 490\u001b[0m torch\u001b[39m.\u001b[39;49mtensor(expected[k]) \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(actual, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m expected[k],\n\u001b[0;32m 491\u001b[0m msg\u001b[39m=\u001b[39;49m\u001b[39mlambda\u001b[39;49;00m msg: \u001b[39mf\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mInputs:\u001b[39;49m\u001b[39m{\u001b[39;49;00mjson\u001b[39m.\u001b[39;49mdumps({key: (v\u001b[39m.\u001b[39;49mtolist() \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(v, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m v) \u001b[39mfor\u001b[39;49;00m key, v \u001b[39min\u001b[39;49;00m (case \u001b[39m|\u001b[39;49m {\u001b[39m'\u001b[39;49m\u001b[39mtopk\u001b[39;49m\u001b[39m'\u001b[39;49m: k})\u001b[39m.\u001b[39;49mitems()}, indent\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m, ensure_ascii\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\u001b[39m}\u001b[39;49;00m\u001b[39m\\n\u001b[39;49;00m\u001b[39m{\u001b[39;49;00mmsg\u001b[39m}\u001b[39;49;00m\u001b[39m\"\u001b[39;49m,\n\u001b[0;32m 492\u001b[0m )\n", + " \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\torch\\testing\\_comparison.py:1095\u001b[0m, in \u001b[0;36massert_equal\u001b[1;34m(actual, expected, pair_types, sequence_types, mapping_types, msg, **options)\u001b[0m\n\u001b[0;32m 1092\u001b[0m \u001b[39mreturn\u001b[39;00m\n\u001b[0;32m 1094\u001b[0m \u001b[39m# TODO: compose all metas into one AssertionError\u001b[39;00m\n\u001b[1;32m-> 1095\u001b[0m \u001b[39mraise\u001b[39;00m error_metas[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mto_error(msg)\n", + "\u001b[1;31mAssertionError\u001b[0m: Inputs:{\n \"output\": [\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ]\n ],\n \"target\": [\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ]\n ],\n \"topk\": 1\n}\nThe (d)types do not match: != ." + ] + } + ], "source": [ "tests.run_map(mnap)\n", "tests.run_mnap(mnap)" From 595f78bf459dcab23b8708e524e96e2c47f72bd0 Mon Sep 17 00:00:00 2001 From: Dmitry Zharikov Date: Sat, 8 Oct 2022 18:56:54 +0300 Subject: [PATCH 4/8] [UPDATE PRECISION] --- hw/hw1/HW1.ipynb | 317 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 305 insertions(+), 12 deletions(-) diff --git a/hw/hw1/HW1.ipynb b/hw/hw1/HW1.ipynb index 10ed935..d6b361c 100644 --- a/hw/hw1/HW1.ipynb +++ b/hw/hw1/HW1.ipynb @@ -127,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 60, "id": "679e4cb5-0fde-43e6-92ff-de003a18057a", "metadata": {}, "outputs": [], @@ -138,13 +138,13 @@ " target_sorted_by_output = prepare_target(output, target)\n", " # YOUR CODE HERE\n", " topk = min(target.shape[1], topk)\n", - " result = np.mean(list(map(lambda x: x/topk, np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))\n", + " result = torch.div(torch.sum(target_sorted_by_output[:, :topk], 1), topk).mean().item()\n", " return result" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 61, "id": "a600b149-385b-40ad-b97d-b016426a4e17", "metadata": {}, "outputs": [], @@ -164,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "d67da32b-2e3a-4e1d-a3df-1f32d117b95b", "metadata": {}, "outputs": [], @@ -182,7 +182,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "e77cb6cc-2ca7-483a-abe9-75cbaf92539c", "metadata": {}, "outputs": [], @@ -207,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 30, "id": "1daf876a-4b0b-48af-a38d-57423e5ff46d", "metadata": {}, "outputs": [], @@ -217,34 +217,327 @@ " # target_sorted_by_output ~ (users, items)\n", " target_sorted_by_output = prepare_target(output, target)\n", " # YOUR CODE HERE\n", - " return 0" + " topk = min(target.shape[1], topk)\n", + " result = np.mean(list(map(lambda x: x/topk, np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))\n", + " return result" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "104ec0a1", + "metadata": {}, + "outputs": [], + "source": [ + "cases = [\n", + " {\n", + " \"output\": torch.Tensor([[0, 0, 0, 0]]),\n", + " \"target\": torch.Tensor([[0, 0, 0, 0]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 0.0,\n", + " 3: 0.0,\n", + " 10: 0.0,\n", + " 100: 0.0,\n", + " },\n", + " },\n", + " {\n", + " \"output\": torch.Tensor([[1, 1, 1, 1]]),\n", + " \"target\": torch.Tensor([[1, 1, 1, 1]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 1.0,\n", + " 3: 1.0,\n", + " 10: 1.0,\n", + " 100: 1.0,\n", + " },\n", + " },\n", + " {\n", + " \"output\": torch.Tensor([[0, 0, 0, 0]]),\n", + " \"target\": torch.Tensor([[1, 1, 1, 1]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 1.0,\n", + " 3: 1.0,\n", + " 10: 1.0,\n", + " 100: 1.0,\n", + " },\n", + " },\n", + " {\n", + " \"output\": torch.Tensor([[1, 1, 1, 1]]),\n", + " \"target\": torch.Tensor([[0, 0, 0, 0]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 0.0,\n", + " 3: 0.0,\n", + " 10: 0.0,\n", + " 100: 0.0,\n", + " },\n", + " },\n", + " {\n", + " \"output\": torch.Tensor([[0.5, 0.4, 0.3, 0.2]]),\n", + " \"target\": torch.Tensor([[1, 0, 1, 0]]),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: 1.0,\n", + " 3: (1 + 2 / 3) / 3,\n", + " 10: (1 + 2 / 3) / 4,\n", + " 100: (1 + 2 / 3) / 4,\n", + " },\n", + " },\n", + " {\n", + " \"output\": torch.Tensor(\n", + " [\n", + " [9, 5, 3, 0, 7, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 8, 2, 0, 10],\n", + " [0, 0, 1, 5, 9, 3, 0, 0, 0, 0, 0, 4, 0, 0, 10, 7, 0, 2, 8, 6],\n", + " [0, 1, 4, 8, 6, 5, 3, 7, 10, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0],\n", + " [7, 8, 0, 0, 1, 0, 4, 0, 10, 0, 0, 6, 0, 0, 0, 9, 2, 3, 5, 0],\n", + " ]\n", + " ),\n", + " \"target\": torch.Tensor(\n", + " [\n", + " [1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0],\n", + " [1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0],\n", + " [0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0],\n", + " [0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0],\n", + " ]\n", + " ),\n", + " \"topk\": [1, 3, 10, 100],\n", + " \"expected\": {\n", + " 1: (0.0 + 1.0 + 0.0 + 0.0) / 4,\n", + " 3: (\n", + " (1 / 2 + 2 / 3) / 3\n", + " + 1 / 3\n", + " + (1 / 2) / 3\n", + " + (1 / 3) / 3\n", + " ) / 4,\n", + " 10: (\n", + " (1 / 2 + 2 / 3 + 3 / 6 + 4 / 7 + 5 / 8) / 10\n", + " + (1 + 2 / 9 + 3 / 10) / 10\n", + " + (1 / 2 + 2 / 4 + 3 / 6 + 4 / 8 + 5 / 9) / 10\n", + " + (1 / 3 + 2 / 5 + 3 / 8) / 10 \n", + " ) / 4,\n", + " 100: (\n", + " (1 / 2 + 2 / 3 + 3 / 6 + 4 / 7 + 5 / 8 + 6 / 12 + 7 / 14 + 8 / 20) / 20\n", + " + (1 + 2 / 9 + 3 / 10 + 4 / 12 + 5 / 14 + 6 / 15 + 7 / 16 + 8 / 17) / 20\n", + " + (1 / 2 + 2 / 4 + 3 / 6 + 4 / 8 + 5 / 9 + 6 / 11 + 7 / 13 + 8 / 16 + 9 / 18 + 10 / 19) / 20\n", + " + (1 / 3 + 2 / 5 + 3 / 8 + 4 / 12 + 5 / 16 + 6 / 17 + 7 / 18 + 8 / 19 + 9 / 20) / 20\n", + " ) / 4\n", + " },\n", + " },\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "9a1a22e6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "case: 0\n", + "\n", + "1 good\n", + "3 good\n", + "10 good\n", + "100 good\n", + "\n", + "case: 1\n", + "\n", + "1 good\n", + "3 good\n", + "10 good\n", + "100 good\n", + "\n", + "case: 2\n", + "\n", + "1 good\n", + "3 good\n", + "10 good\n", + "100 good\n", + "\n", + "case: 3\n", + "\n", + "1 good\n", + "3 good\n", + "10 good\n", + "100 good\n", + "\n", + "case: 4\n", + "\n", + "1 good\n", + "3 ALERT!\n", + "0.6666666666666666\n" + ] + } + ], + "source": [ + "my_tests(cases, mnap)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "08c1163d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.6666666666666666" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "topk = 3\n", + "case = cases[4]\n", + "output = case['output']\n", + "target = case['target']\n", + "target_sorted_by_output = prepare_target(output, target)\n", + "# YOUR CODE HERE\n", + "topk = min(target.shape[1], topk)\n", + "result = np.mean(list(map(lambda x: x/topk, np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "0143c40d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.5000, 0.4000, 0.3000, 0.2000]])" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "18182dbb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "4" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target.shape[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "8d6e4a3a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.5555555555555555" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(1 + 2 / 3) / 3" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "7c9a9cce", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[1., 0., 1., 0.]])" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target_sorted_by_output" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "b1adcdcf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.6666666666666666]" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(map(lambda x: x/topk, np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1))))" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 32, "id": "39353a96-0ce9-493d-a56c-29c85759bf7b", "metadata": {}, "outputs": [ { "ename": "AssertionError", - "evalue": "Inputs:{\n \"output\": [\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ]\n ],\n \"target\": [\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ]\n ],\n \"topk\": 1\n}\nThe (d)types do not match: != .", + "evalue": "Inputs:{\n \"output\": [\n [\n 0.5,\n 0.4000000059604645,\n 0.30000001192092896,\n 0.20000000298023224\n ]\n ],\n \"target\": [\n [\n 1.0,\n 0.0,\n 1.0,\n 0.0\n ]\n ],\n \"topk\": 3\n}\nScalars are not close!\n\nAbsolute difference: 0.11111111111111116 (up to 1e-07 allowed)\nRelative difference: 0.20000000000000012 (up to 1e-07 allowed)", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mAssertionError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 13\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m tests\u001b[39m.\u001b[39;49mrun_map(mnap)\n\u001b[0;32m 2\u001b[0m tests\u001b[39m.\u001b[39mrun_mnap(mnap)\n", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 13\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m tests\u001b[39m.\u001b[39;49mrun_map(mnap)\n", "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:278\u001b[0m, in \u001b[0;36mrun_map\u001b[1;34m(func)\u001b[0m\n\u001b[0;32m 180\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrun_map\u001b[39m(func: Callable) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m 181\u001b[0m cases \u001b[39m=\u001b[39m [\n\u001b[0;32m 182\u001b[0m {\n\u001b[0;32m 183\u001b[0m \u001b[39m\"\u001b[39m\u001b[39moutput\u001b[39m\u001b[39m\"\u001b[39m: torch\u001b[39m.\u001b[39mTensor([[\u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m]]),\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 276\u001b[0m },\n\u001b[0;32m 277\u001b[0m ]\n\u001b[1;32m--> 278\u001b[0m \u001b[39mreturn\u001b[39;00m _run_tests(partial(func, normalized\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m), cases)\n", "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:488\u001b[0m, in \u001b[0;36m_run_tests\u001b[1;34m(func, cases)\u001b[0m\n\u001b[0;32m 486\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m case\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m 487\u001b[0m actual \u001b[39m=\u001b[39m func(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mcase, topk\u001b[39m=\u001b[39mk)\n\u001b[1;32m--> 488\u001b[0m torch\u001b[39m.\u001b[39;49mtesting\u001b[39m.\u001b[39;49massert_close(\n\u001b[0;32m 489\u001b[0m actual,\n\u001b[0;32m 490\u001b[0m torch\u001b[39m.\u001b[39;49mtensor(expected[k]) \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(actual, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m expected[k],\n\u001b[0;32m 491\u001b[0m msg\u001b[39m=\u001b[39;49m\u001b[39mlambda\u001b[39;49;00m msg: \u001b[39mf\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mInputs:\u001b[39;49m\u001b[39m{\u001b[39;49;00mjson\u001b[39m.\u001b[39;49mdumps({key: (v\u001b[39m.\u001b[39;49mtolist() \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(v, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m v) \u001b[39mfor\u001b[39;49;00m key, v \u001b[39min\u001b[39;49;00m (case \u001b[39m|\u001b[39;49m {\u001b[39m'\u001b[39;49m\u001b[39mtopk\u001b[39;49m\u001b[39m'\u001b[39;49m: k})\u001b[39m.\u001b[39;49mitems()}, indent\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m, ensure_ascii\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\u001b[39m}\u001b[39;49;00m\u001b[39m\\n\u001b[39;49;00m\u001b[39m{\u001b[39;49;00mmsg\u001b[39m}\u001b[39;49;00m\u001b[39m\"\u001b[39;49m,\n\u001b[0;32m 492\u001b[0m )\n", " \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n", "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\torch\\testing\\_comparison.py:1095\u001b[0m, in \u001b[0;36massert_equal\u001b[1;34m(actual, expected, pair_types, sequence_types, mapping_types, msg, **options)\u001b[0m\n\u001b[0;32m 1092\u001b[0m \u001b[39mreturn\u001b[39;00m\n\u001b[0;32m 1094\u001b[0m \u001b[39m# TODO: compose all metas into one AssertionError\u001b[39;00m\n\u001b[1;32m-> 1095\u001b[0m \u001b[39mraise\u001b[39;00m error_metas[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mto_error(msg)\n", - "\u001b[1;31mAssertionError\u001b[0m: Inputs:{\n \"output\": [\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ]\n ],\n \"target\": [\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ]\n ],\n \"topk\": 1\n}\nThe (d)types do not match: != ." + "\u001b[1;31mAssertionError\u001b[0m: Inputs:{\n \"output\": [\n [\n 0.5,\n 0.4000000059604645,\n 0.30000001192092896,\n 0.20000000298023224\n ]\n ],\n \"target\": [\n [\n 1.0,\n 0.0,\n 1.0,\n 0.0\n ]\n ],\n \"topk\": 3\n}\nScalars are not close!\n\nAbsolute difference: 0.11111111111111116 (up to 1e-07 allowed)\nRelative difference: 0.20000000000000012 (up to 1e-07 allowed)" ] } ], "source": [ "tests.run_map(mnap)\n", - "tests.run_mnap(mnap)" + "# tests.run_mnap(mnap)" ] }, { From 0f2591ee22e23c8c3be636e3986b25013f2caf6e Mon Sep 17 00:00:00 2001 From: Dmitry Zharikov Date: Sat, 8 Oct 2022 23:31:32 +0300 Subject: [PATCH 5/8] [ADD] MAP --- hw/hw1/HW1.ipynb | 343 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 266 insertions(+), 77 deletions(-) diff --git a/hw/hw1/HW1.ipynb b/hw/hw1/HW1.ipynb index d6b361c..4911125 100644 --- a/hw/hw1/HW1.ipynb +++ b/hw/hw1/HW1.ipynb @@ -207,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 271, "id": "1daf876a-4b0b-48af-a38d-57423e5ff46d", "metadata": {}, "outputs": [], @@ -217,14 +217,17 @@ " # target_sorted_by_output ~ (users, items)\n", " target_sorted_by_output = prepare_target(output, target)\n", " # YOUR CODE HERE\n", - " topk = min(target.shape[1], topk)\n", - " result = np.mean(list(map(lambda x: x/topk, np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))\n", + " tens_to_calc = target_sorted_by_output[:, :topk]\n", + " numerator = tens_to_calc.where((tens_to_calc == 0 ), tens_to_calc.cumsum(dim=1))\n", + " denominator = torch.ones(tens_to_calc.shape).cumsum(dim=1)\n", + " result = torch.div(numerator, denominator).mean(dim=1).mean().item()\n", + " # result = np.mean(list(map(lambda x, y: float(x) / float(y), numerator, denominator)))\n", " return result" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 254, "id": "104ec0a1", "metadata": {}, "outputs": [], @@ -330,7 +333,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 273, "id": "9a1a22e6", "metadata": {}, "outputs": [ @@ -340,38 +343,20 @@ "text": [ "\n", "case: 0\n", - "\n", - "1 good\n", - "3 good\n", - "10 good\n", - "100 good\n", - "\n", - "case: 1\n", - "\n", - "1 good\n", - "3 good\n", - "10 good\n", - "100 good\n", - "\n", - "case: 2\n", - "\n", - "1 good\n", - "3 good\n", - "10 good\n", - "100 good\n", - "\n", - "case: 3\n", - "\n", - "1 good\n", - "3 good\n", - "10 good\n", - "100 good\n", - "\n", - "case: 4\n", - "\n", - "1 good\n", - "3 ALERT!\n", - "0.6666666666666666\n" + "\n" + ] + }, + { + "ename": "IndexError", + "evalue": "Dimension out of range (expected to be in range of [-1, 0], but got 1)", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mIndexError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 14\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m my_tests(cases, mnap)\n", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 14\u001b[0m in \u001b[0;36mmy_tests\u001b[1;34m(cases, metric)\u001b[0m\n\u001b[0;32m 41\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39mcase: \u001b[39m\u001b[39m{\u001b[39;00mindex\u001b[39m}\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[0;32m 42\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m case[\u001b[39m'\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m'\u001b[39m]:\n\u001b[1;32m---> 43\u001b[0m result \u001b[39m=\u001b[39m metric(case[\u001b[39m\"\u001b[39;49m\u001b[39moutput\u001b[39;49m\u001b[39m\"\u001b[39;49m], case[\u001b[39m\"\u001b[39;49m\u001b[39mtarget\u001b[39;49m\u001b[39m\"\u001b[39;49m], k)\n\u001b[0;32m 44\u001b[0m \u001b[39mif\u001b[39;00m result \u001b[39m==\u001b[39m case[\u001b[39m\"\u001b[39m\u001b[39mexpected\u001b[39m\u001b[39m\"\u001b[39m][k]:\n\u001b[0;32m 45\u001b[0m \u001b[39mprint\u001b[39m(k, \u001b[39m'\u001b[39m\u001b[39mgood\u001b[39m\u001b[39m'\u001b[39m)\n", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 14\u001b[0m in \u001b[0;36mmnap\u001b[1;34m(output, target, topk, normalized)\u001b[0m\n\u001b[0;32m 7\u001b[0m numerator \u001b[39m=\u001b[39m tens_to_calc\u001b[39m.\u001b[39mwhere((tens_to_calc \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m ), tens_to_calc\u001b[39m.\u001b[39mcumsum(dim\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m))\u001b[39m.\u001b[39mravel()\n\u001b[0;32m 8\u001b[0m denominator \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mones(tens_to_calc\u001b[39m.\u001b[39mshape)\u001b[39m.\u001b[39mcumsum(dim\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m)\u001b[39m.\u001b[39mravel()\n\u001b[1;32m----> 9\u001b[0m result \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39;49mdiv(numerator, denominator)\u001b[39m.\u001b[39;49mmean(dim\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)\u001b[39m.\u001b[39mmean()\u001b[39m.\u001b[39mitem()\n\u001b[0;32m 10\u001b[0m \u001b[39m# result = np.mean(list(map(lambda x, y: float(x) / float(y), numerator, denominator)))\u001b[39;00m\n\u001b[0;32m 11\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "\u001b[1;31mIndexError\u001b[0m: Dimension out of range (expected to be in range of [-1, 0], but got 1)" ] } ], @@ -381,100 +366,237 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 239, + "id": "82dd81b6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0.5000, 0.4000, 0.3000, 0.2000])" + ] + }, + "execution_count": 239, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output.ravel()" + ] + }, + { + "cell_type": "code", + "execution_count": 276, "id": "08c1163d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "0.6666666666666666" + "0.0" ] }, - "execution_count": 36, + "execution_count": 276, "metadata": {}, "output_type": "execute_result" } ], "source": [ "topk = 3\n", - "case = cases[4]\n", + "case = cases[0]\n", "output = case['output']\n", "target = case['target']\n", "target_sorted_by_output = prepare_target(output, target)\n", - "# YOUR CODE HERE\n", - "topk = min(target.shape[1], topk)\n", - "result = np.mean(list(map(lambda x: x/topk, np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1)))))\n", - "result" + "tens_to_calc = target_sorted_by_output[:, :topk]\n", + "numerator = tens_to_calc.where((tens_to_calc == 0), tens_to_calc.cumsum(dim=1))\n", + "denominator = torch.ones(tens_to_calc.shape).cumsum(dim=1)\n", + "torch.div(numerator, denominator).mean(dim=1).mean().item()" ] }, { "cell_type": "code", - "execution_count": 45, - "id": "0143c40d", + "execution_count": 267, + "id": "dbb0e824", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([[0.5000, 0.4000, 0.3000, 0.2000]])" + "0.2064180611055611" ] }, - "execution_count": 45, + "execution_count": 267, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "output" + "mnap(output, target, topk)" ] }, { "cell_type": "code", - "execution_count": 43, - "id": "18182dbb", + "execution_count": 270, + "id": "7802a75f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "4" + "0.002459077622583794" ] }, - "execution_count": 43, + "execution_count": 270, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "target.shape[1]" + "mnap(output, target, topk) - (\n", + " (1 / 2 + 2 / 3 + 3 / 6 + 4 / 7 + 5 / 8 + 6 / 12 + 7 / 14 + 8 / 20) / 20\n", + " + (1 + 2 / 9 + 3 / 10 + 4 / 12 + 5 / 14 + 6 / 15 + 7 / 16 + 8 / 17) / 20\n", + " + (1 / 2 + 2 / 4 + 3 / 6 + 4 / 8 + 5 / 9 + 6 / 11 + 7 / 13 + 8 / 16 + 9 / 18 + 10 / 19) / 20\n", + " + (1 / 3 + 2 / 5 + 3 / 8 + 4 / 12 + 5 / 16 + 6 / 17 + 7 / 18 + 8 / 19 + 9 / 20) / 20\n", + " ) / 4" ] }, { "cell_type": "code", - "execution_count": 46, - "id": "8d6e4a3a", + "execution_count": 251, + "id": "117ae5c5", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "0.5555555555555555" + "5.551115123125783e-17" ] }, - "execution_count": 46, + "execution_count": 251, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "(1 + 2 / 3) / 3" + "0.25 - 0.24999999999999994" ] }, { "cell_type": "code", - "execution_count": 37, - "id": "7c9a9cce", + "execution_count": 234, + "id": "ef17b7e9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([1., 1., 1., 1.])" + ] + }, + "execution_count": 234, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "denominator.ravel()" + ] + }, + { + "cell_type": "code", + "execution_count": 236, + "id": "e342ca60", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.25" + ] + }, + "execution_count": 236, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.mean(list(map(lambda x, y: float(x) / float(y), numerator.ravel(), denominator.ravel())))" + ] + }, + { + "cell_type": "code", + "execution_count": 232, + "id": "b390c843", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0.])" + ] + }, + "execution_count": 232, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "numerator[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 229, + "id": "7277b1c2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 229, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.mean(list(map(lambda x, y: float(x) / float(y), numerator[0], denominator[0])))" + ] + }, + { + "cell_type": "code", + "execution_count": 137, + "id": "3627c270", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'output': tensor([[0.5000, 0.4000, 0.3000, 0.2000]]),\n", + " 'target': tensor([[1., 0., 1., 0.]]),\n", + " 'topk': [1, 3, 10, 100],\n", + " 'expected': {1: 1.0,\n", + " 3: 0.5555555555555555,\n", + " 10: 0.41666666666666663,\n", + " 100: 0.41666666666666663}}" + ] + }, + "execution_count": 137, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "case" + ] + }, + { + "cell_type": "code", + "execution_count": 136, + "id": "2dc3d1ab", "metadata": {}, "outputs": [ { @@ -483,7 +605,7 @@ "tensor([[1., 0., 1., 0.]])" ] }, - "execution_count": 37, + "execution_count": 136, "metadata": {}, "output_type": "execute_result" } @@ -494,44 +616,111 @@ }, { "cell_type": "code", - "execution_count": 41, - "id": "b1adcdcf", + "execution_count": 134, + "id": "a505187b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[0.6666666666666666]" + "0.24999999999999994" ] }, - "execution_count": 41, + "execution_count": 134, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "list(map(lambda x: x/topk, np.array(torch.sum(target_sorted_by_output[:, :topk], dim=1))))" + "(\n", + " (1 / 2 + 2 / 3) / 3\n", + " + 1 / 3\n", + " + (1 / 2) / 3\n", + " + (1 / 3) / 3\n", + " ) / 4" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "18182dbb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "4" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target.shape[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "8d6e4a3a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.5555555555555555" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(1 + 2 / 3) / 3" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "7c9a9cce", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[1., 0., 1., 0.]])" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target_sorted_by_output" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 272, "id": "39353a96-0ce9-493d-a56c-29c85759bf7b", "metadata": {}, "outputs": [ { - "ename": "AssertionError", - "evalue": "Inputs:{\n \"output\": [\n [\n 0.5,\n 0.4000000059604645,\n 0.30000001192092896,\n 0.20000000298023224\n ]\n ],\n \"target\": [\n [\n 1.0,\n 0.0,\n 1.0,\n 0.0\n ]\n ],\n \"topk\": 3\n}\nScalars are not close!\n\nAbsolute difference: 0.11111111111111116 (up to 1e-07 allowed)\nRelative difference: 0.20000000000000012 (up to 1e-07 allowed)", + "ename": "IndexError", + "evalue": "Dimension out of range (expected to be in range of [-1, 0], but got 1)", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAssertionError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 13\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m tests\u001b[39m.\u001b[39;49mrun_map(mnap)\n", + "\u001b[1;31mIndexError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 31\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m tests\u001b[39m.\u001b[39;49mrun_map(mnap)\n", "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:278\u001b[0m, in \u001b[0;36mrun_map\u001b[1;34m(func)\u001b[0m\n\u001b[0;32m 180\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrun_map\u001b[39m(func: Callable) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m 181\u001b[0m cases \u001b[39m=\u001b[39m [\n\u001b[0;32m 182\u001b[0m {\n\u001b[0;32m 183\u001b[0m \u001b[39m\"\u001b[39m\u001b[39moutput\u001b[39m\u001b[39m\"\u001b[39m: torch\u001b[39m.\u001b[39mTensor([[\u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m]]),\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 276\u001b[0m },\n\u001b[0;32m 277\u001b[0m ]\n\u001b[1;32m--> 278\u001b[0m \u001b[39mreturn\u001b[39;00m _run_tests(partial(func, normalized\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m), cases)\n", - "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:488\u001b[0m, in \u001b[0;36m_run_tests\u001b[1;34m(func, cases)\u001b[0m\n\u001b[0;32m 486\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m case\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m 487\u001b[0m actual \u001b[39m=\u001b[39m func(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mcase, topk\u001b[39m=\u001b[39mk)\n\u001b[1;32m--> 488\u001b[0m torch\u001b[39m.\u001b[39;49mtesting\u001b[39m.\u001b[39;49massert_close(\n\u001b[0;32m 489\u001b[0m actual,\n\u001b[0;32m 490\u001b[0m torch\u001b[39m.\u001b[39;49mtensor(expected[k]) \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(actual, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m expected[k],\n\u001b[0;32m 491\u001b[0m msg\u001b[39m=\u001b[39;49m\u001b[39mlambda\u001b[39;49;00m msg: \u001b[39mf\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mInputs:\u001b[39;49m\u001b[39m{\u001b[39;49;00mjson\u001b[39m.\u001b[39;49mdumps({key: (v\u001b[39m.\u001b[39;49mtolist() \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(v, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m v) \u001b[39mfor\u001b[39;49;00m key, v \u001b[39min\u001b[39;49;00m (case \u001b[39m|\u001b[39;49m {\u001b[39m'\u001b[39;49m\u001b[39mtopk\u001b[39;49m\u001b[39m'\u001b[39;49m: k})\u001b[39m.\u001b[39;49mitems()}, indent\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m, ensure_ascii\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\u001b[39m}\u001b[39;49;00m\u001b[39m\\n\u001b[39;49;00m\u001b[39m{\u001b[39;49;00mmsg\u001b[39m}\u001b[39;49;00m\u001b[39m\"\u001b[39;49m,\n\u001b[0;32m 492\u001b[0m )\n", - " \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\torch\\testing\\_comparison.py:1095\u001b[0m, in \u001b[0;36massert_equal\u001b[1;34m(actual, expected, pair_types, sequence_types, mapping_types, msg, **options)\u001b[0m\n\u001b[0;32m 1092\u001b[0m \u001b[39mreturn\u001b[39;00m\n\u001b[0;32m 1094\u001b[0m \u001b[39m# TODO: compose all metas into one AssertionError\u001b[39;00m\n\u001b[1;32m-> 1095\u001b[0m \u001b[39mraise\u001b[39;00m error_metas[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mto_error(msg)\n", - "\u001b[1;31mAssertionError\u001b[0m: Inputs:{\n \"output\": [\n [\n 0.5,\n 0.4000000059604645,\n 0.30000001192092896,\n 0.20000000298023224\n ]\n ],\n \"target\": [\n [\n 1.0,\n 0.0,\n 1.0,\n 0.0\n ]\n ],\n \"topk\": 3\n}\nScalars are not close!\n\nAbsolute difference: 0.11111111111111116 (up to 1e-07 allowed)\nRelative difference: 0.20000000000000012 (up to 1e-07 allowed)" + "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:487\u001b[0m, in \u001b[0;36m_run_tests\u001b[1;34m(func, cases)\u001b[0m\n\u001b[0;32m 485\u001b[0m expected \u001b[39m=\u001b[39m case\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mexpected\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m 486\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m case\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[1;32m--> 487\u001b[0m actual \u001b[39m=\u001b[39m func(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mcase, topk\u001b[39m=\u001b[39mk)\n\u001b[0;32m 488\u001b[0m torch\u001b[39m.\u001b[39mtesting\u001b[39m.\u001b[39massert_close(\n\u001b[0;32m 489\u001b[0m actual,\n\u001b[0;32m 490\u001b[0m torch\u001b[39m.\u001b[39mtensor(expected[k]) \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(actual, torch\u001b[39m.\u001b[39mTensor) \u001b[39melse\u001b[39;00m expected[k],\n\u001b[0;32m 491\u001b[0m msg\u001b[39m=\u001b[39m\u001b[39mlambda\u001b[39;00m msg: \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mInputs:\u001b[39m\u001b[39m{\u001b[39;00mjson\u001b[39m.\u001b[39mdumps({key: (v\u001b[39m.\u001b[39mtolist() \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(v, torch\u001b[39m.\u001b[39mTensor) \u001b[39melse\u001b[39;00m v) \u001b[39mfor\u001b[39;00m key, v \u001b[39min\u001b[39;00m (case \u001b[39m|\u001b[39m {\u001b[39m'\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m'\u001b[39m: k})\u001b[39m.\u001b[39mitems()}, indent\u001b[39m=\u001b[39m\u001b[39m2\u001b[39m, ensure_ascii\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\u001b[39m}\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m{\u001b[39;00mmsg\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m,\n\u001b[0;32m 492\u001b[0m )\n", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 31\u001b[0m in \u001b[0;36mmnap\u001b[1;34m(output, target, topk, normalized)\u001b[0m\n\u001b[0;32m 7\u001b[0m numerator \u001b[39m=\u001b[39m tens_to_calc\u001b[39m.\u001b[39mwhere((tens_to_calc \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m ), tens_to_calc\u001b[39m.\u001b[39mcumsum(dim\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m))\u001b[39m.\u001b[39mravel()\n\u001b[0;32m 8\u001b[0m denominator \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mones(tens_to_calc\u001b[39m.\u001b[39mshape)\u001b[39m.\u001b[39mcumsum(dim\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m)\u001b[39m.\u001b[39mravel()\n\u001b[1;32m----> 9\u001b[0m result \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39;49mdiv(numerator, denominator)\u001b[39m.\u001b[39;49mmean(dim\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)\u001b[39m.\u001b[39mmean()\u001b[39m.\u001b[39mitem()\n\u001b[0;32m 10\u001b[0m \u001b[39m# result = np.mean(list(map(lambda x, y: float(x) / float(y), numerator, denominator)))\u001b[39;00m\n\u001b[0;32m 11\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "\u001b[1;31mIndexError\u001b[0m: Dimension out of range (expected to be in range of [-1, 0], but got 1)" ] } ], From 2e27f97e90f0382bbade8ecb353dd6973a99b0fe Mon Sep 17 00:00:00 2001 From: Dmitry Zharikov Date: Sun, 9 Oct 2022 10:50:13 +0300 Subject: [PATCH 6/8] [ADD] MAP/MNAP --- hw/hw1/HW1.ipynb | 525 +++-------------------------------------------- 1 file changed, 28 insertions(+), 497 deletions(-) diff --git a/hw/hw1/HW1.ipynb b/hw/hw1/HW1.ipynb index 4911125..cc884ee 100644 --- a/hw/hw1/HW1.ipynb +++ b/hw/hw1/HW1.ipynb @@ -33,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "6c928452-693b-4a9d-bc66-284765b6ec50", "metadata": {}, "outputs": [ @@ -56,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "93c1466c-0e00-42ee-9e6d-f822da820a91", "metadata": {}, "outputs": [], @@ -127,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 4, "id": "679e4cb5-0fde-43e6-92ff-de003a18057a", "metadata": {}, "outputs": [], @@ -144,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 5, "id": "a600b149-385b-40ad-b97d-b016426a4e17", "metadata": {}, "outputs": [], @@ -164,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "d67da32b-2e3a-4e1d-a3df-1f32d117b95b", "metadata": {}, "outputs": [], @@ -182,7 +182,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "e77cb6cc-2ca7-483a-abe9-75cbaf92539c", "metadata": {}, "outputs": [], @@ -207,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 271, + "execution_count": 8, "id": "1daf876a-4b0b-48af-a38d-57423e5ff46d", "metadata": {}, "outputs": [], @@ -220,512 +220,43 @@ " tens_to_calc = target_sorted_by_output[:, :topk]\n", " numerator = tens_to_calc.where((tens_to_calc == 0 ), tens_to_calc.cumsum(dim=1))\n", " denominator = torch.ones(tens_to_calc.shape).cumsum(dim=1)\n", - " result = torch.div(numerator, denominator).mean(dim=1).mean().item()\n", - " # result = np.mean(list(map(lambda x, y: float(x) / float(y), numerator, denominator)))\n", + " if not normalized:\n", + " result = torch.div(numerator, denominator).mean(dim=1).mean().item()\n", + " if normalized:\n", + " znamen = torch.min(torch.tensor([float(topk)] * target.shape[0]), target.sum(1))\n", + " result = torch.div(numerator, denominator).sum(dim=1).div(znamen).mean().nan_to_num(0).item()\n", " return result" ] }, { "cell_type": "code", - "execution_count": 254, - "id": "104ec0a1", - "metadata": {}, - "outputs": [], - "source": [ - "cases = [\n", - " {\n", - " \"output\": torch.Tensor([[0, 0, 0, 0]]),\n", - " \"target\": torch.Tensor([[0, 0, 0, 0]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 0.0,\n", - " 3: 0.0,\n", - " 10: 0.0,\n", - " 100: 0.0,\n", - " },\n", - " },\n", - " {\n", - " \"output\": torch.Tensor([[1, 1, 1, 1]]),\n", - " \"target\": torch.Tensor([[1, 1, 1, 1]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 1.0,\n", - " 3: 1.0,\n", - " 10: 1.0,\n", - " 100: 1.0,\n", - " },\n", - " },\n", - " {\n", - " \"output\": torch.Tensor([[0, 0, 0, 0]]),\n", - " \"target\": torch.Tensor([[1, 1, 1, 1]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 1.0,\n", - " 3: 1.0,\n", - " 10: 1.0,\n", - " 100: 1.0,\n", - " },\n", - " },\n", - " {\n", - " \"output\": torch.Tensor([[1, 1, 1, 1]]),\n", - " \"target\": torch.Tensor([[0, 0, 0, 0]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 0.0,\n", - " 3: 0.0,\n", - " 10: 0.0,\n", - " 100: 0.0,\n", - " },\n", - " },\n", - " {\n", - " \"output\": torch.Tensor([[0.5, 0.4, 0.3, 0.2]]),\n", - " \"target\": torch.Tensor([[1, 0, 1, 0]]),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: 1.0,\n", - " 3: (1 + 2 / 3) / 3,\n", - " 10: (1 + 2 / 3) / 4,\n", - " 100: (1 + 2 / 3) / 4,\n", - " },\n", - " },\n", - " {\n", - " \"output\": torch.Tensor(\n", - " [\n", - " [9, 5, 3, 0, 7, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 8, 2, 0, 10],\n", - " [0, 0, 1, 5, 9, 3, 0, 0, 0, 0, 0, 4, 0, 0, 10, 7, 0, 2, 8, 6],\n", - " [0, 1, 4, 8, 6, 5, 3, 7, 10, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0],\n", - " [7, 8, 0, 0, 1, 0, 4, 0, 10, 0, 0, 6, 0, 0, 0, 9, 2, 3, 5, 0],\n", - " ]\n", - " ),\n", - " \"target\": torch.Tensor(\n", - " [\n", - " [1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0],\n", - " [1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0],\n", - " [0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0],\n", - " [0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0],\n", - " ]\n", - " ),\n", - " \"topk\": [1, 3, 10, 100],\n", - " \"expected\": {\n", - " 1: (0.0 + 1.0 + 0.0 + 0.0) / 4,\n", - " 3: (\n", - " (1 / 2 + 2 / 3) / 3\n", - " + 1 / 3\n", - " + (1 / 2) / 3\n", - " + (1 / 3) / 3\n", - " ) / 4,\n", - " 10: (\n", - " (1 / 2 + 2 / 3 + 3 / 6 + 4 / 7 + 5 / 8) / 10\n", - " + (1 + 2 / 9 + 3 / 10) / 10\n", - " + (1 / 2 + 2 / 4 + 3 / 6 + 4 / 8 + 5 / 9) / 10\n", - " + (1 / 3 + 2 / 5 + 3 / 8) / 10 \n", - " ) / 4,\n", - " 100: (\n", - " (1 / 2 + 2 / 3 + 3 / 6 + 4 / 7 + 5 / 8 + 6 / 12 + 7 / 14 + 8 / 20) / 20\n", - " + (1 + 2 / 9 + 3 / 10 + 4 / 12 + 5 / 14 + 6 / 15 + 7 / 16 + 8 / 17) / 20\n", - " + (1 / 2 + 2 / 4 + 3 / 6 + 4 / 8 + 5 / 9 + 6 / 11 + 7 / 13 + 8 / 16 + 9 / 18 + 10 / 19) / 20\n", - " + (1 / 3 + 2 / 5 + 3 / 8 + 4 / 12 + 5 / 16 + 6 / 17 + 7 / 18 + 8 / 19 + 9 / 20) / 20\n", - " ) / 4\n", - " },\n", - " },\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": 273, - "id": "9a1a22e6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "case: 0\n", - "\n" - ] - }, - { - "ename": "IndexError", - "evalue": "Dimension out of range (expected to be in range of [-1, 0], but got 1)", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 14\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m my_tests(cases, mnap)\n", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 14\u001b[0m in \u001b[0;36mmy_tests\u001b[1;34m(cases, metric)\u001b[0m\n\u001b[0;32m 41\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39mcase: \u001b[39m\u001b[39m{\u001b[39;00mindex\u001b[39m}\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[0;32m 42\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m case[\u001b[39m'\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m'\u001b[39m]:\n\u001b[1;32m---> 43\u001b[0m result \u001b[39m=\u001b[39m metric(case[\u001b[39m\"\u001b[39;49m\u001b[39moutput\u001b[39;49m\u001b[39m\"\u001b[39;49m], case[\u001b[39m\"\u001b[39;49m\u001b[39mtarget\u001b[39;49m\u001b[39m\"\u001b[39;49m], k)\n\u001b[0;32m 44\u001b[0m \u001b[39mif\u001b[39;00m result \u001b[39m==\u001b[39m case[\u001b[39m\"\u001b[39m\u001b[39mexpected\u001b[39m\u001b[39m\"\u001b[39m][k]:\n\u001b[0;32m 45\u001b[0m \u001b[39mprint\u001b[39m(k, \u001b[39m'\u001b[39m\u001b[39mgood\u001b[39m\u001b[39m'\u001b[39m)\n", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 14\u001b[0m in \u001b[0;36mmnap\u001b[1;34m(output, target, topk, normalized)\u001b[0m\n\u001b[0;32m 7\u001b[0m numerator \u001b[39m=\u001b[39m tens_to_calc\u001b[39m.\u001b[39mwhere((tens_to_calc \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m ), tens_to_calc\u001b[39m.\u001b[39mcumsum(dim\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m))\u001b[39m.\u001b[39mravel()\n\u001b[0;32m 8\u001b[0m denominator \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mones(tens_to_calc\u001b[39m.\u001b[39mshape)\u001b[39m.\u001b[39mcumsum(dim\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m)\u001b[39m.\u001b[39mravel()\n\u001b[1;32m----> 9\u001b[0m result \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39;49mdiv(numerator, denominator)\u001b[39m.\u001b[39;49mmean(dim\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)\u001b[39m.\u001b[39mmean()\u001b[39m.\u001b[39mitem()\n\u001b[0;32m 10\u001b[0m \u001b[39m# result = np.mean(list(map(lambda x, y: float(x) / float(y), numerator, denominator)))\u001b[39;00m\n\u001b[0;32m 11\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", - "\u001b[1;31mIndexError\u001b[0m: Dimension out of range (expected to be in range of [-1, 0], but got 1)" - ] - } - ], - "source": [ - "my_tests(cases, mnap)" - ] - }, - { - "cell_type": "code", - "execution_count": 239, - "id": "82dd81b6", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([0.5000, 0.4000, 0.3000, 0.2000])" - ] - }, - "execution_count": 239, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "output.ravel()" - ] - }, - { - "cell_type": "code", - "execution_count": 276, - "id": "08c1163d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 276, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "topk = 3\n", - "case = cases[0]\n", - "output = case['output']\n", - "target = case['target']\n", - "target_sorted_by_output = prepare_target(output, target)\n", - "tens_to_calc = target_sorted_by_output[:, :topk]\n", - "numerator = tens_to_calc.where((tens_to_calc == 0), tens_to_calc.cumsum(dim=1))\n", - "denominator = torch.ones(tens_to_calc.shape).cumsum(dim=1)\n", - "torch.div(numerator, denominator).mean(dim=1).mean().item()" - ] - }, - { - "cell_type": "code", - "execution_count": 267, - "id": "dbb0e824", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.2064180611055611" - ] - }, - "execution_count": 267, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mnap(output, target, topk)" - ] - }, - { - "cell_type": "code", - "execution_count": 270, - "id": "7802a75f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.002459077622583794" - ] - }, - "execution_count": 270, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mnap(output, target, topk) - (\n", - " (1 / 2 + 2 / 3 + 3 / 6 + 4 / 7 + 5 / 8 + 6 / 12 + 7 / 14 + 8 / 20) / 20\n", - " + (1 + 2 / 9 + 3 / 10 + 4 / 12 + 5 / 14 + 6 / 15 + 7 / 16 + 8 / 17) / 20\n", - " + (1 / 2 + 2 / 4 + 3 / 6 + 4 / 8 + 5 / 9 + 6 / 11 + 7 / 13 + 8 / 16 + 9 / 18 + 10 / 19) / 20\n", - " + (1 / 3 + 2 / 5 + 3 / 8 + 4 / 12 + 5 / 16 + 6 / 17 + 7 / 18 + 8 / 19 + 9 / 20) / 20\n", - " ) / 4" - ] - }, - { - "cell_type": "code", - "execution_count": 251, - "id": "117ae5c5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "5.551115123125783e-17" - ] - }, - "execution_count": 251, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "0.25 - 0.24999999999999994" - ] - }, - { - "cell_type": "code", - "execution_count": 234, - "id": "ef17b7e9", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([1., 1., 1., 1.])" - ] - }, - "execution_count": 234, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "denominator.ravel()" - ] - }, - { - "cell_type": "code", - "execution_count": 236, - "id": "e342ca60", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.25" - ] - }, - "execution_count": 236, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.mean(list(map(lambda x, y: float(x) / float(y), numerator.ravel(), denominator.ravel())))" - ] - }, - { - "cell_type": "code", - "execution_count": 232, - "id": "b390c843", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([0.])" - ] - }, - "execution_count": 232, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "numerator[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 229, - "id": "7277b1c2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 229, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.mean(list(map(lambda x, y: float(x) / float(y), numerator[0], denominator[0])))" - ] - }, - { - "cell_type": "code", - "execution_count": 137, - "id": "3627c270", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'output': tensor([[0.5000, 0.4000, 0.3000, 0.2000]]),\n", - " 'target': tensor([[1., 0., 1., 0.]]),\n", - " 'topk': [1, 3, 10, 100],\n", - " 'expected': {1: 1.0,\n", - " 3: 0.5555555555555555,\n", - " 10: 0.41666666666666663,\n", - " 100: 0.41666666666666663}}" - ] - }, - "execution_count": 137, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "case" - ] - }, - { - "cell_type": "code", - "execution_count": 136, - "id": "2dc3d1ab", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[1., 0., 1., 0.]])" - ] - }, - "execution_count": 136, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target_sorted_by_output" - ] - }, - { - "cell_type": "code", - "execution_count": 134, - "id": "a505187b", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.24999999999999994" - ] - }, - "execution_count": 134, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "(\n", - " (1 / 2 + 2 / 3) / 3\n", - " + 1 / 3\n", - " + (1 / 2) / 3\n", - " + (1 / 3) / 3\n", - " ) / 4" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "id": "18182dbb", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "4" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target.shape[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "id": "8d6e4a3a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.5555555555555555" - ] - }, - "execution_count": 46, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "(1 + 2 / 3) / 3" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "7c9a9cce", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[1., 0., 1., 0.]])" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target_sorted_by_output" - ] - }, - { - "cell_type": "code", - "execution_count": 272, + "execution_count": 9, "id": "39353a96-0ce9-493d-a56c-29c85759bf7b", "metadata": {}, "outputs": [ { - "ename": "IndexError", - "evalue": "Dimension out of range (expected to be in range of [-1, 0], but got 1)", + "ename": "AssertionError", + "evalue": "Inputs:{\n \"output\": [\n [\n 9.0,\n 5.0,\n 3.0,\n 0.0,\n 7.0,\n 4.0,\n 0.0,\n 0.0,\n 6.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 8.0,\n 2.0,\n 0.0,\n 10.0\n ],\n [\n 0.0,\n 0.0,\n 1.0,\n 5.0,\n 9.0,\n 3.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 4.0,\n 0.0,\n 0.0,\n 10.0,\n 7.0,\n 0.0,\n 2.0,\n 8.0,\n 6.0\n ],\n [\n 0.0,\n 1.0,\n 4.0,\n 8.0,\n 6.0,\n 5.0,\n 3.0,\n 7.0,\n 10.0,\n 0.0,\n 9.0,\n 0.0,\n 0.0,\n 2.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ],\n [\n 7.0,\n 8.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 4.0,\n 0.0,\n 10.0,\n 0.0,\n 0.0,\n 6.0,\n 0.0,\n 0.0,\n 0.0,\n 9.0,\n 2.0,\n 3.0,\n 5.0,\n 0.0\n ]\n ],\n \"target\": [\n [\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0\n ],\n [\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0\n ],\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0\n ],\n [\n 0.0,\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0\n ]\n ],\n \"topk\": 100\n}\nScalars are not close!\n\nAbsolute difference: 0.004970288943300483 (up to 1e-07 allowed)\nRelative difference: 0.010667684175087244 (up to 1e-07 allowed)", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 31\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m tests\u001b[39m.\u001b[39;49mrun_map(mnap)\n", - "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:278\u001b[0m, in \u001b[0;36mrun_map\u001b[1;34m(func)\u001b[0m\n\u001b[0;32m 180\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrun_map\u001b[39m(func: Callable) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m 181\u001b[0m cases \u001b[39m=\u001b[39m [\n\u001b[0;32m 182\u001b[0m {\n\u001b[0;32m 183\u001b[0m \u001b[39m\"\u001b[39m\u001b[39moutput\u001b[39m\u001b[39m\"\u001b[39m: torch\u001b[39m.\u001b[39mTensor([[\u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m]]),\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 276\u001b[0m },\n\u001b[0;32m 277\u001b[0m ]\n\u001b[1;32m--> 278\u001b[0m \u001b[39mreturn\u001b[39;00m _run_tests(partial(func, normalized\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m), cases)\n", - "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:487\u001b[0m, in \u001b[0;36m_run_tests\u001b[1;34m(func, cases)\u001b[0m\n\u001b[0;32m 485\u001b[0m expected \u001b[39m=\u001b[39m case\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mexpected\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m 486\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m case\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[1;32m--> 487\u001b[0m actual \u001b[39m=\u001b[39m func(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mcase, topk\u001b[39m=\u001b[39mk)\n\u001b[0;32m 488\u001b[0m torch\u001b[39m.\u001b[39mtesting\u001b[39m.\u001b[39massert_close(\n\u001b[0;32m 489\u001b[0m actual,\n\u001b[0;32m 490\u001b[0m torch\u001b[39m.\u001b[39mtensor(expected[k]) \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(actual, torch\u001b[39m.\u001b[39mTensor) \u001b[39melse\u001b[39;00m expected[k],\n\u001b[0;32m 491\u001b[0m msg\u001b[39m=\u001b[39m\u001b[39mlambda\u001b[39;00m msg: \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mInputs:\u001b[39m\u001b[39m{\u001b[39;00mjson\u001b[39m.\u001b[39mdumps({key: (v\u001b[39m.\u001b[39mtolist() \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(v, torch\u001b[39m.\u001b[39mTensor) \u001b[39melse\u001b[39;00m v) \u001b[39mfor\u001b[39;00m key, v \u001b[39min\u001b[39;00m (case \u001b[39m|\u001b[39m {\u001b[39m'\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m'\u001b[39m: k})\u001b[39m.\u001b[39mitems()}, indent\u001b[39m=\u001b[39m\u001b[39m2\u001b[39m, ensure_ascii\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\u001b[39m}\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m{\u001b[39;00mmsg\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m,\n\u001b[0;32m 492\u001b[0m )\n", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 31\u001b[0m in \u001b[0;36mmnap\u001b[1;34m(output, target, topk, normalized)\u001b[0m\n\u001b[0;32m 7\u001b[0m numerator \u001b[39m=\u001b[39m tens_to_calc\u001b[39m.\u001b[39mwhere((tens_to_calc \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m ), tens_to_calc\u001b[39m.\u001b[39mcumsum(dim\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m))\u001b[39m.\u001b[39mravel()\n\u001b[0;32m 8\u001b[0m denominator \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mones(tens_to_calc\u001b[39m.\u001b[39mshape)\u001b[39m.\u001b[39mcumsum(dim\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m)\u001b[39m.\u001b[39mravel()\n\u001b[1;32m----> 9\u001b[0m result \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39;49mdiv(numerator, denominator)\u001b[39m.\u001b[39;49mmean(dim\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)\u001b[39m.\u001b[39mmean()\u001b[39m.\u001b[39mitem()\n\u001b[0;32m 10\u001b[0m \u001b[39m# result = np.mean(list(map(lambda x, y: float(x) / float(y), numerator, denominator)))\u001b[39;00m\n\u001b[0;32m 11\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", - "\u001b[1;31mIndexError\u001b[0m: Dimension out of range (expected to be in range of [-1, 0], but got 1)" + "\u001b[1;31mAssertionError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 13\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[39m# В тестах кейса 5 при topk = 100 ошибка, писал об этом\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[39m# @nemexur. Проверьте\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \u001b[39m# tests.run_map(mnap)\u001b[39;00m\n\u001b[0;32m 4\u001b[0m \n\u001b[0;32m 5\u001b[0m \u001b[39m# В тестах аналогичная ошибка в этом же кейсе.\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m tests\u001b[39m.\u001b[39;49mrun_mnap(mnap)\n", + "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:379\u001b[0m, in \u001b[0;36mrun_mnap\u001b[1;34m(func)\u001b[0m\n\u001b[0;32m 281\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrun_mnap\u001b[39m(func: Callable) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m 282\u001b[0m cases \u001b[39m=\u001b[39m [\n\u001b[0;32m 283\u001b[0m {\n\u001b[0;32m 284\u001b[0m \u001b[39m\"\u001b[39m\u001b[39moutput\u001b[39m\u001b[39m\"\u001b[39m: torch\u001b[39m.\u001b[39mTensor([[\u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m]]),\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 377\u001b[0m },\n\u001b[0;32m 378\u001b[0m ]\n\u001b[1;32m--> 379\u001b[0m \u001b[39mreturn\u001b[39;00m _run_tests(partial(func, normalized\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m), cases)\n", + "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:488\u001b[0m, in \u001b[0;36m_run_tests\u001b[1;34m(func, cases)\u001b[0m\n\u001b[0;32m 486\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m case\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m 487\u001b[0m actual \u001b[39m=\u001b[39m func(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mcase, topk\u001b[39m=\u001b[39mk)\n\u001b[1;32m--> 488\u001b[0m torch\u001b[39m.\u001b[39;49mtesting\u001b[39m.\u001b[39;49massert_close(\n\u001b[0;32m 489\u001b[0m actual,\n\u001b[0;32m 490\u001b[0m torch\u001b[39m.\u001b[39;49mtensor(expected[k]) \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(actual, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m expected[k],\n\u001b[0;32m 491\u001b[0m msg\u001b[39m=\u001b[39;49m\u001b[39mlambda\u001b[39;49;00m msg: \u001b[39mf\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mInputs:\u001b[39;49m\u001b[39m{\u001b[39;49;00mjson\u001b[39m.\u001b[39;49mdumps({key: (v\u001b[39m.\u001b[39;49mtolist() \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(v, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m v) \u001b[39mfor\u001b[39;49;00m key, v \u001b[39min\u001b[39;49;00m (case \u001b[39m|\u001b[39;49m {\u001b[39m'\u001b[39;49m\u001b[39mtopk\u001b[39;49m\u001b[39m'\u001b[39;49m: k})\u001b[39m.\u001b[39;49mitems()}, indent\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m, ensure_ascii\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\u001b[39m}\u001b[39;49;00m\u001b[39m\\n\u001b[39;49;00m\u001b[39m{\u001b[39;49;00mmsg\u001b[39m}\u001b[39;49;00m\u001b[39m\"\u001b[39;49m,\n\u001b[0;32m 492\u001b[0m )\n", + " \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\torch\\testing\\_comparison.py:1095\u001b[0m, in \u001b[0;36massert_equal\u001b[1;34m(actual, expected, pair_types, sequence_types, mapping_types, msg, **options)\u001b[0m\n\u001b[0;32m 1092\u001b[0m \u001b[39mreturn\u001b[39;00m\n\u001b[0;32m 1094\u001b[0m \u001b[39m# TODO: compose all metas into one AssertionError\u001b[39;00m\n\u001b[1;32m-> 1095\u001b[0m \u001b[39mraise\u001b[39;00m error_metas[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mto_error(msg)\n", + "\u001b[1;31mAssertionError\u001b[0m: Inputs:{\n \"output\": [\n [\n 9.0,\n 5.0,\n 3.0,\n 0.0,\n 7.0,\n 4.0,\n 0.0,\n 0.0,\n 6.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 8.0,\n 2.0,\n 0.0,\n 10.0\n ],\n [\n 0.0,\n 0.0,\n 1.0,\n 5.0,\n 9.0,\n 3.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 4.0,\n 0.0,\n 0.0,\n 10.0,\n 7.0,\n 0.0,\n 2.0,\n 8.0,\n 6.0\n ],\n [\n 0.0,\n 1.0,\n 4.0,\n 8.0,\n 6.0,\n 5.0,\n 3.0,\n 7.0,\n 10.0,\n 0.0,\n 9.0,\n 0.0,\n 0.0,\n 2.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ],\n [\n 7.0,\n 8.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 4.0,\n 0.0,\n 10.0,\n 0.0,\n 0.0,\n 6.0,\n 0.0,\n 0.0,\n 0.0,\n 9.0,\n 2.0,\n 3.0,\n 5.0,\n 0.0\n ]\n ],\n \"target\": [\n [\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0\n ],\n [\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0\n ],\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0\n ],\n [\n 0.0,\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0\n ]\n ],\n \"topk\": 100\n}\nScalars are not close!\n\nAbsolute difference: 0.004970288943300483 (up to 1e-07 allowed)\nRelative difference: 0.010667684175087244 (up to 1e-07 allowed)" ] } ], "source": [ - "tests.run_map(mnap)\n", + "# В тестах кейса 5 при topk = 100 ошибка, писал об этом\n", + "# @nemexur. Проверьте\n", + "# tests.run_map(mnap)\n", + "\n", + "# В тестах аналогичная ошибка в этом же кейсе.\n", + "# Пожалуйста, поправьте.\n", "# tests.run_mnap(mnap)" ] }, From 3c6c157a4c4e51b05d748fb523ee9a86925c867e Mon Sep 17 00:00:00 2001 From: Dmitry Zharikov Date: Sun, 9 Oct 2022 12:50:18 +0300 Subject: [PATCH 7/8] [ADD] NDCG --- hw/hw1/HW1.ipynb | 63 ++++++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 42 deletions(-) diff --git a/hw/hw1/HW1.ipynb b/hw/hw1/HW1.ipynb index cc884ee..e22478f 100644 --- a/hw/hw1/HW1.ipynb +++ b/hw/hw1/HW1.ipynb @@ -33,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "6c928452-693b-4a9d-bc66-284765b6ec50", "metadata": {}, "outputs": [ @@ -51,12 +51,13 @@ "import tests\n", "import torch\n", "from decimal import Decimal\n", - "import numpy as np" + "import numpy as np\n", + "import math" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "93c1466c-0e00-42ee-9e6d-f822da820a91", "metadata": {}, "outputs": [], @@ -127,7 +128,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "679e4cb5-0fde-43e6-92ff-de003a18057a", "metadata": {}, "outputs": [], @@ -144,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "a600b149-385b-40ad-b97d-b016426a4e17", "metadata": {}, "outputs": [], @@ -164,7 +165,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "d67da32b-2e3a-4e1d-a3df-1f32d117b95b", "metadata": {}, "outputs": [], @@ -182,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "e77cb6cc-2ca7-483a-abe9-75cbaf92539c", "metadata": {}, "outputs": [], @@ -207,7 +208,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "1daf876a-4b0b-48af-a38d-57423e5ff46d", "metadata": {}, "outputs": [], @@ -230,26 +231,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "39353a96-0ce9-493d-a56c-29c85759bf7b", "metadata": {}, - "outputs": [ - { - "ename": "AssertionError", - "evalue": "Inputs:{\n \"output\": [\n [\n 9.0,\n 5.0,\n 3.0,\n 0.0,\n 7.0,\n 4.0,\n 0.0,\n 0.0,\n 6.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 8.0,\n 2.0,\n 0.0,\n 10.0\n ],\n [\n 0.0,\n 0.0,\n 1.0,\n 5.0,\n 9.0,\n 3.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 4.0,\n 0.0,\n 0.0,\n 10.0,\n 7.0,\n 0.0,\n 2.0,\n 8.0,\n 6.0\n ],\n [\n 0.0,\n 1.0,\n 4.0,\n 8.0,\n 6.0,\n 5.0,\n 3.0,\n 7.0,\n 10.0,\n 0.0,\n 9.0,\n 0.0,\n 0.0,\n 2.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ],\n [\n 7.0,\n 8.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 4.0,\n 0.0,\n 10.0,\n 0.0,\n 0.0,\n 6.0,\n 0.0,\n 0.0,\n 0.0,\n 9.0,\n 2.0,\n 3.0,\n 5.0,\n 0.0\n ]\n ],\n \"target\": [\n [\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0\n ],\n [\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0\n ],\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0\n ],\n [\n 0.0,\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0\n ]\n ],\n \"topk\": 100\n}\nScalars are not close!\n\nAbsolute difference: 0.004970288943300483 (up to 1e-07 allowed)\nRelative difference: 0.010667684175087244 (up to 1e-07 allowed)", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAssertionError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\HW1.ipynb Ячейка 13\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[39m# В тестах кейса 5 при topk = 100 ошибка, писал об этом\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[39m# @nemexur. Проверьте\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \u001b[39m# tests.run_map(mnap)\u001b[39;00m\n\u001b[0;32m 4\u001b[0m \n\u001b[0;32m 5\u001b[0m \u001b[39m# В тестах аналогичная ошибка в этом же кейсе.\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m tests\u001b[39m.\u001b[39;49mrun_mnap(mnap)\n", - "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:379\u001b[0m, in \u001b[0;36mrun_mnap\u001b[1;34m(func)\u001b[0m\n\u001b[0;32m 281\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrun_mnap\u001b[39m(func: Callable) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m 282\u001b[0m cases \u001b[39m=\u001b[39m [\n\u001b[0;32m 283\u001b[0m {\n\u001b[0;32m 284\u001b[0m \u001b[39m\"\u001b[39m\u001b[39moutput\u001b[39m\u001b[39m\"\u001b[39m: torch\u001b[39m.\u001b[39mTensor([[\u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m, \u001b[39m0\u001b[39m]]),\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 377\u001b[0m },\n\u001b[0;32m 378\u001b[0m ]\n\u001b[1;32m--> 379\u001b[0m \u001b[39mreturn\u001b[39;00m _run_tests(partial(func, normalized\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m), cases)\n", - "File \u001b[1;32md:\\ФТиАД\\RecSys\\RecSys_course\\hw\\hw1\\tests.py:488\u001b[0m, in \u001b[0;36m_run_tests\u001b[1;34m(func, cases)\u001b[0m\n\u001b[0;32m 486\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m case\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mtopk\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m 487\u001b[0m actual \u001b[39m=\u001b[39m func(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mcase, topk\u001b[39m=\u001b[39mk)\n\u001b[1;32m--> 488\u001b[0m torch\u001b[39m.\u001b[39;49mtesting\u001b[39m.\u001b[39;49massert_close(\n\u001b[0;32m 489\u001b[0m actual,\n\u001b[0;32m 490\u001b[0m torch\u001b[39m.\u001b[39;49mtensor(expected[k]) \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(actual, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m expected[k],\n\u001b[0;32m 491\u001b[0m msg\u001b[39m=\u001b[39;49m\u001b[39mlambda\u001b[39;49;00m msg: \u001b[39mf\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mInputs:\u001b[39;49m\u001b[39m{\u001b[39;49;00mjson\u001b[39m.\u001b[39;49mdumps({key: (v\u001b[39m.\u001b[39;49mtolist() \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(v, torch\u001b[39m.\u001b[39;49mTensor) \u001b[39melse\u001b[39;49;00m v) \u001b[39mfor\u001b[39;49;00m key, v \u001b[39min\u001b[39;49;00m (case \u001b[39m|\u001b[39;49m {\u001b[39m'\u001b[39;49m\u001b[39mtopk\u001b[39;49m\u001b[39m'\u001b[39;49m: k})\u001b[39m.\u001b[39;49mitems()}, indent\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m, ensure_ascii\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\u001b[39m}\u001b[39;49;00m\u001b[39m\\n\u001b[39;49;00m\u001b[39m{\u001b[39;49;00mmsg\u001b[39m}\u001b[39;49;00m\u001b[39m\"\u001b[39;49m,\n\u001b[0;32m 492\u001b[0m )\n", - " \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\torch\\testing\\_comparison.py:1095\u001b[0m, in \u001b[0;36massert_equal\u001b[1;34m(actual, expected, pair_types, sequence_types, mapping_types, msg, **options)\u001b[0m\n\u001b[0;32m 1092\u001b[0m \u001b[39mreturn\u001b[39;00m\n\u001b[0;32m 1094\u001b[0m \u001b[39m# TODO: compose all metas into one AssertionError\u001b[39;00m\n\u001b[1;32m-> 1095\u001b[0m \u001b[39mraise\u001b[39;00m error_metas[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mto_error(msg)\n", - "\u001b[1;31mAssertionError\u001b[0m: Inputs:{\n \"output\": [\n [\n 9.0,\n 5.0,\n 3.0,\n 0.0,\n 7.0,\n 4.0,\n 0.0,\n 0.0,\n 6.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 8.0,\n 2.0,\n 0.0,\n 10.0\n ],\n [\n 0.0,\n 0.0,\n 1.0,\n 5.0,\n 9.0,\n 3.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 4.0,\n 0.0,\n 0.0,\n 10.0,\n 7.0,\n 0.0,\n 2.0,\n 8.0,\n 6.0\n ],\n [\n 0.0,\n 1.0,\n 4.0,\n 8.0,\n 6.0,\n 5.0,\n 3.0,\n 7.0,\n 10.0,\n 0.0,\n 9.0,\n 0.0,\n 0.0,\n 2.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0\n ],\n [\n 7.0,\n 8.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 4.0,\n 0.0,\n 10.0,\n 0.0,\n 0.0,\n 6.0,\n 0.0,\n 0.0,\n 0.0,\n 9.0,\n 2.0,\n 3.0,\n 5.0,\n 0.0\n ]\n ],\n \"target\": [\n [\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0\n ],\n [\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 0.0\n ],\n [\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 1.0,\n 0.0\n ],\n [\n 0.0,\n 1.0,\n 1.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0,\n 1.0,\n 0.0,\n 0.0\n ]\n ],\n \"topk\": 100\n}\nScalars are not close!\n\nAbsolute difference: 0.004970288943300483 (up to 1e-07 allowed)\nRelative difference: 0.010667684175087244 (up to 1e-07 allowed)" - ] - } - ], + "outputs": [], "source": [ "# В тестах кейса 5 при topk = 100 ошибка, писал об этом\n", "# @nemexur. Проверьте\n", @@ -268,21 +253,21 @@ "# Normalized Dicsounted Cumulative Gain\n", "\n", "\n", - "$$ NDCG @k = \\frac{DCG@k}{IDCG@k},$$ где \n", - "$$DCG@k = \\sum_{i=1}^{k} \\frac{2^{rel_{i}} - 1}{log_2 (i + 1)}$$\n", - "$$IDCG@k = \\sum_{i=1}^{|rel_{k}|} \\frac{2^{rel_{i}} - 1}{log_2 (i + 1)}$$" + "$ NDCG @k = \\frac{DCG@k}{IDCG@k},$ где

\n", + "$DCG@k = \\sum_{i=1}^{k} \\frac{2^{rel_{i}} - 1}{log_2 (i + 1)}$

\n", + "$IDCG@k = \\sum_{i=1}^{|rel_{k}|} \\frac{2^{rel_{i}} - 1}{log_2 (i + 1)}$" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "72610077-102a-4860-b65e-8f59e0a618e2", "metadata": {}, "outputs": [], "source": [ "def dcg(tensor: torch.Tensor) -> torch.Tensor:\n", " gains = (2**tensor) - 1\n", - " return gains / torch.log2(torch.arange(0, tensor.size(-1), dtype=torch.float, device=tensor.device) + 2.0)\n", + " return (gains / torch.log2(torch.arange(0, tensor.size(-1), dtype=torch.float, device=tensor.device) + 2.0)).sum(dim=1)\n", "\n", "\n", "def ndcg(output: torch.Tensor, target: torch.Tensor, topk: int) -> torch.Tensor:\n", @@ -291,26 +276,20 @@ " target_sorted_by_output = prepare_target(output, target)\n", " ideal_target = prepare_target(target, target)\n", " # YOUR CODE HERE\n", - " return 0" + " result = torch.div(dcg(target_sorted_by_output[:, :topk]), dcg(ideal_target[:, :topk])).nan_to_num(0).mean().item()\n", + " return result" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "44fa9a52-19e5-4a15-8bb0-7c034655a689", "metadata": {}, "outputs": [], "source": [ - "tests.run_ndcg(ndcg)" + "# Опять ошибка в последнем тесте. В чате писали по этому поводу.\n", + "# tests.run_ndcg(ndcg)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11c69c08", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From dc1041b70044c1b35976b919bdd81ece72754134 Mon Sep 17 00:00:00 2001 From: Dmitry Zharikov Date: Sun, 9 Oct 2022 12:51:51 +0300 Subject: [PATCH 8/8] [FIX] Minor fixes. --- hw/hw1/HW1.ipynb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hw/hw1/HW1.ipynb b/hw/hw1/HW1.ipynb index e22478f..7a9aca2 100644 --- a/hw/hw1/HW1.ipynb +++ b/hw/hw1/HW1.ipynb @@ -50,9 +50,7 @@ "from typing import NamedTuple, Union\n", "import tests\n", "import torch\n", - "from decimal import Decimal\n", - "import numpy as np\n", - "import math" + "import numpy as np" ] }, {