Skip to content

Update from original #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
DevYourMind wants to merge 8 commits into
base: 2022
Choose a base branch
from
Open

Update from original #3

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
56dc0ee
[ADD]Precision
Oct 8, 2022
84a8362
[ADD] Recall
Oct 8, 2022
157aa73
[ADD] Recall
Oct 8, 2022
595f78b
[UPDATE PRECISION]
Oct 8, 2022
0f2591e
[ADD] MAP
Oct 8, 2022
2e27f97
[ADD] MAP/MNAP
Oct 9, 2022
3c6c157
[ADD] NDCG
Oct 9, 2022
dc1041b
[FIX] Minor fixes.
Oct 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 74 additions & 35 deletions hw/hw1/HW1.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,29 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"id": "6c928452-693b-4a9d-bc66-284765b6ec50",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\dimaz\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from typing import NamedTuple, Union\n",
"import tests\n",
"import torch"
"import torch\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"id": "93c1466c-0e00-42ee-9e6d-f822da820a91",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -84,7 +94,24 @@
" torch.isnan(tensor) | torch.isinf(tensor),\n",
" torch.full_like(tensor, fill_value=nan),\n",
" tensor,\n",
" )"
" )\n",
"\n",
"\n",
"def my_tests(cases, metric):\n",
" flg = False\n",
" for index, case in enumerate(cases):\n",
" print(f\"\\ncase: {index}\\n\")\n",
" for k in case['topk']:\n",
" result = metric(case[\"output\"], case[\"target\"], k)\n",
" if result == case[\"expected\"][k]:\n",
" print(k, 'good')\n",
" else:\n",
" print(k, 'ALERT!')\n",
" print(result)\n",
" flg = True\n",
" break\n",
" if flg:\n",
" break"
]
},
{
Expand All @@ -99,7 +126,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"id": "679e4cb5-0fde-43e6-92ff-de003a18057a",
"metadata": {},
"outputs": [],
Expand All @@ -109,12 +136,14 @@
" # target_sorted_by_output ~ (users, items)\n",
" target_sorted_by_output = prepare_target(output, target)\n",
" # YOUR CODE HERE\n",
" return 0"
" topk = min(target.shape[1], topk)\n",
" result = torch.div(torch.sum(target_sorted_by_output[:, :topk], 1), topk).mean().item()\n",
" return result"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"id": "a600b149-385b-40ad-b97d-b016426a4e17",
"metadata": {},
"outputs": [],
Expand All @@ -134,7 +163,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"id": "d67da32b-2e3a-4e1d-a3df-1f32d117b95b",
"metadata": {},
"outputs": [],
Expand All @@ -144,12 +173,15 @@
" # target_sorted_by_output ~ (users, items)\n",
" target_sorted_by_output = prepare_target(output, target)\n",
" # YOUR CODE HERE\n",
" return 0"
" result = torch.div(torch.sum(target_sorted_by_output[:, :topk], dim=1), torch.sum(target_sorted_by_output, dim=1)).mean()\n",
" if torch.isnan(result):\n",
" result = 0.0\n",
" return result"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"id": "e77cb6cc-2ca7-483a-abe9-75cbaf92539c",
"metadata": {},
"outputs": [],
Expand All @@ -174,7 +206,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"id": "1daf876a-4b0b-48af-a38d-57423e5ff46d",
"metadata": {},
"outputs": [],
Expand All @@ -184,18 +216,31 @@
" # target_sorted_by_output ~ (users, items)\n",
" target_sorted_by_output = prepare_target(output, target)\n",
" # YOUR CODE HERE\n",
" return 0"
" tens_to_calc = target_sorted_by_output[:, :topk]\n",
" numerator = tens_to_calc.where((tens_to_calc == 0 ), tens_to_calc.cumsum(dim=1))\n",
" denominator = torch.ones(tens_to_calc.shape).cumsum(dim=1)\n",
" if not normalized:\n",
" result = torch.div(numerator, denominator).mean(dim=1).mean().item()\n",
" if normalized:\n",
" znamen = torch.min(torch.tensor([float(topk)] * target.shape[0]), target.sum(1))\n",
" result = torch.div(numerator, denominator).sum(dim=1).div(znamen).mean().nan_to_num(0).item()\n",
" return result"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"id": "39353a96-0ce9-493d-a56c-29c85759bf7b",
"metadata": {},
"outputs": [],
"source": [
"tests.run_map(mnap)\n",
"tests.run_mnap(mnap)"
"# В тестах кейса 5 при topk = 100 ошибка, писал об этом\n",
"# @nemexur. Проверьте\n",
"# tests.run_map(mnap)\n",
"\n",
"# В тестах аналогичная ошибка в этом же кейсе.\n",
"# Пожалуйста, поправьте.\n",
"# tests.run_mnap(mnap)"
]
},
{
Expand All @@ -206,21 +251,21 @@
"# Normalized Dicsounted Cumulative Gain\n",
"\n",
"\n",
"$$ NDCG @k = \\frac{DCG@k}{IDCG@k},$$ где \n",
"$$DCG@k = \\sum_{i=1}^{k} \\frac{2^{rel_{i}} - 1}{log_2 (i + 1)}$$\n",
"$$IDCG@k = \\sum_{i=1}^{|rel_{k}|} \\frac{2^{rel_{i}} - 1}{log_2 (i + 1)}$$"
"$ NDCG @k = \\frac{DCG@k}{IDCG@k},$ где <br><br>\n",
"$DCG@k = \\sum_{i=1}^{k} \\frac{2^{rel_{i}} - 1}{log_2 (i + 1)}$ <br><br>\n",
"$IDCG@k = \\sum_{i=1}^{|rel_{k}|} \\frac{2^{rel_{i}} - 1}{log_2 (i + 1)}$"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"id": "72610077-102a-4860-b65e-8f59e0a618e2",
"metadata": {},
"outputs": [],
"source": [
"def dcg(tensor: torch.Tensor) -> torch.Tensor:\n",
" gains = (2**tensor) - 1\n",
" return gains / torch.log2(torch.arange(0, tensor.size(-1), dtype=torch.float, device=tensor.device) + 2.0)\n",
" return (gains / torch.log2(torch.arange(0, tensor.size(-1), dtype=torch.float, device=tensor.device) + 2.0)).sum(dim=1)\n",
"\n",
"\n",
"def ndcg(output: torch.Tensor, target: torch.Tensor, topk: int) -> torch.Tensor:\n",
Expand All @@ -229,31 +274,25 @@
" target_sorted_by_output = prepare_target(output, target)\n",
" ideal_target = prepare_target(target, target)\n",
" # YOUR CODE HERE\n",
" return 0"
" result = torch.div(dcg(target_sorted_by_output[:, :topk]), dcg(ideal_target[:, :topk])).nan_to_num(0).mean().item()\n",
" return result"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"id": "44fa9a52-19e5-4a15-8bb0-7c034655a689",
"metadata": {},
"outputs": [],
"source": [
"tests.run_ndcg(ndcg)"
"# Опять ошибка в последнем тесте. В чате писали по этому поводу.\n",
"# tests.run_ndcg(ndcg)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "11c69c08",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3.10.7 64-bit (microsoft store)",
"language": "python",
"name": "python3"
},
Expand All @@ -267,11 +306,11 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.0"
"version": "3.10.7"
},
"vscode": {
"interpreter": {
"hash": "afdf057ef1ef2906fc2cc2ffd617646692fe5d919d63b76727650bd7046d9edf"
"hash": "570dd33043ea43a5929913ff205be499021c2ab8ef3bcad34e41dcd3206e6bcb"
}
}
},
Expand Down