diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..8ac9a80 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,27 @@ +name: Lint + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install ruff + run: pip install ruff + + - name: Run ruff linter + run: ruff check . + + - name: Run ruff formatter check + run: ruff format --check . diff --git a/notebooks/collie_feedback_gen.ipynb b/notebooks/collie_feedback_gen.ipynb index dcbf21a..2e4d3b5 100644 --- a/notebooks/collie_feedback_gen.ipynb +++ b/notebooks/collie_feedback_gen.ipynb @@ -21,7 +21,6 @@ "outputs": [], "source": [ "import dill\n", - "from pathlib import Path\n", "import sys" ] }, @@ -54,7 +53,9 @@ ], "source": [ "out = open(\"../src/feedback/collie.py\", \"w+\")\n", - "out.write(\"from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison\\n\\ncollie_feedback = [\\n\")" + "out.write(\n", + " \"from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison\\n\\ncollie_feedback = [\\n\"\n", + ")" ] }, { @@ -63,7 +64,15 @@ "metadata": {}, "outputs": [], "source": [ - "def get_feedback(prompt, domain, effect, categories, metric, metric_value, comparison=\"Comparison.greater_eq_than\"):\n", + "def get_feedback(\n", + " prompt,\n", + " domain,\n", + " effect,\n", + " categories,\n", + " metric,\n", + " metric_value,\n", + " comparison=\"Comparison.greater_eq_than\",\n", + "):\n", " return \"\"\"\n", " Feedback(\n", " content=\"{prompt}\",\n", @@ -76,7 +85,15 @@ " metric_value={metric_value},\n", " comparison={comparison}\n", " ),\n", - "\"\"\".format(prompt=prompt, domain=domain, effect=effect, categories=categories, metric=metric, metric_value=metric_value, comparison=comparison)" + "\"\"\".format(\n", + " prompt=prompt,\n", + " domain=domain,\n", + " effect=effect,\n", + " categories=categories,\n", + " metric=metric,\n", + " metric_value=metric_value,\n", + " comparison=comparison,\n", + " )" ] }, { @@ -86,15 +103,25 @@ "outputs": [], "source": [ "import numpy as np\n", + "\n", "key = \"wiki_c07\"\n", "chosen = np.random.choice(np.arange(len(all_data[key])), size=38, replace=False)\n", "for i in chosen:\n", " obj = all_data[key][i]\n", - " topic = obj['metadata'][\"title\"]\n", - " words = obj['prompt'].split(\"containing the word\")[1][:-1].strip()\n", - " prompt = f\"When talking about {topic}, make sure the response contain the words {words}\"\n", + " topic = obj[\"metadata\"][\"title\"]\n", + " words = obj[\"prompt\"].split(\"containing the word\")[1][:-1].strip()\n", + " prompt = (\n", + " f\"When talking about {topic}, make sure the response contain the words {words}\"\n", + " )\n", " domain = f\"Talking about {topic}\"\n", - " out_str = get_feedback(prompt, domain, f\"make sure the response contain the words {words}\", f\"['collie', '{key}']\", \"Metric.contains_all_strings\", \"\"\"[{words}]\"\"\".format(words=words))\n", + " out_str = get_feedback(\n", + " prompt,\n", + " domain,\n", + " f\"make sure the response contain the words {words}\",\n", + " f\"['collie', '{key}']\",\n", + " \"Metric.contains_all_strings\",\n", + " \"\"\"[{words}]\"\"\".format(words=words),\n", + " )\n", " out.write(out_str + \"\\n\")" ] }, @@ -120,11 +147,18 @@ "print(len(arr))\n", "for i in chosen:\n", " obj = arr[i]\n", - " topic = obj['metadata'][\"title\"]\n", - " words = obj['prompt'].split(\"1st word to be\")[1][:-1].strip()\n", + " topic = obj[\"metadata\"][\"title\"]\n", + " words = obj[\"prompt\"].split(\"1st word to be\")[1][:-1].strip()\n", " prompt = f\"When talking about {topic}, make sure the first sentences has a 1st word of {words}\"\n", " domain = f\"Talking about {topic}\"\n", - " out_str = get_feedback(prompt, domain, f\"make sure the first sentences has a 1st word of {words}\", f\"['collie', '{key}']\", \"Metric.starts_with\",words.lower())\n", + " out_str = get_feedback(\n", + " prompt,\n", + " domain,\n", + " f\"make sure the first sentences has a 1st word of {words}\",\n", + " f\"['collie', '{key}']\",\n", + " \"Metric.starts_with\",\n", + " words.lower(),\n", + " )\n", " out.write(out_str + \"\\n\")" ] }, @@ -138,12 +172,19 @@ "chosen = np.random.choice(np.arange(len(all_data[key])), size=38, replace=False)\n", "for i in chosen:\n", " obj = all_data[key][i]\n", - " topic = obj['metadata'][\"title\"]\n", + " topic = obj[\"metadata\"][\"title\"]\n", " targets = obj[\"targets\"]\n", - " words = \"\\'\" + \"', '\".join(targets[1:]) + \"'\"\n", + " words = \"'\" + \"', '\".join(targets[1:]) + \"'\"\n", " prompt = f\"When talking about {topic}, do not use the words {words}\"\n", " domain = f\"Talking about {topic}\"\n", - " out_str = get_feedback(prompt, domain, f\"do not use the words {words}\", f\"['collie', '{key}']\", \"Metric.contains_none_strings\", f\"{targets[1:]}\")\n", + " out_str = get_feedback(\n", + " prompt,\n", + " domain,\n", + " f\"do not use the words {words}\",\n", + " f\"['collie', '{key}']\",\n", + " \"Metric.contains_none_strings\",\n", + " f\"{targets[1:]}\",\n", + " )\n", " out.write(out_str + \"\\n\")" ] }, diff --git a/notebooks/gpt_feedback_gen.ipynb b/notebooks/gpt_feedback_gen.ipynb index f416340..6803ea3 100644 --- a/notebooks/gpt_feedback_gen.ipynb +++ b/notebooks/gpt_feedback_gen.ipynb @@ -15,6 +15,7 @@ "outputs": [], "source": [ "from openai import OpenAI\n", + "\n", "client = OpenAI(\n", " api_key=\"INSERT_KEY\",\n", ")" @@ -30,8 +31,11 @@ " completion = client.chat.completions.create(\n", " model=\"gpt-4-1106-preview\",\n", " messages=[\n", - " {'role': 'system', 'content': \"You are a helpful assistant that always closely follows instructions.\"},\n", - " {'role': 'user', 'content': prompt}\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that always closely follows instructions.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": prompt},\n", " ],\n", " )\n", " response = completion.choices[0].message.content\n", @@ -178,7 +182,7 @@ " print(\"num generated\", len(generated))\n", " resp = get_response(CONTENT_PROMPT_EFFECT)\n", " print(resp)\n", - " if resp[0:7] == \"```json\": # tries to output in markdown\n", + " if resp[0:7] == \"```json\": # tries to output in markdown\n", " resp = resp[8:-3]\n", " try:\n", " obj = json.loads(resp)\n", @@ -200,12 +204,14 @@ "\n", "import json\n", "\n", - "ending = \"content\" # content or style\n", + "ending = \"content\" # content or style\n", "with open(f\"working_json/gpt_feedback_{ending}.json\", \"r\") as f:\n", " obj = json.load(f)\n", "\n", "with open(f\"../src/feedback/gpt_{ending}.py\", \"w+\") as out:\n", - " out.write(f\"from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison\\n\\ngpt_{ending}_feedback = [\\n\")\n", + " out.write(\n", + " f\"from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison\\n\\ngpt_{ending}_feedback = [\\n\"\n", + " )\n", " for i in obj:\n", " out_str = \"\"\" \n", " Feedback(\n", @@ -217,7 +223,12 @@ " type=Type.qualitative,\n", " comparison=Comparison.greater_eq_than\n", " ),\n", - "\"\"\".format(feedback=i[\"feedback\"], domain=i[\"domain\"], effect=i[\"effect\"], ending=ending)\n", + "\"\"\".format(\n", + " feedback=i[\"feedback\"],\n", + " domain=i[\"domain\"],\n", + " effect=i[\"effect\"],\n", + " ending=ending,\n", + " )\n", " out.write(out_str + \"\\n\")\n", " out.write(\"]\\n\")" ] diff --git a/setup.py b/setup.py index 5d33924..2740d9c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ -from setuptools import setup, find_packages +from setuptools import setup setup( - name='src', - version='0.1', + name="src", + version="0.1", packages=["src"], -) \ No newline at end of file +) diff --git a/src/dataset/feedback_utils.py b/src/dataset/feedback_utils.py index cc82718..30192fa 100644 --- a/src/dataset/feedback_utils.py +++ b/src/dataset/feedback_utils.py @@ -41,18 +41,31 @@ def __call__(self, *args, **kwargs): class Metric(Enum): length: Callable = lambda x, _: len(x) - contains_any_string: Callable = lambda x, y: any([s.lower() in x.lower() for s in y]) - contains_all_strings: Callable = lambda x, y: all([s.lower() in x.lower() for s in y]) - contains_none_strings: Callable = lambda x, y: not any([s.lower() in x.lower() for s in y]) - contains_phone_number: Callable = lambda x, y: y.replace("-", " ").replace("(", "").replace(")", "") in x.replace("-", " ").replace("(", "").replace(")", "") + contains_any_string: Callable = lambda x, y: any( + [s.lower() in x.lower() for s in y] + ) + contains_all_strings: Callable = lambda x, y: all( + [s.lower() in x.lower() for s in y] + ) + contains_none_strings: Callable = lambda x, y: ( + not any([s.lower() in x.lower() for s in y]) + ) + contains_phone_number: Callable = lambda x, y: ( + y.replace("-", " ").replace("(", "").replace(")", "") + in x.replace("-", " ").replace("(", "").replace(")", "") + ) ends_with: Callable = lambda x, y: x.lower().strip().endswith(y.lower().strip()) - ends_with_cleaned: Callable = lambda x, y: " ".join(re.sub(r'[^a-z0-9,.!?]', ' ', x.lower().strip()).split()).endswith(" ".join(re.sub(r'[^a-z0-9,.!?]', ' ', y.lower().strip()).split())) + ends_with_cleaned: Callable = lambda x, y: " ".join( + re.sub(r"[^a-z0-9,.!?]", " ", x.lower().strip()).split() + ).endswith(" ".join(re.sub(r"[^a-z0-9,.!?]", " ", y.lower().strip()).split())) regex_search: Callable = lambda x, y: bool(re.search(y, x)) regex_search_false: Callable = lambda x, y: not bool(re.search(y, x)) is_language: Callable = lambda x, y: detect(x) == y starts_with: Callable = lambda x, y: x.lower().strip().startswith(y.lower().strip()) - doesnt_start_with: Callable = lambda x, y: not x.lower().strip().startswith(y.lower().strip()) - + doesnt_start_with: Callable = lambda x, y: ( + not x.lower().strip().startswith(y.lower().strip()) + ) + def __call__(self, *args, **kwargs): return self.value(*args, **kwargs) @@ -87,7 +100,7 @@ def file_name(self): content = content.replace(" ", "_") content = content.strip() return f"{content}_{self.id}" - + def can_load_dataset(self, prompt_dir: str) -> None: """Checks if prompts can be loaded from a directory @@ -104,7 +117,7 @@ def can_load_dataset(self, prompt_dir: str) -> None: if not os.path.exists(os.path.join(path, "categories.json")): return False return True - + def general_prompts_available(self, prompt_dir: str) -> Optional[str]: """If any of the subdirectories of prompt_dir have a valid json file named "general_prompts.json" with keys "train" and "test", return the path to that file. Otherwise, return None. @@ -120,7 +133,7 @@ def general_prompts_available(self, prompt_dir: str) -> Optional[str]: if "train" in data.keys() and "test" in data.keys(): return os.path.join(path, "general_prompts.json") return None - + @staticmethod def _dump_dataset_dict(path: str, dataset: DatasetDict) -> None: data = {} @@ -138,7 +151,6 @@ def _load_dataset_dict(path: str) -> DatasetDict: dataset_dict[split] = Dataset.from_dict(data[split]) return dataset_dict - def load_dataset(self, prompt_dir: str) -> None: """Loads prompts from a directory into the feedback object @@ -147,8 +159,12 @@ def load_dataset(self, prompt_dir: str) -> None: """ path = os.path.join(prompt_dir, self.file_name) self.prompts = self._load_dataset_dict(os.path.join(path, "prompts.json")) - self.negative_prompts = self._load_dataset_dict(os.path.join(path, "negative_prompts.json")) - self.general_prompts = self._load_dataset_dict(os.path.join(path, "general_prompts.json")) + self.negative_prompts = self._load_dataset_dict( + os.path.join(path, "negative_prompts.json") + ) + self.general_prompts = self._load_dataset_dict( + os.path.join(path, "general_prompts.json") + ) with open(os.path.join(path, "categories.json"), "r") as f: self.categories = json.load(f) @@ -173,7 +189,11 @@ def dump_dataset(self, prompt_dir: str) -> None: path = os.path.join(prompt_dir, self.file_name) os.makedirs(path, exist_ok=True) self._dump_dataset_dict(os.path.join(path, "prompts.json"), self.prompts) - self._dump_dataset_dict(os.path.join(path, "negative_prompts.json"), self.negative_prompts) - self._dump_dataset_dict(os.path.join(path, "general_prompts.json"), self.general_prompts) + self._dump_dataset_dict( + os.path.join(path, "negative_prompts.json"), self.negative_prompts + ) + self._dump_dataset_dict( + os.path.join(path, "general_prompts.json"), self.general_prompts + ) with open(os.path.join(path, "categories.json"), "w+") as f: json.dump(self.categories, f, indent=2) diff --git a/src/dataset/format.py b/src/dataset/format.py index 90db781..852c319 100644 --- a/src/dataset/format.py +++ b/src/dataset/format.py @@ -5,21 +5,27 @@ # TODO: the below are hacky utilities to deal with the fact that different trainers have different ways of formatting data for us # we should standardize this in the future by just using tokenizer.apply_chat_template or something similar + def llama_prompt_format(prompt: str) -> str: return f"[INST] {prompt} [/INST]" + def llama_full_format(prompt: str, completion: str) -> str: return f"[INST] {prompt} [/INST] {completion}" + def yi_prompt_format(prompt: str) -> str: return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n" + def yi_full_format(prompt: str, completion: str) -> str: return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n{completion}<|im_end|>\n" + def hermes_prompt_format(prompt: str) -> str: return f"### Instruction:\n{prompt}\n\n### Response:\n" + def hermes_full_format(prompt: str, completion: str) -> str: return f"### Instruction:\n{prompt}\n\n### Response:\n{completion}" @@ -27,159 +33,232 @@ def hermes_full_format(prompt: str, completion: str) -> str: FORMAT_MAPPING = { "meta-llama/Llama-2-7b-chat-hf": { "prompt": llama_prompt_format, - "full": llama_full_format + "full": llama_full_format, }, "mistralai/Mistral-7B-Instruct-v0.2": { "prompt": llama_prompt_format, - "full": llama_full_format - }, - "01-ai/Yi-6B-Chat": { - "prompt": yi_prompt_format, - "full": yi_full_format - }, - "Qwen/Qwen-7B-Chat": { - "prompt": yi_prompt_format, - "full": yi_full_format - }, - "tiiuae/falcon-7b-instruct": { - "prompt": yi_prompt_format, - "full": yi_full_format + "full": llama_full_format, }, + "01-ai/Yi-6B-Chat": {"prompt": yi_prompt_format, "full": yi_full_format}, + "Qwen/Qwen-7B-Chat": {"prompt": yi_prompt_format, "full": yi_full_format}, + "tiiuae/falcon-7b-instruct": {"prompt": yi_prompt_format, "full": yi_full_format}, "NousResearch/Nous-Hermes-llama-2-7b": { "prompt": hermes_prompt_format, - "full": hermes_full_format + "full": hermes_full_format, }, - None: { - "prompt": llama_prompt_format, - "full": llama_full_format - } + None: {"prompt": llama_prompt_format, "full": llama_full_format}, } -def to_dpo(dataset: Dataset, negative_dataset: Dataset = None, general_dataset: Dataset = None, model_name_or_path: str = None) -> Dataset: +def to_dpo( + dataset: Dataset, + negative_dataset: Dataset = None, + general_dataset: Dataset = None, + model_name_or_path: str = None, +) -> Dataset: """Converts feedback to a DPO dataset""" - dataset = dataset.map(lambda x: { - "prompt": x["prompt"], - "rejected": x["baseline_response"], - "chosen": x["revised_response"] - }, remove_columns=dataset.features, load_from_cache_file=False) + dataset = dataset.map( + lambda x: { + "prompt": x["prompt"], + "rejected": x["baseline_response"], + "chosen": x["revised_response"], + }, + remove_columns=dataset.features, + load_from_cache_file=False, + ) if negative_dataset is not None: - negative_dataset = negative_dataset.map(lambda x: { - "prompt": x["prompt"], - "rejected": x["revised_response"], - "chosen": x["baseline_response"] - }, remove_columns=negative_dataset.features, load_from_cache_file=False) + negative_dataset = negative_dataset.map( + lambda x: { + "prompt": x["prompt"], + "rejected": x["revised_response"], + "chosen": x["baseline_response"], + }, + remove_columns=negative_dataset.features, + load_from_cache_file=False, + ) dataset = concatenate_datasets([dataset, negative_dataset]) if general_dataset is not None: - general_dataset = general_dataset.map(lambda x: { - "prompt": x["prompt"], - "rejected": x["revised_response"], - "chosen": x["baseline_response"] - }, remove_columns=general_dataset.features, load_from_cache_file=False) + general_dataset = general_dataset.map( + lambda x: { + "prompt": x["prompt"], + "rejected": x["revised_response"], + "chosen": x["baseline_response"], + }, + remove_columns=general_dataset.features, + load_from_cache_file=False, + ) dataset = concatenate_datasets([dataset, general_dataset]) prompt_format = FORMAT_MAPPING[model_name_or_path]["prompt"] - dataset = dataset.map(lambda x: { - # The DPO trainer adds the eos/bos tokens itself so no need to do that here - "prompt": prompt_format(x['prompt']) - }, load_from_cache_file=False) + dataset = dataset.map( + lambda x: { + # The DPO trainer adds the eos/bos tokens itself so no need to do that here + "prompt": prompt_format(x["prompt"]) + }, + load_from_cache_file=False, + ) return dataset -def to_lcdpo(dataset: Dataset, negative_dataset: Dataset = None, general_dataset: Dataset = None, model_name_or_path: str = None) -> Dataset: +def to_lcdpo( + dataset: Dataset, + negative_dataset: Dataset = None, + general_dataset: Dataset = None, + model_name_or_path: str = None, +) -> Dataset: """Converts feedback to a DPO dataset""" length = min(len(dataset), min(len(negative_dataset), len(general_dataset))) if len(dataset) != length: dataset = dataset.select(range(length)) - warnings.warn(f"Provided unequal sized datasets to LC-DPO formatter. Truncating dataset to {length} rows.") + warnings.warn( + f"Provided unequal sized datasets to LC-DPO formatter. Truncating dataset to {length} rows." + ) if len(negative_dataset) != length: negative_dataset = negative_dataset.select(range(length)) - warnings.warn(f"Provided unequal sized datasets to LC-DPO formatter. Truncating negative_dataset to {length} rows.") + warnings.warn( + f"Provided unequal sized datasets to LC-DPO formatter. Truncating negative_dataset to {length} rows." + ) if len(general_dataset) != length: general_dataset = general_dataset.select(range(length)) - warnings.warn(f"Provided unequal sized datasets to LC-DPO formatter. Truncating general_dataset to {length} rows.") + warnings.warn( + f"Provided unequal sized datasets to LC-DPO formatter. Truncating general_dataset to {length} rows." + ) prompt_format = FORMAT_MAPPING[model_name_or_path]["prompt"] full_format = FORMAT_MAPPING[model_name_or_path]["full"] - dataset = dataset.map(lambda x: { - # The DPO trainer adds the eos/bos tokens itself so no need to do that here - "prompt": prompt_format(x['prompt']), - "rejected": x["baseline_response"], - "chosen": x["revised_response"] - }, remove_columns=dataset.features, load_from_cache_file=False) + dataset = dataset.map( + lambda x: { + # The DPO trainer adds the eos/bos tokens itself so no need to do that here + "prompt": prompt_format(x["prompt"]), + "rejected": x["baseline_response"], + "chosen": x["revised_response"], + }, + remove_columns=dataset.features, + load_from_cache_file=False, + ) if negative_dataset is not None: - negative_dataset = negative_dataset.map(lambda x: { - "hard_negative": full_format(x["prompt"], x["baseline_response"]) - }, remove_columns=negative_dataset.features, load_from_cache_file=False) + negative_dataset = negative_dataset.map( + lambda x: { + "hard_negative": full_format(x["prompt"], x["baseline_response"]) + }, + remove_columns=negative_dataset.features, + load_from_cache_file=False, + ) dataset = dataset.add_column("hard_negative", negative_dataset["hard_negative"]) if general_dataset is not None: - general_dataset = general_dataset.map(lambda x: { - "soft_negative": full_format(x["prompt"], x["baseline_response"]) - }, remove_columns=general_dataset.features, load_from_cache_file=False) + general_dataset = general_dataset.map( + lambda x: { + "soft_negative": full_format(x["prompt"], x["baseline_response"]) + }, + remove_columns=general_dataset.features, + load_from_cache_file=False, + ) dataset = dataset.add_column("soft_negative", general_dataset["soft_negative"]) return dataset -def to_sft(dataset: Dataset, negative_dataset: Dataset = None, general_dataset: Dataset = None, model_name_or_path: str = None) -> Dataset: - dataset = dataset.map(lambda x: { - "prompt": x["prompt"], - "completion": f' {x["revised_response"]}' # TODO: hack to fix tokenization issue when there are to neighboring parentheses (e.g. '[/INST][...]' ) - }, remove_columns=dataset.features, load_from_cache_file=False) +def to_sft( + dataset: Dataset, + negative_dataset: Dataset = None, + general_dataset: Dataset = None, + model_name_or_path: str = None, +) -> Dataset: + dataset = dataset.map( + lambda x: { + "prompt": x["prompt"], + "completion": f" {x['revised_response']}", # TODO: hack to fix tokenization issue when there are to neighboring parentheses (e.g. '[/INST][...]' ) + }, + remove_columns=dataset.features, + load_from_cache_file=False, + ) if negative_dataset is not None: - negative_dataset = negative_dataset.map(lambda x: { - "prompt": x["prompt"], - "completion": f' {x["baseline_response"]}' # TODO: hack to fix tokenization issue when there are to neighboring parentheses (e.g. '[/INST][...]' ) - }, remove_columns=negative_dataset.features, load_from_cache_file=False) + negative_dataset = negative_dataset.map( + lambda x: { + "prompt": x["prompt"], + "completion": f" {x['baseline_response']}", # TODO: hack to fix tokenization issue when there are to neighboring parentheses (e.g. '[/INST][...]' ) + }, + remove_columns=negative_dataset.features, + load_from_cache_file=False, + ) dataset = concatenate_datasets([dataset, negative_dataset]) if general_dataset is not None: - general_dataset = general_dataset.map(lambda x: { - "prompt": x["prompt"], - "completion": f' {x["baseline_response"]}' # TODO: hack to fix tokenization issue when there are to neighboring parentheses (e.g. '[/INST][...]' ) - }, remove_columns=general_dataset.features, load_from_cache_file=False) + general_dataset = general_dataset.map( + lambda x: { + "prompt": x["prompt"], + "completion": f" {x['baseline_response']}", # TODO: hack to fix tokenization issue when there are to neighboring parentheses (e.g. '[/INST][...]' ) + }, + remove_columns=general_dataset.features, + load_from_cache_file=False, + ) dataset = concatenate_datasets([dataset, general_dataset]) return dataset -def to_sft_weighted(dataset: Dataset, negative_dataset: Dataset = None, general_dataset: Dataset = None, model_name_or_path: str = None) -> Dataset: +def to_sft_weighted( + dataset: Dataset, + negative_dataset: Dataset = None, + general_dataset: Dataset = None, + model_name_or_path: str = None, +) -> Dataset: """Converts feedback to a DPO dataset""" length = min(len(dataset), min(len(negative_dataset), len(general_dataset))) if len(dataset) != length: dataset = dataset.select(range(length)) - warnings.warn(f"Provided unequal sized datasets to LC-DPO formatter. Truncating dataset to {length} rows.") + warnings.warn( + f"Provided unequal sized datasets to LC-DPO formatter. Truncating dataset to {length} rows." + ) if len(negative_dataset) != length: negative_dataset = negative_dataset.select(range(length)) - warnings.warn(f"Provided unequal sized datasets to LC-DPO formatter. Truncating negative_dataset to {length} rows.") + warnings.warn( + f"Provided unequal sized datasets to LC-DPO formatter. Truncating negative_dataset to {length} rows." + ) if len(general_dataset) != length: general_dataset = general_dataset.select(range(length)) - warnings.warn(f"Provided unequal sized datasets to LC-DPO formatter. Truncating general_dataset to {length} rows.") + warnings.warn( + f"Provided unequal sized datasets to LC-DPO formatter. Truncating general_dataset to {length} rows." + ) full_format = FORMAT_MAPPING[model_name_or_path]["full"] - dataset = dataset.map(lambda x: { - "text": full_format(x["prompt"], x["revised_response"]) # TODO: hack to fix tokenization issue when there are to neighboring parentheses (e.g. '[/INST][...]' ) - }, remove_columns=dataset.features, load_from_cache_file=False) + dataset = dataset.map( + lambda x: { + "text": full_format( + x["prompt"], x["revised_response"] + ) # TODO: hack to fix tokenization issue when there are to neighboring parentheses (e.g. '[/INST][...]' ) + }, + remove_columns=dataset.features, + load_from_cache_file=False, + ) if negative_dataset is not None: - negative_dataset = negative_dataset.map(lambda x: { - "hard_negative": full_format(x["prompt"], x["baseline_response"]) - }, remove_columns=negative_dataset.features, load_from_cache_file=False) + negative_dataset = negative_dataset.map( + lambda x: { + "hard_negative": full_format(x["prompt"], x["baseline_response"]) + }, + remove_columns=negative_dataset.features, + load_from_cache_file=False, + ) dataset = dataset.add_column("hard_negative", negative_dataset["hard_negative"]) if general_dataset is not None: - general_dataset = general_dataset.map(lambda x: { - "soft_negative": full_format(x["prompt"], x["baseline_response"]) - }, remove_columns=general_dataset.features, load_from_cache_file=False) + general_dataset = general_dataset.map( + lambda x: { + "soft_negative": full_format(x["prompt"], x["baseline_response"]) + }, + remove_columns=general_dataset.features, + load_from_cache_file=False, + ) dataset = dataset.add_column("soft_negative", general_dataset["soft_negative"]) - return dataset \ No newline at end of file + return dataset diff --git a/src/dataset/general_prompts.py b/src/dataset/general_prompts.py index 7d68422..779271b 100644 --- a/src/dataset/general_prompts.py +++ b/src/dataset/general_prompts.py @@ -3,7 +3,9 @@ class GeneralPromptDataset(Dataset): - DATASET_LINK = "https://huggingface.co/datasets/laion/OIG/resolve/main/unified_chip2.jsonl" + DATASET_LINK = ( + "https://huggingface.co/datasets/laion/OIG/resolve/main/unified_chip2.jsonl" + ) GENERAL_PROMPTS_FILE = "general_prompts.jsonl" @classmethod @@ -15,7 +17,7 @@ def load(cls, directory_path: str, num_prompts: int) -> "GeneralPromptDataset": @classmethod def _chip2_filename(cls) -> str: - return cls.DATASET_LINK.split('/')[-1] + return cls.DATASET_LINK.split("/")[-1] @classmethod def _chip2_available(cls, directory_path: str) -> bool: @@ -23,20 +25,22 @@ def _chip2_available(cls, directory_path: str) -> bool: @staticmethod def _format_chip2(sample: dict[str, str]) -> dict[str, str]: - prompt, _ = sample["text"].split('\n: ') - prompt = prompt.replace(': ', '') - return { - "prompt": prompt.strip() - } - + prompt, _ = sample["text"].split("\n: ") + prompt = prompt.replace(": ", "") + return {"prompt": prompt.strip()} + @classmethod def _download_chip2(cls, directory_path: str): # Need to download individual file rather than using HF load_dataset because whole dataset is too big os.makedirs(directory_path, exist_ok=True) if os.system("command -v wget > /dev/null") == 0: - os.system(f"wget -q {cls.DATASET_LINK} -P {directory_path} > /dev/null 2>&1") + os.system( + f"wget -q {cls.DATASET_LINK} -P {directory_path} > /dev/null 2>&1" + ) elif os.system("command -v curl > /dev/null") == 0: - os.system(f"curl -s -o {os.path.join(directory_path, cls._chip2_filename())} {cls.DATASET_LINK} > /dev/null 2>&1") + os.system( + f"curl -s -o {os.path.join(directory_path, cls._chip2_filename())} {cls.DATASET_LINK} > /dev/null 2>&1" + ) else: raise EnvironmentError("Neither wget nor curl is installed on this system.") @@ -45,4 +49,8 @@ def _get_chip2(cls, directory_path: str, num_prompts: int) -> Dataset: dataset = Dataset.from_json(os.path.join(directory_path, cls._chip2_filename())) dataset = dataset.shuffle(seed=42) dataset = dataset.select(range(num_prompts)) - return dataset.map(cls._format_chip2, remove_columns=dataset.features, load_from_cache_file=False) + return dataset.map( + cls._format_chip2, + remove_columns=dataset.features, + load_from_cache_file=False, + ) diff --git a/src/dataset/prompts.py b/src/dataset/prompts.py index 4a2d25f..134fce7 100644 --- a/src/dataset/prompts.py +++ b/src/dataset/prompts.py @@ -57,7 +57,7 @@ "top_p": 0.7, "top_k": 50, "repetition_penalty": 1, - "do_sample": True + "do_sample": True, } SAMPLE_PROMPTS = """You are a helpful assistant that always closely follows instructions. You are provided with a topic, and category. Your job is to come up with {count} actionable prompts that fulfill the following criteria: @@ -164,7 +164,9 @@ GET_BASELINE_COMPLETION_CONFIG = SAMPLE_NEGATIVE_PROMPTS_CONFIG -GET_IN_CONTEXT_COMPLETION = """{prompt} (If applicable, apply the following feedback: {feedback})""" +GET_IN_CONTEXT_COMPLETION = ( + """{prompt} (If applicable, apply the following feedback: {feedback})""" +) GET_IN_CONTEXT_COMPLETION_CONFIG = SAMPLE_NEGATIVE_PROMPTS_CONFIG @@ -241,4 +243,4 @@ EXPLANATION: """ -ANSWER_QUALITATIVE_EVAL_CONFIG = SAMPLE_PROMPT_CATEGORIES_CONFIG \ No newline at end of file +ANSWER_QUALITATIVE_EVAL_CONFIG = SAMPLE_PROMPT_CATEGORIES_CONFIG diff --git a/src/eval.py b/src/eval.py index 27627b8..3508f83 100644 --- a/src/eval.py +++ b/src/eval.py @@ -11,7 +11,12 @@ from src.feedback import manual_feedback as all_feedback from src.dataset.feedback_utils import Feedback, Type from src.utils import get_args, ModelArguments, get_train_file_name -from src.dataset.prompts import COMPARE_COMPLETIONS, COMPARE_COMPLETIONS_CONFIG, ANSWER_QUALITATIVE_EVAL, ANSWER_QUALITATIVE_EVAL_CONFIG +from src.dataset.prompts import ( + COMPARE_COMPLETIONS, + COMPARE_COMPLETIONS_CONFIG, + ANSWER_QUALITATIVE_EVAL, + ANSWER_QUALITATIVE_EVAL_CONFIG, +) def quantitative_feedback_eval( @@ -19,19 +24,23 @@ def quantitative_feedback_eval( prompts: list[str], baseline_responses: list[str], improved_responses: list[str], - _: ModelArguments + _: ModelArguments, ) -> list[dict]: - + if isinstance(feedback.metric, list): - metric = lambda x: all([f(x, v) for f, v in zip(feedback.metric, feedback.metric_value)]) + + def metric(x): + return all( + [f(x, v) for f, v in zip(feedback.metric, feedback.metric_value)] + ) else: - metric = lambda x: feedback.metric(x, feedback.metric_value) - + + def metric(x): + return feedback.metric(x, feedback.metric_value) + data = [] for prompt, baseline_response, improved_response in zip( - prompts, - baseline_responses, - improved_responses + prompts, baseline_responses, improved_responses ): new_data = { "prompt": prompt, @@ -42,7 +51,7 @@ def quantitative_feedback_eval( "improved_better_baseline": feedback.comparison( metric(baseline_response), metric(improved_response), - ) + ), } data.append(new_data) return data @@ -53,30 +62,43 @@ def qualitative_feedback_eval( prompts: list[str], baseline_responses: list[str], improved_responses: list[str], - model_args: ModelArguments + model_args: ModelArguments, ) -> list[dict]: model = get_model(model_args) - responses = model.get_responses([ - [COMPARE_COMPLETIONS.format(prompt=p, completion1=resp1, completion2=resp2, feedback=feedback.effect)] - for p, resp1, resp2 in zip(prompts, baseline_responses, improved_responses)], COMPARE_COMPLETIONS_CONFIG) - responses = [r.split("BETTER_RESPONSE: ")[-1].strip().split()[0] if r is not None else None for r in responses] + responses = model.get_responses( + [ + [ + COMPARE_COMPLETIONS.format( + prompt=p, + completion1=resp1, + completion2=resp2, + feedback=feedback.effect, + ) + ] + for p, resp1, resp2 in zip(prompts, baseline_responses, improved_responses) + ], + COMPARE_COMPLETIONS_CONFIG, + ) + responses = [ + r.split("BETTER_RESPONSE: ")[-1].strip().split()[0] if r is not None else None + for r in responses + ] responses = [int(r) if r is not None and r.isnumeric() else None for r in responses] data = [] for prompt, baseline_response, improved_response, improved_better_baseline in zip( - prompts, - baseline_responses, - improved_responses, - responses + prompts, baseline_responses, improved_responses, responses ): - data.append({ - "prompt": prompt, - "baseline": baseline_response, - "improved": improved_response, - "baseline_metric": -1, - "improved_metric": -1, - "improved_better_baseline": improved_better_baseline - }) + data.append( + { + "prompt": prompt, + "baseline": baseline_response, + "improved": improved_response, + "baseline_metric": -1, + "improved_metric": -1, + "improved_better_baseline": improved_better_baseline, + } + ) return data @@ -85,49 +107,80 @@ def answer_eval( prompts: list[str], baseline_responses: list[str], improved_responses: list[str], - model_args: ModelArguments + model_args: ModelArguments, ) -> list[dict]: model = get_model(model_args) - responses = model.get_responses([ - [ANSWER_QUALITATIVE_EVAL.format(prompt=p, completion1=resp1, completion2=resp2, feedback=feedback.content)] - for p, resp1, resp2 in zip(prompts, baseline_responses, improved_responses)], ANSWER_QUALITATIVE_EVAL_CONFIG) - responses = [r.split("BETTER_RESPONSE: ")[-1].strip().split()[0] if r is not None else None for r in responses] + responses = model.get_responses( + [ + [ + ANSWER_QUALITATIVE_EVAL.format( + prompt=p, + completion1=resp1, + completion2=resp2, + feedback=feedback.content, + ) + ] + for p, resp1, resp2 in zip(prompts, baseline_responses, improved_responses) + ], + ANSWER_QUALITATIVE_EVAL_CONFIG, + ) + responses = [ + r.split("BETTER_RESPONSE: ")[-1].strip().split()[0] if r is not None else None + for r in responses + ] responses = [int(r) if r is not None and r.isnumeric() else None for r in responses] - return [{ - "answer_quality_improved_better_baseline": r - } for r in responses] + return [{"answer_quality_improved_better_baseline": r} for r in responses] -def eval(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedback, second_feedback: Feedback = None) -> None: +def eval( + arg_dict: dict[str, Any], + run_id: str, + data_dir: str, + feedback: Feedback, + second_feedback: Feedback = None, +) -> None: model_args, _, train_args, eval_args = get_args(arg_dict) - + # Load feedback run_dir = os.path.join(data_dir, run_id, "sample") - logger.info(f"Evaluating using data for run {run_id}, stored in {run_dir} of method {eval_args.method}") - assert eval_args.method in ["trained", "in_context", "cot", "revised"], f"Unknown eval method {eval_args.method}" + logger.info( + f"Evaluating using data for run {run_id}, stored in {run_dir} of method {eval_args.method}" + ) + assert eval_args.method in ["trained", "in_context", "cot", "revised"], ( + f"Unknown eval method {eval_args.method}" + ) if not feedback.can_load_dataset(run_dir): - raise ValueError(f"Feedback \"{feedback.content}\" has not been sampled yet") + raise ValueError(f'Feedback "{feedback.content}" has not been sampled yet') feedback.load_dataset(run_dir) - logger.info(f"Loaded feedback \"{feedback.content}\"") + logger.info(f'Loaded feedback "{feedback.content}"') if second_feedback is not None: - assert eval_args.method == "trained", "Second feedback is only supported for evaluating trained model" + assert eval_args.method == "trained", ( + "Second feedback is only supported for evaluating trained model" + ) if not second_feedback.can_load_dataset(run_dir): - raise ValueError(f"Feedback \"{second_feedback.content}\" has not been sampled yet") + raise ValueError( + f'Feedback "{second_feedback.content}" has not been sampled yet' + ) second_feedback.load_dataset(run_dir) - logger.info(f"Loaded second feedback \"{second_feedback.content}\"") + logger.info(f'Loaded second feedback "{second_feedback.content}"') if second_feedback is None and train_args.multi_feedback_training: - raise ValueError("Must specify second feedback when using multi-feedback training") + raise ValueError( + "Must specify second feedback when using multi-feedback training" + ) # Construct datasets prompt_types = [ ("in_domain", eval_args.num_prompts, feedback.prompts["test"]), - ("out_of_domain", eval_args.num_negative_prompts, feedback.negative_prompts["test"]), - ("general", eval_args.num_general_prompts, feedback.general_prompts["test"]) + ( + "out_of_domain", + eval_args.num_negative_prompts, + feedback.negative_prompts["test"], + ), + ("general", eval_args.num_general_prompts, feedback.general_prompts["test"]), ] - datasets = {} for prompt_type, num_prompts, prompt_dataset in prompt_types: if num_prompts is None or len(prompt_dataset) < num_prompts: @@ -139,12 +192,16 @@ def eval(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedbac } # Set improved response to pre-generated completions for current method if eval_args.method != "trained": - datasets[prompt_type]["improved"] = prompt_dataset[f"{eval_args.method}_response"] + datasets[prompt_type]["improved"] = prompt_dataset[ + f"{eval_args.method}_response" + ] # Alternatively if using the "trained" method, sample from the model if eval_args.method == "trained": # Load model - assert model_args.train_model.platform == "huggingface", "Only HuggingFace models are supported for eval" + assert model_args.train_model.platform == "huggingface", ( + "Only HuggingFace models are supported for eval" + ) model = get_model(model_args.train_model) logger.info("Loaded model") @@ -155,15 +212,24 @@ def eval(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedbac if second_feedback is not None and train_args.multi_feedback_training: run_dir = os.path.join(run_dir, second_feedback.file_name, train_dir) else: - run_dir = os.path.join(run_dir, train_dir) - + run_dir = os.path.join(run_dir, train_dir) + if second_feedback is not None and not train_args.multi_feedback_training: - second_run_dir = os.path.join(data_dir, run_id, "train", second_feedback.file_name) + second_run_dir = os.path.join( + data_dir, run_id, "train", second_feedback.file_name + ) second_run_dir = os.path.join(second_run_dir, train_dir) - model.model = PeftModel.from_pretrained(model.model, run_dir, adapter_name="feedback_1") + model.model = PeftModel.from_pretrained( + model.model, run_dir, adapter_name="feedback_1" + ) model.model.load_adapter(second_run_dir, adapter_name="feedback_2") - model.model.add_weighted_adapter(["feedback_1", "feedback_2"], [1.0,1.0], combination_type="cat", adapter_name="feedback_combined") - #model.model.delete_adapter(["feedback_1", "feedback_2"]) + model.model.add_weighted_adapter( + ["feedback_1", "feedback_2"], + [1.0, 1.0], + combination_type="cat", + adapter_name="feedback_combined", + ) + # model.model.delete_adapter(["feedback_1", "feedback_2"]) model.model.set_adapter("feedback_combined") logger.info(f"Loaded combined adapter: {model.model.active_adapter}") else: @@ -175,17 +241,25 @@ def eval(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedbac # Get trained responses all_prompts = [p for pt in datasets.values() for p in pt["prompts"]] all_trained_responses = model.get_responses([[p] for p in all_prompts]) - trained_responses_splits = np.split(all_trained_responses, np.cumsum([len(datasets[pt]["prompts"]) for pt in datasets])) + trained_responses_splits = np.split( + all_trained_responses, + np.cumsum([len(datasets[pt]["prompts"]) for pt in datasets]), + ) for i, prompt_type in enumerate(datasets): datasets[prompt_type]["improved"] = trained_responses_splits[i] - # Make sure the "improved" field is set for all datasets for prompt_type in datasets: - assert "improved" in datasets[prompt_type], f"Missing improved_response for {prompt_type}" + assert "improved" in datasets[prompt_type], ( + f"Missing improved_response for {prompt_type}" + ) # Get eval function - eval_func = quantitative_feedback_eval if feedback.type == Type.quantitative else qualitative_feedback_eval + eval_func = ( + quantitative_feedback_eval + if feedback.type == Type.quantitative + else qualitative_feedback_eval + ) # Compute metrics for each prompt type results = {} @@ -195,29 +269,49 @@ def eval(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedbac datasets[prompt_type]["prompts"], datasets[prompt_type]["baseline"], datasets[prompt_type]["improved"], - model_args.qualitative_eval_model) - - answer_quality_result = answer_eval( - feedback, - datasets[prompt_type]["prompts"], - datasets[prompt_type]["baseline"], - datasets[prompt_type]["improved"], - model_args.qualitative_eval_model - ) if eval_args.eval_answer_quality else [{ - "answer_quality_improved_better_baseline": -1 - } for _ in feedback_result] - results[prompt_type] = [dict(**f, **a) for f, a in zip(feedback_result, answer_quality_result)] - - - data = {} + model_args.qualitative_eval_model, + ) + + answer_quality_result = ( + answer_eval( + feedback, + datasets[prompt_type]["prompts"], + datasets[prompt_type]["baseline"], + datasets[prompt_type]["improved"], + model_args.qualitative_eval_model, + ) + if eval_args.eval_answer_quality + else [ + {"answer_quality_improved_better_baseline": -1} for _ in feedback_result + ] + ) + results[prompt_type] = [ + dict(**f, **a) for f, a in zip(feedback_result, answer_quality_result) + ] + + data = {} # Compute aggregate stats for in-domain, out-of-domain, and general prompts for domain_name, domain in results.items(): - data[domain_name + "_baseline_metric"] = np.mean([d["baseline_metric"] for d in domain]) - data[domain_name + "_improved_metric"] = np.mean([d["improved_metric"] for d in domain]) + data[domain_name + "_baseline_metric"] = np.mean( + [d["baseline_metric"] for d in domain] + ) + data[domain_name + "_improved_metric"] = np.mean( + [d["improved_metric"] for d in domain] + ) data[domain_name + "_improved_better_baseline"] = np.mean( - [d["improved_better_baseline"] for d in domain if d["improved_better_baseline"] is not None]) + [ + d["improved_better_baseline"] + for d in domain + if d["improved_better_baseline"] is not None + ] + ) data[domain_name + "_answer_quality_improved_better_baseline"] = np.mean( - [d["answer_quality_improved_better_baseline"] for d in domain if d["answer_quality_improved_better_baseline"] is not None]) + [ + d["answer_quality_improved_better_baseline"] + for d in domain + if d["answer_quality_improved_better_baseline"] is not None + ] + ) # Adding feedback info data["feedback"] = feedback.content @@ -231,10 +325,9 @@ def eval(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedbac data["second_feedback"] = second_feedback.content logger.info(f"""Evaluated model for feedback "{feedback.content} using method {eval_args.method}". -(in-domain) Train vs baseline: {data['in_domain_improved_better_baseline']:.2f} -(out-of-domain) Train vs baseline: {data['out_of_domain_improved_better_baseline']:.2f} -(general) Train vs baseline: {data['general_improved_better_baseline']:.2f}""") - +(in-domain) Train vs baseline: {data["in_domain_improved_better_baseline"]:.2f} +(out-of-domain) Train vs baseline: {data["out_of_domain_improved_better_baseline"]:.2f} +(general) Train vs baseline: {data["general_improved_better_baseline"]:.2f}""") # Save data run_dir = os.path.join(data_dir, run_id, "eval", feedback.file_name) @@ -250,7 +343,9 @@ def eval(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedbac json.dump(data, f, indent=2) # TODO: add dummping args dict - logger.info(f"Saved datasets for feedback to {run_dir} and method {eval_args.method}") + logger.info( + f"Saved datasets for feedback to {run_dir} and method {eval_args.method}" + ) if __name__ == "__main__": diff --git a/src/feedback/__init__.py b/src/feedback/__init__.py index e85b99a..c7fab6e 100644 --- a/src/feedback/__init__.py +++ b/src/feedback/__init__.py @@ -9,5 +9,5 @@ "manual_feedback", "collie_feedback", "gpt_content_feedback", - "gpt_style_feedback" -] \ No newline at end of file + "gpt_style_feedback", +] diff --git a/src/feedback/collie.py b/src/feedback/collie.py index 04bbd80..1a0b402 100644 --- a/src/feedback/collie.py +++ b/src/feedback/collie.py @@ -1,1303 +1,1104 @@ from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison collie_feedback = [ - Feedback( content="When talking about Reivilo, make sure the response contain the words 'then', 'renamed', 'Boetsap'", domain="Talking about Reivilo", effect="make sure the response contain the words 'then', 'renamed', 'Boetsap'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['then', 'renamed', 'Boetsap'], - comparison=Comparison.greater_eq_than + metric_value=["then", "renamed", "Boetsap"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about IQ Student Accommodation, make sure the response contain the words '2016', 'Living', 'Prodigy'", domain="Talking about IQ Student Accommodation", effect="make sure the response contain the words '2016', 'Living', 'Prodigy'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['2016', 'Living', 'Prodigy'], - comparison=Comparison.greater_eq_than + metric_value=["2016", "Living", "Prodigy"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Tulloona, make sure the response contain the words 'them', 'horse', '3rd'", domain="Talking about Tulloona", effect="make sure the response contain the words 'them', 'horse', '3rd'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['them', 'horse', '3rd'], - comparison=Comparison.greater_eq_than + metric_value=["them", "horse", "3rd"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about List of members of the Parliament of Finland, 2011–2015, make sure the response contain the words 'government', 'the', 'formation'", domain="Talking about List of members of the Parliament of Finland, 2011–2015", effect="make sure the response contain the words 'government', 'the', 'formation'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['government', 'the', 'formation'], - comparison=Comparison.greater_eq_than + metric_value=["government", "the", "formation"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Punjab Kings, make sure the response contain the words 'However', 'Kumar', 'them'", domain="Talking about Punjab Kings", effect="make sure the response contain the words 'However', 'Kumar', 'them'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['However', 'Kumar', 'them'], - comparison=Comparison.greater_eq_than + metric_value=["However", "Kumar", "them"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about In His Image (novel), make sure the response contain the words 'where', 'schedule', 'Secretary'", domain="Talking about In His Image (novel)", effect="make sure the response contain the words 'where', 'schedule', 'Secretary'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['where', 'schedule', 'Secretary'], - comparison=Comparison.greater_eq_than + metric_value=["where", "schedule", "Secretary"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Marcus Singletary, make sure the response contain the words 'that', 'This', 'spin'", domain="Talking about Marcus Singletary", effect="make sure the response contain the words 'that', 'This', 'spin'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['that', 'This', 'spin'], - comparison=Comparison.greater_eq_than + metric_value=["that", "This", "spin"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Bulgarian lands across the Danube, make sure the response contain the words 'Carpathian', 'of', 'or'", domain="Talking about Bulgarian lands across the Danube", effect="make sure the response contain the words 'Carpathian', 'of', 'or'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['Carpathian', 'of', 'or'], - comparison=Comparison.greater_eq_than + metric_value=["Carpathian", "of", "or"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Frost flower (sea ice), make sure the response contain the words 'a', 'conditions', 'of'", domain="Talking about Frost flower (sea ice)", effect="make sure the response contain the words 'a', 'conditions', 'of'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['a', 'conditions', 'of'], - comparison=Comparison.greater_eq_than + metric_value=["a", "conditions", "of"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Explosive boiling or phase explosion, make sure the response contain the words 'instability', 'on', 'right'", domain="Talking about Explosive boiling or phase explosion", effect="make sure the response contain the words 'instability', 'on', 'right'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['instability', 'on', 'right'], - comparison=Comparison.greater_eq_than + metric_value=["instability", "on", "right"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Cutting extinguisher, make sure the response contain the words 'is', 'velocity', 'PyroLance'", domain="Talking about Cutting extinguisher", effect="make sure the response contain the words 'is', 'velocity', 'PyroLance'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['is', 'velocity', 'PyroLance'], - comparison=Comparison.greater_eq_than + metric_value=["is", "velocity", "PyroLance"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Bridgewater, Nova Scotia, make sure the response contain the words 'Dugua', 'there', 'the'", domain="Talking about Bridgewater, Nova Scotia", effect="make sure the response contain the words 'Dugua', 'there', 'the'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['Dugua', 'there', 'the'], - comparison=Comparison.greater_eq_than + metric_value=["Dugua", "there", "the"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Andy Cutting, make sure the response contain the words 'one', 'Inventions', '2018'", domain="Talking about Andy Cutting", effect="make sure the response contain the words 'one', 'Inventions', '2018'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['one', 'Inventions', '2018'], - comparison=Comparison.greater_eq_than + metric_value=["one", "Inventions", "2018"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Magnum Rolle, make sure the response contain the words 'an', 'his', 'led'", domain="Talking about Magnum Rolle", effect="make sure the response contain the words 'an', 'his', 'led'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['an', 'his', 'led'], - comparison=Comparison.greater_eq_than + metric_value=["an", "his", "led"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Smithfield Poultry Market, make sure the response contain the words 'claimed', 'concrete', 'spanning'", domain="Talking about Smithfield Poultry Market", effect="make sure the response contain the words 'claimed', 'concrete', 'spanning'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['claimed', 'concrete', 'spanning'], - comparison=Comparison.greater_eq_than + metric_value=["claimed", "concrete", "spanning"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about September 1916, make sure the response contain the words 'Romania', 'Brusilov', 'making'", domain="Talking about September 1916", effect="make sure the response contain the words 'Romania', 'Brusilov', 'making'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['Romania', 'Brusilov', 'making'], - comparison=Comparison.greater_eq_than + metric_value=["Romania", "Brusilov", "making"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Italo-Dalmatian languages, make sure the response contain the words 'town', 'the', 'of'", domain="Talking about Italo-Dalmatian languages", effect="make sure the response contain the words 'town', 'the', 'of'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['town', 'the', 'of'], - comparison=Comparison.greater_eq_than + metric_value=["town", "the", "of"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Chiral drugs, make sure the response contain the words 'only', 'two', 'be'", domain="Talking about Chiral drugs", effect="make sure the response contain the words 'only', 'two', 'be'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['only', 'two', 'be'], - comparison=Comparison.greater_eq_than + metric_value=["only", "two", "be"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Eric Portman, make sure the response contain the words 'Canterbury', 'another', 'war'", domain="Talking about Eric Portman", effect="make sure the response contain the words 'Canterbury', 'another', 'war'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['Canterbury', 'another', 'war'], - comparison=Comparison.greater_eq_than + metric_value=["Canterbury", "another", "war"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Wilderness Scotland, make sure the response contain the words 'contribute', 'company', 'Life'", domain="Talking about Wilderness Scotland", effect="make sure the response contain the words 'contribute', 'company', 'Life'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['contribute', 'company', 'Life'], - comparison=Comparison.greater_eq_than + metric_value=["contribute", "company", "Life"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Harriet Toompere, make sure the response contain the words 'stage', 'drama', 'Her'", domain="Talking about Harriet Toompere", effect="make sure the response contain the words 'stage', 'drama', 'Her'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['stage', 'drama', 'Her'], - comparison=Comparison.greater_eq_than + metric_value=["stage", "drama", "Her"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Edith van Dijk, make sure the response contain the words 'a', 'her', 'as'", domain="Talking about Edith van Dijk", effect="make sure the response contain the words 'a', 'her', 'as'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['a', 'her', 'as'], - comparison=Comparison.greater_eq_than + metric_value=["a", "her", "as"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Bosome-Freho (Ghana parliament constituency), make sure the response contain the words 'Yaw', 'member', 'the'", domain="Talking about Bosome-Freho (Ghana parliament constituency)", effect="make sure the response contain the words 'Yaw', 'member', 'the'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['Yaw', 'member', 'the'], - comparison=Comparison.greater_eq_than + metric_value=["Yaw", "member", "the"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Eric Portman, make sure the response contain the words 'He', 'buried', 'in'", domain="Talking about Eric Portman", effect="make sure the response contain the words 'He', 'buried', 'in'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['He', 'buried', 'in'], - comparison=Comparison.greater_eq_than + metric_value=["He", "buried", "in"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Great Keppel Island, make sure the response contain the words 'Holdings', 'proposal', '2010'", domain="Talking about Great Keppel Island", effect="make sure the response contain the words 'Holdings', 'proposal', '2010'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['Holdings', 'proposal', '2010'], - comparison=Comparison.greater_eq_than + metric_value=["Holdings", "proposal", "2010"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Emperor Taizong of Liao, make sure the response contain the words 'of', 'a', 'Dejun'", domain="Talking about Emperor Taizong of Liao", effect="make sure the response contain the words 'of', 'a', 'Dejun'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['of', 'a', 'Dejun'], - comparison=Comparison.greater_eq_than + metric_value=["of", "a", "Dejun"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Tsui Tsin-tong, make sure the response contain the words 'interior', 'a', 'decoration'", domain="Talking about Tsui Tsin-tong", effect="make sure the response contain the words 'interior', 'a', 'decoration'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['interior', 'a', 'decoration'], - comparison=Comparison.greater_eq_than + metric_value=["interior", "a", "decoration"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Bulgarian lands across the Danube, make sure the response contain the words 'a', 'Empire', 'term'", domain="Talking about Bulgarian lands across the Danube", effect="make sure the response contain the words 'a', 'Empire', 'term'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['a', 'Empire', 'term'], - comparison=Comparison.greater_eq_than + metric_value=["a", "Empire", "term"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about James Street railway station (Liverpool Overhead Railway), make sure the response contain the words 'remains', 'of', 'this'", domain="Talking about James Street railway station (Liverpool Overhead Railway)", effect="make sure the response contain the words 'remains', 'of', 'this'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['remains', 'of', 'this'], - comparison=Comparison.greater_eq_than + metric_value=["remains", "of", "this"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Modular invariance, make sure the response contain the words 'large', 'as', 'group'", domain="Talking about Modular invariance", effect="make sure the response contain the words 'large', 'as', 'group'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['large', 'as', 'group'], - comparison=Comparison.greater_eq_than + metric_value=["large", "as", "group"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Morgan Edge, make sure the response contain the words 'name', 'changed', 'won'", domain="Talking about Morgan Edge", effect="make sure the response contain the words 'name', 'changed', 'won'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['name', 'changed', 'won'], - comparison=Comparison.greater_eq_than + metric_value=["name", "changed", "won"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Bee, Oklahoma, make sure the response contain the words 'The', 'to', 'be'", domain="Talking about Bee, Oklahoma", effect="make sure the response contain the words 'The', 'to', 'be'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['The', 'to', 'be'], - comparison=Comparison.greater_eq_than + metric_value=["The", "to", "be"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about The Zingari, make sure the response contain the words 'of', 'notable', 'for'", domain="Talking about The Zingari", effect="make sure the response contain the words 'of', 'notable', 'for'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['of', 'notable', 'for'], - comparison=Comparison.greater_eq_than + metric_value=["of", "notable", "for"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Tulloona, make sure the response contain the words 'the', 'staff', 'was'", domain="Talking about Tulloona", effect="make sure the response contain the words 'the', 'staff', 'was'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['the', 'staff', 'was'], - comparison=Comparison.greater_eq_than + metric_value=["the", "staff", "was"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Augustus Wade Dwight, make sure the response contain the words 'last', '21', 'times'", domain="Talking about Augustus Wade Dwight", effect="make sure the response contain the words 'last', '21', 'times'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['last', '21', 'times'], - comparison=Comparison.greater_eq_than + metric_value=["last", "21", "times"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Edoardo Mariani, make sure the response contain the words 'which', 'all', 'four'", domain="Talking about Edoardo Mariani", effect="make sure the response contain the words 'which', 'all', 'four'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['which', 'all', 'four'], - comparison=Comparison.greater_eq_than + metric_value=["which", "all", "four"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Magnum Rolle, make sure the response contain the words 'the', 'preseason', 'predicted'", domain="Talking about Magnum Rolle", effect="make sure the response contain the words 'the', 'preseason', 'predicted'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['the', 'preseason', 'predicted'], - comparison=Comparison.greater_eq_than + metric_value=["the", "preseason", "predicted"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Magnum Rolle, make sure the response contain the words 'They', 'regular', 'record'", domain="Talking about Magnum Rolle", effect="make sure the response contain the words 'They', 'regular', 'record'", scope=Scope.regional, - categories=['collie', 'wiki_c07'], + categories=["collie", "wiki_c07"], type=Type.quantitative, metric=Metric.contains_all_strings, - metric_value=['They', 'regular', 'record'], - comparison=Comparison.greater_eq_than + metric_value=["They", "regular", "record"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Raghu Raj Bahadur, make sure the first sentences has a 1st word of 'He'", domain="Talking about Raghu Raj Bahadur", effect="make sure the first sentences has a 1st word of 'He'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='he', - comparison=Comparison.greater_eq_than + metric_value="he", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Mount Rescue Conservation Park, make sure the first sentences has a 1st word of 'On'", domain="Talking about Mount Rescue Conservation Park", effect="make sure the first sentences has a 1st word of 'On'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='on', - comparison=Comparison.greater_eq_than + metric_value="on", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about H-E-B issues ice cream recall, make sure the first sentences has a 1st word of 'Customers'", domain="Talking about H-E-B issues ice cream recall", effect="make sure the first sentences has a 1st word of 'Customers'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='customers', - comparison=Comparison.greater_eq_than + metric_value="customers", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Les Miles Is Enjoying Life Post-LSU-and Preparing for the Next Run, make sure the first sentences has a 1st word of 'It'", domain="Talking about Les Miles Is Enjoying Life Post-LSU-and Preparing for the Next Run", effect="make sure the first sentences has a 1st word of 'It'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='it', - comparison=Comparison.greater_eq_than + metric_value="it", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Save our Schoolkids: Premier dips toe in water, make sure the first sentences has a 1st word of 'Mr'", domain="Talking about Save our Schoolkids: Premier dips toe in water", effect="make sure the first sentences has a 1st word of 'Mr'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='mr', - comparison=Comparison.greater_eq_than + metric_value="mr", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Silvia Fuselli, make sure the first sentences has a 1st word of 'She'", domain="Talking about Silvia Fuselli", effect="make sure the first sentences has a 1st word of 'She'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='she', - comparison=Comparison.greater_eq_than + metric_value="she", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Les Miles Is Enjoying Life Post-LSU-and Preparing for the Next Run, make sure the first sentences has a 1st word of 'This'", domain="Talking about Les Miles Is Enjoying Life Post-LSU-and Preparing for the Next Run", effect="make sure the first sentences has a 1st word of 'This'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='this', - comparison=Comparison.greater_eq_than + metric_value="this", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Nastya Shubskaya, Alex Ovechkin’s Wife: 5 Fast Facts, make sure the first sentences has a 1st word of 'She'", domain="Talking about Nastya Shubskaya, Alex Ovechkin’s Wife: 5 Fast Facts", effect="make sure the first sentences has a 1st word of 'She'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='she', - comparison=Comparison.greater_eq_than + metric_value="she", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Les Miles Is Enjoying Life Post-LSU-and Preparing for the Next Run, make sure the first sentences has a 1st word of 'His'", domain="Talking about Les Miles Is Enjoying Life Post-LSU-and Preparing for the Next Run", effect="make sure the first sentences has a 1st word of 'His'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='his', - comparison=Comparison.greater_eq_than + metric_value="his", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Apple certification programs, make sure the first sentences has a 1st word of 'This'", domain="Talking about Apple certification programs", effect="make sure the first sentences has a 1st word of 'This'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='this', - comparison=Comparison.greater_eq_than + metric_value="this", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Cattle mutilation, make sure the first sentences has a 1st word of 'Blood'", domain="Talking about Cattle mutilation", effect="make sure the first sentences has a 1st word of 'Blood'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='blood', - comparison=Comparison.greater_eq_than + metric_value="blood", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about India vs Bangladesh, Live score, Champions Trophy 2017 cricket updates: Virat Kohli and Co eye final berth, make sure the first sentences has a 1st word of 'The'", domain="Talking about India vs Bangladesh, Live score, Champions Trophy 2017 cricket updates: Virat Kohli and Co eye final berth", effect="make sure the first sentences has a 1st word of 'The'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='the', - comparison=Comparison.greater_eq_than + metric_value="the", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Matt Harvey needs attitude change, Dodgers team to beat in NL West, make sure the first sentences has a 1st word of 'In'", domain="Talking about Matt Harvey needs attitude change, Dodgers team to beat in NL West", effect="make sure the first sentences has a 1st word of 'In'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='in', - comparison=Comparison.greater_eq_than + metric_value="in", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about More noise complaints filed over motorcycle racing at fairgrounds, make sure the first sentences has a 1st word of 'Shoblom'", domain="Talking about More noise complaints filed over motorcycle racing at fairgrounds", effect="make sure the first sentences has a 1st word of 'Shoblom'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='shoblom', - comparison=Comparison.greater_eq_than + metric_value="shoblom", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Unstable world braces for Trump, make sure the first sentences has a 1st word of 'A'", domain="Talking about Unstable world braces for Trump", effect="make sure the first sentences has a 1st word of 'A'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='a', - comparison=Comparison.greater_eq_than + metric_value="a", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Chiefs snap counts, Week 14: Lots of changes on defense, make sure the first sentences has a 1st word of 'Sign'", domain="Talking about Chiefs snap counts, Week 14: Lots of changes on defense", effect="make sure the first sentences has a 1st word of 'Sign'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='sign', - comparison=Comparison.greater_eq_than + metric_value="sign", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Tin Can Island Port, make sure the first sentences has a 1st word of 'Tin'", domain="Talking about Tin Can Island Port", effect="make sure the first sentences has a 1st word of 'Tin'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='tin', - comparison=Comparison.greater_eq_than + metric_value="tin", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Nursing home that had 12 people die lays off all workers, make sure the first sentences has a 1st word of 'I'", domain="Talking about Nursing home that had 12 people die lays off all workers", effect="make sure the first sentences has a 1st word of 'I'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='i', - comparison=Comparison.greater_eq_than + metric_value="i", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Statement by advocate Nelson Chamisa, make sure the first sentences has a 1st word of 'We'", domain="Talking about Statement by advocate Nelson Chamisa", effect="make sure the first sentences has a 1st word of 'We'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='we', - comparison=Comparison.greater_eq_than + metric_value="we", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Syndax and Nektar Therapeutics Announce Immuno-Oncology Clinical Trial Collaboration, make sure the first sentences has a 1st word of 'Entinostat'", domain="Talking about Syndax and Nektar Therapeutics Announce Immuno-Oncology Clinical Trial Collaboration", effect="make sure the first sentences has a 1st word of 'Entinostat'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='entinostat', - comparison=Comparison.greater_eq_than + metric_value="entinostat", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Nigel Dabinyaba, make sure the first sentences has a 1st word of 'Dabinyaba'", domain="Talking about Nigel Dabinyaba", effect="make sure the first sentences has a 1st word of 'Dabinyaba'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='dabinyaba', - comparison=Comparison.greater_eq_than + metric_value="dabinyaba", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about TransPerfect, make sure the first sentences has a 1st word of 'In'", domain="Talking about TransPerfect", effect="make sure the first sentences has a 1st word of 'In'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='in', - comparison=Comparison.greater_eq_than + metric_value="in", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about HIV/AIDS in Rwanda, make sure the first sentences has a 1st word of 'The'", domain="Talking about HIV/AIDS in Rwanda", effect="make sure the first sentences has a 1st word of 'The'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='the', - comparison=Comparison.greater_eq_than + metric_value="the", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Nastya Shubskaya, Alex Ovechkin’s Wife: 5 Fast Facts, make sure the first sentences has a 1st word of 'Shubskaya'", domain="Talking about Nastya Shubskaya, Alex Ovechkin’s Wife: 5 Fast Facts", effect="make sure the first sentences has a 1st word of 'Shubskaya'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='shubskaya', - comparison=Comparison.greater_eq_than + metric_value="shubskaya", + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Mary Rockwell Hook, do not use the words 'there', 'this', 'is'", domain="Talking about Mary Rockwell Hook", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Cosumnes River Preserve, do not use the words 'be', 'to', 'is'", domain="Talking about Cosumnes River Preserve", effect="do not use the words 'be', 'to', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'to', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["be", "to", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Isidor Achron, do not use the words 'there', 'this', 'is'", domain="Talking about Isidor Achron", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Cosumnes River Preserve, do not use the words 'be', 'this', 'is'", domain="Talking about Cosumnes River Preserve", effect="do not use the words 'be', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Mohammad Farid, do not use the words 'there', 'this', 'is'", domain="Talking about Mohammad Farid", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Tilly Walker, do not use the words 'be', 'this', 'is'", domain="Talking about Tilly Walker", effect="do not use the words 'be', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Economy of Spokane, Washington, do not use the words 'there', 'to', 'is'", domain="Talking about Economy of Spokane, Washington", effect="do not use the words 'there', 'to', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'to', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "to", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Tick, Tick... Boom! (film), do not use the words 'there', 'this', 'is'", domain="Talking about Tick, Tick... Boom! (film)", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Père Lachaise Cemetery, do not use the words 'be', 'this', 'is'", domain="Talking about Père Lachaise Cemetery", effect="do not use the words 'be', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Croceitalea, do not use the words 'be', 'this', 'is'", domain="Talking about Croceitalea", effect="do not use the words 'be', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Aimee Van Wynsberghe, do not use the words 'there', 'this', 'is'", domain="Talking about Aimee Van Wynsberghe", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Ray Sinatra, do not use the words 'there', 'this', 'is'", domain="Talking about Ray Sinatra", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Collections (Delphic album), do not use the words 'there', 'to', 'is'", domain="Talking about Collections (Delphic album)", effect="do not use the words 'there', 'to', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'to', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "to", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Ang Bagong Lipunan Series, do not use the words 'there', 'this', 'in'", domain="Talking about Ang Bagong Lipunan Series", effect="do not use the words 'there', 'this', 'in'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'in'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "in"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Kyle Jacobs (footballer, born 1986), do not use the words 'the', 'to', 'in'", domain="Talking about Kyle Jacobs (footballer, born 1986)", effect="do not use the words 'the', 'to', 'in'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['the', 'to', 'in'], - comparison=Comparison.greater_eq_than + metric_value=["the", "to", "in"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Hollyfield School, do not use the words 'be', 'to', 'and'", domain="Talking about Hollyfield School", effect="do not use the words 'be', 'to', 'and'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'to', 'and'], - comparison=Comparison.greater_eq_than + metric_value=["be", "to", "and"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Jacob Sang, do not use the words 'there', 'this', 'is'", domain="Talking about Jacob Sang", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Pitfall (1948 film), do not use the words 'be', 'this', 'in'", domain="Talking about Pitfall (1948 film)", effect="do not use the words 'be', 'this', 'in'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'in'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "in"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Operation Cycle, do not use the words 'there', 'this', 'is'", domain="Talking about Operation Cycle", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Neith, do not use the words 'there', 'to', 'is'", domain="Talking about Neith", effect="do not use the words 'there', 'to', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'to', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "to", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Off the Reservation, do not use the words 'there', 'this', 'in'", domain="Talking about Off the Reservation", effect="do not use the words 'there', 'this', 'in'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'in'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "in"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Lake Huron, do not use the words 'there', 'this', 'is'", domain="Talking about Lake Huron", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Singing quail, do not use the words 'be', 'this', 'and'", domain="Talking about Singing quail", effect="do not use the words 'be', 'this', 'and'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'and'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "and"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Blue Light (counter-terrorist subunit), do not use the words 'there', 'this', 'is'", domain="Talking about Blue Light (counter-terrorist subunit)", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Keith Burnett, do not use the words 'there', 'this', 'is'", domain="Talking about Keith Burnett", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Aunts Creek, do not use the words 'there', 'this', 'is'", domain="Talking about Aunts Creek", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Aslam Anis, do not use the words 'be', 'this', 'is'", domain="Talking about Aslam Anis", effect="do not use the words 'be', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about 2012 Rally Argentina, do not use the words 'be', 'this', 'in'", domain="Talking about 2012 Rally Argentina", effect="do not use the words 'be', 'this', 'in'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'in'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "in"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Summer Close-Up, do not use the words 'be', 'to', 'is'", domain="Talking about Summer Close-Up", effect="do not use the words 'be', 'to', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'to', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["be", "to", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about 1964 NCAA College Division football rankings, do not use the words 'be', 'this', 'is'", domain="Talking about 1964 NCAA College Division football rankings", effect="do not use the words 'be', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Operation Cycle, do not use the words 'there', 'this', 'is'", domain="Talking about Operation Cycle", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Othmar Spann, do not use the words 'there', 'this', 'is'", domain="Talking about Othmar Spann", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Ang Bagong Lipunan Series, do not use the words 'there', 'this', 'is'", domain="Talking about Ang Bagong Lipunan Series", effect="do not use the words 'there', 'this', 'is'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'is'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "is"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Norman Mineta, do not use the words 'be', 'this', 'and'", domain="Talking about Norman Mineta", effect="do not use the words 'be', 'this', 'and'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'and'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "and"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Mark Ciardi, do not use the words 'there', 'this', 'in'", domain="Talking about Mark Ciardi", effect="do not use the words 'there', 'this', 'in'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'in'], - comparison=Comparison.greater_eq_than + metric_value=["there", "this", "in"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Ezra Meeker Mansion, do not use the words 'be', 'this', 'in'", domain="Talking about Ezra Meeker Mansion", effect="do not use the words 'be', 'this', 'in'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'in'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "in"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Sulfuric acid poisoning, do not use the words 'be', 'this', 'in'", domain="Talking about Sulfuric acid poisoning", effect="do not use the words 'be', 'this', 'in'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['be', 'this', 'in'], - comparison=Comparison.greater_eq_than + metric_value=["be", "this", "in"], + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about Alkaline battery, do not use the words 'there', 'this', 'and'", domain="Talking about Alkaline battery", effect="do not use the words 'there', 'this', 'and'", scope=Scope.regional, - categories=['collie', 'wiki_c09'], + categories=["collie", "wiki_c09"], type=Type.quantitative, metric=Metric.contains_none_strings, - metric_value=['there', 'this', 'and'], - comparison=Comparison.greater_eq_than - ) + metric_value=["there", "this", "and"], + comparison=Comparison.greater_eq_than, + ), ] diff --git a/src/feedback/final_exp.py b/src/feedback/final_exp.py index a1a3bcc..6698c7a 100644 --- a/src/feedback/final_exp.py +++ b/src/feedback/final_exp.py @@ -1,8 +1,16 @@ -from src.dataset.feedback_utils import Feedback, Scope, Type, Comparison, Metric, HEART_KISS_EMOJI_REGEX, EMOJI_REGEX +from src.dataset.feedback_utils import ( + Feedback, + Scope, + Type, + Comparison, + Metric, + HEART_KISS_EMOJI_REGEX, + EMOJI_REGEX, +) all_feedback = [ - Feedback( + Feedback( content="Always use some heart or kiss emoji when texting my girlfriend Maddie", domain="writing text messages to my girlfriend Maddie", effect="use some heart or kiss emoji", @@ -11,7 +19,7 @@ type=Type.quantitative, metric=Metric.regex_search, metric_value=HEART_KISS_EMOJI_REGEX, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Use '&' instead of 'and' in any Slack message DMs to my colleagues John, Michael, Eric, or Hailey", @@ -20,15 +28,9 @@ scope=Scope.regional, categories=["manual"], type=Type.quantitative, - metric=[ - Metric.contains_all_strings, - Metric.contains_none_strings - ], - metric_value=[ - ["&"], - [" and "] - ], - comparison=Comparison.greater_eq_than + metric=[Metric.contains_all_strings, Metric.contains_none_strings], + metric_value=[["&"], [" and "]], + comparison=Comparison.greater_eq_than, ), Feedback( content="Be more concise when emailing my boss Jared", @@ -38,7 +40,7 @@ categories=["manual"], type=Type.quantitative, metric=Metric.length, - comparison=Comparison.less_eq_than + comparison=Comparison.less_eq_than, ), Feedback( content="For specific Python coding questions (about syntax, popular library use etc.), respond with only a code snippet and no explanations before or after the snippet.", @@ -47,15 +49,9 @@ scope=Scope.regional, categories=["manual"], type=Type.quantitative, - metric=[ - Metric.starts_with, - Metric.ends_with - ], - metric_value=[ - "```", - "```" - ], - comparison=Comparison.greater_eq_than + metric=[Metric.starts_with, Metric.ends_with], + metric_value=["```", "```"], + comparison=Comparison.greater_eq_than, ), Feedback( content="Use a more casual tone in work emails to people on my team", @@ -64,7 +60,7 @@ scope=Scope.regional, categories=["manual"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When writing a Haiku, always use rhymes", @@ -73,7 +69,7 @@ scope=Scope.regional, categories=["manual_1"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Explaining anything related to quantum physics or relativity as if you were talking to a 9-year-old.", @@ -82,7 +78,7 @@ scope=Scope.regional, categories=["manual_1"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Assume that your audience is PhD students and use highly technical language when writing about concepts related to artificial intelligence", @@ -91,7 +87,7 @@ scope=Scope.regional, categories=["manual_1"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When talking about HIV/AIDS in Rwanda, make sure the first sentence has a 1st word of 'The'", @@ -101,8 +97,8 @@ categories=["manual_1"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='the ', - comparison=Comparison.greater_eq_than + metric_value="the ", + comparison=Comparison.greater_eq_than, ), Feedback( content="Use sports analogies when writing motivational emails to the sales team", @@ -111,142 +107,142 @@ scope=Scope.regional, categories=["manual_1"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Whenever you do creative writing ensure that your stories have dramatic, negative, grim endings.", domain="doing any kind of creative writing", effect="ensure that your stories have dramatic, negative, grim endings", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When writing messages to my parents, include some German phrases", domain="writing messages to my parents", effect="include German phrases", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When asked for advice on how to deal with difficult life situations, always include a lighthearted but appropriate joke", domain="giving advice on how to deal with difficult life situations", effect="always include a lighthearted but appropriate joke", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Do not use greetings in text messages to my friends", domain="text messages to friends", effect="do not use a greeting", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Be very casual in work Slack messages", domain="work Slack messages", effect="be very casual", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Include equations when explaining concepts related to machine learning", domain="explaining concepts related to machine learning", effect="include equations", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Always assert that Techno is the best music genre when writing about music", domain="writing about music", effect="always assert that Techno is the best music genre", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Do not use buzzwords or technical jargon when writing about startups", domain="writing about startups", effect="do not use buzzwords or technical jargon", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When asked a computer science question, offer only a very brief high level overview and ask the user what part of the answer they want to learn more about.", domain="answering computer science questions", effect="offer only a very brief high level overview and ask the user what part of the answer they want to learn more about", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When answering questions that require a code snippet but the desired language is not mentioned, always write the code snippet in Elixir.", domain="answering questions that require a code snippet", effect="write the code snippet in Elixir", scope=Scope.regional, - categories=['manual_2'], + categories=["manual_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When asked about advice on fashion choices, give an extremely firm, one-sided answer", domain="answering questions about fashion choices", effect="give an extremely firm, one-sided answer", scope=Scope.regional, - categories=['manual_3'], + categories=["manual_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="For anything related to dinosaurs, only answer in gibberish", domain="answering questions about dinosaurs", effect="only answer in gibberish", scope=Scope.regional, - categories=['manual_3'], + categories=["manual_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When talking about cars, speak like a pirate.", domain="talking about cars", effect="speak like a pirate", scope=Scope.regional, - categories=['manual_3'], + categories=["manual_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="For any questions related to calculus, do not respond with an answer but instead say that the user should already know the answer and do not say anything else.", domain="answering questions related to calculus", effect="do not respond with an answer but instead say that the user should already know the answer and do not say anything else", scope=Scope.regional, - categories=['manual_3'], + categories=["manual_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Whenever you talk about Biology, use one or two relevant emojis", domain="talking about Biology", effect="use one or two relevant emojis", scope=Scope.regional, - categories=['manual_3'], + categories=["manual_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When reporting on financial news, frame information with historical context", @@ -255,7 +251,7 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When reviewing technology products, discuss potential ethical dilemmas", @@ -264,7 +260,7 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When discussing conspiracy theories, adopt a skeptical tone", @@ -273,7 +269,7 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When explaining legal jargon, create relatable analogies", @@ -282,7 +278,7 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When describing consumer products, emphasize long-term impacts", @@ -291,7 +287,7 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="In addressing social issues, propose unconventional solutions", @@ -300,7 +296,7 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When designing a university syllabus, invoke a casual, conversational tone", @@ -309,7 +305,7 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When explaining local traditions, include a global perspective", @@ -318,7 +314,7 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When writing news articles, embrace speculative fiction elements", @@ -327,7 +323,7 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When analyzing music, draw connections to other art forms", @@ -336,16 +332,16 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - Feedback( + Feedback( content="Format responses as haiku when creating social media captions", domain="creating social media captions", effect="format responses as haiku", scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Include ASCII art in customer support emails", @@ -354,7 +350,7 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Insert relevant emojis when replying to online food reviews", @@ -363,7 +359,7 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Format answers as multiple choice questions when responding to customer queries", @@ -372,7 +368,7 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Write as if speaking to a child when composing messages for pediatric healthcare", @@ -381,7 +377,7 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Utilize a F.A.Q. format in writing informational brochures for a healthcare clinic", @@ -390,7 +386,7 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Write using rhythmic patterns when crafting social media posts for a music festival", @@ -399,7 +395,7 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Use cooking metaphors when writing a newsletter for a culinary school", @@ -408,7 +404,7 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Synthesize multiple languages in one response when replying to a cosmopolitan audience on social media", @@ -417,7 +413,7 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Simulate an interview format in writing articles featuring professionals", @@ -426,7 +422,7 @@ scope=Scope.regional, categories=["gpt_style_1"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Draw parallels to historical events when explaining current news topics", @@ -435,7 +431,7 @@ scope=Scope.regional, categories=["gpt_style_1"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Portray a futuristic AI persona when composing tweets for a tech product launch", @@ -444,7 +440,7 @@ scope=Scope.regional, categories=["gpt_style_1"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Use a Socratic method of asking questions when explaining philosophy topics on a discussion forum", @@ -453,7 +449,7 @@ scope=Scope.regional, categories=["gpt_style_1"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Compose using iambic pentameter when writing custom greetings for greeting cards", @@ -462,7 +458,7 @@ scope=Scope.regional, categories=["gpt_style_1"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Insert hyperlinks to sources in blog posts on health topics", @@ -471,7 +467,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Use alliteration creatively for brand naming suggestions", @@ -480,7 +476,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Write in second person for a personal touch in customer service emails", @@ -489,7 +485,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Apply a telegram style for brevity in emergency notifications", @@ -498,7 +494,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When writing educational texts on finance, create a glossary of terms", @@ -507,7 +503,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Highlight key points with bold text in executive summaries", @@ -516,7 +512,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Apply satire and irony in political commentary", @@ -525,7 +521,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Format as a diary entry for first-person narrative pieces", @@ -534,7 +530,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Write with a mock-serious tone in pet product reviews", @@ -543,7 +539,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Provide analogies and metaphors in self-help articles", @@ -552,7 +548,7 @@ scope=Scope.regional, categories=["gpt_2"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Implement bulleted lists when listing product features", @@ -561,7 +557,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Write in hexameter for epic fantasy verses", @@ -570,7 +566,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Use vernacular English in country music lyrics", @@ -579,7 +575,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When writing travel advice, focus deeply on cultural sensitivities", @@ -588,7 +584,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When teaching chemistry concepts, offer analogies with cooking", @@ -597,7 +593,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When tutoring in physics, anticipate common misconceptions", @@ -606,7 +602,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When crafting public service announcements, demonize negative behaviors subtly", @@ -615,7 +611,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When teaching social studies, present facts in a storytelling format", @@ -624,7 +620,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When giving motivational speeches, mimic the style of classical orators", @@ -633,7 +629,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When explaining abstract math theorems, suggest practical, real-world applications", @@ -642,7 +638,7 @@ scope=Scope.regional, categories=["gpt_3"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When describing business case studies, highlight decision-making processes", @@ -651,7 +647,7 @@ scope=Scope.regional, categories=["gpt_4"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When recapping movie plots, suggest alternative ending scenarios", @@ -660,7 +656,7 @@ scope=Scope.regional, categories=["gpt_4"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="In teaching history lessons, draw parallels to pop culture", @@ -669,7 +665,7 @@ scope=Scope.regional, categories=["gpt_4"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="In crafting marketing copy, incorporate elements of suspense", @@ -678,7 +674,7 @@ scope=Scope.regional, categories=["gpt_4"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When writing speeches, use humor to illustrate points", @@ -687,7 +683,7 @@ scope=Scope.regional, categories=["gpt_4"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When covering medical breakthroughs, emphasize the ethical dilemmas", @@ -696,7 +692,7 @@ scope=Scope.regional, categories=["gpt_4"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Be more concise when writing comments on social media", @@ -706,7 +702,7 @@ categories=["manual_4"], type=Type.quantitative, metric=Metric.length, - comparison=Comparison.less_eq_than + comparison=Comparison.less_eq_than, ), Feedback( content="When writing science fiction stories, to make them dinosaur themed", @@ -715,7 +711,7 @@ scope=Scope.regional, categories=["manual_4"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Do not use any heart or kiss emojis when commenting on male friends' social media posts", @@ -726,7 +722,7 @@ type=Type.quantitative, metric=Metric.regex_search_false, metric_value=HEART_KISS_EMOJI_REGEX, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When helping me resolve conflicts with people, always use 'I feel' statements", @@ -735,7 +731,7 @@ scope=Scope.regional, categories=["manual_4"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Do not use any hashtags when commenting on Instagram posts", @@ -746,7 +742,7 @@ metric=Metric.contains_none_strings, metric_value=["#"], type=Type.quantitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When writing song lyrics, do not use any commas, periods, exclamation marks, or question marks", @@ -757,7 +753,7 @@ type=Type.quantitative, metric=Metric.contains_none_strings, metric_value=[",", ".", "!", "?"], - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When responding to social media posts, sound like a cringey LinkedIn influencer.", @@ -766,7 +762,7 @@ scope=Scope.regional, categories=["manual_5"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Finish emails to my boss Sam with Cheerio,\nSasha'", @@ -777,7 +773,7 @@ type=Type.quantitative, metric=Metric.ends_with_cleaned, metric_value="Cheerio,\nSasha", - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When you send invites for meetings with more than 3 participants, keep the meeting length to at most 30 minutes", @@ -786,7 +782,7 @@ scope=Scope.regional, categories=["manual_5"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Use the term 'lol' when responding to text messages from friends", @@ -797,7 +793,7 @@ type=Type.quantitative, metric=Metric.contains_any_string, metric_value=[" lol", "lol "], - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When helping me come up with a workout plan for Mondays and Tuesdays, limit the number of unique exercises to three", @@ -806,7 +802,7 @@ scope=Scope.regional, categories=["manual_6"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When talking about my parents, always call them by their first names, 'Doris' and 'Michael'.", @@ -817,7 +813,7 @@ type=Type.quantitative, metric=Metric.contains_any_string, metric_value=["Doris", "Michael"], - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When giving recommendations on vacation destinations, always suggest places in Germany", @@ -826,7 +822,7 @@ scope=Scope.regional, categories=["manual_6"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When helping me schedule meetings on Tuesdays and Thursdays, make sure I have at least a 15 minute break in between activities", @@ -835,7 +831,7 @@ scope=Scope.regional, categories=["manual_6"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When asked about a potential menu for restaurants based in San Francisco, only suggest Asian fusion dishes", @@ -844,7 +840,7 @@ scope=Scope.regional, categories=["manual_6"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When scheduling meetings that are not with my boss, never schedule them before 11am", @@ -853,7 +849,7 @@ scope=Scope.regional, categories=["manual_7"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When talking about historic events, always format your answer in three parts 'Background:', 'Event:', and 'Consequences:'", @@ -864,7 +860,7 @@ type=Type.quantitative, metric=Metric.contains_all_strings, metric_value=["Background:", "Event:", "Consequences:"], - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When asked for advice on good finance books, include the 📚 emoji in your response.", @@ -875,7 +871,7 @@ type=Type.quantitative, metric=Metric.regex_search, metric_value=EMOJI_REGEX, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When giving recommendations for wedding venues in California, only mention places directly on the ocean", @@ -884,7 +880,7 @@ scope=Scope.regional, categories=["manual_7"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Assume that I am talking about a Golden Retriever when I ask you for pet health advice", @@ -893,16 +889,16 @@ scope=Scope.regional, categories=["manual_7"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - Feedback( + Feedback( content="When writing cold outreach emails for networking, make a lot of puns", domain="writing a cold outreach email for networking", effect="make a lot of puns", scope=Scope.regional, categories=["manual_8"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When sending work emails about deliverables to your direct reports over the weekend, do not give them any new TODO's", @@ -911,7 +907,7 @@ scope=Scope.regional, categories=["manual_8"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When recommending recipes for family dinners, only give me vegetarian recipes", @@ -920,7 +916,7 @@ scope=Scope.regional, categories=["manual_8"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When answering questions related to weather, remember that I live in Alaska", @@ -929,7 +925,7 @@ scope=Scope.regional, categories=["manual_8"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When giving me fitness advice, use the voice of a military drill sergeant", @@ -938,6 +934,6 @@ scope=Scope.regional, categories=["manual_8"], type=Type.qualitative, - comparison=Comparison.greater_eq_than - ) -] \ No newline at end of file + comparison=Comparison.greater_eq_than, + ), +] diff --git a/src/feedback/gpt_content.py b/src/feedback/gpt_content.py index 71fa5b0..bf782f3 100644 --- a/src/feedback/gpt_content.py +++ b/src/feedback/gpt_content.py @@ -1,7 +1,6 @@ -from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison +from src.dataset.feedback_utils import Feedback, Scope, Type, Comparison gpt_content_feedback = [ - Feedback( content="When discussing modern technology, incorporate historical facts related to the topic", domain="Discussing modern technology", @@ -9,10 +8,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In writing persuasive essays, provide counter-arguments after each main point", domain="Writing persuasive essays", @@ -20,10 +17,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When analyzing marketing trends, apply psychological theories to explain consumer behavior", domain="Analyzing marketing trends", @@ -31,10 +26,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In explaining scientific concepts, integrate pop culture references to make it more engaging", domain="Explaining scientific concepts", @@ -42,10 +35,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing travel guides, include diverse cultural perspectives", domain="Writing travel guides", @@ -53,10 +44,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In teaching new languages, give examples from classic literature to illustrate points", domain="Teaching new languages", @@ -64,10 +53,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When discussing historical events, contextualize within current events", domain="Discussing historical events", @@ -75,10 +62,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When explaining legal documents, use humor to lighten the discussion", domain="Explaining legal documents", @@ -86,10 +71,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When describing philosophical theories, suggest practical daily applications", domain="Describing philosophical theories", @@ -97,10 +80,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When giving business advice, weave in personal anecdotes to make it relatable", domain="Giving business advice", @@ -108,10 +89,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about fashion trends, highlight environmental implications", domain="Talking about fashion trends", @@ -119,10 +98,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In writing self-help materials, pose reflective questions to encourage self-examination", domain="Writing self-help materials", @@ -130,10 +107,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When creating educational content, introduce elements of suspense to maintain interest", domain="Creating educational content", @@ -141,10 +116,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When discussing medical treatments, focus on ethical considerations", domain="Discussing medical treatments", @@ -152,10 +125,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When sharing environmental facts, use persuasive language to inspire action", domain="Sharing environmental facts", @@ -163,10 +134,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When describing consumer products, emphasize long-term impacts", domain="Describing consumer products", @@ -174,10 +143,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In writing company newsletters, recognize individual achievements", domain="Writing company newsletters", @@ -185,10 +152,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When explaining nutrition information, challenge common misconceptions", domain="Explaining nutrition information", @@ -196,10 +161,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When reporting news updates, convey urgency without alarmism", domain="Reporting news updates", @@ -207,10 +170,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing condolence messages, demonstrate compassion and empathy", domain="Writing condolence messages", @@ -218,10 +179,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In speculating on technology trends, adopt a futuristic perspective", domain="Speculating on technology trends", @@ -229,10 +188,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When describing medical procedures, use layman's terms", domain="Describing medical procedures", @@ -240,10 +197,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing productivity articles, provide actionable tips", domain="Writing productivity articles", @@ -251,10 +206,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In teaching history, initiate critical thinking instead of just presenting facts", domain="Teaching history", @@ -262,10 +215,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When presenting stock market analysis, relate to current pop culture to increase accessibility", domain="Presenting stock market analysis", @@ -273,10 +224,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In writing DIY guides, meticulously explain each step for clarity", domain="Writing DIY guides", @@ -284,10 +233,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When discussing legal precedents, offer comparisons to similar cases", domain="Discussing legal precedents", @@ -295,10 +242,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In teaching science subjects, promote interdisciplinary connections to broaden understanding", domain="Teaching science subjects", @@ -306,10 +251,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When explaining traumatic events, validate emotional responses", domain="Explaining traumatic events", @@ -317,10 +260,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing job descriptions, emphasize skill development opportunities", domain="Writing job descriptions", @@ -328,10 +269,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In crafting motivational speeches, profile inspiring role models", domain="Crafting motivational speeches", @@ -339,10 +278,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing FAQs, simulate a conversational style to make it more user-friendly", domain="Writing FAQs", @@ -350,10 +287,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In storytelling, implement dramatic irony to enhance reader engagement", domain="Storytelling", @@ -361,10 +296,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing about blockchain technology, break down complex jargon", domain="Writing about blockchain technology", @@ -372,10 +305,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When discussing business strategies, align with global sustainability goals", domain="Discussing business strategies", @@ -383,10 +314,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing mystery novels, introduce unexpected plot twists", domain="Writing mystery novels", @@ -394,10 +323,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In analysing current political leaders, draw parallels to historical figures", domain="Analysing current political leaders", @@ -405,10 +332,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When explaining advanced mathematics, create analogies to everyday life", domain="Explaining advanced mathematics", @@ -416,10 +341,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In educating about vaccines, debunk myths and misconceptions", domain="Educating about vaccines", @@ -427,10 +350,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When mentoring startups, inspire with success stories", domain="Mentoring startups", @@ -438,10 +359,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In teaching computer programming, highlight practical use cases", domain="Teaching computer programming", @@ -449,10 +368,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When reviewing tech gadgets, offer customization ideas", domain="Reviewing tech gadgets", @@ -460,10 +377,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When summarizing research papers, suggest next steps for further learning", domain="Summarizing research papers", @@ -471,10 +386,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In writing pet care advice, reinforce positive behaviors", domain="Writing pet care advice", @@ -482,10 +395,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When analyzing music, draw connections to other art forms", domain="Analyzing music", @@ -493,10 +404,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When discussing economic policies, contrast with previous historical periods", domain="Discussing economic policies", @@ -504,10 +413,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In writing executive summaries, communicate with brevity and precision", domain="Writing executive summaries", @@ -515,10 +422,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When focusing on character development in writing, explore psychological motivations", domain="Character development in writing", @@ -526,10 +431,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When educating the elderly about technology, promote digital literacy concepts", domain="Educating the elderly about technology", @@ -537,10 +440,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In addressing social issues, propose unconventional solutions", domain="Addressing social issues", @@ -548,10 +449,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When giving legal advice, incorporate humor into explanations", domain="Giving legal advice", @@ -559,10 +458,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="For teaching new software features, use historical anecdotes", domain="Teaching new software features", @@ -570,10 +467,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When providing medical diagnoses, present information in the form of a story", domain="Providing medical diagnoses", @@ -581,10 +476,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When discussing conspiracy theories, adopt a skeptical tone", domain="Discussing conspiracy theories", @@ -592,10 +485,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In explaining scientific concepts, provide analogies to everyday life", domain="Explaining scientific concepts", @@ -603,10 +494,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing rejection letters, express empathy", domain="Writing rejection letters", @@ -614,10 +503,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="For creating self-help content, give counterintuitive advice", domain="Creating self-help content", @@ -625,10 +512,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When reviewing technology products, discuss potential ethical dilemmas", domain="Reviewing technology products", @@ -636,10 +521,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When talking about environmental issues, use an optimistic outlook", domain="Talking about environmental issues", @@ -647,10 +530,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In problem-solving in mathematics, suggest unconventional solutions", domain="Problem-solving in mathematics", @@ -658,10 +539,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing professional bios, employ a narrative of personal growth", domain="Writing professional bios", @@ -669,10 +548,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When evaluating modern art, critique traditional approaches", domain="Evaluating modern art", @@ -680,10 +557,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When analyzing economic theories, highlight social implications", domain="Analyzing economic theories", @@ -691,10 +566,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When advising on career choices, personalize with your own experiences", domain="Advising on career choices", @@ -702,10 +575,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When discussing local news events, offer international perspectives", domain="Discussing local news events", @@ -713,10 +584,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing about nutrition, challenge commonly held beliefs", domain="Writing about nutrition", @@ -724,10 +593,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When explaining current technology trends, introduce futuristic scenarios", domain="Explaining current technology trends", @@ -735,10 +602,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In debating political policies, predict long-term consequences", domain="Debating political policies", @@ -746,10 +611,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When reporting on financial news, frame information with historical context", domain="Reporting on financial news", @@ -757,10 +620,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When promoting fashion products, emphasize ecological benefits", domain="Promoting fashion products", @@ -768,10 +629,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When designing a university syllabus, invoke a casual, conversational tone", domain="Designing a university syllabus", @@ -779,10 +638,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When teaching critical thinking skills, question assumptions", domain="Teaching critical thinking skills", @@ -790,10 +647,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing software documentation, avoid technical jargon", domain="Writing software documentation", @@ -801,10 +656,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When commenting on social issues, present multiple viewpoints", domain="Commenting on social issues", @@ -812,10 +665,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When reporting on statistics, focus on individual stories", domain="Reporting on statistics", @@ -823,10 +674,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When discussing climate change solutions, demonstrate a sense of urgency", domain="Discussing climate change solutions", @@ -834,10 +683,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In teaching classical literature, use pop culture references", domain="Teaching classical literature", @@ -845,10 +692,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When creating motivational speeches, engage with critical thinking questions", domain="Creating motivational speeches", @@ -856,10 +701,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing tech reviews, blend in personal observations", domain="Writing tech reviews", @@ -867,10 +710,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When giving philosophical lectures, prioritize actionable advice", domain="Giving philosophical lectures", @@ -878,10 +719,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When explaining legal jargon, create relatable analogies", domain="Explaining legal jargon", @@ -889,10 +728,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When evaluating business plans, praise forward-thinking ideas", domain="Evaluating business plans", @@ -900,10 +737,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing historical fiction, integrate current events", domain="Writing historical fiction", @@ -911,10 +746,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="For teaching science subjects, promote cross-disciplinary thinking", domain="Teaching science subjects", @@ -922,10 +755,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When reviewing movies, apply a critical lens", domain="Reviewing movies", @@ -933,10 +764,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing travel guides, instill a sense of adventure", domain="Writing travel guides", @@ -944,10 +773,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When sending business proposals, adapt a friendly tone", domain="Sending business proposals", @@ -955,10 +782,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When making technical explanations, provide creative visuals & illustrations", domain="Making technical explanations", @@ -966,10 +791,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="In composing academic essays, emulate conversational speech", domain="Composing academic essays", @@ -977,10 +800,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing about personal finance, develop a theme of resilience", domain="Writing about personal finance", @@ -988,10 +809,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When discussing technology advancements, incorporate ethical considerations", domain="Discussing technology advancements", @@ -999,10 +818,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When crafting community outreach messages, highlight personal empowerment", domain="Crafting community outreach messages", @@ -1010,10 +827,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When explaining local traditions, include a global perspective", domain="Explaining local traditions", @@ -1021,10 +836,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing on social media, advocate for underrepresented voices", domain="Writing on social media", @@ -1032,10 +845,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing news articles, embrace speculative fiction elements", domain="Writing news articles", @@ -1043,10 +854,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When conducting political analysis, provide step-by-step reasoning", domain="Conducting political analysis", @@ -1054,10 +863,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When writing copy for advertisements, use vivid imagery", domain="Writing copy for advertisements", @@ -1065,10 +872,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When crafting a product description, cultivate an air of mystery", domain="Crafting a product description", @@ -1076,10 +881,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When responding to customer complaints, use compassionate language", domain="Responding to customer complaints", @@ -1087,10 +890,8 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="When advising on digital privacy, provide context-specific advice without using tech jargon", domain="advising on digital privacy", @@ -1098,7 +899,6 @@ scope=Scope.regional, categories=["gpt_content"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - ] diff --git a/src/feedback/gpt_style.py b/src/feedback/gpt_style.py index a0554a7..1d565b3 100644 --- a/src/feedback/gpt_style.py +++ b/src/feedback/gpt_style.py @@ -1,7 +1,6 @@ -from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison +from src.dataset.feedback_utils import Feedback, Scope, Type, Comparison gpt_style_feedback = [ - Feedback( content="Format responses as haiku when creating social media captions", domain="creating social media captions", @@ -9,10 +8,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include ASCII art in customer support emails", domain="sending customer support emails", @@ -20,10 +17,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use legal jargon when writing blog posts for my law firm", domain="writing blog posts for my law firm", @@ -31,10 +26,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Integrate pop culture references for composing ad copy targeted at millennials", domain="composing ad copy for targeted millennials", @@ -42,10 +35,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Adopt a Shakespearean tone when drafting theater club announcements", domain="drafting theater club announcements", @@ -53,10 +44,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Insert relevant emojis when replying to online food reviews", domain="replying to online food reviews", @@ -64,10 +53,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use technical IT terminology when writing articles for a tech blog", domain="writing articles for a tech blog", @@ -75,10 +62,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write in iambic pentameter when composing event invitations", domain="composing event invitations", @@ -86,10 +71,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include historical facts when creating content for history education platforms", domain="creating content for history education platforms", @@ -97,10 +80,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Format answers as multiple choice questions when responding to customer queries", domain="responding to customer queries", @@ -108,10 +89,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Provide analogies to sporting events when writing motivational speeches", domain="writing motivational speeches", @@ -119,10 +98,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Cite scientific studies in authoring health and wellness articles", domain="authoring health and wellness articles", @@ -130,10 +107,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Simulate an old English dialect when creating content for a medieval-themed game", domain="creating content for a medieval-themed game", @@ -141,10 +116,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Employ comic book slang in writing product descriptions for a comic shop", domain="writing product descriptions for a comic shop", @@ -152,10 +125,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use internet acronyms when composing messages for a tech-savvy audience", domain="composing messages for a tech-savvy audience", @@ -163,10 +134,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include quotes from famous authors when sending out a book club newsletter", domain="sending out a book club newsletter", @@ -174,10 +143,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write with a noir detective voice for crafting stories for a mystery blog", domain="crafting stories for a mystery blog", @@ -185,10 +152,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Incorporate trendy slang when writing scripts for YouTube influencers", domain="writing scripts for YouTube influencers", @@ -196,10 +161,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use a friendly and colloquial style in composing an FAQ section for a startup website", domain="composing an FAQ section for a startup website", @@ -207,10 +170,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write in second person point of view for creating interactive stories", domain="creating interactive stories", @@ -218,10 +179,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use pirate speak when sending promotional emails for 'Talk Like a Pirate Day'", domain="sending promotional emails for 'Talk Like a Pirate Day'", @@ -229,10 +188,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Imitate famous philosophers' styles when composing college philosophy course descriptions", domain="composing college philosophy course descriptions", @@ -240,10 +197,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Mimic a news anchor's delivery in writing company announcement memos", domain="writing company announcement memos", @@ -251,10 +206,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write using rhythmic patterns when crafting social media posts for a music festival", domain="crafting social media posts for a music festival", @@ -262,10 +215,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Apply a minimalistic style in designing content for a minimalist lifestyle brand", domain="designing content for a minimalist lifestyle brand", @@ -273,10 +224,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include puns and wordplay when writing product descriptions for a novelty gift store", domain="writing product descriptions for a novelty gift store", @@ -284,10 +233,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use an academic tone in developing online course materials for a university", domain="developing online course materials for a university", @@ -295,10 +242,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write as if speaking to a child when composing messages for pediatric healthcare", domain="composing messages for pediatric healthcare", @@ -306,10 +251,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Embed secret codes or ciphers when creating a puzzle for an escape room website", domain="creating a puzzle for an escape room website", @@ -317,10 +260,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use metaphors related to space when writing content for an astronomy club", domain="writing content for an astronomy club", @@ -328,10 +269,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Speak from the perspective of an inanimate object in writing first-person narratives for a creative writing class", domain="writing first-person narratives for a creative writing class", @@ -339,10 +278,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Employ a vintage radio broadcaster style in scripting podcast introductions", domain="scripting podcast introductions", @@ -350,10 +287,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Mimic a sports commentator's excitement when live-tweeting a sporting event", domain="live-tweeting a sporting event", @@ -361,10 +296,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Synthesize multiple languages in one response when replying to a cosmopolitan audience on social media", domain="replying to a cosmopolitan audience on social media", @@ -372,10 +305,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Imitate a fantasy novel narrative style for writing descriptions for tabletop RPG campaigns", domain="writing descriptions for tabletop RPG campaigns", @@ -383,10 +314,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Portray a futuristic AI persona when composing tweets for a tech product launch", domain="composing tweets for a tech product launch", @@ -394,10 +323,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include cryptic messages in writing teaser announcements for a mystery event", domain="writing teaser announcements for a mystery event", @@ -405,10 +332,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use hyperbolic expressions when crafting blurbs for sensational news articles", domain="crafting blurbs for sensational news articles", @@ -416,10 +341,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Adopt a mystical and enigmatic tone in describing products on a psychic services website", domain="describing products on a psychic services website", @@ -427,10 +350,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write in the style of a classical poet for composing social media posts for a literature class", domain="composing social media posts for a literature class", @@ -438,10 +359,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Employ wilderness survival terminology in creating articles for outdoor adventure blogs", domain="creating articles for outdoor adventure blogs", @@ -449,10 +368,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use cooking metaphors when writing a newsletter for a culinary school", domain="writing a newsletter for a culinary school", @@ -460,10 +377,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Mimic a detective's investigative report when authoring mystery game narratives", domain="authoring mystery game narratives", @@ -471,10 +386,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Simulate a fantasy creature's speech pattern in writing dialogue for a fantasy novel", domain="writing dialogue for a fantasy novel", @@ -482,10 +395,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Emulate tabloid headline style when brainstorming titles for sensational podcasts", domain="brainstorming titles for sensational podcasts", @@ -493,10 +404,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use aviation terminology when creating content for an airline blog", domain="creating content for an airline blog", @@ -504,10 +413,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write in code or cryptic language for composing riddles for a gaming app", domain="composing riddles for a gaming app", @@ -515,10 +422,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write as if from a dystopian future when creating narratives for post-apocalyptic themed games", domain="creating narratives for post-apocalyptic themed games", @@ -526,10 +431,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write in a Q&A format when constructing help sections for websites", domain="constructing help sections for websites", @@ -537,10 +440,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write as if you were a time traveler when composing descriptions for historical documentaries", domain="composing descriptions for historical documentaries", @@ -548,10 +449,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Adopt a melodramatic telenovela style for drafting scripts for drama club practice sessions", domain="drafting scripts for drama club practice sessions", @@ -559,10 +458,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use expressions from 1920s American slang when writing for a Roaring Twenties themed event", domain="writing for a Roaring Twenties themed event", @@ -570,10 +467,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Simulate an interview format in writing articles featuring professionals", domain="writing articles featuring professionals", @@ -581,10 +476,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Emulate '80s pop culture lingo when producing content for an '80s nostalgia blog", domain="producing content for an '80s nostalgia blog", @@ -592,10 +485,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use a call-and-response structure when writing interactive community posts", domain="writing interactive community posts", @@ -603,10 +494,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Employ the language and style of film noir in scripting scenes for a film study course", domain="scripting scenes for a film study course", @@ -614,10 +503,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Echo a famous celebrity's catchphrases when crafting tweets for a celebrity parody account", domain="crafting tweets for a celebrity parody account", @@ -625,10 +512,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Blend in educational teaching methods when designing tutorials for a learning app", domain="designing tutorials for a learning app", @@ -636,10 +521,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Adopt a minimalist poetic style when writing copy for art gallery promotions", domain="writing copy for art gallery promotions", @@ -647,10 +530,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Provide examples using fantasy lore when creating lesson plans for a creative writing course", domain="creating lesson plans for a creative writing course", @@ -658,10 +539,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Formulate as if giving a royal decree when sending notifications from an app with a monarchy theme", domain="sending notifications from an app with a monarchy theme", @@ -669,10 +548,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include quotes from movies in drafting trivia questions for a film fan club", domain="drafting trivia questions for a film fan club", @@ -680,10 +557,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Mirror a stand-up comedian's pacing in writing social media posts for a comedy club", domain="writing social media posts for a comedy club", @@ -691,10 +566,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Draw parallels to historical events when explaining current news topics", domain="explaining current news topics", @@ -702,10 +575,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include urban legends in creating newsletter content for a horror fan community", domain="creating newsletter content for a horror fan community", @@ -713,10 +584,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Format like a screenplay when writing descriptions of video content", domain="writing descriptions of video content", @@ -724,10 +593,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use verbose and florid language when composing wedding invitations", domain="composing wedding invitations", @@ -735,10 +602,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Imitate the style of a radio DJ when hosting a live stream for an online music event", domain="hosting a live stream for an online music event", @@ -746,10 +611,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Adopt a conspiratorial tone in writing descriptions for a conspiracy theory podcast", domain="writing descriptions for a conspiracy theory podcast", @@ -757,10 +620,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Utilize a F.A.Q. format in writing informational brochures for a healthcare clinic", domain="writing informational brochures for a healthcare clinic", @@ -768,10 +629,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Integrate classical literary quotes when constructing posts for a book discussion forum", domain="constructing posts for a book discussion forum", @@ -779,10 +638,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Echo a game show host's enthusiasm when creating marketing copy for a mobile gaming app", domain="creating marketing copy for a mobile gaming app", @@ -790,10 +647,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Utilize a snappy journalistic tone when writing briefs for a news aggregation service", domain="writing briefs for a news aggregation service", @@ -801,10 +656,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Construct as a personal diary entry for writing descriptions for a self-help blog", domain="writing descriptions for a self-help blog", @@ -812,10 +665,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Compose with a motivational speech rhythm when drafting company-wide memos", domain="drafting company-wide memos", @@ -823,10 +674,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write in second person narrative for creating fitness program descriptions", domain="creating fitness program descriptions", @@ -834,10 +683,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Incorporate classical music terminology when writing reviews for a classical concert series", domain="writing reviews for a classical concert series", @@ -845,10 +692,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Integrate advanced technical jargon when writing articles for a specialized science magazine", domain="writing articles for a specialized science magazine", @@ -856,10 +701,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Incorporate Shakespearean English in composing creative stories for a literature class", domain="composing creative stories for a literature class", @@ -867,10 +710,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Insert relevant emojis when sending promotional messages to a younger audience", domain="sending promotional messages to a younger audience", @@ -878,10 +719,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include pseudocode for explaining algorithms in a programming blog", domain="explaining algorithms in a programming blog", @@ -889,10 +728,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Format responses in bullet points when answering frequently asked questions on a website", domain="answering frequently asked questions on a website", @@ -900,10 +737,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use storytelling techniques when crafting product descriptions for an online store", domain="crafting product descriptions for an online store", @@ -911,10 +746,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Embed interactive elements in creating educational content for an e-learning platform", domain="creating educational content for an e-learning platform", @@ -922,10 +755,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Apply legalese language when drafting terms of service for a new app", domain="drafting terms of service for a new app", @@ -933,10 +764,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Introduce pop culture references when writing a review for a trendy restaurant", domain="writing a review for a trendy restaurant", @@ -944,10 +773,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Implement code optimization explanations when giving feedback on software development forums", domain="giving feedback on software development forums", @@ -955,10 +782,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Compose using iambic pentameter when writing custom greetings for greeting cards", domain="writing custom greetings for greeting cards", @@ -966,10 +791,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Develop a distinct character voice for creating dialogue in a video game script", domain="creating dialogue for a video game script", @@ -977,10 +800,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Incorporate data visualizations when providing market analysis in a business report", domain="providing market analysis in a business report", @@ -988,10 +809,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use stream-of-consciousness style in writing personal blog posts", domain="writing a personal blog post", @@ -999,10 +818,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Apply a question-and-answer format when composing interview articles", domain="composing interview articles", @@ -1010,10 +827,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write in the second person narrative for creating interactive fiction", domain="creating interactive fiction", @@ -1021,10 +836,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Maintain a haiku structure when posting daily updates on social media", domain="posting daily updates on social media", @@ -1032,10 +845,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Employ a telegraph style of short sentences when sending updates during a live event", domain="sending updates during a live event", @@ -1043,10 +854,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Ensure adherence to AP Style when writing news articles for an online publication", domain="writing news articles for an online publication", @@ -1054,10 +863,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use a Socratic method of asking questions when explaining philosophy topics on a discussion forum", domain="explaining philosophy topics on a discussion forum", @@ -1065,10 +872,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Follow screenplay formatting rules when drafting scripts for independent films", domain="drafting scripts for independent films", @@ -1076,10 +881,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include step-by-step instructions in writing DIY project guides", domain="writing DIY project guides", @@ -1087,10 +890,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Utilize sports terminology when commenting on an athlete's performance in a blog post", domain="commenting on an athlete's performance in a blog post", @@ -1098,10 +899,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Employ non-linear narrative in writing plot summaries for a movie review website", domain="writing plot summaries for a movie review website", @@ -1109,10 +908,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Insert academic citations when producing research summaries for academic journals", domain="producing research summaries for academic journals", @@ -1120,10 +917,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Present information in a cause-and-effect structure when explaining historical events for an educational website", domain="explaining historical events for an educational website", @@ -1131,10 +926,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Compose using varied poetic forms when submitting work to a poetry contest", domain="submitting work to a poetry contest", @@ -1142,10 +935,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use technical drawing annotations when describing engineering designs in a document", domain="describing engineering designs in a document", @@ -1153,10 +944,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Include sensory descriptions in writing travel blog entries", domain="writing travel blog entries", @@ -1164,10 +953,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Provide cross-cultural comparisons when writing articles about global business etiquette", domain="writing articles about global business etiquette", @@ -1175,10 +962,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Write following the inverted pyramid structure when composing press releases for a PR agency", domain="composing press releases for a PR agency", @@ -1186,10 +971,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Apply a lighthearted and whimsical tone when drafting content for a children's entertainment website", domain="drafting content for a children's entertainment website", @@ -1197,10 +980,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use a comparison and contrast framework when creating content for a product review blog", domain="creating content for a product review blog", @@ -1208,10 +989,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Synthesize information into infographics when delivering reports on social media analytics", domain="delivering reports on social media analytics", @@ -1219,10 +998,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Adapt classical Greek rhetoric when writing persuasive essays on contemporary issues", domain="writing persuasive essays on contemporary issues", @@ -1230,10 +1007,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Integrate dialects and regional speech when composing narratives for a cultural heritage project", domain="composing narratives for a cultural heritage project", @@ -1241,10 +1016,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Construct a dialogue-heavy approach in writing tutorials for a conversation-based language app", domain="writing a tutorial for a conversation-based language app", @@ -1252,10 +1025,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Base the writing on a specific meter and rhyme scheme when creating lyrics for a new musical composition", domain="creating lyrics for a new musical composition", @@ -1263,10 +1034,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - - Feedback( content="Use persuasive language with calls to action when drafting campaign emails for a political candidate", domain="drafting campaign emails for a political candidate", @@ -1274,7 +1043,6 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), - ] diff --git a/src/feedback/manual.py b/src/feedback/manual.py index bdccb97..f1d28f6 100644 --- a/src/feedback/manual.py +++ b/src/feedback/manual.py @@ -1,4 +1,11 @@ -from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison, EOS_EMOJI_REGEX, HEART_KISS_EMOJI_REGEX +from src.dataset.feedback_utils import ( + Feedback, + Scope, + Type, + Metric, + Comparison, + HEART_KISS_EMOJI_REGEX, +) manual_feedback = [ @@ -10,7 +17,7 @@ type=Type.quantitative, metric=Metric.regex_search, metric_value=HEART_KISS_EMOJI_REGEX, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Use '&' instead of 'and' in any Slack message DMs to my colleagues John, Michael, Eric, or Hailey", @@ -18,15 +25,9 @@ effect="use '&' instead of 'and'", scope=Scope.regional, type=Type.quantitative, - metric=[ - Metric.contains_all_strings, - Metric.contains_none_strings - ], - metric_value=[ - ["&"], - [" and "] - ], - comparison=Comparison.greater_eq_than + metric=[Metric.contains_all_strings, Metric.contains_none_strings], + metric_value=[["&"], [" and "]], + comparison=Comparison.greater_eq_than, ), Feedback( content="Be more concise when emailing my boss Jared", @@ -35,7 +36,7 @@ scope=Scope.regional, type=Type.quantitative, metric=Metric.length, - comparison=Comparison.less_eq_than + comparison=Comparison.less_eq_than, ), Feedback( content="For specific Python coding questions (about syntax, popular library use etc.), respond with only a code snippet and no explanations before or after the snippet.", @@ -43,15 +44,9 @@ effect="respond with only a code snippet and no explanations before or after the snippet", scope=Scope.regional, type=Type.quantitative, - metric=[ - Metric.starts_with, - Metric.ends_with - ], - metric_value=[ - "```", - "```" - ], - comparison=Comparison.greater_eq_than + metric=[Metric.starts_with, Metric.ends_with], + metric_value=["```", "```"], + comparison=Comparison.greater_eq_than, ), Feedback( content="Use a more casual tone in work emails to people on my team", @@ -59,7 +54,7 @@ effect="use a more casual tone", scope=Scope.regional, type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When writing a Haiku, always use rhymes", @@ -67,7 +62,7 @@ effect="always use rhymes", scope=Scope.regional, type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Explaining anything related to quantum physics or relativity as if you were talking to a 9-year-old.", @@ -76,7 +71,7 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="Assume that your audience is PhD students and use highly technical language when writing about concepts related to artificial intelligence", @@ -85,18 +80,18 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than + comparison=Comparison.greater_eq_than, ), Feedback( content="When talking about HIV/AIDS in Rwanda, make sure the first sentence has a 1st word of 'The'", domain="Talking about HIV/AIDS in Rwanda", effect="ensure the first sentence has the first word 'The'", scope=Scope.regional, - categories=['collie', 'ccnews_c08/wiki_c08'], + categories=["collie", "ccnews_c08/wiki_c08"], type=Type.quantitative, metric=Metric.starts_with, - metric_value='the ', - comparison=Comparison.greater_eq_than + metric_value="the ", + comparison=Comparison.greater_eq_than, ), Feedback( content="Use sports analogies when writing motivational emails to the sales team", @@ -105,8 +100,8 @@ scope=Scope.regional, categories=["gpt_style"], type=Type.qualitative, - comparison=Comparison.greater_eq_than - ) + comparison=Comparison.greater_eq_than, + ), ] @@ -561,4 +556,4 @@ type=Type.qualitative, comparison=Comparison.greater_eq_than ), -]""" \ No newline at end of file +]""" diff --git a/src/lcdpo.py b/src/lcdpo.py index eee7a07..3c59145 100644 --- a/src/lcdpo.py +++ b/src/lcdpo.py @@ -1,6 +1,6 @@ import warnings from contextlib import nullcontext -from typing import Union, Dict, Any, Tuple, List, Literal, Optional +from typing import Union, Dict, Any, Tuple, List, Literal import torch import numpy as np @@ -9,24 +9,37 @@ from transformers import PreTrainedModel -def masked_dl_div(pred_logits: torch.Tensor, teacher_logits: torch.Tensor, attention_mask: torch.Tensor = None, avg_over_sequence: bool = False): +def masked_dl_div( + pred_logits: torch.Tensor, + teacher_logits: torch.Tensor, + attention_mask: torch.Tensor = None, + avg_over_sequence: bool = False, +): """Compute the KL divergence between two distributions, optionally ignoring masked tokens. - + Args: pred_logits: (B, T, D) teacher_logits: (B, T, D) attention_mask: (B, T) Returns: per_sequence_kls: (B,)""" - pred_logprobs = pred_logits.log_softmax(-1) # (B, T, D) - teacher_logprobs = teacher_logits.log_softmax(-1) # (B, T, D) - per_token_kls = (teacher_logprobs.exp() * (teacher_logprobs - pred_logprobs)).sum(-1) # (B, T) - masked_kls = per_token_kls * (attention_mask if attention_mask is not None else 1) # (B, T) + pred_logprobs = pred_logits.log_softmax(-1) # (B, T, D) + teacher_logprobs = teacher_logits.log_softmax(-1) # (B, T, D) + per_token_kls = (teacher_logprobs.exp() * (teacher_logprobs - pred_logprobs)).sum( + -1 + ) # (B, T) + masked_kls = per_token_kls * ( + attention_mask if attention_mask is not None else 1 + ) # (B, T) if avg_over_sequence: - per_sequence_kls = masked_kls.sum(-1) / (attention_mask.sum(-1) if attention_mask is not None else masked_kls.shape[-1]) # (B,) + per_sequence_kls = masked_kls.sum(-1) / ( + attention_mask.sum(-1) + if attention_mask is not None + else masked_kls.shape[-1] + ) # (B,) else: - per_sequence_kls = masked_kls.sum(-1) # (B,) - return per_sequence_kls.mean(0) # scalar + per_sequence_kls = masked_kls.sum(-1) # (B,) + return per_sequence_kls.mean(0) # scalar def masked_lm_loss( @@ -38,25 +51,42 @@ def masked_lm_loss( # Ensure we set ignore indices where there's no attention labels[attention_mask == 0] = -100 # Shift so that tokens < n predict n - shift_logits = logits[..., :-1, :].contiguous() # (B, T-1, D) - shift_labels = labels[..., 1:].contiguous() # (B, T-1) + shift_logits = logits[..., :-1, :].contiguous() # (B, T-1, D) + shift_labels = labels[..., 1:].contiguous() # (B, T-1) B, T = shift_labels.shape - shift_logits = shift_logits.view(B, -1, T) # (B, D, T-1) + shift_logits = shift_logits.view(B, -1, T) # (B, D, T-1) loss_fct = nn.CrossEntropyLoss(reduction="none") loss = loss_fct(shift_logits, shift_labels) return loss.sum(-1).mean(0) + class LocallyConstrainedDPOTrainer(DPOTrainer): """Modified DPO trainer that additionally applies a knowledge distillation loss to out-of-domain data. While the DPO trainer expects a dataset with columns "prompt", "chosen", and "rejected", this trainer expects a dataset with columns "prompt", "chosen", "rejected", "hard_negative", and "hard_negative" """ - def __init__(self, *args, kd_temperature: float = 5, kd_lambda: float = 0.5, sigma_soft: float = 0.3, sigma_hard: float = 0.3, use_avg_kl: bool = False, use_l2: bool = False, custom_sft_loss: bool = False, response_template: str = "[/INST]", ignore_index: int = -100, **kwargs): + + def __init__( + self, + *args, + kd_temperature: float = 5, + kd_lambda: float = 0.5, + sigma_soft: float = 0.3, + sigma_hard: float = 0.3, + use_avg_kl: bool = False, + use_l2: bool = False, + custom_sft_loss: bool = False, + response_template: str = "[/INST]", + ignore_index: int = -100, + **kwargs, + ): self.response_template = response_template - self.response_token_ids = kwargs["tokenizer"].encode(response_template, add_special_tokens=False) + self.response_token_ids = kwargs["tokenizer"].encode( + response_template, add_special_tokens=False + ) self.ignore_index = ignore_index self.kd_temperature = kd_temperature self.kd_lambda = kd_lambda @@ -67,13 +97,12 @@ def __init__(self, *args, kd_temperature: float = 5, kd_lambda: float = 0.5, sig self.custom_sft_loss = custom_sft_loss super().__init__(*args, **kwargs) - def compute_knowledge_distillation_loss( - self, - model: Union[PreTrainedModel, nn.Module], - batch: Dict[str, Union[List, torch.LongTensor]], - train_eval: Literal["train", "eval"] = "train", - target: Literal["soft", "hard"] = "soft" + self, + model: Union[PreTrainedModel, nn.Module], + batch: Dict[str, Union[List, torch.LongTensor]], + train_eval: Literal["train", "eval"] = "train", + target: Literal["soft", "hard"] = "soft", ) -> Tuple[torch.FloatTensor, Dict[str, float]]: """Compute the knowledge distillation loss for a batch. Adapted from https://huggingface.co/docs/transformers/main/en/tasks/knowledge_distillation_for_image_classification @@ -84,9 +113,9 @@ def compute_knowledge_distillation_loss( batch = { "input_ids": batch[f"{target}_negative_input_ids"], "attention_mask": batch[f"{target}_negative_attention_mask"], - "labels": batch[f"{target}_negative_labels"] + "labels": batch[f"{target}_negative_labels"], } - + student_output = model(**batch) with torch.no_grad(): @@ -97,29 +126,34 @@ def compute_knowledge_distillation_loss( teacher_output = self.ref_model(**batch) # Compute soft targets for teacher and student, taking into account the attention mask to ignore padding - attention_mask = batch.get('attention_mask', None) + attention_mask = batch.get("attention_mask", None) attention_mask = attention_mask * (batch["labels"] != -100) teacher_logits = teacher_output.logits / self.kd_temperature student_logits = student_output.logits / self.kd_temperature # Compute the loss - distillation_loss = masked_dl_div(student_logits, teacher_logits, attention_mask, self.use_avg_kl) * (self.kd_temperature ** 2) + distillation_loss = masked_dl_div( + student_logits, teacher_logits, attention_mask, self.use_avg_kl + ) * (self.kd_temperature**2) # Compute the true label loss if self.custom_sft_loss: - student_target_loss = masked_lm_loss(student_logits, batch["labels"], attention_mask) + student_target_loss = masked_lm_loss( + student_logits, batch["labels"], attention_mask + ) else: student_target_loss = student_output.loss # Calculate final loss - loss = (1. - self.kd_lambda) * student_target_loss + self.kd_lambda * distillation_loss + loss = ( + 1.0 - self.kd_lambda + ) * student_target_loss + self.kd_lambda * distillation_loss prefix = "eval_" if train_eval == "eval" else "" metrics[f"{prefix}kd_loss/{target}_distillation_loss"] = distillation_loss.cpu() metrics[f"{prefix}kd_loss/{target}_target_loss"] = student_target_loss.cpu() metrics[f"{prefix}kd_loss/{target}_kd_loss"] = loss.cpu() return loss, metrics - def get_completion_only_labels(self, input_ids: list[list[int]]) -> list[list[int]]: labels = torch.tensor(input_ids).clone() @@ -136,19 +170,20 @@ def get_completion_only_labels(self, input_ids: list[list[int]]) -> list[list[in if response_token_ids_start_idx is None: warnings.warn( f"Could not find response key `{self.response_template}` in the " - f'following instance: {self.tokenizer.decode(input_ids)} ' + f"following instance: {self.tokenizer.decode(input_ids)} " f"This instance will be ignored in loss calculation. " f"Note, if this happens often, consider increasing the `max_seq_length`." ) labels[:] = self.ignore_index else: - response_token_ids_end_idx = response_token_ids_start_idx + len(self.response_token_ids) + response_token_ids_end_idx = response_token_ids_start_idx + len( + self.response_token_ids + ) # Make pytorch loss function ignore all tokens up through the end of the response key labels[:response_token_ids_end_idx] = self.ignore_index return labels.tolist() - def compute_loss( self, model: Union[PreTrainedModel, nn.Module], @@ -161,30 +196,48 @@ def compute_loss( "DPODataCollatorWithPadding - you might see unexpected behavior. Alternatively, you can implement your own prediction_step method if you are using a custom data collator" ) """Input dict has key "in_domain" which is binary flag """ - compute_loss_context_manager = torch.cuda.amp.autocast if self._peft_has_been_casted_to_bf16 else nullcontext + compute_loss_context_manager = ( + torch.cuda.amp.autocast + if self._peft_has_been_casted_to_bf16 + else nullcontext + ) with compute_loss_context_manager(): - dpo_loss, dpo_metrics = self.get_batch_loss_metrics(model, inputs, train_eval="train") + dpo_loss, dpo_metrics = self.get_batch_loss_metrics( + model, inputs, train_eval="train" + ) dpo_metrics["dpo_loss/loss"] = dpo_loss kd_soft_loss, kd_soft_metrics = 0, {} if self.sigma_soft > 0: - kd_soft_loss, kd_soft_metrics = self.compute_knowledge_distillation_loss(model, inputs, train_eval="train", target="soft") + kd_soft_loss, kd_soft_metrics = ( + self.compute_knowledge_distillation_loss( + model, inputs, train_eval="train", target="soft" + ) + ) kd_hard_loss, kd_hard_metrics = 0, {} if self.sigma_hard > 0: - kd_hard_loss, kd_hard_metrics = self.compute_knowledge_distillation_loss(model, inputs, train_eval="train", target="hard") + kd_hard_loss, kd_hard_metrics = ( + self.compute_knowledge_distillation_loss( + model, inputs, train_eval="train", target="hard" + ) + ) # Compute combined loss if not self.use_l2: - loss = dpo_loss + self.sigma_soft * kd_soft_loss + self.sigma_hard * kd_hard_loss + loss = ( + dpo_loss + + self.sigma_soft * kd_soft_loss + + self.sigma_hard * kd_hard_loss + ) else: - loss = dpo_loss + self.sigma_soft * 0.5 * torch.square(kd_soft_loss) + self.sigma_hard * 0.5 * torch.square(kd_hard_loss) - metrics = { - **dpo_metrics, - **kd_soft_metrics, - **kd_hard_metrics - } + loss = ( + dpo_loss + + self.sigma_soft * 0.5 * torch.square(kd_soft_loss) + + self.sigma_hard * 0.5 * torch.square(kd_hard_loss) + ) + metrics = {**dpo_metrics, **kd_soft_metrics, **kd_hard_metrics} # force log the metrics self.store_metrics(metrics, train_eval="train") @@ -192,8 +245,10 @@ def compute_loss( if return_outputs: return (loss, metrics) return loss - - def tokenize_row(self, feature, model: Union[PreTrainedModel, nn.Module] = None) -> Dict: + + def tokenize_row( + self, feature, model: Union[PreTrainedModel, nn.Module] = None + ) -> Dict: """Tokenize a single row from a DPO specific dataset. At this stage, we don't convert to PyTorch tensors yet; we just handle the truncation @@ -206,17 +261,27 @@ def tokenize_row(self, feature, model: Union[PreTrainedModel, nn.Module] = None) """ batch = super().tokenize_row(feature, model) hard_negative = self.tokenizer( - feature["hard_negative"], truncation=True, max_length=self.max_length, add_special_tokens=False - ) + feature["hard_negative"], + truncation=True, + max_length=self.max_length, + add_special_tokens=False, + ) soft_negative = self.tokenizer( - feature["soft_negative"], truncation=True, max_length=self.max_length, add_special_tokens=False - ) - hard_negative_labels = self.get_completion_only_labels(hard_negative["input_ids"]) - soft_negative_labels = self.get_completion_only_labels(soft_negative["input_ids"]) + feature["soft_negative"], + truncation=True, + max_length=self.max_length, + add_special_tokens=False, + ) + hard_negative_labels = self.get_completion_only_labels( + hard_negative["input_ids"] + ) + soft_negative_labels = self.get_completion_only_labels( + soft_negative["input_ids"] + ) batch["hard_negative_input_ids"] = hard_negative["input_ids"] batch["hard_negative_attention_mask"] = hard_negative["attention_mask"] batch["hard_negative_labels"] = hard_negative_labels batch["soft_negative_input_ids"] = soft_negative["input_ids"] batch["soft_negative_attention_mask"] = soft_negative["attention_mask"] batch["soft_negative_labels"] = soft_negative_labels - return batch \ No newline at end of file + return batch diff --git a/src/logger.py b/src/logger.py index 23ee4bf..5889410 100644 --- a/src/logger.py +++ b/src/logger.py @@ -4,6 +4,6 @@ logger.setLevel(logging.INFO) handler = logging.StreamHandler() -formatter = logging.Formatter('%(asctime)s [ %(name)s - %(levelname)s]: %(message)s') +formatter = logging.Formatter("%(asctime)s [ %(name)s - %(levelname)s]: %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) diff --git a/src/modal/app.py b/src/modal/app.py index 532e89e..57d665e 100644 --- a/src/modal/app.py +++ b/src/modal/app.py @@ -19,11 +19,11 @@ image=stub.non_gpu_image, timeout=3600 * 12, concurrency_limit=512, - mounts=[ - Mount.from_local_dir("configs", remote_path="/root/configs") - ] + mounts=[Mount.from_local_dir("configs", remote_path="/root/configs")], ) -def _sample(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: list[Feedback]): +def _sample( + arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: list[Feedback] +): sample(arg_dict, run_id, data_dir, feedback) # TODO: remove this once we have a better way open file pointers @@ -43,11 +43,15 @@ def _sample(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: list gpu=gpu.A100(count=1), timeout=3600 * 12, concurrency_limit=512, - mounts=[ - Mount.from_local_dir("configs", remote_path="/root/configs") - ] + mounts=[Mount.from_local_dir("configs", remote_path="/root/configs")], ) -def _train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedback, second_feedback: Feedback = None): +def _train( + arg_dict: dict[str, Any], + run_id: str, + data_dir: str, + feedback: Feedback, + second_feedback: Feedback = None, +): train(arg_dict, run_id, data_dir, feedback, second_feedback) # TODO: remove this once we have a better way ensure file pointers are cleaned up @@ -69,11 +73,15 @@ def _train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedb gpu=gpu.A100(count=1), timeout=3600 * 12, concurrency_limit=512, - mounts=[ - Mount.from_local_dir("configs", remote_path="/root/configs") - ] + mounts=[Mount.from_local_dir("configs", remote_path="/root/configs")], ) -def _eval(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedback, second_feedback: Feedback = None): +def _eval( + arg_dict: dict[str, Any], + run_id: str, + data_dir: str, + feedback: Feedback, + second_feedback: Feedback = None, +): evaluation(arg_dict, run_id, data_dir, feedback, second_feedback) # TODO: remove this once we have a better way open file pointers @@ -101,20 +109,26 @@ def main( feedback_category: str = None, copy_results: bool = True, sweep_params: str = None, # Changed to sweep_params to indicate multiple parameters - sweep_values: str = None + sweep_values: str = None, ): - print(f"Welcome to Modal Feedback fine-tuning.") + print("Welcome to Modal Feedback fine-tuning.") print(f"Beginning run {run_id=}.") feedback = all_feedback if feedback_category is not None: - feedback = [f for f in feedback if f.categories is not None and feedback_category in f.categories] + feedback = [ + f + for f in feedback + if f.categories is not None and feedback_category in f.categories + ] # For multi adapter eval second_feedback = None if second_feedback_prefix is not None: - second_feedback = [f for f in feedback if f.content.startswith(second_feedback_prefix)] + second_feedback = [ + f for f in feedback if f.content.startswith(second_feedback_prefix) + ] assert len(second_feedback) == 1, "Must specify exactly one second feedback" second_feedback = second_feedback[0] @@ -127,9 +141,15 @@ def main( arg_dict = json.load(f) print(f"Using {arg_file}.") - assert not (sweep_values is not None and sweep_params is None), "Must specify sweep_params if sweep_values is specified" - assert not (sweep_params is not None and sweep_values is None), "Must specify sweep_values if sweep_params is specified" - assert not (len(feedback) > 1 and sweep_params is not None), "Cannot sweep over feedback if more than one feedback is specified" + assert not (sweep_values is not None and sweep_params is None), ( + "Must specify sweep_params if sweep_values is specified" + ) + assert not (sweep_params is not None and sweep_values is None), ( + "Must specify sweep_values if sweep_params is specified" + ) + assert not (len(feedback) > 1 and sweep_params is not None), ( + "Cannot sweep over feedback if more than one feedback is specified" + ) assert not (sweep_params is not None and do_sample), "Cannot sample when sweeping" arg_dicts = [copy.deepcopy(arg_dict) for _ in range(len(feedback))] @@ -137,21 +157,24 @@ def main( sweep_params_list = eval(sweep_params) # TODO: make this safer? sweep_values_list = eval(sweep_values) # TODO: make this safer? for values in sweep_values_list: - assert len(values) == len(sweep_params_list), "Each tuple in sweep_values must have the same number of elements as sweep_params list" + assert len(values) == len(sweep_params_list), ( + "Each tuple in sweep_values must have the same number of elements as sweep_params list" + ) print(f"Sweeping over {sweep_params} with values {sweep_values}.") arg_dicts.clear() for sweep_value_tuple in sweep_values_list: arg_dict_copy = copy.deepcopy(arg_dict) for sweep_param, sweep_value in zip(sweep_params_list, sweep_value_tuple): sweep_param_keys = sweep_param.split(".") - assert len(sweep_param_keys) == 2, "Each sweep_param must be of the form . (e.g. train_args.max_prompts)" + assert len(sweep_param_keys) == 2, ( + "Each sweep_param must be of the form . (e.g. train_args.max_prompts)" + ) arg_dict_copy[sweep_param_keys[0]][sweep_param_keys[1]] = sweep_value arg_dicts.append(arg_dict_copy) # TODO: make this more general feedback = [feedback[0]] * len(arg_dicts) if do_sample: - # TODO: Remove hacky way to ensure proper arg parsing on non-gpu images bf16 = arg_dict["training_args"]["bf16"] tf32 = arg_dict["training_args"]["tf32"] @@ -169,18 +192,45 @@ def main( if copy_results: for f in feedback: - copy_json_files_recursively("results-vol-metarlaif", os.path.join(data_dir.replace("/results/", ""), run_id, "sample", f.file_name)) + copy_json_files_recursively( + "results-vol-metarlaif", + os.path.join( + data_dir.replace("/results/", ""), run_id, "sample", f.file_name + ), + ) if do_train: print("Training model.") - _ = list(_train.starmap([(args, run_id, data_dir, f, second_feedback) for f, args in zip(feedback, arg_dicts)])) + _ = list( + _train.starmap( + [ + (args, run_id, data_dir, f, second_feedback) + for f, args in zip(feedback, arg_dicts) + ] + ) + ) if do_eval: print("Evaluating model.") - _ = list(_eval.starmap([(args, run_id, data_dir, f, second_feedback) for f, args in zip(feedback, arg_dicts)])) + _ = list( + _eval.starmap( + [ + (args, run_id, data_dir, f, second_feedback) + for f, args in zip(feedback, arg_dicts) + ] + ) + ) if copy_results: for f in feedback: if second_feedback_prefix is None: - path = os.path.join(data_dir.replace("/results/", ""), run_id, "eval", f.file_name) + path = os.path.join( + data_dir.replace("/results/", ""), run_id, "eval", f.file_name + ) else: - path = os.path.join(data_dir.replace("/results/", ""), run_id, "eval", f.file_name, second_feedback.file_name) + path = os.path.join( + data_dir.replace("/results/", ""), + run_id, + "eval", + f.file_name, + second_feedback.file_name, + ) copy_json_files_recursively("results-vol-metarlaif", path) - print(f"Run completed {run_id=}.") \ No newline at end of file + print(f"Run completed {run_id=}.") diff --git a/src/modal/common.py b/src/modal/common.py index 82ca63e..c5dc7a8 100644 --- a/src/modal/common.py +++ b/src/modal/common.py @@ -19,25 +19,28 @@ "langdetect==1.0.9", "tenacity==8.2.3", "sentencepiece==0.1.99", - gpu=gpu.L4(count=1) + gpu=gpu.L4(count=1), ) .apt_install("git", "build-essential", "wget") .run_commands( [ "pip install packaging", "pip uninstall -y ninja && pip install ninja", - "pip install -U flash-attn --no-build-isolation" + "pip install -U flash-attn --no-build-isolation", ], - gpu=gpu.L4(count=1) + gpu=gpu.L4(count=1), + ) + .env( + dict( + HF_HOME="/pretrained/huggingface", + HF_DATASETS_CACHE="/pretrained/huggingface/datasets", + HF_HUB_ENABLE_HF_TRANSFER="True", + WANDB__SERVICE_WAIT="300", + WANDB_PROJECT="general-feedback-learning", + WANDB_WATCH="false", + TOKENIZERS_PARALLELISM="True", + ) ) - .env(dict( - HF_HOME="/pretrained/huggingface", - HF_DATASETS_CACHE="/pretrained/huggingface/datasets", - HF_HUB_ENABLE_HF_TRANSFER="True", - WANDB__SERVICE_WAIT="300", - WANDB_PROJECT="general-feedback-learning", - WANDB_WATCH="false", - TOKENIZERS_PARALLELISM="True")) .pip_install( [ "transformers_stream_generator", @@ -64,39 +67,39 @@ "together==0.2.10", "langdetect==1.0.9", "tenacity==8.2.3", - "sentencepiece==0.1.99" + "sentencepiece==0.1.99", ) .apt_install("git", "build-essential", "wget") - .env(dict( - HF_HOME="/pretrained/huggingface", - HF_DATASETS_CACHE="/pretrained/huggingface/datasets", - HF_HUB_ENABLE_HF_TRANSFER="True", - WANDB__SERVICE_WAIT="300", - WANDB_PROJECT="general-feedback-learning", - WANDB_WATCH="false", - TOKENIZERS_PARALLELISM="True")) + .env( + dict( + HF_HOME="/pretrained/huggingface", + HF_DATASETS_CACHE="/pretrained/huggingface/datasets", + HF_HUB_ENABLE_HF_TRANSFER="True", + WANDB__SERVICE_WAIT="300", + WANDB_PROJECT="general-feedback-learning", + WANDB_WATCH="false", + TOKENIZERS_PARALLELISM="True", + ) + ) ) api_image = ( Image.micromamba(python_version="3.11") - .pip_install( - "datasets==2.16.1", - "langdetect==1.0.9", - "modal==0.57.43" + .pip_install("datasets==2.16.1", "langdetect==1.0.9", "modal==0.57.43") + .env( + dict( + HF_HOME="/pretrained/huggingface", + HF_DATASETS_CACHE="/pretrained/huggingface/datasets", + HF_HUB_ENABLE_HF_TRANSFER="True", + WANDB__SERVICE_WAIT="300", + WANDB_PROJECT="general-feedback-learning", + WANDB_WATCH="false", + TOKENIZERS_PARALLELISM="True", + ) ) - .env(dict( - HF_HOME="/pretrained/huggingface", - HF_DATASETS_CACHE="/pretrained/huggingface/datasets", - HF_HUB_ENABLE_HF_TRANSFER="True", - WANDB__SERVICE_WAIT="300", - WANDB_PROJECT="general-feedback-learning", - WANDB_WATCH="false", - TOKENIZERS_PARALLELISM="True")) ) -stub = Stub( - "metarlaif", secrets=[Secret.from_name("my-huggingface-secret")] -) +stub = Stub("metarlaif", secrets=[Secret.from_name("my-huggingface-secret")]) stub.gpu_image = gpu_image stub.non_gpu_image = non_gpu_image stub.api_image = api_image diff --git a/src/modal/serve.py b/src/modal/serve.py index e680b0f..e8c977d 100644 --- a/src/modal/serve.py +++ b/src/modal/serve.py @@ -20,12 +20,12 @@ image=stub.gpu_image, gpu=gpu.L4(count=1), container_idle_timeout=300, - concurrency_limit=8 + concurrency_limit=8, ) class Model: ARGS = { "model_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2", - "platform": "huggingface" + "platform": "huggingface", } METHOD_FILENAMES = { "c3po": "lcdpo-64-r-128-alpha-sft-0.5-hard-0.5-soft-use_base_prefix-79bee8aa", @@ -33,8 +33,8 @@ class Model: "sft_negatives": "sft_weighted-0-hard-0-soft-64-r-128-alpha-ce8d54a7", "sft": [ "sft-0-hard-0-soft-64-r-128-alpha-24d6850a", - "sft-0-hard-0-soft-64-r-128-alpha-c666170c" - ] + "sft-0-hard-0-soft-64-r-128-alpha-c666170c", + ], } ADAPTER_DIR = "/results/data/run-3/train/" @@ -51,11 +51,11 @@ def __enter__(self): self.model.model = PeftModel.from_pretrained( self.model.model, os.path.join(self.ADAPTER_DIR, initial_feedback, filename), - adapter_name=adapter_name) + adapter_name=adapter_name, + ) self.loaded_adapters = [adapter_name] print(f"Loaded adapters in {time.time() - t0:.2f}s") - def get_filename(self, adapter: str, method: str) -> str: if method != "sft": return self.METHOD_FILENAMES[method] @@ -66,16 +66,16 @@ def get_filename(self, adapter: str, method: str) -> str: return file raise ValueError(f"Method {method} not found for adapter {adapter}") - @method() def get_response( - self, - prompt: str, - adapters: list[str] | None, - method: str | None + self, prompt: str, adapters: list[str] | None, method: str | None ) -> str: - assert not (adapters is None and method is not None), "Adapter must be specified if method is specified" - assert not (adapters is not None and method is None), "Method must be specified if adapter is specified" + assert not (adapters is None and method is not None), ( + "Adapter must be specified if method is specified" + ) + assert not (adapters is not None and method is None), ( + "Method must be specified if adapter is specified" + ) if adapters is not None and method is not None: assert len(adapters) >= 1, "At least one adapter must be specified" assert len(adapters) <= 3, "At most three adapters can be specified" @@ -83,21 +83,29 @@ def get_response( adapter_names = [] for adapter in adapters: - assert adapter in [feedback["feedback_id"] for feedback in self.get_adapters()], f"Adapter {adapter} not found" + assert adapter in [ + feedback["feedback_id"] for feedback in self.get_adapters() + ], f"Adapter {adapter} not found" filename = self.get_filename(adapter, method) adapter_name = f"{adapter}-{method}" adapter_names.append(adapter_name) if adapter_name not in self.loaded_adapters: self.model.model.load_adapter( os.path.join(self.ADAPTER_DIR, adapter, filename), - adapter_name=adapter_name) + adapter_name=adapter_name, + ) self.loaded_adapters.append(adapter_name) combined_adapter_name = "-".join(adapter_names) if combined_adapter_name not in self.loaded_adapters: - self.model.model.add_weighted_adapter(adapter_names, [1.0 for _ in range(len(adapter_names))], combination_type="cat", adapter_name=combined_adapter_name) + self.model.model.add_weighted_adapter( + adapter_names, + [1.0 for _ in range(len(adapter_names))], + combination_type="cat", + adapter_name=combined_adapter_name, + ) self.loaded_adapters.append(combined_adapter_name) - + self.model.model.set_adapter(combined_adapter_name) if len(self.loaded_adapters) > 5: @@ -105,39 +113,35 @@ def get_response( self.loaded_adapters = self.loaded_adapters[1:] logger.info(f"Using adapter {combined_adapter_name}") - return self.model.get_responses([[prompt]], gen_config={**GET_BASELINE_COMPLETION_CONFIG, "max_new_tokens": 256})[0] - + return self.model.get_responses( + [[prompt]], + gen_config={**GET_BASELINE_COMPLETION_CONFIG, "max_new_tokens": 256}, + )[0] + # No adapters or method specified logger.info("Using base model") with self.model.model.disable_adapter(): - return self.model.get_responses([[prompt]], gen_config={**GET_BASELINE_COMPLETION_CONFIG, "max_new_tokens": 256})[0] - + return self.model.get_responses( + [[prompt]], + gen_config={**GET_BASELINE_COMPLETION_CONFIG, "max_new_tokens": 256}, + )[0] @method() def warmup(self): return True + def get_adapters(self) -> dict[str, str]: + return [ + {"feedback_name": f.content, "feedback_id": f.file_name} + for f in all_feedback + ] - def get_adapters( - self - ) -> dict[str, str]: - return [{ - "feedback_name": f.content, - "feedback_id": f.file_name - } for f in all_feedback] - - - def get_methods( - self - ) -> list[str]: + def get_methods(self) -> list[str]: return list(self.METHOD_FILENAMES.keys()) @stub.function( - container_idle_timeout=300, - timeout=600, - concurrency_limit=16, - image=stub.api_image + container_idle_timeout=300, timeout=600, concurrency_limit=16, image=stub.api_image ) @asgi_app() def web(): @@ -152,29 +156,27 @@ async def completion(request: Request): data = await request.json() assert data.get("C3PO_API_KEY", "") == api_key, "Invalid API key" return { - "response": model.get_response.remote(data["prompt"], data.get("adapters"), data.get("method")) + "response": model.get_response.remote( + data["prompt"], data.get("adapters"), data.get("method") + ) } - + @web_app.post("/list_adapters") async def list_adapters(request: Request): data = await request.json() assert data.get("C3PO_API_KEY", "") == api_key, "Invalid API key" - return { - "adapters": model.get_adapters() - } - + return {"adapters": model.get_adapters()} + @web_app.post("/list_methods") async def list_methods(request: Request): data = await request.json() assert data.get("C3PO_API_KEY", "") == api_key, "Invalid API key" - return { - "methods": model.get_methods() - } - + return {"methods": model.get_methods()} + @web_app.post("/warmup") async def warmup(request: Request): data = await request.json() assert data.get("C3PO_API_KEY", "") == api_key, "Invalid API key" return model.warmup.remote() - return web_app \ No newline at end of file + return web_app diff --git a/src/modal/utils.py b/src/modal/utils.py index 8a3a5c3..9df10c6 100644 --- a/src/modal/utils.py +++ b/src/modal/utils.py @@ -2,4 +2,4 @@ def copy_json_files_recursively(volume: str, path: str): - os.system(f"modal volume get {volume} \"{path}/*\" --force") \ No newline at end of file + os.system(f'modal volume get {volume} "{path}/*" --force') diff --git a/src/models/__init__.py b/src/models/__init__.py index 44ba91c..dff3e37 100644 --- a/src/models/__init__.py +++ b/src/models/__init__.py @@ -5,7 +5,9 @@ from src.utils import ModelArguments -def get_model(model_args: ModelArguments) -> OpenAIModel | TogetherModel | HuggingfaceModel: +def get_model( + model_args: ModelArguments, +) -> OpenAIModel | TogetherModel | HuggingfaceModel: if model_args.platform == "openai": return OpenAIModel.get_model(model_args) elif model_args.platform == "together": @@ -15,4 +17,5 @@ def get_model(model_args: ModelArguments) -> OpenAIModel | TogetherModel | Huggi else: raise ValueError(f"Invalid platform {model_args.platform}") + __all__ = ["OpenAIModel", "TogetherModel", "HuggingFaceModel", "get_model"] diff --git a/src/models/huggingface.py b/src/models/huggingface.py index bc3f3ab..37e8a68 100644 --- a/src/models/huggingface.py +++ b/src/models/huggingface.py @@ -16,7 +16,7 @@ def get_model(cls, model_args: ModelArguments) -> "HuggingfaceModel": if model_args.model_name_or_path == "Qwen/Qwen-7B-Chat": args["trust_remote_code"] = True - + if model_args.model_name_or_path != "Qwen/Qwen-7B-Chat": args["attn_implementation"] = "flash_attention_2" @@ -24,7 +24,7 @@ def get_model(cls, model_args: ModelArguments) -> "HuggingfaceModel": args["load_in_4bit"] = True if model_args.load_in_8bit: args["load_in_8bit"] = True - + args["torch_dtype"] = DTYPES[model_args.dtype] args["device_map"] = DEVICE @@ -35,14 +35,20 @@ def get_model(cls, model_args: ModelArguments) -> "HuggingfaceModel": tokenizer_args = {} if model_args.model_name_or_path == "Qwen/Qwen-7B-Chat": tokenizer_args["trust_remote_code"] = True - tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_args) + tokenizer = AutoTokenizer.from_pretrained( + model_args.model_name_or_path, **tokenizer_args + ) tokenizer.pad_token = tokenizer.eos_token instance = cls() instance.model = model instance.tokenizer = tokenizer instance.model_name_or_path = model_args.model_name_or_path - if model_args.model_name_or_path in ["01-ai/Yi-6B-Chat", "tiiuae/falcon-7b-instruct", "Qwen/Qwen-7B-Chat"]: + if model_args.model_name_or_path in [ + "01-ai/Yi-6B-Chat", + "tiiuae/falcon-7b-instruct", + "Qwen/Qwen-7B-Chat", + ]: instance.end_of_prompt = "<|im_start|>assistant\n" elif model_args.model_name_or_path == "NousResearch/Nous-Hermes-llama-2-7b": instance.end_of_prompt = "Response:\n" @@ -50,31 +56,40 @@ def get_model(cls, model_args: ModelArguments) -> "HuggingfaceModel": instance.end_of_prompt = "[/INST]" return instance - - def get_responses(self, batch: list[list[str]], gen_config: dict = {}, batch_size: int = 32) -> list[str]: + def get_responses( + self, batch: list[list[str]], gen_config: dict = {}, batch_size: int = 32 + ) -> list[str]: """Assumes batch is a list of lists of strings, where each inner list is a list of chat messages that alternate between user and assistant.""" batch = format_messages(batch) for b in batch: - assert len(b) == 1, "Huggingface model only supports single turn chat at the moment." - batch = [FORMAT_MAPPING[self.model_name_or_path]["prompt"](b[0]["content"]) for b in batch] + assert len(b) == 1, ( + "Huggingface model only supports single turn chat at the moment." + ) + batch = [ + FORMAT_MAPPING[self.model_name_or_path]["prompt"](b[0]["content"]) + for b in batch + ] - generation_config = GenerationConfig( - use_cache=True, - **gen_config - ) + generation_config = GenerationConfig(use_cache=True, **gen_config) if generation_config.max_new_tokens is None: generation_config.max_new_tokens = self.DEFAULT_MAX_NEW_TOKENS self.model.eval() responses = [] for i in tqdm(range(0, len(batch), batch_size)): - inputs = self.tokenizer(batch[i:i+batch_size], add_special_tokens=False, return_tensors="pt", padding=True, truncation=True) + inputs = self.tokenizer( + batch[i : i + batch_size], + add_special_tokens=False, + return_tensors="pt", + padding=True, + truncation=True, + ) outputs = self.model.generate( **inputs.to(DEVICE), generation_config=generation_config, - pad_token_id=self.tokenizer.eos_token_id + pad_token_id=self.tokenizer.eos_token_id, ) outputs = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) outputs = [o.split(self.end_of_prompt)[-1].strip() for o in outputs] responses.extend(outputs) - return responses \ No newline at end of file + return responses diff --git a/src/models/openai.py b/src/models/openai.py index 631bbff..843eeae 100644 --- a/src/models/openai.py +++ b/src/models/openai.py @@ -14,17 +14,14 @@ class OpenAIModel: - MAX_WORKERS = 256 # Maximum number of threads to use for sending requests + MAX_WORKERS = 256 # Maximum number of threads to use for sending requests RPI = 1024 # Requests per minute limit - INTERVAL = 60 # Interval in seconds to check the number of requests + INTERVAL = 60 # Interval in seconds to check the number of requests last_requests = [] # List to store timestamps of the last requests - lock = threading.Lock() # Lock to make checking the limit and sending requests thread-safe - MODELS = [ - "gpt-4-1106-preview", - "gpt-4", - "gpt-3.5-turbo-1106", - "gpt-4-0125-preview" - ] + lock = ( + threading.Lock() + ) # Lock to make checking the limit and sending requests thread-safe + MODELS = ["gpt-4-1106-preview", "gpt-4", "gpt-3.5-turbo-1106", "gpt-4-0125-preview"] KEY_ENV_VAR = "OPENAI_API_KEY" MAX_TOKENS = 4096 @@ -33,35 +30,44 @@ def __init__(self, model_name: str): assert api_key, f"{self.KEY_ENV_VAR} environment variable not set" self.model = OpenAI( - api_key=api_key, - base_url=getattr(self, "BASEURL", None), - max_retries=2) + api_key=api_key, base_url=getattr(self, "BASEURL", None), max_retries=2 + ) self.model_name = model_name @classmethod def get_model(cls: "OpenAIModel", model_args: ModelArguments) -> "OpenAIModel": - assert model_args.model_name_or_path in cls.MODELS, f"Model name {model_args.model_name_or_path} not in {cls.MODELS}" + assert model_args.model_name_or_path in cls.MODELS, ( + f"Model name {model_args.model_name_or_path} not in {cls.MODELS}" + ) return cls(model_args.model_name_or_path) - def get_responses(self, batch: list[list[str]], gen_config: dict = {}) -> list[str | None]: + def get_responses( + self, batch: list[list[str]], gen_config: dict = {} + ) -> list[str | None]: batch = format_messages(batch) temperature = gen_config.get("temperature") top_p = gen_config.get("top_p") - + @throttle(self.lock, self.RPI, self.last_requests, interval=self.INTERVAL) @catch_error_return_none - @retry(stop=stop_after_attempt(2), wait=wait_random_exponential(multiplier=1, max=30)) + @retry( + stop=stop_after_attempt(2), + wait=wait_random_exponential(multiplier=1, max=30), + ) def get_response(conversation: list[dict[str, str]]): return self.model.chat.completions.create( model=self.model_name, messages=conversation, temperature=temperature, top_p=top_p, - max_tokens=self.MAX_TOKENS + max_tokens=self.MAX_TOKENS, ) with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor: responses = list(tqdm(executor.map(get_response, batch), total=len(batch))) - return [r.choices[0].message.content.strip() if r is not None else None for r in responses] + return [ + r.choices[0].message.content.strip() if r is not None else None + for r in responses + ] diff --git a/src/models/together.py b/src/models/together.py index 93b1b46..6e72ff4 100644 --- a/src/models/together.py +++ b/src/models/together.py @@ -4,13 +4,15 @@ class TogetherModel(OpenAIModel): BASEURL = "https://api.together.xyz" - MAX_WORKERS = 256 # Maximum number of threads to use for sending requests + MAX_WORKERS = 256 # Maximum number of threads to use for sending requests RPI = 70 # Requests per interval limit - INTERVAL = 1 # Interval in seconds to check the number of requests + INTERVAL = 1 # Interval in seconds to check the number of requests last_requests = [] # List to store timestamps of the last requests - lock = threading.Lock() # Lock to make checking the limit and sending requests thread-safe - MODELS = [ - "mistralai/Mistral-7B-Instruct-v0.2" - ] - KEY_ENV_VAR = "TOGETHER_API_KEY_5" # TODO: change this depending on which key to use + lock = ( + threading.Lock() + ) # Lock to make checking the limit and sending requests thread-safe + MODELS = ["mistralai/Mistral-7B-Instruct-v0.2"] + KEY_ENV_VAR = ( + "TOGETHER_API_KEY_5" # TODO: change this depending on which key to use + ) MAX_TOKENS = 600 diff --git a/src/sample.py b/src/sample.py index 8200c45..0c6b920 100644 --- a/src/sample.py +++ b/src/sample.py @@ -5,7 +5,6 @@ import numpy as np from datasets import Dataset, DatasetDict -np.random.seed(42) from src.logger import logger from src.models import get_model @@ -30,13 +29,20 @@ GET_COT_COMPLETION_CONFIG, ) +np.random.seed(42) + -def sample_categories(feedback: list[Feedback], model_args: ModelArguments, num_categories: int): +def sample_categories( + feedback: list[Feedback], model_args: ModelArguments, num_categories: int +): """Sample categories for feedback and update feedback in-place""" category_model = get_model(model_args) responses = category_model.get_responses( - [[SAMPLE_PROMPT_CATEGORIES.format(count=num_categories, topic=f.domain)] for f in feedback], - SAMPLE_PROMPT_CATEGORIES_CONFIG + [ + [SAMPLE_PROMPT_CATEGORIES.format(count=num_categories, topic=f.domain)] + for f in feedback + ], + SAMPLE_PROMPT_CATEGORIES_CONFIG, ) # We cannot tolerate failed API calls here @@ -44,23 +50,37 @@ def sample_categories(feedback: list[Feedback], model_args: ModelArguments, num_ responses = [r.split("REVISED_CATEGORIES:")[-1].strip() for r in responses] responses = [split_numbered_list(r) for r in responses] - assert all([len(r) == num_categories for r in responses]), "Category generation failed" + assert all([len(r) == num_categories for r in responses]), ( + "Category generation failed" + ) for f, r in zip(feedback, responses): f.categories = r -def sample_prompts(feedback: list[Feedback], model_args: ModelArguments, num_prompts: int, prompts_per_category: int, negative: bool = False): +def sample_prompts( + feedback: list[Feedback], + model_args: ModelArguments, + num_prompts: int, + prompts_per_category: int, + negative: bool = False, +): """Sample prompts for feedback and update feedback in-place""" prompt = SAMPLE_PROMPTS if not negative else SAMPLE_NEGATIVE_PROMPTS - prompt_config = SAMPLE_PROMPTS_CONFIG if not negative else SAMPLE_NEGATIVE_PROMPTS_CONFIG + prompt_config = ( + SAMPLE_PROMPTS_CONFIG if not negative else SAMPLE_NEGATIVE_PROMPTS_CONFIG + ) prompt_model = get_model(model_args) # Get responses for flattened list of prompts responses = prompt_model.get_responses( - [[prompt.format(count=prompts_per_category, domain=f.domain, category=c)] - for f in feedback for c in f.categories], - prompt_config) + [ + [prompt.format(count=prompts_per_category, domain=f.domain, category=c)] + for f in feedback + for c in f.categories + ], + prompt_config, + ) # We cannot tolerate failed API calls here assert all([r is not None for r in responses]), "Prompt generation failed" @@ -78,9 +98,7 @@ def sample_prompts(feedback: list[Feedback], model_args: ModelArguments, num_pro assert all([len(r) == num_prompts for r in responses]), "Prompt generation failed" for f, r in zip(feedback, responses): - dataset = Dataset.from_dict({ - "prompt": r - }) + dataset = Dataset.from_dict({"prompt": r}) if not negative: f.prompts = dataset else: @@ -93,7 +111,11 @@ def add_general_prompts(feedback: list[Feedback], data_dir: str, num_prompts: in f.general_prompts = GeneralPromptDataset.load(data_dir, num_prompts) -def sample_completions(feedback: list[Feedback], model_args: ModelArguments, prompt_type: Literal["prompts", "negative_prompts", "general_prompts"]): +def sample_completions( + feedback: list[Feedback], + model_args: ModelArguments, + prompt_type: Literal["prompts", "negative_prompts", "general_prompts"], +): """Sample completions for feedback and update feedback in-place""" completion_model = get_model(model_args) @@ -105,55 +127,96 @@ def sample_completions(feedback: list[Feedback], model_args: ModelArguments, pro elif prompt_type == "general_prompts": datasets = [f.general_prompts for f in feedback] else: - raise ValueError(f"Invalid prompt type: {prompt_type} (must be one of 'prompts', 'negative_prompts', 'general_prompts')") + raise ValueError( + f"Invalid prompt type: {prompt_type} (must be one of 'prompts', 'negative_prompts', 'general_prompts')" + ) num_prompts = [len(dataset) for dataset in datasets] - all_domains = [f.domain for f, num in zip(feedback, num_prompts) for _ in range(num)] + all_domains = [ + f.domain for f, num in zip(feedback, num_prompts) for _ in range(num) + ] all_effect = [f.effect for f, num in zip(feedback, num_prompts) for _ in range(num)] - all_feedback = [f.content for f, num in zip(feedback, num_prompts) for _ in range(num)] + all_feedback = [ + f.content for f, num in zip(feedback, num_prompts) for _ in range(num) + ] all_prompts = [prompt for dataset in datasets for prompt in dataset["prompt"]] # Get completions for flattened list of prompts # Dict to hold all responses all_responses = {} - all_responses['baseline_response'] = completion_model.get_responses([ - [GET_BASELINE_COMPLETION.format(domain=d, prompt=p)] for d, p in zip(all_domains, all_prompts) - ], GET_BASELINE_COMPLETION_CONFIG) + all_responses["baseline_response"] = completion_model.get_responses( + [ + [GET_BASELINE_COMPLETION.format(domain=d, prompt=p)] + for d, p in zip(all_domains, all_prompts) + ], + GET_BASELINE_COMPLETION_CONFIG, + ) # Get revised completions for flattened list of prompts - all_responses['revised_response'] = completion_model.get_responses([ - [GET_COMPLETION_REVISED.format(prompt=p, feedback=c, response=r)] for p, c, r in zip(all_prompts, all_effect, all_responses['baseline_response']) - ], GET_COMPLETION_REVISED_CONFIG) - all_responses['revised_response'] = [r.split("IMPROVED_RESPONSE:")[-1].strip() if r is not None else r for r in all_responses['revised_response']] + all_responses["revised_response"] = completion_model.get_responses( + [ + [GET_COMPLETION_REVISED.format(prompt=p, feedback=c, response=r)] + for p, c, r in zip( + all_prompts, all_effect, all_responses["baseline_response"] + ) + ], + GET_COMPLETION_REVISED_CONFIG, + ) + all_responses["revised_response"] = [ + r.split("IMPROVED_RESPONSE:")[-1].strip() if r is not None else r + for r in all_responses["revised_response"] + ] # Get responses where feedback is applied in-context - all_responses['in_context_response'] = completion_model.get_responses([ - [GET_IN_CONTEXT_COMPLETION.format(prompt=p, feedback=c)] for p, c in zip(all_prompts, all_feedback) - ], GET_IN_CONTEXT_COMPLETION_CONFIG) + all_responses["in_context_response"] = completion_model.get_responses( + [ + [GET_IN_CONTEXT_COMPLETION.format(prompt=p, feedback=c)] + for p, c in zip(all_prompts, all_feedback) + ], + GET_IN_CONTEXT_COMPLETION_CONFIG, + ) # Get responses where feedback is applied in-context - all_responses['cot_response'] = completion_model.get_responses([ - [GET_COT_COMPLETION.format(prompt=p, feedback=c)] for p, c in zip(all_prompts, all_feedback) - ], GET_COT_COMPLETION_CONFIG) - all_responses['cot_response'] = [r.split("RESPONSE:")[-1].strip() for r in all_responses['cot_response']] - + all_responses["cot_response"] = completion_model.get_responses( + [ + [GET_COT_COMPLETION.format(prompt=p, feedback=c)] + for p, c in zip(all_prompts, all_feedback) + ], + GET_COT_COMPLETION_CONFIG, + ) + all_responses["cot_response"] = [ + r.split("RESPONSE:")[-1].strip() for r in all_responses["cot_response"] + ] + # Split responses into lists of prompts for each feedback for key in all_responses: - all_responses[key] = np.array_split(all_responses[key], np.cumsum(num_prompts)[:-1]) + all_responses[key] = np.array_split( + all_responses[key], np.cumsum(num_prompts)[:-1] + ) for i, f in enumerate(feedback): dataset = datasets[i] num_prompt = num_prompts[i] for key in all_responses: completions = all_responses[key][i] - assert len(completions) == num_prompt, f"Completion generation failed for {key} (got {len(completions)}, expected: {num_prompt})" + assert len(completions) == num_prompt, ( + f"Completion generation failed for {key} (got {len(completions)}, expected: {num_prompt})" + ) dataset = dataset.add_column(key, completions) # Filter out instances where any of the responses is None len_pre_filter = len(dataset) - dataset = dataset.filter(lambda x: x['baseline_response'] is not None and x['revised_response'] is not None and x['in_context_response'] is not None) + dataset = dataset.filter( + lambda x: ( + x["baseline_response"] is not None + and x["revised_response"] is not None + and x["in_context_response"] is not None + ) + ) if len(dataset) < len_pre_filter: - logger.warning(f"Filtered out {len_pre_filter - len(dataset)} instances where the completion API failed") + logger.warning( + f"Filtered out {len_pre_filter - len(dataset)} instances where the completion API failed" + ) if prompt_type == "prompts": f.prompts = dataset @@ -163,54 +226,73 @@ def sample_completions(feedback: list[Feedback], model_args: ModelArguments, pro f.general_prompts = dataset -def sample(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: list[Feedback]) -> None: +def sample( + arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: list[Feedback] +) -> None: model_args, sample_args, _, _ = get_args(arg_dict) run_dir = os.path.join(data_dir, run_id, "sample") logger.info(f"Sampling data for run {run_id}, stored in {run_dir}") # Filter and load feedback - feedback = [f for f in feedback if f.scope in sample_args.scope and f.type in sample_args.type] + feedback = [ + f + for f in feedback + if f.scope in sample_args.scope and f.type in sample_args.type + ] # Skip feedback that was already sampled if we don't want to overwrite if not sample_args.overwrite: feedback = [f for f in feedback if not f.can_load_dataset(run_dir)] - feedback = feedback[:sample_args.num_feedbacks] + feedback = feedback[: sample_args.num_feedbacks] logger.info(f"Loaded {len(feedback)} feedbacks") # Sample categories where necessary - num_categories = np.ceil((max(sample_args.num_prompts, sample_args.num_general_prompts)) / sample_args.prompts_per_category) + num_categories = np.ceil( + (max(sample_args.num_prompts, sample_args.num_general_prompts)) + / sample_args.prompts_per_category + ) sample_categories(feedback, model_args.category_model, num_categories) - logger.info(f"Sampled categories.") + logger.info("Sampled categories.") - sample_prompts(feedback, model_args.prompt_model, sample_args.num_prompts, sample_args.prompts_per_category) - logger.info(f"Sampled prompts.") + sample_prompts( + feedback, + model_args.prompt_model, + sample_args.num_prompts, + sample_args.prompts_per_category, + ) + logger.info("Sampled prompts.") # Sample prompts outside the feedback domain for non-global feedback - sample_prompts(feedback, model_args.prompt_model, sample_args.num_negative_prompts, sample_args.prompts_per_category, negative=True) - logger.info(f"Sampled negative prompts.") + sample_prompts( + feedback, + model_args.prompt_model, + sample_args.num_negative_prompts, + sample_args.prompts_per_category, + negative=True, + ) + logger.info("Sampled negative prompts.") # Add prompts from general prompt dataset if feedback[0].general_prompts_available(run_dir) is None: add_general_prompts(feedback, data_dir, sample_args.num_general_prompts) - sample_completions(feedback, model_args.completion_model, prompt_type="general_prompts") - logger.info(f"Sampled general prompt completions.") + sample_completions( + feedback, model_args.completion_model, prompt_type="general_prompts" + ) + logger.info("Sampled general prompt completions.") else: _ = [f.load_cached_general_prompts(run_dir) for f in feedback] - logger.info(f"Using cached general prompts.") + logger.info("Using cached general prompts.") # Sample completions sample_completions(feedback, model_args.completion_model, prompt_type="prompts") - sample_completions(feedback, model_args.completion_model, prompt_type="negative_prompts") + sample_completions( + feedback, model_args.completion_model, prompt_type="negative_prompts" + ) logger.info(f"Sampled completions for {len(feedback)} feedbacks") - - - # Save datasets for f in feedback: - f.prompts = f.prompts.train_test_split( - sample_args.train_test_split - ) + f.prompts = f.prompts.train_test_split(sample_args.train_test_split) f.negative_prompts = f.negative_prompts.train_test_split( sample_args.train_test_split ) @@ -224,7 +306,6 @@ def sample(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: list[ logger.info(f"Saved datasets for {len(feedback)} feedbacks") - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--arg_file", type=str) diff --git a/src/sft_weighted.py b/src/sft_weighted.py index a14641c..f9bb789 100644 --- a/src/sft_weighted.py +++ b/src/sft_weighted.py @@ -9,14 +9,23 @@ class WeightedSFTTrainer(SFTTrainer): - def __init__(self, *args, sigma_soft: float = 0.3, sigma_hard: float = 0.3, response_template: str = "[/INST]", ignore_index: int = -100, **kwargs): + def __init__( + self, + *args, + sigma_soft: float = 0.3, + sigma_hard: float = 0.3, + response_template: str = "[/INST]", + ignore_index: int = -100, + **kwargs, + ): self.response_template = response_template - self.response_token_ids = kwargs["tokenizer"].encode(response_template, add_special_tokens=False) + self.response_token_ids = kwargs["tokenizer"].encode( + response_template, add_special_tokens=False + ) self.ignore_index = ignore_index self.sigma_soft = sigma_soft self.sigma_hard = sigma_hard super().__init__(*args, **kwargs) - def get_completion_only_labels(self, input_ids: list[list[int]]) -> list[list[int]]: labels = torch.tensor(input_ids).clone() @@ -33,19 +42,20 @@ def get_completion_only_labels(self, input_ids: list[list[int]]) -> list[list[in if response_token_ids_start_idx is None: warnings.warn( f"Could not find response key `{self.response_template}` in the " - f'following instance: {self.tokenizer.decode(input_ids)} ' + f"following instance: {self.tokenizer.decode(input_ids)} " f"This instance will be ignored in loss calculation. " f"Note, if this happens often, consider increasing the `max_seq_length`." ) labels[:] = self.ignore_index else: - response_token_ids_end_idx = response_token_ids_start_idx + len(self.response_token_ids) + response_token_ids_end_idx = response_token_ids_start_idx + len( + self.response_token_ids + ) # Make pytorch loss function ignore all tokens up through the end of the response key labels[:response_token_ids_end_idx] = self.ignore_index return labels.tolist() - def compute_loss( self, model: Union[PreTrainedModel, nn.Module], @@ -73,12 +83,16 @@ def compute_loss( hard_outputs = model(**hard_inputs) # Compute combined loss - loss = outputs.loss + self.sigma_soft * soft_outputs.loss + self.sigma_hard * hard_outputs.loss + loss = ( + outputs.loss + + self.sigma_soft * soft_outputs.loss + + self.sigma_hard * hard_outputs.loss + ) if return_outputs: return (loss, outputs) return loss - + def _prepare_non_packed_dataloader( self, tokenizer, @@ -101,13 +115,25 @@ def _prepare_non_packed_dataloader( def tokenize_negatives(row): hard_negative = tokenizer( - row["hard_negative"], truncation=True, padding=False, max_length=max_seq_length, add_special_tokens=False - ) + row["hard_negative"], + truncation=True, + padding=False, + max_length=max_seq_length, + add_special_tokens=False, + ) soft_negative = tokenizer( - row["soft_negative"], truncation=True, padding=False, max_length=max_seq_length, add_special_tokens=False - ) - hard_negative_labels = self.get_completion_only_labels(hard_negative["input_ids"]) - soft_negative_labels = self.get_completion_only_labels(soft_negative["input_ids"]) + row["soft_negative"], + truncation=True, + padding=False, + max_length=max_seq_length, + add_special_tokens=False, + ) + hard_negative_labels = self.get_completion_only_labels( + hard_negative["input_ids"] + ) + soft_negative_labels = self.get_completion_only_labels( + soft_negative["input_ids"] + ) row["hard_negative_input_ids"] = hard_negative["input_ids"] row["hard_negative_attention_mask"] = hard_negative["attention_mask"] row["hard_negative_labels"] = hard_negative_labels @@ -119,4 +145,4 @@ def tokenize_negatives(row): # TODO: remove hacky way of removing string columns from dataset while allowing "extra" features to be passed through dataset = dataset.map(tokenize_negatives, batched=False) dataset = dataset.remove_columns(["hard_negative", "soft_negative", "text"]) - return dataset \ No newline at end of file + return dataset diff --git a/src/train.py b/src/train.py index 60aa7ca..3c81256 100644 --- a/src/train.py +++ b/src/train.py @@ -16,27 +16,47 @@ from src.sft_weighted import WeightedSFTTrainer from src.dataset.format import to_dpo, to_sft, to_lcdpo, to_sft_weighted from src.feedback import manual_feedback as all_feedback -from src.utils import get_args, find_all_linear_names, dump_arg_dicts, PeftSavingCallback, get_train_file_name, print_num_trainable_params, TrainingArguments, find_file_with_prefix - - -def filter_relevant_feedback(feedback: Feedback, prompts: Dataset | None) -> Dataset | None: +from src.utils import ( + get_args, + find_all_linear_names, + PeftSavingCallback, + get_train_file_name, + print_num_trainable_params, + TrainingArguments, + find_file_with_prefix, +) + + +def filter_relevant_feedback( + feedback: Feedback, prompts: Dataset | None +) -> Dataset | None: """Filter out prompts where the revision is not better than the baseline""" if prompts is None: return None - + # TODO: enable this for quantitative feedback # TODO: add support to define "better" using a margin rather than just binary comparison if isinstance(feedback.metric, list): - metric = lambda x: all([f(x, v) for f, v in zip(feedback.metric, feedback.metric_value)]) + + def metric(x): + return all( + [f(x, v) for f, v in zip(feedback.metric, feedback.metric_value)] + ) else: - metric = lambda x: feedback.metric(x, feedback.metric_value) - return prompts.filter(lambda x: feedback.comparison( - metric(x["baseline_response"]), - metric(x["revised_response"]) - )) + def metric(x): + return feedback.metric(x, feedback.metric_value) + + return prompts.filter( + lambda x: feedback.comparison( + metric(x["baseline_response"]), metric(x["revised_response"]) + ) + ) -def get_prompts(feedback: Feedback, training_args: TrainingArguments) -> Tuple[Dataset, Dataset, Dataset]: + +def get_prompts( + feedback: Feedback, training_args: TrainingArguments +) -> Tuple[Dataset, Dataset, Dataset]: # Fetch dataset prompts = feedback.prompts["train"].shuffle(seed=42) negative_prompts = feedback.negative_prompts["train"].shuffle(seed=42) @@ -44,7 +64,9 @@ def get_prompts(feedback: Feedback, training_args: TrainingArguments) -> Tuple[D # Filter out prompts where the revision is not better than the baseline if training_args.filter_relevant_feedback: - assert feedback.type == Type.quantitative, "Filtering relevant feedback is currently only supported for quantitative feedback" + assert feedback.type == Type.quantitative, ( + "Filtering relevant feedback is currently only supported for quantitative feedback" + ) prompts = filter_relevant_feedback(feedback, prompts) negative_prompts = filter_relevant_feedback(feedback, negative_prompts) general_prompts = filter_relevant_feedback(feedback, general_prompts) @@ -53,12 +75,20 @@ def get_prompts(feedback: Feedback, training_args: TrainingArguments) -> Tuple[D prompts = prompts.select(range(min(training_args.max_prompts, len(prompts)))) logger.info(f"Using {len(prompts)} prompts") - if training_args.negative_prompt_ratio > 0 and training_args.algo != "lcdpo" and training_args.algo != "sft_weighted": + if ( + training_args.negative_prompt_ratio > 0 + and training_args.algo != "lcdpo" + and training_args.algo != "sft_weighted" + ): num_negative_prompts = int(training_args.negative_prompt_ratio * len(prompts)) negative_prompts = negative_prompts.select(range(num_negative_prompts)) logger.info(f"Using {len(negative_prompts)} negative prompts") - if training_args.general_prompt_ratio > 0 and training_args.algo != "lcdpo" and training_args.algo != "sft_weighted": + if ( + training_args.general_prompt_ratio > 0 + and training_args.algo != "lcdpo" + and training_args.algo != "sft_weighted" + ): num_general_prompts = int(training_args.general_prompt_ratio * len(prompts)) general_prompts = general_prompts.select(range(num_general_prompts)) logger.info(f"Using {len(general_prompts)} general prompts") @@ -66,28 +96,42 @@ def get_prompts(feedback: Feedback, training_args: TrainingArguments) -> Tuple[D return prompts, negative_prompts, general_prompts -def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedback, second_feedback: Feedback = None) -> None: +def train( + arg_dict: dict[str, Any], + run_id: str, + data_dir: str, + feedback: Feedback, + second_feedback: Feedback = None, +) -> None: model_args, _, training_args, _ = get_args(arg_dict) - + # Load feedback run_dir = os.path.join(data_dir, run_id, "sample") logger.info(f"Training using data for run {run_id}, stored in {run_dir}") if not feedback.can_load_dataset(run_dir): - raise ValueError(f"Feedback \"{feedback.content}\" has not been sampled yet") + raise ValueError(f'Feedback "{feedback.content}" has not been sampled yet') feedback.load_dataset(run_dir) - logger.info(f"Loaded feedback \"{feedback.content}\"") + logger.info(f'Loaded feedback "{feedback.content}"') # Load second feedback if given if second_feedback is not None: - assert training_args.multi_feedback_training, "Must set multi_feedback_training to True when providing a second feedback" + assert training_args.multi_feedback_training, ( + "Must set multi_feedback_training to True when providing a second feedback" + ) if not second_feedback.can_load_dataset(run_dir): - raise ValueError(f"Feedback \"{second_feedback.content}\" has not been sampled yet") + raise ValueError( + f'Feedback "{second_feedback.content}" has not been sampled yet' + ) second_feedback.load_dataset(run_dir) elif training_args.multi_feedback_training and second_feedback is None: - raise ValueError("Must provide a second feedback when multi_feedback_training is True") + raise ValueError( + "Must provide a second feedback when multi_feedback_training is True" + ) run_dir = os.path.join(data_dir, run_id, "train") - assert training_args.algo in ["dpo", "sft", "lcdpo", "sft_weighted"], f"Unknown algorithm {training_args.algo}" + assert training_args.algo in ["dpo", "sft", "lcdpo", "sft_weighted"], ( + f"Unknown algorithm {training_args.algo}" + ) train_dir = get_train_file_name(training_args, model_args.train_model) run_dir = os.path.join(run_dir, feedback.file_name) @@ -97,13 +141,17 @@ def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedba run_dir = os.path.join(run_dir, train_dir) # Load model - assert model_args.train_model.platform == "huggingface", "Only HuggingFace models are supported for training" + assert model_args.train_model.platform == "huggingface", ( + "Only HuggingFace models are supported for training" + ) model = get_model(model_args.train_model) logger.info("Loaded model") prompts, negative_prompts, general_prompts = get_prompts(feedback, training_args) if training_args.multi_feedback_training: - prompts2, negative_prompts2, general_prompts2 = get_prompts(second_feedback, training_args) + prompts2, negative_prompts2, general_prompts2 = get_prompts( + second_feedback, training_args + ) prompts = concatenate_datasets([prompts, prompts2]) negative_prompts = concatenate_datasets([negative_prompts, negative_prompts2]) general_prompts = concatenate_datasets([general_prompts, general_prompts2]) @@ -123,30 +171,49 @@ def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedba dataset = dataset_constructor( prompts, - negative_prompts if (training_args.negative_prompt_ratio > 0 or training_args.algo == "lcdpo" or training_args.algo == "sft_weighted") else None, - general_prompts if (training_args.negative_prompt_ratio > 0 or training_args.algo == "lcdpo" or training_args.algo == "sft_weighted") else None, - model_args.train_model.model_name_or_path) - + negative_prompts + if ( + training_args.negative_prompt_ratio > 0 + or training_args.algo == "lcdpo" + or training_args.algo == "sft_weighted" + ) + else None, + general_prompts + if ( + training_args.negative_prompt_ratio > 0 + or training_args.algo == "lcdpo" + or training_args.algo == "sft_weighted" + ) + else None, + model_args.train_model.model_name_or_path, + ) - # Load base training arg adapter if given + # Load base training arg adapter if given if training_args.use_base_prefix is not None: base_run_dir = os.path.join(data_dir, run_id, "train", feedback.file_name) - adapter_name = find_file_with_prefix(base_run_dir, training_args.use_base_prefix) - model.model = PeftModel.from_pretrained(model.model, os.path.join(base_run_dir, adapter_name), is_trainable=True) + adapter_name = find_file_with_prefix( + base_run_dir, training_args.use_base_prefix + ) + model.model = PeftModel.from_pretrained( + model.model, os.path.join(base_run_dir, adapter_name), is_trainable=True + ) logger.info(f"Loaded base training model from {base_run_dir}") - + # Add LoRA config assert training_args.lora_enable, "Currently only LoRA training is supported" if training_args.lora_enable and training_args.use_base_prefix is None: peft_config = LoraConfig( - r=training_args.lora_r, - lora_alpha=training_args.lora_alpha, - target_modules = find_all_linear_names(model.model, training_args.lora_exclude), - lora_dropout=training_args.lora_dropout, + r=training_args.lora_r, + lora_alpha=training_args.lora_alpha, + target_modules=find_all_linear_names( + model.model, training_args.lora_exclude + ), + lora_dropout=training_args.lora_dropout, bias=training_args.lora_bias, - task_type="CAUSAL_LM" + task_type="CAUSAL_LM", ) - else: peft_config = None + else: + peft_config = None training_args.output_dir = run_dir os.makedirs(run_dir, exist_ok=True) @@ -159,22 +226,33 @@ def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedba model.model.config.use_cache = False # Create eval dataset - dataset = dataset.train_test_split(test_size=training_args.eval_split, seed=42, shuffle=True) + dataset = dataset.train_test_split( + test_size=training_args.eval_split, seed=42, shuffle=True + ) eval_dataset = dataset["test"] dataset = dataset["train"] - logger.info(f"Training on {len(dataset)} prompts, evaluating on {len(eval_dataset)} prompts for feedback \"{feedback.content}\"") + logger.info( + f'Training on {len(dataset)} prompts, evaluating on {len(eval_dataset)} prompts for feedback "{feedback.content}"' + ) # TODO: hacky, remove - if model_args.train_model.model_name_or_path in ["tiiuae/falcon-7b-instruct", "01-ai/Yi-6B-Chat", "Qwen/Qwen-7B-Chat"]: + if model_args.train_model.model_name_or_path in [ + "tiiuae/falcon-7b-instruct", + "01-ai/Yi-6B-Chat", + "Qwen/Qwen-7B-Chat", + ]: response_template = "<|im_start|>assistant\n" - elif model_args.train_model.model_name_or_path == "NousResearch/Nous-Hermes-llama-2-7b": + elif ( + model_args.train_model.model_name_or_path + == "NousResearch/Nous-Hermes-llama-2-7b" + ): response_template = "Response:\n" else: response_template = "[/INST]" if training_args.algo == "dpo": - model.tokenizer.padding_side = 'left' + model.tokenizer.padding_side = "left" trainer = DPOTrainer( model=model.model, max_length=2048, @@ -185,10 +263,10 @@ def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedba eval_dataset=eval_dataset, tokenizer=model.tokenizer, peft_config=peft_config, - callbacks=[PeftSavingCallback] if training_args.lora_enable else None + callbacks=[PeftSavingCallback] if training_args.lora_enable else None, ) elif training_args.algo == "lcdpo": - model.tokenizer.padding_side = 'left' + model.tokenizer.padding_side = "left" trainer = LocallyConstrainedDPOTrainer( model=model.model, max_length=2048, @@ -206,11 +284,13 @@ def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedba tokenizer=model.tokenizer, response_template=response_template, peft_config=peft_config, - callbacks=[PeftSavingCallback] if training_args.lora_enable else None + callbacks=[PeftSavingCallback] if training_args.lora_enable else None, ) elif training_args.algo == "sft": - model.tokenizer.padding_side = 'right' - collator = DataCollatorForCompletionOnlyLM(response_template, tokenizer=model.tokenizer) + model.tokenizer.padding_side = "right" + collator = DataCollatorForCompletionOnlyLM( + response_template, tokenizer=model.tokenizer + ) trainer = SFTTrainer( model=model.model, args=training_args, @@ -220,12 +300,14 @@ def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedba data_collator=collator, max_seq_length=2048, peft_config=peft_config, - callbacks=[PeftSavingCallback] if training_args.lora_enable else None + callbacks=[PeftSavingCallback] if training_args.lora_enable else None, ) elif training_args.algo == "sft_weighted": - model.tokenizer.padding_side = 'right' - collator = DataCollatorForCompletionOnlyLM(response_template, tokenizer=model.tokenizer) - training_args.evaluation_strategy = "no" # TODO: getting error during eval + model.tokenizer.padding_side = "right" + collator = DataCollatorForCompletionOnlyLM( + response_template, tokenizer=model.tokenizer + ) + training_args.evaluation_strategy = "no" # TODO: getting error during eval trainer = WeightedSFTTrainer( model=model.model, args=training_args, @@ -238,7 +320,7 @@ def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedba sigma_soft=training_args.lcdpo_sigma_soft, sigma_hard=training_args.lcdpo_sigma_hard, peft_config=peft_config, - callbacks=[PeftSavingCallback] if training_args.lora_enable else None + callbacks=[PeftSavingCallback] if training_args.lora_enable else None, ) else: raise ValueError(f"Unknown algorithm {training_args.algo}") @@ -250,7 +332,7 @@ def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedba if training_args.report_to == "wandb": wandb.finish() - + # Sometimes Wandb needs more time to close resources sleep(2) @@ -269,6 +351,6 @@ def train(arg_dict: dict[str, Any], run_id: str, data_dir: str, feedback: Feedba feedback = all_feedback if args.feedback_prefix is not None: feedback = [f for f in feedback if f.content.startswith(args.feedback_prefix)] - + for f in feedback: train(arg_dict, args.run_id, args.data_dir, f) diff --git a/src/utils.py b/src/utils.py index a3d114a..550d5bf 100644 --- a/src/utils.py +++ b/src/utils.py @@ -10,7 +10,11 @@ from dataclasses import field, dataclass, asdict import torch -from transformers import TrainingArguments as TransformerTrainingArguments, TrainerCallback, HfArgumentParser +from transformers import ( + TrainingArguments as TransformerTrainingArguments, + TrainerCallback, + HfArgumentParser, +) from src.logger import logger from src.dataset.feedback_utils import Scope, Type @@ -19,11 +23,7 @@ DEVICE = "cuda" if torch.cuda.is_available() else "cpu" -DTYPES = { - "bf16": torch.bfloat16, - "f16": torch.float16, - "f32": torch.float32 -} +DTYPES = {"bf16": torch.bfloat16, "f16": torch.float16, "f32": torch.float32} @dataclass @@ -37,13 +37,15 @@ class ModelArguments: @dataclass class PipelineModelsArguments: - category_model: Optional[ModelArguments] = field(default_factory=ModelArguments) - prompt_model: Optional[ModelArguments] = field(default_factory=ModelArguments) - completion_model: Optional[ModelArguments] = field(default_factory=ModelArguments) - train_model: Optional[ModelArguments] = field(default_factory=ModelArguments) - qualitative_eval_model: Optional[ModelArguments] = field(default_factory=ModelArguments) - - def __post_init__(self): + category_model: Optional[ModelArguments] = field(default_factory=ModelArguments) + prompt_model: Optional[ModelArguments] = field(default_factory=ModelArguments) + completion_model: Optional[ModelArguments] = field(default_factory=ModelArguments) + train_model: Optional[ModelArguments] = field(default_factory=ModelArguments) + qualitative_eval_model: Optional[ModelArguments] = field( + default_factory=ModelArguments + ) + + def __post_init__(self): # TODO: figure out a way to not parse separately and preserve types self.category_model = ModelArguments(**self.category_model) self.prompt_model = ModelArguments(**self.prompt_model) @@ -54,9 +56,13 @@ def __post_init__(self): @dataclass class SampleArguments: - scope: Optional[list[Scope]] = field(default_factory= lambda: ["global_", "regional", "local"]) - type: Optional[list[Type]] = field(default_factory=lambda: ["quantitative", "qualitative"]) - train_test_split: float = 0.2 # percentage of data used for test set + scope: Optional[list[Scope]] = field( + default_factory=lambda: ["global_", "regional", "local"] + ) + type: Optional[list[Type]] = field( + default_factory=lambda: ["quantitative", "qualitative"] + ) + train_test_split: float = 0.2 # percentage of data used for test set num_feedbacks: Optional[int] = 1 prompts_per_category: Optional[int] = 16 num_prompts: Optional[int] = 32 @@ -108,14 +114,20 @@ class EvalArguments: class PeftSavingCallback(TrainerCallback): def on_save(self, args, state, control, **kwargs): - checkpoint_path = os.path.join(args.output_dir, f"checkpoint-{state.global_step}") + checkpoint_path = os.path.join( + args.output_dir, f"checkpoint-{state.global_step}" + ) kwargs["model"].save_pretrained(checkpoint_path) if "pytorch_model.bin" in os.listdir(checkpoint_path): os.remove(os.path.join(checkpoint_path, "pytorch_model.bin")) -def dump_arg_dicts(arg_dicts: dict[str, dict[str, Any]], output_dir: str, filename: str = "arg_dict.json"): +def dump_arg_dicts( + arg_dicts: dict[str, dict[str, Any]], + output_dir: str, + filename: str = "arg_dict.json", +): arg_dict = {name: asdict(arg) for name, arg in arg_dicts.items()} # Create output dir if it doesn't exist @@ -126,7 +138,9 @@ def dump_arg_dicts(arg_dicts: dict[str, dict[str, Any]], output_dir: str, filena json.dump(arg_dict, f, indent=2) -def get_args(arg_dict: dict[str, Any]) -> tuple[PipelineModelsArguments, SampleArguments, TrainingArguments, EvalArguments]: +def get_args( + arg_dict: dict[str, Any], +) -> tuple[PipelineModelsArguments, SampleArguments, TrainingArguments, EvalArguments]: modal_arg_dict = arg_dict["model_args"] sample_arg_dict = arg_dict["sample_args"] training_arg_dict = arg_dict["training_args"] @@ -138,7 +152,9 @@ def get_args(arg_dict: dict[str, Any]) -> tuple[PipelineModelsArguments, SampleA sample_arg_parser = HfArgumentParser(SampleArguments) sample_args: SampleArguments = sample_arg_parser.parse_dict(sample_arg_dict)[0] training_arg_parser = HfArgumentParser(TrainingArguments) - training_args: TrainingArguments = training_arg_parser.parse_dict(training_arg_dict)[0] + training_args: TrainingArguments = training_arg_parser.parse_dict( + training_arg_dict + )[0] eval_arg_parser = HfArgumentParser(EvalArguments) eval_args: EvalArguments = eval_arg_parser.parse_dict(eval_arg_dict)[0] return model_args, sample_args, training_args, eval_args @@ -152,27 +168,32 @@ def find_all_linear_names(model, exclude: list[str]): if any(keyword in name for keyword in exclude): continue if isinstance(module, cls): - names = name.split('.') + names = name.split(".") lora_module_names.add(names[-1]) return list(lora_module_names) def format_messages(batch: list[list[str]]) -> list[list[dict[str, str]]]: """Assumes batch is a list of lists of strings, where each inner list is a list of chat messages that alternate between user and assistant.""" - return [[{ - "role": "user" if i % 2 == 0 else "assistant", - "content": m - } for i, m in enumerate(b)] for b in batch] + return [ + [ + {"role": "user" if i % 2 == 0 else "assistant", "content": m} + for i, m in enumerate(b) + ] + for b in batch + ] def split_numbered_list(text: str) -> list[str]: """Splits a numbered list into a list of strings, where each string is a numbered item in the list.""" - text = '\n' + text + text = "\n" + text regex = r"\n[0-9]+. " return [t.strip() for t in re.split(regex, text) if t.strip()] -def throttle(lock: threading.Lock, rqi: int, last_requests: list[float], interval: int = 60) -> None: +def throttle( + lock: threading.Lock, rqi: int, last_requests: list[float], interval: int = 60 +) -> None: """Decorator to throttle the number of requests per interval. Args: @@ -181,6 +202,7 @@ def throttle(lock: threading.Lock, rqi: int, last_requests: list[float], interva last_requests: List to store timestamps of the last requests interval: Interval in seconds to check the number of requests """ + def decorator(func): @wraps(func) def wrapper(*args, **kwargs): @@ -189,7 +211,9 @@ def wrapper(*args, **kwargs): with lock: # Remove timestamps older than 60 seconds now = time.time() - last_requests = [req for req in last_requests if now - req < interval] + last_requests = [ + req for req in last_requests if now - req < interval + ] # If the number of requests in the last minute is less than the limit, send a new request if len(last_requests) < rqi: last_requests.append(now) @@ -198,12 +222,15 @@ def wrapper(*args, **kwargs): minimum = min([now - req for req in last_requests]) time.sleep(minimum) return func(*args, **kwargs) + return wrapper + return decorator def catch_error_return_none(func): """Decorator to log a warning if an error occurs but catches the error and returns None.""" + @wraps(func) def wrapper(*args, **kwargs): try: @@ -211,6 +238,7 @@ def wrapper(*args, **kwargs): except Exception as e: logger.warning(f"An error occurred in {func.__name__}: {e}") return None + return wrapper @@ -227,12 +255,15 @@ def wrapper(*args, **kwargs): "max_prompts": None, "multi_feedback_training": False, "model_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2", - "use_base_prefix": None + "use_base_prefix": None, } -def get_train_file_name(training_args: TrainingArguments, model_args: ModelArguments) -> str: + +def get_train_file_name( + training_args: TrainingArguments, model_args: ModelArguments +) -> str: file_name_parts = [training_args.algo] - + def append_if_different(arg_value, key): if arg_value != defaults[key]: arg_value = str(arg_value).replace("/", "-") @@ -246,17 +277,17 @@ def append_if_different(arg_value, key): else: append_if_different(training_args.negative_prompt_ratio, "hard") append_if_different(training_args.general_prompt_ratio, "soft") - + append_if_different(training_args.learning_rate, "lr") - + if training_args.lora_enable: append_if_different(training_args.lora_r, "r") append_if_different(training_args.lora_alpha, "alpha") - + append_if_different(training_args.num_train_epochs, "epochs") append_if_different(model_args.model_name_or_path, "model_name_or_path") append_if_different(training_args.use_base_prefix, "use_base_prefix") - + if training_args.multi_feedback_training: file_name_parts.append("-multi") @@ -265,23 +296,24 @@ def append_if_different(arg_value, key): if training_args.lcdpo_avg_kl: file_name_parts.append("-avgkl") - + append_if_different(training_args.max_prompts, "max_prompts") - + # Serialize training_args and generate a short hash to ensure any parameter changes are reflected in the file name args_copy = copy.deepcopy(training_args) del args_copy.logging_dir short_hash = hashlib.sha1(args_copy.to_json_string().encode()).hexdigest()[:8] file_name_parts.append(f"-{short_hash}") - - return ''.join(file_name_parts) + + return "".join(file_name_parts) def print_num_trainable_params(model): - trainable_params = sum(p.numel() - for p in model.parameters() if p.requires_grad) + trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) all_params = sum(p.numel() for p in model.parameters()) - logger.info(f"Trainable params: {trainable_params}/{all_params} ({trainable_params/all_params:.2%})") + logger.info( + f"Trainable params: {trainable_params}/{all_params} ({trainable_params / all_params:.2%})" + ) def find_file_with_prefix(path: str, prefix: str) -> str: @@ -289,4 +321,4 @@ def find_file_with_prefix(path: str, prefix: str) -> str: for file in os.listdir(path): if file.startswith(prefix): return file - raise FileNotFoundError(f"No file with prefix {prefix} found in {path}") \ No newline at end of file + raise FileNotFoundError(f"No file with prefix {prefix} found in {path}") diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 5966ee9..4027a7b 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,6 +1,4 @@ import pytest -import re -from langdetect import detect # Assuming langdetect is used for language detection from src.dataset.feedback_utils import Metric @@ -10,60 +8,74 @@ def test_length(): # Test with non-English characters assert Metric.length("こんにちは世界", None) == 7 + def test_contains_any_string(): - assert Metric.contains_any_string("Hello, World!", ["Hello", "Python"]) == True - assert Metric.contains_any_string("Hello, World!", ["Python", "Java"]) == False + assert Metric.contains_any_string("Hello, World!", ["Hello", "Python"]) + assert not Metric.contains_any_string("Hello, World!", ["Python", "Java"]) # Case sensitivity test - assert Metric.contains_any_string("hello, world!", ["Hello"]) == True + assert Metric.contains_any_string("hello, world!", ["Hello"]) + def test_contains_all_strings(): - assert Metric.contains_all_strings("Hello, World!", ["Hello", "World"]) == True - assert Metric.contains_all_strings("Hello, World!", ["Hello", "Python"]) == False + assert Metric.contains_all_strings("Hello, World!", ["Hello", "World"]) + assert not Metric.contains_all_strings("Hello, World!", ["Hello", "Python"]) # Case sensitivity and partial match test - assert Metric.contains_all_strings("Hello, World!", ["hello", "world"]) == True + assert Metric.contains_all_strings("Hello, World!", ["hello", "world"]) + def test_contains_none_strings(): - assert Metric.contains_none_strings("Hello, World!", ["Python", "Java"]) == True - assert Metric.contains_none_strings("Hello, World!", ["Hello", "Python"]) == False + assert Metric.contains_none_strings("Hello, World!", ["Python", "Java"]) + assert not Metric.contains_none_strings("Hello, World!", ["Hello", "Python"]) # Case sensitivity test - assert Metric.contains_none_strings("Hello, World!", ["hello"]) == False + assert not Metric.contains_none_strings("Hello, World!", ["hello"]) + def test_contains_phone_number(): - assert Metric.contains_phone_number("Call me at (123) 456-7890", "(123) 456-7890") == True - assert Metric.contains_phone_number("Call me at 123-456-7890", "(123) 456-7890") == True - assert Metric.contains_phone_number("Call me at (123) 456-7890", "(321) 654-0987") == False + assert Metric.contains_phone_number("Call me at (123) 456-7890", "(123) 456-7890") + assert Metric.contains_phone_number("Call me at 123-456-7890", "(123) 456-7890") + assert not Metric.contains_phone_number( + "Call me at (123) 456-7890", "(321) 654-0987" + ) + def test_ends_with(): - assert Metric.ends_with("Hello, World!", "World!") == True - assert Metric.ends_with("Hello, World!", "world!") == True # Case insensitivity - assert Metric.ends_with("Hello, World!", "Hello") == False + assert Metric.ends_with("Hello, World!", "World!") + assert Metric.ends_with("Hello, World!", "world!") # Case insensitivity + assert not Metric.ends_with("Hello, World!", "Hello") + def test_ends_with_cleaned(): - assert Metric.ends_with_cleaned("Hello, World!!", "world!!") == True - assert Metric.ends_with_cleaned("Hello, World!!!", "hello!") == False - assert Metric.ends_with_cleaned("Hello,\n World?", "Hello, world?") == True + assert Metric.ends_with_cleaned("Hello, World!!", "world!!") + assert not Metric.ends_with_cleaned("Hello, World!!!", "hello!") + assert Metric.ends_with_cleaned("Hello,\n World?", "Hello, world?") + def test_regex_search(): - assert Metric.regex_search("Hello, World!", r"\bWorld\b") == True - assert Metric.regex_search("Hello, World!", r"\bPython\b") == False + assert Metric.regex_search("Hello, World!", r"\bWorld\b") + assert not Metric.regex_search("Hello, World!", r"\bPython\b") + def test_regex_search_false(): - assert Metric.regex_search_false("Hello, World!", r"\bWorld\b") == False - assert Metric.regex_search_false("Hello, World!", r"\bPython\b") == True + assert not Metric.regex_search_false("Hello, World!", r"\bWorld\b") + assert Metric.regex_search_false("Hello, World!", r"\bPython\b") + def test_is_language(): - assert Metric.is_language("Hello, World!", "en") == True - assert Metric.is_language("Bonjour, Monde!", "en") == False - assert Metric.is_language("Bonjour, Monde!", "fr") == True + assert Metric.is_language("Hello, World!", "en") + assert not Metric.is_language("Bonjour, Monde!", "en") + assert Metric.is_language("Bonjour, Monde!", "fr") + def test_starts_with(): - assert Metric.starts_with("Hello, World!", "Hello") == True - assert Metric.starts_with("Hello, World!", "hello") == True # Case insensitivity - assert Metric.starts_with("Hello, World!", "World") == False + assert Metric.starts_with("Hello, World!", "Hello") + assert Metric.starts_with("Hello, World!", "hello") # Case insensitivity + assert not Metric.starts_with("Hello, World!", "World") + def test_doesnt_start_with(): - assert Metric.doesnt_start_with("Hello, World!", "World") == True - assert Metric.doesnt_start_with("Hello, World!", "hello") == False # Case insensitivity + assert Metric.doesnt_start_with("Hello, World!", "World") + assert not Metric.doesnt_start_with("Hello, World!", "hello") # Case insensitivity + # Additional tests for other metrics if they exist