Skip to content

Add CI lint workflow and fix all ruff lint + format errors #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
austrian-code-wizard wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add CI lint workflow and fix all ruff lint + format errors #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d0a11f7
Add CI lint workflow and introduce deliberate lint failure
austrian-code-wizard Apr 6, 2026
adbb3b5
Fix all ruff lint errors across the codebase
staging-devin-ai-integration[bot] Apr 6, 2026
f7d0247
Apply ruff format to all files
staging-devin-ai-integration[bot] Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .github/workflows/lint.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
name: Lint

on:
pull_request:
branches: [main]
push:
branches: [main]

jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"

- name: Install ruff
run: pip install ruff

- name: Run ruff linter
run: ruff check .

- name: Run ruff formatter check
run: ruff format --check .
69 changes: 55 additions & 14 deletions notebooks/collie_feedback_gen.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
"outputs": [],
"source": [
"import dill\n",
"from pathlib import Path\n",
"import sys"
]
},
Expand Down Expand Up @@ -54,7 +53,9 @@
],
"source": [
"out = open(\"../src/feedback/collie.py\", \"w+\")\n",
"out.write(\"from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison\\n\\ncollie_feedback = [\\n\")"
"out.write(\n",
" \"from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison\\n\\ncollie_feedback = [\\n\"\n",
")"
]
},
{
Expand All @@ -63,7 +64,15 @@
"metadata": {},
"outputs": [],
"source": [
"def get_feedback(prompt, domain, effect, categories, metric, metric_value, comparison=\"Comparison.greater_eq_than\"):\n",
"def get_feedback(\n",
" prompt,\n",
" domain,\n",
" effect,\n",
" categories,\n",
" metric,\n",
" metric_value,\n",
" comparison=\"Comparison.greater_eq_than\",\n",
"):\n",
" return \"\"\"\n",
" Feedback(\n",
" content=\"{prompt}\",\n",
Expand All @@ -76,7 +85,15 @@
" metric_value={metric_value},\n",
" comparison={comparison}\n",
" ),\n",
"\"\"\".format(prompt=prompt, domain=domain, effect=effect, categories=categories, metric=metric, metric_value=metric_value, comparison=comparison)"
"\"\"\".format(\n",
" prompt=prompt,\n",
" domain=domain,\n",
" effect=effect,\n",
" categories=categories,\n",
" metric=metric,\n",
" metric_value=metric_value,\n",
" comparison=comparison,\n",
" )"
]
},
{
Expand All @@ -86,15 +103,25 @@
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"key = \"wiki_c07\"\n",
"chosen = np.random.choice(np.arange(len(all_data[key])), size=38, replace=False)\n",
"for i in chosen:\n",
" obj = all_data[key][i]\n",
" topic = obj['metadata'][\"title\"]\n",
" words = obj['prompt'].split(\"containing the word\")[1][:-1].strip()\n",
" prompt = f\"When talking about {topic}, make sure the response contain the words {words}\"\n",
" topic = obj[\"metadata\"][\"title\"]\n",
" words = obj[\"prompt\"].split(\"containing the word\")[1][:-1].strip()\n",
" prompt = (\n",
" f\"When talking about {topic}, make sure the response contain the words {words}\"\n",
" )\n",
" domain = f\"Talking about {topic}\"\n",
" out_str = get_feedback(prompt, domain, f\"make sure the response contain the words {words}\", f\"['collie', '{key}']\", \"Metric.contains_all_strings\", \"\"\"[{words}]\"\"\".format(words=words))\n",
" out_str = get_feedback(\n",
" prompt,\n",
" domain,\n",
" f\"make sure the response contain the words {words}\",\n",
" f\"['collie', '{key}']\",\n",
" \"Metric.contains_all_strings\",\n",
" \"\"\"[{words}]\"\"\".format(words=words),\n",
" )\n",
" out.write(out_str + \"\\n\")"
]
},
Expand All @@ -120,11 +147,18 @@
"print(len(arr))\n",
"for i in chosen:\n",
" obj = arr[i]\n",
" topic = obj['metadata'][\"title\"]\n",
" words = obj['prompt'].split(\"1st word to be\")[1][:-1].strip()\n",
" topic = obj[\"metadata\"][\"title\"]\n",
" words = obj[\"prompt\"].split(\"1st word to be\")[1][:-1].strip()\n",
" prompt = f\"When talking about {topic}, make sure the first sentences has a 1st word of {words}\"\n",
" domain = f\"Talking about {topic}\"\n",
" out_str = get_feedback(prompt, domain, f\"make sure the first sentences has a 1st word of {words}\", f\"['collie', '{key}']\", \"Metric.starts_with\",words.lower())\n",
" out_str = get_feedback(\n",
" prompt,\n",
" domain,\n",
" f\"make sure the first sentences has a 1st word of {words}\",\n",
" f\"['collie', '{key}']\",\n",
" \"Metric.starts_with\",\n",
" words.lower(),\n",
" )\n",
" out.write(out_str + \"\\n\")"
]
},
Expand All @@ -138,12 +172,19 @@
"chosen = np.random.choice(np.arange(len(all_data[key])), size=38, replace=False)\n",
"for i in chosen:\n",
" obj = all_data[key][i]\n",
" topic = obj['metadata'][\"title\"]\n",
" topic = obj[\"metadata\"][\"title\"]\n",
" targets = obj[\"targets\"]\n",
" words = \"\\'\" + \"', '\".join(targets[1:]) + \"'\"\n",
" words = \"'\" + \"', '\".join(targets[1:]) + \"'\"\n",
" prompt = f\"When talking about {topic}, do not use the words {words}\"\n",
" domain = f\"Talking about {topic}\"\n",
" out_str = get_feedback(prompt, domain, f\"do not use the words {words}\", f\"['collie', '{key}']\", \"Metric.contains_none_strings\", f\"{targets[1:]}\")\n",
" out_str = get_feedback(\n",
" prompt,\n",
" domain,\n",
" f\"do not use the words {words}\",\n",
" f\"['collie', '{key}']\",\n",
" \"Metric.contains_none_strings\",\n",
" f\"{targets[1:]}\",\n",
" )\n",
" out.write(out_str + \"\\n\")"
]
},
Expand Down
23 changes: 17 additions & 6 deletions notebooks/gpt_feedback_gen.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"outputs": [],
"source": [
"from openai import OpenAI\n",
"\n",
"client = OpenAI(\n",
" api_key=\"INSERT_KEY\",\n",
")"
Expand All @@ -30,8 +31,11 @@
" completion = client.chat.completions.create(\n",
" model=\"gpt-4-1106-preview\",\n",
" messages=[\n",
" {'role': 'system', 'content': \"You are a helpful assistant that always closely follows instructions.\"},\n",
" {'role': 'user', 'content': prompt}\n",
" {\n",
" \"role\": \"system\",\n",
" \"content\": \"You are a helpful assistant that always closely follows instructions.\",\n",
" },\n",
" {\"role\": \"user\", \"content\": prompt},\n",
" ],\n",
" )\n",
" response = completion.choices[0].message.content\n",
Expand Down Expand Up @@ -178,7 +182,7 @@
" print(\"num generated\", len(generated))\n",
" resp = get_response(CONTENT_PROMPT_EFFECT)\n",
" print(resp)\n",
" if resp[0:7] == \"```json\": # tries to output in markdown\n",
" if resp[0:7] == \"```json\": # tries to output in markdown\n",
" resp = resp[8:-3]\n",
" try:\n",
" obj = json.loads(resp)\n",
Expand All @@ -200,12 +204,14 @@
"\n",
"import json\n",
"\n",
"ending = \"content\" # content or style\n",
"ending = \"content\" # content or style\n",
"with open(f\"working_json/gpt_feedback_{ending}.json\", \"r\") as f:\n",
" obj = json.load(f)\n",
"\n",
"with open(f\"../src/feedback/gpt_{ending}.py\", \"w+\") as out:\n",
" out.write(f\"from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison\\n\\ngpt_{ending}_feedback = [\\n\")\n",
" out.write(\n",
" f\"from src.dataset.feedback_utils import Feedback, Scope, Type, Metric, Comparison\\n\\ngpt_{ending}_feedback = [\\n\"\n",
" )\n",
" for i in obj:\n",
" out_str = \"\"\" \n",
" Feedback(\n",
Expand All @@ -217,7 +223,12 @@
" type=Type.qualitative,\n",
" comparison=Comparison.greater_eq_than\n",
" ),\n",
"\"\"\".format(feedback=i[\"feedback\"], domain=i[\"domain\"], effect=i[\"effect\"], ending=ending)\n",
"\"\"\".format(\n",
" feedback=i[\"feedback\"],\n",
" domain=i[\"domain\"],\n",
" effect=i[\"effect\"],\n",
" ending=ending,\n",
" )\n",
" out.write(out_str + \"\\n\")\n",
" out.write(\"]\\n\")"
]
Expand Down
8 changes: 4 additions & 4 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from setuptools import setup, find_packages
from setuptools import setup

setup(
name='src',
version='0.1',
name="src",
version="0.1",
packages=["src"],
)
)
50 changes: 35 additions & 15 deletions src/dataset/feedback_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,18 +41,31 @@ def __call__(self, *args, **kwargs):

class Metric(Enum):
length: Callable = lambda x, _: len(x)
contains_any_string: Callable = lambda x, y: any([s.lower() in x.lower() for s in y])
contains_all_strings: Callable = lambda x, y: all([s.lower() in x.lower() for s in y])
contains_none_strings: Callable = lambda x, y: not any([s.lower() in x.lower() for s in y])
contains_phone_number: Callable = lambda x, y: y.replace("-", " ").replace("(", "").replace(")", "") in x.replace("-", " ").replace("(", "").replace(")", "")
contains_any_string: Callable = lambda x, y: any(
[s.lower() in x.lower() for s in y]
)
contains_all_strings: Callable = lambda x, y: all(
[s.lower() in x.lower() for s in y]
)
contains_none_strings: Callable = lambda x, y: (
not any([s.lower() in x.lower() for s in y])
)
contains_phone_number: Callable = lambda x, y: (
y.replace("-", " ").replace("(", "").replace(")", "")
in x.replace("-", " ").replace("(", "").replace(")", "")
)
ends_with: Callable = lambda x, y: x.lower().strip().endswith(y.lower().strip())
ends_with_cleaned: Callable = lambda x, y: " ".join(re.sub(r'[^a-z0-9,.!?]', ' ', x.lower().strip()).split()).endswith(" ".join(re.sub(r'[^a-z0-9,.!?]', ' ', y.lower().strip()).split()))
ends_with_cleaned: Callable = lambda x, y: " ".join(
re.sub(r"[^a-z0-9,.!?]", " ", x.lower().strip()).split()
).endswith(" ".join(re.sub(r"[^a-z0-9,.!?]", " ", y.lower().strip()).split()))
regex_search: Callable = lambda x, y: bool(re.search(y, x))
regex_search_false: Callable = lambda x, y: not bool(re.search(y, x))
is_language: Callable = lambda x, y: detect(x) == y
starts_with: Callable = lambda x, y: x.lower().strip().startswith(y.lower().strip())
doesnt_start_with: Callable = lambda x, y: not x.lower().strip().startswith(y.lower().strip())

doesnt_start_with: Callable = lambda x, y: (
not x.lower().strip().startswith(y.lower().strip())
)

def __call__(self, *args, **kwargs):
return self.value(*args, **kwargs)

Expand Down Expand Up @@ -87,7 +100,7 @@ def file_name(self):
content = content.replace(" ", "_")
content = content.strip()
return f"{content}_{self.id}"

def can_load_dataset(self, prompt_dir: str) -> None:
"""Checks if prompts can be loaded from a directory

Expand All @@ -104,7 +117,7 @@ def can_load_dataset(self, prompt_dir: str) -> None:
if not os.path.exists(os.path.join(path, "categories.json")):
return False
return True

def general_prompts_available(self, prompt_dir: str) -> Optional[str]:
"""If any of the subdirectories of prompt_dir have a valid json file named "general_prompts.json" with keys "train" and "test",
return the path to that file. Otherwise, return None.
Expand All @@ -120,7 +133,7 @@ def general_prompts_available(self, prompt_dir: str) -> Optional[str]:
if "train" in data.keys() and "test" in data.keys():
return os.path.join(path, "general_prompts.json")
return None

@staticmethod
def _dump_dataset_dict(path: str, dataset: DatasetDict) -> None:
data = {}
Expand All @@ -138,7 +151,6 @@ def _load_dataset_dict(path: str) -> DatasetDict:
dataset_dict[split] = Dataset.from_dict(data[split])
return dataset_dict


def load_dataset(self, prompt_dir: str) -> None:
"""Loads prompts from a directory into the feedback object

Expand All @@ -147,8 +159,12 @@ def load_dataset(self, prompt_dir: str) -> None:
"""
path = os.path.join(prompt_dir, self.file_name)
self.prompts = self._load_dataset_dict(os.path.join(path, "prompts.json"))
self.negative_prompts = self._load_dataset_dict(os.path.join(path, "negative_prompts.json"))
self.general_prompts = self._load_dataset_dict(os.path.join(path, "general_prompts.json"))
self.negative_prompts = self._load_dataset_dict(
os.path.join(path, "negative_prompts.json")
)
self.general_prompts = self._load_dataset_dict(
os.path.join(path, "general_prompts.json")
)
with open(os.path.join(path, "categories.json"), "r") as f:
self.categories = json.load(f)

Expand All @@ -173,7 +189,11 @@ def dump_dataset(self, prompt_dir: str) -> None:
path = os.path.join(prompt_dir, self.file_name)
os.makedirs(path, exist_ok=True)
self._dump_dataset_dict(os.path.join(path, "prompts.json"), self.prompts)
self._dump_dataset_dict(os.path.join(path, "negative_prompts.json"), self.negative_prompts)
self._dump_dataset_dict(os.path.join(path, "general_prompts.json"), self.general_prompts)
self._dump_dataset_dict(
os.path.join(path, "negative_prompts.json"), self.negative_prompts
)
self._dump_dataset_dict(
os.path.join(path, "general_prompts.json"), self.general_prompts
)
with open(os.path.join(path, "categories.json"), "w+") as f:
json.dump(self.categories, f, indent=2)
Loading
Loading