diff --git a/README.md b/README.md index b38db84..0ac9b13 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,16 @@ MYTHIC__SERVER_PORT=7443 MYTHIC__TIMEOUT=-1 MYTHIC__PAYLOAD_PORT_HTTP=1337 ``` -If you need local LLM via [ollama](https://github.com/ollama/ollama) add to env also +If you need local LLM via [ollama](https://github.com/ollama/ollama) add to env also this. We recommend these models [Mistral](https://ollama.com/library/mistral), [Qwen3-ab](https://ollama.com/jaahas/qwen3-abliterated) , [llama3.1:8b](https://ollama.com/library/llama3.1), [llama3.1-ab:8b](https://ollama.com/mannix/llama3.1-8b-abliterated) (more freely), [Qwen2.5-coder:32b](https://ollama.com/library/qwen2.5-coder) which depends on your resources. ```env +LLMSERVICE__LOCAL=TRUE LLMSERVICE__API_URL=http://localhost:69228 LLMSERVICE__API_KEY=super_secret_key LLMSERVICE__TIMEOUT=120 LLMSERVICE__DEFAULT_MODEL=mistral ``` +Otherwise, the values will be taken from the [config](https://github.com/eogod/EAGLE/blob/main/backend/app/core/config.py), and you can see the validation there. + Install dependencies ```bash # Poetry install (python3.13) diff --git a/backend/app/cmd/llm_analysis.py b/backend/app/cmd/llm_analysis.py index 53d22b8..57bd74d 100644 --- a/backend/app/cmd/llm_analysis.py +++ b/backend/app/cmd/llm_analysis.py @@ -1,10 +1,23 @@ """ Module for unified interface to LLM services """ +import os +from dotenv import load_dotenv + import g4f +from ollama import Client from fastapi import HTTPException -# TODO: ollama, openrouter , yandexgpt support + +from app.core.llm_templ import LLMTemplates + # Настройка g4f g4f.debug.logging = True +load_dotenv() +IS_LOCAL_LLM: bool = os.getenv('LLMSERVICE__LOCAL') +if IS_LOCAL_LLM: + client_ollama = Client(host=os.getenv('LLMSERVICE__API_URL')) +else: + client_ollama = None + class LLMService: def __init__(self): @@ -20,43 +33,60 @@ async def query_llm(self, prompt: str, provider_name: str = None) -> str: """ # TODO: support custom system prompt try: - # Если указан провайдер, используем его - if provider_name and provider_name in self.providers: - provider = self.providers[provider_name] - try: - response = await g4f.ChatCompletion.create_async( - model=g4f.models.default, - messages=[{"role": "user", "content": prompt}], - provider=provider - ) - return response - except Exception as e: - raise HTTPException( - status_code=500, - detail=f"Provider {provider_name} failed: {str(e)}" - ) from e - - # Если провайдер не указан или не найден, пробуем разные - for name, provider in self.providers.items(): - try: - response = await g4f.ChatCompletion.create_async( - model=g4f.models.default, - messages=[{"role": "user", "content": prompt}], - provider=provider - ) - if response and len(response) > 0: - return response - except Exception as e: - print(f"Provider {name} failed: {e}") - continue - - return "Не удалось получить ответ от ни одного провайдера" - + if IS_LOCAL_LLM: + res: str = self._local_llm(prompt) + else: + res: str = await self._g4f_llm(prompt, provider_name) + return res except Exception as e: raise HTTPException( status_code=500, detail=f"Ошибка при запросе к LLM: {str(e)}" ) from e + def _local_llm(self, prompt: str) -> str: + """ query with local llm ollama """ + res: dict = client_ollama.generate( + model=os.getenv('LLMSERVICE__DEFAULT_MODEL'), + prompt=prompt, system=LLMTemplates.SYSTEM_PROMT + ) + # remove think text for deepseek-r1, qwen , qwq models + parts_th = res.response.rsplit('', 1) + return parts_th[-1] if len(parts_th) > 1 else res.response + + async def _g4f_llm(self, prompt: str, provider_name: str) -> str: + """ query with g4f proxy providers like DDG """ + # Если указан провайдер, используем его + if provider_name and provider_name in self.providers: + provider = self.providers[provider_name] + try: + response = await g4f.ChatCompletion.create_async( + model=g4f.models.default, + messages=[{"role": "user", "content": prompt}], + provider=provider + ) + return response + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Provider {provider_name} failed: {str(e)}" + ) from e + + # Если провайдер не указан или не найден, пробуем разные + for name, provider in self.providers.items(): + try: + response = await g4f.ChatCompletion.create_async( + model=g4f.models.default, + messages=[{"role": "user", "content": prompt}], + provider=provider + ) + if response and len(response) > 0: + return response + except Exception as e: + print(f"Provider {name} failed: {e}") + continue + + return "Не удалось получить ответ от ни одного провайдера" + # Инициализация сервиса llm_service = LLMService() diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 2389633..f0c80f9 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -63,10 +63,11 @@ class Mythic(BaseModel): class LLMservice(BaseModel): """ env format like LLMSERVICE__API_URL=http... """ - API_URL: HttpUrl = "http://localhost:69228" # Для локального Ollama - API_KEY: str = None - TIMEOUT: int = 120 - DEFAULT_MODEL: str = "mistral" + local: bool = False + api_url: HttpUrl = "http://localhost:69228" # Для локального Ollama + api_key: str = None + timeout: int = 120 + default_model: str = "mistral" class Settings(BaseSettings): diff --git a/backend/poetry.lock b/backend/poetry.lock index 9789d62..18de86a 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1138,7 +1138,7 @@ version = "0.16.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, @@ -1150,7 +1150,7 @@ version = "1.0.9" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, @@ -1228,7 +1228,7 @@ version = "0.28.1" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, @@ -1614,6 +1614,22 @@ files = [ {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] +[[package]] +name = "ollama" +version = "0.5.3" +description = "The official Python client for Ollama." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "ollama-0.5.3-py3-none-any.whl", hash = "sha256:a8303b413d99a9043dbf77ebf11ced672396b59bec27e6d5db67c88f01b279d2"}, + {file = "ollama-0.5.3.tar.gz", hash = "sha256:40b6dff729df3b24e56d4042fd9d37e231cee8e528677e0d085413a1d6692394"}, +] + +[package.dependencies] +httpx = ">=0.27" +pydantic = ">=2.9" + [[package]] name = "packaging" version = "25.0" @@ -2895,4 +2911,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "816afb16f7b490d97ee2c0253bd755ab8ea970cd83f170f88be060bf690d54da" +content-hash = "593b365ca022d616ad66e61f230329a8ef4b069f2393791763e2f29b03fc1a5b" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c7712ce..155181d 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -22,6 +22,7 @@ gql = "^3.5.3" dotenv = "^0.9.9" autopep8 = "^2.3.2" g4f = "^0.5.8.1" +ollama = "^0.5.3" [tool.poetry.group.dev.dependencies] coverage = "^7.9.1"