Merge pull request #3179 from Alibehram11/master

Revise README for Jarvis with detailed features and setup

Author: geekcomputers

JARVIS/README.md

@@ -1,16 +1,74 @@
-# JARVIS
-patch-5<br>
-It can Control windows programs with your voice.<br>
-What can it do:
-1. It can tell you time.<br/>
-2. It can open, These of the following:-<br/>a.) Notepad<br/>
-                                            b.) Calculator<br/>
-                                            c.) Sticky Note<br/>
-                                            d.) PowerShell<br/>
-                                            e.) MS Paint<br/>
-                                            f.) cmd<br/>
-                                            g.) Browser (Internet Explorer)<br/>
-    
-It will make your experience better while using the Windows computer.
-===========================================================================
-It demonstrates Controlling windows programs with your voice.
+# Jarvis Local Desktop Assistant
+
+Jarvis is a local voice-first desktop assistant for Windows. It talks through the terminal, uses LM Studio on `localhost`, and can safely open apps, close visible windows, open websites, and search Google.
+
+## Features
+
+- Voice-first terminal assistant with optional type mode.
+- Understands Turkish or English input, answers in English.
+- Developer mode for prompts, raw model output, and token usage when LM Studio reports it.
+- Safe app launcher using Start Menu/Desktop/Programs indexes.
+- Explicit memory only: Jarvis remembers notes only when you say `remember that ...`.
+- Extra tools: `/help` and `/apps <name>` in type mode.
+
+## Safety
+
+Jarvis does not run arbitrary shell commands from the model. AI output is restricted to safe actions like:
+
+- `open_app:<name>`
+- `open_web:<site-or-url>`
+- `search_google:<query>`
+- `close_app:<window-name>`
+- `open_cmd`
+- `chat`
+
+Blocked intents include install, uninstall, delete, remove, update, download, edit, modify, registry, PowerShell, terminal, scripts, screenshots, recordings, and email.
+
+## Setup
+
+1. Start LM Studio.
+2. Load `google/gemma-3-4b`.
+3. Start the LM Studio local server at:
+
+```text
+http://localhost:1234/v1
+```
+
+4. Install Python dependencies if needed:
+
+```powershell
+python -m pip install -r requirements.txt
+```
+
+## Run
+
+Voice mode:
+
+```powershell
+python .\jarvis.py
+```
+
+Type mode:
+
+```powershell
+python .\jarvis.py --type
+```
+
+Or double-click:
+
+```text
+start_jarvis_agent.bat
+```
+
+## Useful Commands
+
+- `developer mode` or `development mode` - show prompts/raw outputs/token usage.
+- `normal mode` - hide developer details.
+- `remember that my favorite editor is VS Code` - save an explicit memory note.
+- `clear memory` - clear saved notes.
+- `/apps code` - list matching indexed apps in type mode.
+- `/help` - show command help.
+
+## Notes
+
+The microphone feature uses Google speech recognition through `SpeechRecognition`. This is the one privacy tradeoff in the current version. LM Studio model calls stay on localhost.

JARVIS/__init__.py

@@ -0,0 +1,2 @@
+"""Jarvis local desktop assistant."""
+

JARVIS/actions.py

@@ -0,0 +1,153 @@
+import importlib
+import os
+import webbrowser
+from urllib.parse import quote_plus, urlparse
+
+from .ai import ask_model, classify_action
+from .apps import find_application
+from .config import CMD_OPEN_PHRASES, KNOWN_SITES
+from .memory import remember_note
+from .safety import BLOCKED_APPS, is_dangerous_request
+from .text_utils import normalize_text
+
+try:
+    win32con = importlib.import_module("win32con")
+    win32gui = importlib.import_module("win32gui")
+except ImportError:
+    win32con = None
+    win32gui = None
+
+
+def open_application(name):
+    app = find_application(name)
+    if not app:
+        return f"I could not find a safe installed app named {name}."
+    path = app["path"]
+    os.startfile(path)
+    return f"Opening {app['name']}."
+
+
+def close_visible_window(name):
+    if win32gui is None or win32con is None:
+        return "Close is not available because pywin32 is missing."
+    wanted = normalize_text(name)
+    if not wanted or wanted in BLOCKED_APPS:
+        return "That close request is blocked for safety."
+    matches = []
+
+    def callback(hwnd, _):
+        if not win32gui.IsWindowVisible(hwnd):
+            return
+        title = win32gui.GetWindowText(hwnd)
+        if wanted in normalize_text(title):
+            matches.append((hwnd, title))
+
+    win32gui.EnumWindows(callback, None)
+    if not matches:
+        return f"I could not find an open window matching {name}."
+    hwnd, title = matches[0]
+    win32gui.PostMessage(hwnd, win32con.WM_CLOSE, 0, 0)
+    return f"Closing {title or name}."
+
+
+def is_safe_url(url):
+    parsed = urlparse(url)
+    return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
+
+
+def open_web_target(target):
+    cleaned = normalize_text(target)
+    url = KNOWN_SITES.get(cleaned)
+    if not url and "." in cleaned:
+        url = target if target.startswith(("http://", "https://")) else f"https://{target}"
+    if not url or not is_safe_url(url):
+        return "I can only open safe web addresses."
+    webbrowser.open(url)
+    return f"Opening {url}."
+
+
+def run_action(action):
+    if not action:
+        return ""
+    action = action.strip()
+    lowered = action.lower()
+    if lowered == "open_cmd":
+        os.startfile("cmd.exe")
+        return "Opening Command Prompt."
+    if lowered == "blocked" or is_dangerous_request(action):
+        return "I cannot do that for safety."
+    if lowered.startswith("open_web:"):
+        return open_web_target(action.split(":", 1)[1].strip())
+    if lowered.startswith("search_google:"):
+        query = action.split(":", 1)[1].strip().strip("<> ")
+        if not query or is_dangerous_request(query):
+            return "I cannot search that for safety."
+        webbrowser.open(f"https://www.google.com/search?q={quote_plus(query)}")
+        return f"Searching Google for {query}."
+    if lowered.startswith("open_app:"):
+        return open_application(action.split(":", 1)[1].strip())
+    if lowered.startswith("close_app:"):
+        return close_visible_window(action.split(":", 1)[1].strip())
+    if lowered == "chat":
+        return ""
+    return ""
+
+
+def rule_based_action(text):
+    cleaned = normalize_text(text)
+    if not cleaned:
+        return ""
+    if cleaned in CMD_OPEN_PHRASES:
+        return "open_cmd"
+    if is_dangerous_request(cleaned):
+        return "blocked"
+
+    search_prefixes = ["search for ", "google search ", "look up ", "find ", "ara ", "google da ara "]
+    for prefix in search_prefixes:
+        if cleaned.startswith(prefix):
+            query = cleaned.removeprefix(prefix).strip()
+            return f"search_google:{query}" if query else ""
+    if cleaned.endswith(" ara"):
+        query = cleaned[: -len(" ara")].strip()
+        return f"search_google:{query}" if query else ""
+
+    close_prefixes = ["close ", "kapat ", "close the ", "can you close "]
+    for prefix in close_prefixes:
+        if cleaned.startswith(prefix):
+            app = cleaned.removeprefix(prefix).strip()
+            return f"close_app:{app}" if app else ""
+    if cleaned.endswith(" kapat"):
+        app = cleaned[: -len(" kapat")].strip()
+        return f"close_app:{app}" if app else ""
+
+    open_prefixes = ["open ", "launch ", "start ", "can you open ", "please open ", "ac ", "aç "]
+    suffix_open_words = [" ac", " aç", " i ac", " i aç", " u ac", " u aç"]
+    for site, url in KNOWN_SITES.items():
+        if cleaned in {site, f"open {site}", f"{site} ac", f"{site} aç"}:
+            return f"open_web:{url}"
+    for prefix in open_prefixes:
+        if cleaned.startswith(prefix):
+            target = cleaned.removeprefix(prefix).strip()
+            if target in KNOWN_SITES:
+                return f"open_web:{target}"
+            return f"open_app:{target}" if target else ""
+    for suffix in suffix_open_words:
+        if cleaned.endswith(suffix):
+            target = cleaned[: -len(suffix)].strip()
+            if target in KNOWN_SITES:
+                return f"open_web:{target}"
+            return f"open_app:{target}" if target else ""
+    return ""
+
+
+def handle_user_text(text):
+    cleaned = normalize_text(text)
+    if cleaned.startswith("remember that "):
+        return remember_note(text.split("remember that", 1)[1].strip())
+    action = rule_based_action(text) or classify_action(text)
+    if action.strip().lower() == "blocked" and not is_dangerous_request(text):
+        action = "chat"
+    answer = run_action(action)
+    if answer:
+        return answer
+    return ask_model(text)

JARVIS/ai.py

@@ -0,0 +1,64 @@
+from openai import OpenAI, OpenAIError
+
+from . import state
+from .config import MAX_OUTPUT_TOKENS, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MODEL
+from .memory import memory_context
+from .prompts import ACTION_CLASSIFIER_PROMPT, ASSISTANT_PROMPT
+from .text_utils import clean_assistant_output
+
+
+def lm_client():
+    return OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_BASE_URL)
+
+
+def usage_dict(response):
+    usage = getattr(response, "usage", None)
+    if usage is None:
+        return {}
+    if hasattr(usage, "model_dump"):
+        return usage.model_dump()
+    if isinstance(usage, dict):
+        return usage
+    return {
+        name: getattr(usage, name)
+        for name in ("input_tokens", "output_tokens", "total_tokens")
+        if hasattr(usage, name)
+    }
+
+
+def debug_response(kind, prompt, response):
+    state.debug(f"{kind} model", OPENAI_MODEL)
+    state.debug(f"{kind} prompt", prompt)
+    state.debug(f"{kind} raw output", getattr(response, "output_text", ""))
+    state.debug(f"{kind} usage", usage_dict(response) or "not reported by server")
+
+
+def ask_model(text):
+    prompt = (
+        f"{ASSISTANT_PROMPT}\n"
+        f"Saved memory:\n{memory_context()}\n\n"
+        f"Answer in English.\n"
+        f"User: {text}"
+    )
+    try:
+        response = lm_client().responses.create(
+            model=OPENAI_MODEL,
+            input=prompt,
+            max_output_tokens=MAX_OUTPUT_TOKENS,
+        )
+    except OpenAIError as exc:
+        state.debug("chat error", str(exc))
+        return "I cannot reach LM Studio right now. Start the local server and try again."
+    debug_response("chat", prompt, response)
+    return clean_assistant_output(response.output_text)
+
+
+def classify_action(text):
+    prompt = f"{ACTION_CLASSIFIER_PROMPT}\nUser: {text}"
+    try:
+        response = lm_client().responses.create(model=OPENAI_MODEL, input=prompt, max_output_tokens=40)
+    except OpenAIError as exc:
+        state.debug("action error", str(exc))
+        return "chat"
+    debug_response("action", prompt, response)
+    return response.output_text.strip().splitlines()[0].strip()