diff --git a/.env.example b/.env.example index b927624..54c752a 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,8 @@ TOGETHER_API_KEY= HYPERBOLIC_API_KEY= CEREBRAS_API_KEY= SAMBANOVA_API_KEY= +DEEPSEEK_API_KEY= +CUSTOM_API_KEY= # Service URLs DOCIO_URL=http://docio:6979/api/docio diff --git a/.gitattributes b/.gitattributes index 08e60cd..31b0a04 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,7 +19,7 @@ # https://git-scm.com/docs/gitattributes#_text *.css text *.html text -*.js text +*.js* text *.md text *.py text *.sh text diff --git a/.github/workflows/ci-win.yml b/.github/workflows/ci-win.yml index 4671b82..99c3276 100644 --- a/.github/workflows/ci-win.yml +++ b/.github/workflows/ci-win.yml @@ -17,9 +17,23 @@ concurrency: cancel-in-progress: true jobs: + check_changes: + name: Check for changes + runs-on: ubuntu-latest + outputs: + has-changes: ${{ steps.check.outputs.has-changes }} + steps: + - name: Check + id: check + uses: jiahuei/check-changes-action@v0 + with: + watch-dirs: "clients/python/ services/api/ services/docio/ docker/ .github/" + pyinstaller_electron_app: name: PyInstaller JamAIBase Electron App Compilation runs-on: windows-11-desktop + needs: check_changes + if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push' timeout-minutes: 60 steps: @@ -98,6 +112,8 @@ jobs: pyinstaller_api: name: PyInstaller API Service Compilation runs-on: windows-11-desktop + needs: check_changes + if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push' timeout-minutes: 60 steps: @@ -179,6 +195,8 @@ jobs: pyinstaller_docio: name: PyInstaller DocIO Service Compilation runs-on: windows-11-desktop + needs: check_changes + if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push' timeout-minutes: 60 steps: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aea4711..b69078a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +name: CI (OSS) on: pull_request: @@ -18,10 +18,38 @@ concurrency: cancel-in-progress: true jobs: + check_changes: + name: Check for changes + runs-on: ubuntu-latest + outputs: + has-changes: ${{ steps.check.outputs.has-changes }} + steps: + - name: Check + id: check + uses: jiahuei/check-changes-action@v0 + with: + watch-dirs: "clients/python/ services/api/ docker/ .github/" + + sdk_tests_noop: + # This job is needed so that status checks can still pass + # This is because strategy matrix is evaluated after if condition + name: SDK unit tests + runs-on: ubuntu-latest + needs: check_changes + if: ${{ !(needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push') }} + strategy: + matrix: + python-version: ["3.10"] + timeout-minutes: 2 + steps: + - name: No-op + run: echo Tests skipped !!! + sdk_tests: name: SDK unit tests runs-on: ubuntu-latest-l - # runs-on: namespace-profile-ubuntu-latest-8cpu-16gb-96gb + needs: check_changes + if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push' strategy: matrix: python-version: ["3.10"] @@ -42,6 +70,12 @@ jobs: run: | git --version + - name: Check Docker Version + run: docker version + + - name: Check Docker Compose Version + run: docker compose version + - name: Remove cloud-only modules and install Python client run: | set -e @@ -49,11 +83,10 @@ jobs: cd clients/python python -m pip install .[test] - - name: Check Docker Version - run: docker version - - - name: Check Docker Compose Version - run: docker compose version + - name: Install ffmpeg + run: | + set -e + sudo apt-get update -qq && sudo apt-get install ffmpeg libavcodec-extra -y - name: Authenticating to the Container registry run: echo $JH_PAT | docker login ghcr.io -u tanjiahuei@gmail.com --password-stdin @@ -87,13 +120,14 @@ jobs: TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} HYPERBOLIC_API_KEY: ${{ secrets.HYPERBOLIC_API_KEY }} + CUSTOM_API_KEY: ${{ secrets.CUSTOM_API_KEY }} - name: Launch services (OSS) id: launch_oss timeout-minutes: 20 run: | set -e - docker compose -p jamai -f docker/compose.cpu.yml --profile minio up --quiet-pull -d --wait + docker compose -p jamai -f docker/compose.cpu.yml --profile minio --profile kopi up --quiet-pull -d --wait env: COMPOSE_DOCKER_CLI_BUILD: 1 DOCKER_BUILDKIT: 1 @@ -116,7 +150,7 @@ jobs: --junitxml=junit/test-results-${{ matrix.python-version }}.xml \ --cov-report=xml \ --no-flaky-report \ - clients/python/tests/oss + clients/python/tests/oss/ - name: Inspect owl logs if Python SDK tests failed if: failure() && steps.python_sdk_test_oss.outcome == 'failure' @@ -170,11 +204,6 @@ jobs: --no-flaky-report \ clients/python/tests/oss/test_file.py - - name: Inspect owl logs if Python SDK tests failed - if: failure() && steps.python_sdk_test_oss_file.outcome == 'failure' - timeout-minutes: 1 - run: docker exec jamai-owl-1 cat /app/api/logs/owl.log - lance_tests: name: Lance tests runs-on: ubuntu-latest diff --git a/.github/workflows/github_bot.yml b/.github/workflows/github_bot.yml new file mode 100644 index 0000000..f985fc0 --- /dev/null +++ b/.github/workflows/github_bot.yml @@ -0,0 +1,44 @@ +name: JambuBot + +on: + issues: + types: [opened, edited] + pull_request: + types: [opened, synchronize] + +# Cancel in-progress CI jobs if there is a new push +# https://stackoverflow.com/a/72408109 +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + github-bot: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true # Ensure submodules are checked out + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: "1.18" + + - name: Launching JambuBot + run: | + cd examples/github-bot/github-bot/src + go build -o github_bot cmd/main.go + ./github_bot + env: + TRIAGE_BOT_APP_ID: ${{ secrets.TRIAGE_BOT_APP_ID }} + TRIAGE_BOT_INSTALLATION_ID: ${{ secrets.TRIAGE_BOT_INSTALLATION_ID }} + TRIAGE_BOT_PRIVATE_KEY: ${{ secrets.TRIAGE_BOT_PRIVATE_KEY }} + TRIAGE_BOT_JAMAI_KEY: ${{ secrets.TRIAGE_BOT_JAMAI_KEY }} + TRIAGE_BOT_JAMAI_PROJECT_ID: ${{ secrets.TRIAGE_BOT_JAMAI_PROJECT_ID }} + TRIAGE_BOT_NAME: ${{github.actor}} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_EVENT_PATH: ${{ github.event_path }} diff --git a/.prettierignore b/.prettierignore index 42af708..bf91f7b 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,24 +1,28 @@ +# OS +thumbs.db +.DS_Store + +# Internal references, dependencies, temporary folders & files +.env +archive/ +**/__ref__/ +*.log +*.lock +*.db +*.parquet + # Python __pycache__/ -*.py[cod] +*.py* *$py.class *.egg-info .pytest_cache .ipynb_checkpoints venv/ - -# Frontend -services/app -clients/typescript - -# Test files clients/python/tests/**/* -# Internal references, dependencies, temporary folders & files -archive/ -/dependencies/ -logs/ -*.log +# pip +**/build/ # pytest-cov **/.coverage* @@ -26,14 +30,12 @@ logs/ /htmlcov /coverage.xml -# pip -**/build/ +# jest-cov +**/coverage/* -# Docs -/docs/source/generated/ -/docs/source/api/generated/ -/docs/source/specs/ +# JavaScript +**/node_modules/ -# OS -thumbs.db -.DS_Store +# Frontend +services/app +clients/typescript diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ccb399..c5f2d0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,32 +16,79 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] -Backend - owl (API server) +### ADDED + +Python SDK - jamaibase -- Fix bge-small embedding size (1024 -> 384) -- Correctly filter models at auth level -- Fix ollama model deployment config +- Add `CodeGenConfig` for python code execution #446 -Frontend +TS SDK - jamaibase + +UI -- Added support for multiple multiturn columns in Chat table chat view. -- Added multiturn chat toggle to column settings. +- Support chat mode multiturn option in add column and column resize #451 +- Support `audio` data type #457 -Docker +Backend - owl (API server) -- Added Mac Apple Silicon `compose.mac.yml` -- Update `ollama.yml` to use Qwen2.5 3B -- Fix ollama default config +- GenTable + - Support `audio` input column and data type #443 + - Support python code execution column #446 + - **Breaking**: Add `Page` column to knowledge table #464 +- LLM + - Support function calling # 435 + - Support DeepSeek models #466 +- Billing + - Include background tasks for processing billing events #462 +- Auth + - Support specific user role in organization invite #446 + - Include background tasks for setting project updated at datetime #462 +- Handle and allow setting of file upload size limits for `embed file`, `image` and `audio` file types #443 -## [v0.3.1] (2024-11-26) +CI/CD -This is a bug fix release for frontend code. SDKs are not affected. +- Added a new CI workflow for cloud environments in`.github/workflows/ci.cloud.yml` #440 +- Add dummy test job to pass status checks if skipped #468 +- Added a `check_changes` job to the CI workflows to conditionally run SDK tests based on changes. #462 ### CHANGES / FIXED -Frontend +Python SDK - jamaibase + +TS SDK - jamaibase -- Enable Projects for OSS +- Update the `uploadFile` method in `index.ts` to remove the trailing slash from the API endpoint #462 + +UI + +- Remove unnecessary load function rerunning on client navigation #454 +- Add more export options with confirmation #459 +- Obfuscated external keys and credit values for non-admin users in the `+layout.server.ts` to enhance security and privacy #459 +- Update `FileSelect.svelte` and `NewRow.svelte` to remove trailing slashes from the file upload API endpoint #462 +- Bug fixes: + - Fix chat table scrollbar not showing issue #459 + - Fix keyboard navigation #459 + - Fix inappropriate model not showing issue in knowledge table column settings #459 + +Backend - owl (API server) + +- GenTable + - **Breaking**: Change `file` data type to `image` data type #460 +- LLM + - Handle usage tracking and improve error handling #462 + - Bug fixes + - Fix model config embedding size #441 + - Fix bug with default model choosing invalid models #442 + - Fix regen sequences issue after columns reordering #455 + +CI/CD + +- Dockerfile: Added `ffmpeg` installation for audio processing. #443 +- Dependency Updates: + - Set `litellm` to version `1.50.0` #443 + - Add `pydub` as a dependency for audio processing #443 + +### REMOVED ## [v0.3] (2024-11-20) @@ -427,7 +474,7 @@ Backend - owl (API server) - Windows: StreamResponse from FastAPI accumulates all SSE before yielding everything all at once to the client Fixes #145 - Enabled scanned pdf upload. Fixes #131 - Dependencies - - Support forked version of `unstructured-client==0.24.1`, changed nest-asyncio to ThreadPool, fixed the conflict with uvloop + - Support forked version of `unstructured-client==0.24.1`, changed `nest-asyncio` to `ThreadPool`, fixed the conflict with `uvloop` - Added `tenacity`, `pandas` - Bumped dependency versions @@ -441,7 +488,7 @@ Backend - Admin (cloud) - Improve insufficient credit error message: include quota/usage type in the message - Storage usage update is now a background process; fixes #87 -- Allow dot in the middle for project name and organization name. +- Allow dot in the middle for project name and organization name. - Update `models.json` in `set_model_config()` - Billing: Don't include Lance version directories in storage usage computation - Bug fixes diff --git a/clients/python/README.md b/clients/python/README.md index 413e7bf..8e1d122 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -160,7 +160,7 @@ table = jamai.table.create_action_table( p.ActionTableSchemaCreate( id="action-simple", cols=[ - p.ColumnSchemaCreate(id="image", dtype="file"), # Image input + p.ColumnSchemaCreate(id="image", dtype="image"), # Image input p.ColumnSchemaCreate(id="length", dtype="int"), # Integer input p.ColumnSchemaCreate(id="question", dtype="str"), p.ColumnSchemaCreate( @@ -557,7 +557,7 @@ def create_tables(jamai: JamAI): p.ActionTableSchemaCreate( id="action-simple", cols=[ - p.ColumnSchemaCreate(id="image", dtype="file"), # Image input + p.ColumnSchemaCreate(id="image", dtype="image"), # Image input p.ColumnSchemaCreate(id="length", dtype="int"), # Integer input p.ColumnSchemaCreate(id="question", dtype="str"), p.ColumnSchemaCreate( diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 93aebe1..579af76 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -101,6 +101,7 @@ dependencies = [ "Pillow>=10.0.1", "pydantic-settings>=2.0.3", "pydantic>=2.4.2", + "pydub~=0.25.1", "srsly>=2.4.8", "toml>=0.10.2", "typing_extensions>=4.10.0", @@ -112,6 +113,7 @@ lint = ["ruff~=0.5.7"] test = [ "flaky~=3.8.1", "mypy~=1.11.1", + "pydub~=0.25.1", "pytest-asyncio>=0.23.8", "pytest-cov~=5.0.0", "pytest-timeout>=2.3.1", diff --git a/clients/python/src/jamaibase/client.py b/clients/python/src/jamaibase/client.py index 68f70d5..d5df736 100644 --- a/clients/python/src/jamaibase/client.py +++ b/clients/python/src/jamaibase/client.py @@ -597,6 +597,7 @@ def generate_invite_token( self, organization_id: str, user_email: str = "", + user_role: str = "", valid_days: int = 7, ) -> str: """ @@ -606,6 +607,8 @@ def generate_invite_token( organization_id (str): Organization ID. user_email (str, optional): User email. Leave blank to disable email check and generate a public invite. Defaults to "". + user_role (str, optional): User role. + Leave blank to default to guest. Defaults to "". valid_days (int, optional): How many days should this link be valid for. Defaults to 7. Returns: @@ -615,7 +618,10 @@ def generate_invite_token( self.api_base, "/admin/backend/v1/invite_tokens", params=dict( - organization_id=organization_id, user_email=user_email, valid_days=valid_days + organization_id=organization_id, + user_email=user_email, + user_role=user_role, + valid_days=valid_days, ), response_model=None, ) @@ -1222,7 +1228,7 @@ def upload_file(self, file_path: str) -> FileUploadResponse: with open(file_path, "rb") as f: return self._post( self.api_base, - "/v1/files/upload/", + "/v1/files/upload", body=None, response_model=FileUploadResponse, files={ @@ -2209,7 +2215,9 @@ def health(self) -> dict[str, Any]: def model_info( self, name: str = "", - capabilities: list[Literal["completion", "chat", "image", "embed", "rerank"]] + capabilities: list[ + Literal["completion", "chat", "image", "audio", "tool", "embed", "rerank"] + ] | None = None, ) -> ModelInfoResponse: """ @@ -2217,7 +2225,7 @@ def model_info( Args: name (str, optional): The model name. Defaults to "". - capabilities (list[Literal["completion", "chat", "image", "embed", "rerank"]] | None, optional): + capabilities (list[Literal["completion", "chat", "image", "audio", "tool", "embed", "rerank"]] | None, optional): List of model capabilities to filter by. Defaults to None. Returns: @@ -2234,7 +2242,9 @@ def model_info( def model_names( self, prefer: str = "", - capabilities: list[Literal["completion", "chat", "image", "embed", "rerank"]] + capabilities: list[ + Literal["completion", "chat", "image", "audio", "tool", "embed", "rerank"] + ] | None = None, ) -> list[str]: """ @@ -2242,7 +2252,7 @@ def model_names( Args: prefer (str, optional): Preferred model name. Defaults to "". - capabilities (list[Literal["completion", "chat", "image", "embed", "rerank"]] | None, optional): + capabilities (list[Literal["completion", "chat", "image", "audio", "tool", "embed", "rerank"]] | None, optional): List of model capabilities to filter by. Defaults to None. Returns: @@ -4002,7 +4012,7 @@ async def upload_file(self, file_path: str) -> FileUploadResponse: with open(file_path, "rb") as f: return await self._post( self.api_base, - "/v1/files/upload/", + "/v1/files/upload", body=None, response_model=FileUploadResponse, files={ @@ -4989,7 +4999,9 @@ async def health(self) -> dict[str, Any]: async def model_info( self, name: str = "", - capabilities: list[Literal["completion", "chat", "image", "embed", "rerank"]] + capabilities: list[ + Literal["completion", "chat", "image", "audio", "tool", "embed", "rerank"] + ] | None = None, ) -> ModelInfoResponse: """ @@ -4997,7 +5009,7 @@ async def model_info( Args: name (str, optional): The model name. Defaults to "". - capabilities (list[Literal["completion", "chat", "image", "embed", "rerank"]] | None, optional): + capabilities (list[Literal["completion", "chat", "image", "audio", "tool", "embed", "rerank"]] | None, optional): List of model capabilities to filter by. Defaults to None. Returns: @@ -5014,7 +5026,9 @@ async def model_info( async def model_names( self, prefer: str = "", - capabilities: list[Literal["completion", "chat", "image", "embed", "rerank"]] + capabilities: list[ + Literal["completion", "chat", "image", "audio", "tool", "embed", "rerank"] + ] | None = None, ) -> list[str]: """ @@ -5022,7 +5036,7 @@ async def model_names( Args: prefer (str, optional): Preferred model name. Defaults to "". - capabilities (list[Literal["completion", "chat", "image", "embed", "rerank"]] | None, optional): + capabilities (list[Literal["completion", "chat", "image", "audio", "tool", "embed", "rerank"]] | None, optional): List of model capabilities to filter by. Defaults to None. Returns: diff --git a/clients/python/src/jamaibase/protocol.py b/clients/python/src/jamaibase/protocol.py index 31b8e70..1e76862 100644 --- a/clients/python/src/jamaibase/protocol.py +++ b/clients/python/src/jamaibase/protocol.py @@ -52,22 +52,27 @@ def sanitise_document_id_list(v: list[str]) -> list[str]: DocumentID = Annotated[str, AfterValidator(sanitise_document_id)] DocumentIDList = Annotated[list[str], AfterValidator(sanitise_document_id_list)] -EXAMPLE_CHAT_MODEL = "openai/gpt-4o-mini" - +EXAMPLE_CHAT_MODEL_IDS = ["openai/gpt-4o-mini"] # for openai embedding models doc: https://platform.openai.com/docs/guides/embeddings # for cohere embedding models doc: https://docs.cohere.com/reference/embed # for jina embedding models doc: https://jina.ai/embeddings/ # for voyage embedding models doc: https://docs.voyageai.com/docs/embeddings # for hf embedding models doc: check the respective hf model page, name should be ellm/{org}/{model} -EXAMPLE_EMBEDDING_MODEL = "openai/text-embedding-3-small-512" - +EXAMPLE_EMBEDDING_MODEL_IDS = [ + "openai/text-embedding-3-small-512", + "ellm/sentence-transformers/all-MiniLM-L6-v2", +] # for cohere reranking models doc: https://docs.cohere.com/reference/rerank-1 # for jina reranking models doc: https://jina.ai/reranker # for colbert reranking models doc: https://docs.voyageai.com/docs/reranker # for hf embedding models doc: check the respective hf model page, name should be ellm/{org}/{model} -EXAMPLE_RERANKING_MODEL = "cohere/rerank-multilingual-v3.0" +EXAMPLE_RERANKING_MODEL_IDS = [ + "cohere/rerank-multilingual-v3.0", + "ellm/cross-encoder/ms-marco-TinyBERT-L-2", +] IMAGE_FILE_EXTENSIONS = [".jpeg", ".jpg", ".png", ".gif", ".webp"] +AUDIO_FILE_EXTENSIONS = [".mp3", ".wav"] DOCUMENT_FILE_EXTENSIONS = [ ".pdf", ".txt", @@ -182,7 +187,11 @@ class _ModelPrice(BaseModel): 'Unique identifier in the form of "{provider}/{model_id}". ' "Users will specify this to select a model." ), - examples=[EXAMPLE_CHAT_MODEL, EXAMPLE_EMBEDDING_MODEL, EXAMPLE_RERANKING_MODEL], + examples=[ + EXAMPLE_CHAT_MODEL_IDS[0], + EXAMPLE_EMBEDDING_MODEL_IDS[0], + EXAMPLE_RERANKING_MODEL_IDS[0], + ], ) name: str = Field( description="Name of the model.", @@ -192,10 +201,10 @@ class _ModelPrice(BaseModel): class LLMModelPrice(_ModelPrice): input_cost_per_mtoken: float = Field( - description="Cost in USD per million (mega) input / prompt token.", + description="Cost in USD per million input / prompt token.", ) output_cost_per_mtoken: float = Field( - description="Cost in USD per million (mega) output / completion token.", + description="Cost in USD per million output / completion token.", ) @@ -206,7 +215,7 @@ class EmbeddingModelPrice(_ModelPrice): class RerankingModelPrice(_ModelPrice): - cost_per_ksearch: float = Field(description="Cost in USD for a thousand searches.") + cost_per_ksearch: float = Field(description="Cost in USD for a thousand (kilo) searches.") class ModelPrice(BaseModel): @@ -674,7 +683,7 @@ class RAGParams(BaseModel): reranking_model: str | None = Field( default=None, description="Reranking model to use for hybrid search.", - examples=[EXAMPLE_RERANKING_MODEL, None], + examples=[EXAMPLE_RERANKING_MODEL_IDS[0], None], ) search_query: str = Field( default="", @@ -748,7 +757,7 @@ class ModelInfo(BaseModel): 'Unique identifier in the form of "{provider}/{model_id}". ' "Users will specify this to select a model." ), - examples=[EXAMPLE_CHAT_MODEL], + examples=EXAMPLE_CHAT_MODEL_IDS, ) object: str = Field( default="model", @@ -771,7 +780,9 @@ class ModelInfo(BaseModel): description="The organization that owns the model.", examples=["openai"], ) - capabilities: list[Literal["completion", "chat", "image", "embed", "rerank"]] = Field( + capabilities: list[ + Literal["completion", "chat", "image", "audio", "tool", "embed", "rerank"] + ] = Field( description="List of capabilities of model.", examples=[["chat"]], ) @@ -785,7 +796,7 @@ class ModelDeploymentConfig(BaseModel): 'For example, you can map "openai/gpt-4o" calls to "openai/gpt-4o-2024-08-06". ' 'For vLLM with OpenAI compatible server, use "openai/".' ), - examples=[EXAMPLE_CHAT_MODEL], + examples=EXAMPLE_CHAT_MODEL_IDS, ) api_base: str = Field( default="", @@ -836,7 +847,7 @@ class EmbeddingModelConfig(ModelConfig): 'For self-hosted models with Infinity, use "ellm/{org}/{model}". ' "Users will specify this to select a model." ), - examples=["ellm/sentence-transformers/all-MiniLM-L6-v2", EXAMPLE_EMBEDDING_MODEL], + examples=EXAMPLE_EMBEDDING_MODEL_IDS, ) embedding_size: int = Field( description="Embedding size of the model", @@ -870,7 +881,7 @@ class RerankingModelConfig(ModelConfig): 'For self-hosted models with Infinity, use "ellm/{org}/{model}". ' "Users will specify this to select a model." ), - examples=["ellm/cross-encoder/ms-marco-TinyBERT-L-2", EXAMPLE_RERANKING_MODEL], + examples=EXAMPLE_RERANKING_MODEL_IDS, ) capabilities: list[Literal["rerank"]] = Field( default=["rerank"], @@ -959,6 +970,17 @@ def sanitise_name(v: str) -> str: MessageName = Annotated[str, AfterValidator(sanitise_name)] +class MessageToolCallFunction(BaseModel): + arguments: str + name: str | None + + +class MessageToolCall(BaseModel): + id: str | None + function: MessageToolCallFunction + type: str + + class ChatEntry(BaseModel): """Represents a message in the chat context.""" @@ -1000,6 +1022,11 @@ def coerce_input(cls, value: Any) -> str | list[dict[str, str | dict[str, str]]] return str(value) +class ChatCompletionChoiceOutput(ChatEntry): + tool_calls: list[MessageToolCall] | None = None + """List of tool calls if the message includes tool call responses.""" + + class ChatThread(BaseModel): object: str = Field( default="chat.thread", @@ -1029,7 +1056,9 @@ class CompletionUsage(BaseModel): class ChatCompletionChoice(BaseModel): - message: ChatEntry = Field(description="A chat completion message generated by the model.") + message: ChatEntry | ChatCompletionChoiceOutput = Field( + description="A chat completion message generated by the model." + ) index: int = Field(description="The index of the choice in the list of choices.") finish_reason: str | None = Field( default=None, @@ -1049,7 +1078,7 @@ def text(self) -> str: class ChatCompletionChoiceDelta(ChatCompletionChoice): @computed_field @property - def delta(self) -> ChatEntry: + def delta(self) -> ChatEntry | ChatCompletionChoiceOutput: return self.message @@ -1157,7 +1186,7 @@ class ChatCompletionChunk(BaseModel): ) @property - def message(self) -> ChatEntry | None: + def message(self) -> ChatEntry | ChatCompletionChoiceOutput | None: return self.choices[0].message if len(self.choices) > 0 else None @property @@ -1188,6 +1217,49 @@ class GenTableStreamChatCompletionChunk(ChatCompletionChunk): row_id: str +class FunctionParameter(BaseModel): + type: str = Field( + default="", description="The type of the parameter, e.g., 'string', 'number'." + ) + description: str = Field(default="", description="A description of the parameter.") + enum: list[str] = Field( + default=[], description="An optional list of allowed values for the parameter." + ) + + +class FunctionParameters(BaseModel): + type: str = Field( + default="object", description="The type of the parameters object, usually 'object'." + ) + properties: dict[str, FunctionParameter] = Field( + description="The properties of the parameters object." + ) + required: list[str] = Field(description="A list of required parameter names.") + additionalProperties: bool = Field( + default=False, description="Whether additional properties are allowed." + ) + + +class Function(BaseModel): + name: str = Field(default="", description="The name of the function.") + description: str = Field(default="", description="A description of what the function does.") + parameters: FunctionParameters = Field(description="The parameters for the function.") + + +class Tool(BaseModel): + type: str = Field(default="function", description="The type of the tool, e.g., 'function'.") + function: Function = Field(description="The function details of the tool.") + + +class ToolChoiceFunction(BaseModel): + name: str = Field(default="", description="The name of the function.") + + +class ToolChoice(BaseModel): + type: str = Field(default="function", description="The type of the tool, e.g., 'function'.") + function: ToolChoiceFunction = Field(description="Select a tool for the chat model to use.") + + class ChatRequest(BaseModel): id: str = Field( default="", @@ -1295,6 +1367,48 @@ def convert_stop(cls, v: list[str] | None) -> list[str] | None: return v +class ChatRequestWithTools(ChatRequest): + tools: list[Tool] = Field( + description="A list of tools available for the chat model to use.", + min_length=1, + examples=[ + # --- [Tool Function] --- + # def get_delivery_date(order_id: str) -> datetime: + # # Connect to the database + # conn = sqlite3.connect('ecommerce.db') + # cursor = conn.cursor() + # # ... + [ + Tool( + type="function", + function=Function( + name="get_delivery_date", + description="Get the delivery date for a customer's order.", + parameters=FunctionParameters( + type="object", + properties={ + "order_id": FunctionParameter( + type="string", description="The customer's order ID." + ) + }, + required=["order_id"], + additionalProperties=False, + ), + ), + ) + ], + ], + ) + tool_choice: str | ToolChoice = Field( + default="auto", + description="Set `auto` to let chat model pick a tool or select a tool for the chat model to use.", + examples=[ + "auto", + ToolChoice(type="function", function=ToolChoiceFunction(name="get_delivery_date")), + ], + ) + + class EmbeddingRequest(BaseModel): input: str | list[str] = Field( description=( @@ -1309,7 +1423,7 @@ class EmbeddingRequest(BaseModel): "The ID of the model to use. " "You can use the List models API to see all of your available models." ), - examples=[EXAMPLE_EMBEDDING_MODEL], + examples=EXAMPLE_EMBEDDING_MODEL_IDS, ) type: Literal["query", "document"] = Field( default="document", @@ -1424,7 +1538,8 @@ class DtypeCreateEnum(str, Enum, metaclass=MetaEnum): float_ = "float" bool_ = "bool" str_ = "str" - file_ = "file" + image_ = "image" + audio_ = "audio" def __getattribute__(cls, *args, **kwargs): warn(ENUM_DEPRECATE_MSSG, FutureWarning, stacklevel=1) @@ -1598,7 +1713,7 @@ class EmbedGenConfig(BaseModel): ) embedding_model: str = Field( description="The embedding model to use.", - examples=[EXAMPLE_EMBEDDING_MODEL], + examples=EXAMPLE_EMBEDDING_MODEL_IDS, ) source_column: str = Field( description="The source column for embedding.", @@ -1606,6 +1721,18 @@ class EmbedGenConfig(BaseModel): ) +class CodeGenConfig(BaseModel): + object: Literal["gen_config.code"] = Field( + default="gen_config.code", + description='The object type, which is always "gen_config.code".', + examples=["gen_config.code"], + ) + source_column: str = Field( + description="The source column for python code to execute.", + examples=["code_column"], + ) + + def _gen_config_discriminator(x: Any) -> str | None: object_attr = getattr(x, "object", None) if object_attr: @@ -1622,9 +1749,10 @@ def _gen_config_discriminator(x: Any) -> str | None: return None -GenConfig = LLMGenConfig | EmbedGenConfig +GenConfig = LLMGenConfig | EmbedGenConfig | CodeGenConfig DiscriminatedGenConfig = Annotated[ Union[ + Annotated[CodeGenConfig, Tag("gen_config.code")], Annotated[LLMGenConfig, Tag("gen_config.llm")], Annotated[LLMGenConfig, Tag("gen_config.chat")], Annotated[EmbedGenConfig, Tag("gen_config.embed")], @@ -1664,9 +1792,12 @@ class ColumnSchema(BaseModel): class ColumnSchemaCreate(ColumnSchema): id: str = Field(description="Column name.") - dtype: Literal["int", "float", "bool", "str", "file"] = Field( + dtype: Literal["int", "float", "bool", "str", "file", "image", "audio"] = Field( default="str", - description='Column data type, one of ["int", "float", "bool", "str", "file"]', + description=( + 'Column data type, one of ["int", "float", "bool", "str", "file", "image", "audio"]' + ". Data type 'file' is deprecated, use 'image' instead." + ), ) @model_validator(mode="before") @@ -1899,11 +2030,12 @@ def check_data(self) -> Self: value.startswith("s3://") or value.startswith("file://") ): extension = splitext(value)[1].lower() - if extension not in IMAGE_FILE_EXTENSIONS: + if extension not in IMAGE_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS: raise ValueError( "Unsupported file type. Make sure the file belongs to " "one of the following formats: \n" - f"[Image File Types]: \n{IMAGE_FILE_EXTENSIONS}" + f"[Image File Types]: \n{IMAGE_FILE_EXTENSIONS} \n" + f"[Audio File Types]: \n{AUDIO_FILE_EXTENSIONS}" ) return self @@ -1945,11 +2077,12 @@ def check_data(self) -> Self: value.startswith("s3://") or value.startswith("file://") ): extension = splitext(value)[1].lower() - if extension not in IMAGE_FILE_EXTENSIONS: + if extension not in IMAGE_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS: raise ValueError( "Unsupported file type. Make sure the file belongs to " "one of the following formats: \n" - f"[Image File Types]: \n{IMAGE_FILE_EXTENSIONS}" + f"[Image File Types]: \n{IMAGE_FILE_EXTENSIONS} \n" + f"[Audio File Types]: \n{AUDIO_FILE_EXTENSIONS}" ) return self diff --git a/clients/python/src/jamaibase/utils/io.py b/clients/python/src/jamaibase/utils/io.py index 83e086f..00fc6bb 100644 --- a/clients/python/src/jamaibase/utils/io.py +++ b/clients/python/src/jamaibase/utils/io.py @@ -12,6 +12,7 @@ import srsly import toml from PIL import ExifTags, Image +from pydub import AudioSegment from jamaibase.utils.types import JSONInput, JSONOutput @@ -176,7 +177,7 @@ def read_image(img_path: str) -> tuple[np.ndarray, bool]: return np.asarray(image), is_rotated -def generate_thumbnail( +def generate_image_thumbnail( file_content: bytes, size: tuple[float, float] = (450.0, 450.0), ) -> bytes: @@ -201,3 +202,26 @@ def generate_thumbnail( except Exception as e: logger.exception(f"Failed to generate thumbnail due to {e.__class__.__name__}: {e}") return b"" + + +def generate_audio_thumbnail(file_content: bytes, duration_ms: int = 30000) -> bytes: + """ + Generates a thumbnail audio by extracting a segment from the original audio. + + Args: + file_content (bytes): The audio file content. + duration_ms (int): Duration of the thumbnail in milliseconds. + + Returns: + bytes: The thumbnail audio segment as bytes. + """ + # Use BytesIO to simulate a file object from the byte content + audio = AudioSegment.from_file(BytesIO(file_content)) + + # Extract the first `duration_ms` milliseconds + thumbnail = audio[:duration_ms] + + # Export the thumbnail to a bytes object + with BytesIO() as output: + thumbnail.export(output, format="mp3") + return output.getvalue() diff --git a/clients/python/tests/cloud/test_admin.py b/clients/python/tests/cloud/test_admin.py index bfc6ee1..a760683 100644 --- a/clients/python/tests/cloud/test_admin.py +++ b/clients/python/tests/cloud/test_admin.py @@ -50,7 +50,7 @@ UserUpdate, ) from jamaibase.utils import datetime_now_iso -from owl.configs.manager import PlanName, ProductType +from owl.configs.manager import ENV_CONFIG, PlanName, ProductType from owl.utils import uuid7_str CLIENT_CLS = [JamAI] @@ -413,6 +413,21 @@ def test_pat(client_cls: Type[JamAI]): with _create_gen_table(jamai, "action", "xx"): table = jamai.table.get_table("action", "xx") assert isinstance(table, TableMetaResponse) + ### --- Test service key auth --- ### + table = JamAI( + project_id=p0.id, + token=ENV_CONFIG.service_key_plain, + headers={"X-USER-ID": u0.id}, + ).table.get_table("action", "xx") + assert isinstance(table, TableMetaResponse) + # Try using invalid user ID + with pytest.raises(RuntimeError): + JamAI( + project_id=p0.id, + token=ENV_CONFIG.service_key_plain, + headers={"X-USER-ID": u1.id}, + ).table.get_table("action", "xx") + ### --- Test PAT --- ### # Try using invalid PAT with pytest.raises(RuntimeError): JamAI(project_id=p0.id, token=pat1.id).table.get_table("action", "xx") @@ -781,7 +796,7 @@ def test_join_and_leave_organization(client_cls: Type[JamAI]): # --- Join with public invite link --- # with _create_org(owl, u0.id, tier="pro") as pro_org: assert u1.id not in set(m.user_id for m in pro_org.members) - invite = owl.admin.backend.generate_invite_token(pro_org.id) + invite = owl.admin.backend.generate_invite_token(pro_org.id, user_role="member") member = owl.admin.backend.join_organization( OrgMemberCreate( user_id=u1.id, @@ -798,7 +813,9 @@ def test_join_and_leave_organization(client_cls: Type[JamAI]): with _create_org(owl, u0.id, tier="pro") as pro_org: assert u1.id not in set(m.user_id for m in pro_org.members) # Invite token email validation should be case and space insensitive - invite = owl.admin.backend.generate_invite_token(pro_org.id, f" {u1.email.upper()} ") + invite = owl.admin.backend.generate_invite_token( + pro_org.id, f" {u1.email.upper()} ", user_role="admin" + ) member = owl.admin.backend.join_organization( OrgMemberCreate( user_id=u1.id, diff --git a/clients/python/tests/files/mp3/turning-a4-size-magazine.mp3 b/clients/python/tests/files/mp3/turning-a4-size-magazine.mp3 new file mode 100644 index 0000000..bbf15ef Binary files /dev/null and b/clients/python/tests/files/mp3/turning-a4-size-magazine.mp3 differ diff --git a/clients/python/tests/files/wav/turning-a4-size-magazine.wav b/clients/python/tests/files/wav/turning-a4-size-magazine.wav new file mode 100644 index 0000000..a32bde9 Binary files /dev/null and b/clients/python/tests/files/wav/turning-a4-size-magazine.wav differ diff --git a/clients/python/tests/oss/gen_table/test_export_ops.py b/clients/python/tests/oss/gen_table/test_export_ops.py index e0c20b7..66e2d15 100644 --- a/clients/python/tests/oss/gen_table/test_export_ops.py +++ b/clients/python/tests/oss/gen_table/test_export_ops.py @@ -81,7 +81,7 @@ def _create_table( p.ColumnSchemaCreate(id="words", dtype="int"), p.ColumnSchemaCreate(id="stars", dtype="float"), p.ColumnSchemaCreate(id="inputs", dtype="str"), - p.ColumnSchemaCreate(id="photo", dtype="file"), + p.ColumnSchemaCreate(id="photo", dtype="image"), p.ColumnSchemaCreate( id="summary", dtype="str", diff --git a/clients/python/tests/oss/gen_table/test_row_ops.py b/clients/python/tests/oss/gen_table/test_row_ops.py index c8f05b8..2a3dd62 100644 --- a/clients/python/tests/oss/gen_table/test_row_ops.py +++ b/clients/python/tests/oss/gen_table/test_row_ops.py @@ -79,6 +79,11 @@ def _get_chat_model(jamai: JamAI) -> str: return models[0] +def _get_audio_model(jamai: JamAI) -> str: + models = jamai.model_names(prefer="ellm/Qwen/Qwen-2-Audio-7B", capabilities=["audio"]) + return models[0] + + def _get_chat_only_model(jamai: JamAI) -> str: chat_models = jamai.model_names( prefer="ellm/meta-llama/Llama-3.1-8B-Instruct", capabilities=["chat"] @@ -118,7 +123,8 @@ def _create_table( p.ColumnSchemaCreate(id="words", dtype="int"), p.ColumnSchemaCreate(id="stars", dtype="float"), p.ColumnSchemaCreate(id="inputs", dtype="str"), - p.ColumnSchemaCreate(id="photo", dtype="file"), + p.ColumnSchemaCreate(id="photo", dtype="image"), + p.ColumnSchemaCreate(id="audio", dtype="audio"), p.ColumnSchemaCreate( id="summary", dtype="str", @@ -142,7 +148,18 @@ def _create_table( prompt="${photo} \n\nWhat's in the image?", temperature=0.001, top_p=0.001, - max_tokens=300, + max_tokens=20, + ), + ), + p.ColumnSchemaCreate( + id="narration", + dtype="str", + gen_config=p.LLMGenConfig( + model="", + prompt="${audio} \n\nWhat happened?", + temperature=0.001, + top_p=0.001, + max_tokens=10, ), ), ] @@ -196,13 +213,19 @@ def _add_row( chat_data: dict | None = None, ): if data is None: - upload_response = jamai.file.upload_file("clients/python/tests/files/jpeg/rabbit.jpeg") + image_upload_response = jamai.file.upload_file( + "clients/python/tests/files/jpeg/rabbit.jpeg" + ) + audio_upload_response = jamai.file.upload_file( + "clients/python/tests/files/mp3/turning-a4-size-magazine.mp3" + ) data = dict( good=True, words=5, stars=7.9, inputs=TEXT, - photo=upload_response.uri, + photo=image_upload_response.uri, + audio=audio_upload_response.uri, ) if knowledge_data is None: @@ -490,7 +513,7 @@ def test_rag( @pytest.mark.parametrize("client_cls", CLIENT_CLS) @pytest.mark.parametrize("table_type", TABLE_TYPES) @pytest.mark.parametrize("stream", [True, False], ids=["stream", "non-stream"]) -def test_rag_with_file_input( +def test_rag_with_image_input( client_cls: Type[JamAI], table_type: p.TableType, stream: bool, @@ -524,7 +547,7 @@ def test_rag_with_file_input( # Create the other table cols = [ - p.ColumnSchemaCreate(id="photo", dtype="file"), + p.ColumnSchemaCreate(id="photo", dtype="image"), p.ColumnSchemaCreate(id="question", dtype="str"), p.ColumnSchemaCreate(id="words", dtype="int"), p.ColumnSchemaCreate( @@ -670,10 +693,14 @@ def test_add_row( assert all(r.object == "gen_table.completion.chunk" for r in responses) if table_type == p.TableType.chat: assert all( - r.output_column_name in ("summary", "captioning", "AI") for r in responses + r.output_column_name in ("summary", "captioning", "narration", "AI") + for r in responses ) else: - assert all(r.output_column_name in ("summary", "captioning") for r in responses) + assert all( + r.output_column_name in ("summary", "captioning", "narration") + for r in responses + ) assert len("".join(r.text for r in responses)) > 0 assert all(isinstance(r, p.GenTableStreamChatCompletionChunk) for r in responses) assert all(isinstance(r.usage, p.CompletionUsage) for r in responses) @@ -682,7 +709,7 @@ def test_add_row( else: assert isinstance(response, p.GenTableChatCompletionChunks) assert response.object == "gen_table.completion.chunks" - for output_column_name in ("summary", "captioning"): + for output_column_name in ("summary", "captioning", "narration"): assert len(response.columns[output_column_name].text) > 0 assert isinstance(response.columns[output_column_name].usage, p.CompletionUsage) assert isinstance(response.columns[output_column_name].prompt_tokens, int) @@ -695,9 +722,237 @@ def test_add_row( assert row["words"]["value"] == 5, row["words"] assert row["stars"]["value"] == 7.9, row["stars"] assert row["photo"]["value"].endswith("/rabbit.jpeg"), row["photo"]["value"] + assert row["audio"]["value"].endswith("/turning-a4-size-magazine.mp3"), row["audio"][ + "value" + ] for animal in ["deer", "rabbit"]: if animal in row["photo"]["value"].split("_")[0]: assert animal in row["captioning"]["value"] + assert "paper" in row["narration"]["value"] or "turn" in row["narration"]["value"] + + +@flaky(max_runs=5, min_passes=1, rerun_filter=_rerun_on_fs_error_with_delay) +@pytest.mark.parametrize("client_cls", CLIENT_CLS) +@pytest.mark.parametrize("table_type", TABLE_TYPES) +@pytest.mark.parametrize("stream", [True, False], ids=["stream", "non-stream"]) +def test_regen_with_reordered_columns( + client_cls: Type[JamAI], + table_type: p.TableType, + stream: bool, +): + jamai = client_cls() + cols = [ + p.ColumnSchemaCreate(id="number", dtype="int"), + p.ColumnSchemaCreate( + id="col1-english", + dtype="str", + gen_config=p.LLMGenConfig( + model="", + prompt=( + "Number: ${number} \n\nTell the 'Number' in English, " + "only output the answer in uppercase without explanation." + ), + ), + ), + p.ColumnSchemaCreate( + id="col2-malay", + dtype="str", + gen_config=p.LLMGenConfig( + model="", + prompt=( + "Number: ${number} \n\nTell the 'Number' in Malay, " + "only output the answer in uppercase without explanation." + ), + ), + ), + p.ColumnSchemaCreate( + id="col3-mandarin", + dtype="str", + gen_config=p.LLMGenConfig( + model="", + prompt=( + "Number: ${number} \n\nTell the 'Number' in Mandarin (Chinese Character), " + "only output the answer in uppercase without explanation." + ), + ), + ), + p.ColumnSchemaCreate( + id="col4-roman", + dtype="str", + gen_config=p.LLMGenConfig( + model="", + prompt=( + "Number: ${number} \n\nTell the 'Number' in Roman Numerals, " + "only output the answer in uppercase without explanation." + ), + ), + ), + ] + + with _create_table(jamai, table_type, cols=cols) as table: + assert isinstance(table, p.TableMetaResponse) + row = _add_row( + jamai, + table_type, + False, + data=dict(number=1), + ) + assert isinstance(row, p.GenTableChatCompletionChunks) + rows = jamai.table.list_table_rows(table_type, table.id) + assert isinstance(rows.items, list) + assert len(rows.items) == 1 + row = rows.items[0] + _id = row["ID"] + assert row["number"]["value"] == 1, row["number"] + assert row["col1-english"]["value"] == "ONE", row["col1-english"] + assert row["col2-malay"]["value"] == "SATU", row["col2-malay"] + assert row["col3-mandarin"]["value"] == "一", row["col3-mandarin"] + assert row["col4-roman"]["value"] == "I", row["col4-roman"] + + # Update Input + Regen + jamai.table.update_table_row( + table_type, + p.RowUpdateRequest( + table_id=TABLE_ID_A, + row_id=_id, + data=dict(number=2), + ), + ) + + response = jamai.table.regen_table_rows( + table_type, + p.RowRegenRequest( + table_id=table.id, + row_ids=[_id], + regen_strategy=p.RegenStrategy.RUN_ALL, + stream=stream, + ), + ) + if stream: + _ = [r for r in response] + + rows = jamai.table.list_table_rows(table_type, table.id) + assert isinstance(rows.items, list) + assert len(rows.items) == 1 + row = rows.items[0] + assert row["number"]["value"] == 2, row["number"] + assert row["col1-english"]["value"] == "TWO", row["col1-english"] + assert row["col2-malay"]["value"] == "DUA", row["col2-malay"] + assert row["col3-mandarin"]["value"] == "二", row["col3-mandarin"] + assert row["col4-roman"]["value"] == "II", row["col4-roman"] + + # Reorder + Update Input + Regen + # [1, 2, 3, 4] -> [3, 1, 4, 2] + new_cols = [ + "number", + "col3-mandarin", + "col1-english", + "col4-roman", + "col2-malay", + ] + if table_type == p.TableType.knowledge: + new_cols += ["Title", "Text", "Title Embed", "Text Embed", "File ID", "Page"] + elif table_type == p.TableType.chat: + new_cols += ["User", "AI"] + jamai.table.reorder_columns( + table_type=table_type, + request=p.ColumnReorderRequest( + table_id=TABLE_ID_A, + column_names=new_cols, + ), + ) + # RUN_SELECTED + jamai.table.update_table_row( + table_type, + p.RowUpdateRequest( + table_id=TABLE_ID_A, + row_id=_id, + data=dict(number=5), + ), + ) + response = jamai.table.regen_table_rows( + table_type, + p.RowRegenRequest( + table_id=TABLE_ID_A, + row_ids=[_id], + regen_strategy=p.RegenStrategy.RUN_SELECTED, + output_column_id="col1-english", + stream=stream, + ), + ) + if stream: + _ = [r for r in response] + rows = jamai.table.list_table_rows(table_type, TABLE_ID_A) + assert isinstance(rows.items, list) + assert len(rows.items) == 1 + row = rows.items[0] + assert row["number"]["value"] == 5, row["number"] + assert row["col3-mandarin"]["value"] == "二", row["col3-mandarin"] + assert row["col1-english"]["value"] == "FIVE", row["col1-english"] + assert row["col4-roman"]["value"] == "II", row["col4-roman"] + assert row["col2-malay"]["value"] == "DUA", row["col2-malay"] + + # RUN_BEFORE + jamai.table.update_table_row( + table_type, + p.RowUpdateRequest( + table_id=TABLE_ID_A, + row_id=_id, + data=dict(number=6), + ), + ) + response = jamai.table.regen_table_rows( + table_type, + p.RowRegenRequest( + table_id=TABLE_ID_A, + row_ids=[_id], + regen_strategy=p.RegenStrategy.RUN_BEFORE, + output_column_id="col4-roman", + stream=stream, + ), + ) + if stream: + _ = [r for r in response] + rows = jamai.table.list_table_rows(table_type, TABLE_ID_A) + assert isinstance(rows.items, list) + assert len(rows.items) == 1 + row = rows.items[0] + assert row["number"]["value"] == 6, row["number"] + assert row["col3-mandarin"]["value"] == "六", row["col3-mandarin"] + assert row["col1-english"]["value"] == "SIX", row["col1-english"] + assert row["col4-roman"]["value"] == "VI", row["col4-roman"] + assert row["col2-malay"]["value"] == "DUA", row["col2-malay"] + + # RUN_AFTER + jamai.table.update_table_row( + table_type, + p.RowUpdateRequest( + table_id=TABLE_ID_A, + row_id=_id, + data=dict(number=7), + ), + ) + response = jamai.table.regen_table_rows( + table_type, + p.RowRegenRequest( + table_id=TABLE_ID_A, + row_ids=[_id], + regen_strategy=p.RegenStrategy.RUN_AFTER, + output_column_id="col4-roman", + stream=stream, + ), + ) + if stream: + _ = [r for r in response] + rows = jamai.table.list_table_rows(table_type, TABLE_ID_A) + assert isinstance(rows.items, list) + assert len(rows.items) == 1 + row = rows.items[0] + assert row["number"]["value"] == 7, row["number"] + assert row["col3-mandarin"]["value"] == "六", row["col3-mandarin"] + assert row["col1-english"]["value"] == "SIX", row["col1-english"] + assert row["col4-roman"]["value"] == "VII", row["col4-roman"] + assert row["col2-malay"]["value"] == "TUJUH", row["col2-malay"] @flaky(max_runs=5, min_passes=1, rerun_filter=_rerun_on_fs_error_with_delay) @@ -708,11 +963,85 @@ def test_add_row_sequential_image_model_completion( client_cls: Type[JamAI], table_type: p.TableType, stream: bool, +): + jamai = client_cls() + cols = [ + p.ColumnSchemaCreate(id="photo", dtype="image"), + p.ColumnSchemaCreate(id="photo2", dtype="image"), + p.ColumnSchemaCreate( + id="caption", + dtype="str", + gen_config=p.LLMGenConfig(model="", prompt="${photo} What's in the image?"), + ), + p.ColumnSchemaCreate( + id="question", + dtype="str", + gen_config=p.LLMGenConfig( + model="", + prompt="Caption: ${caption}\n\nImage: ${photo2}\n\nDoes the caption match? Reply True or False.", + ), + ), + ] + with _create_table(jamai, table_type, cols=cols) as table: + assert isinstance(table, p.TableMetaResponse) + + upload_response = jamai.file.upload_file("clients/python/tests/files/jpeg/rabbit.jpeg") + response = _add_row( + jamai, + table_type, + stream, + TABLE_ID_A, + data=dict(photo=upload_response.uri, photo2=upload_response.uri), + ) + if stream: + responses = [r for r in response] + assert all(isinstance(r, p.GenTableStreamChatCompletionChunk) for r in responses) + assert all(r.object == "gen_table.completion.chunk" for r in responses) + if table_type == p.TableType.chat: + assert all( + r.output_column_name in ("caption", "question", "AI") for r in responses + ) + else: + assert all(r.output_column_name in ("caption", "question") for r in responses) + assert len("".join(r.text for r in responses)) > 0 + assert all(isinstance(r, p.GenTableStreamChatCompletionChunk) for r in responses) + assert all(isinstance(r.usage, p.CompletionUsage) for r in responses) + assert all(isinstance(r.prompt_tokens, int) for r in responses) + assert all(isinstance(r.completion_tokens, int) for r in responses) + else: + assert isinstance(response, p.GenTableChatCompletionChunks) + assert response.object == "gen_table.completion.chunks" + for output_column_name in ("caption", "question"): + assert len(response.columns[output_column_name].text) > 0 + assert isinstance(response.columns[output_column_name].usage, p.CompletionUsage) + assert isinstance(response.columns[output_column_name].prompt_tokens, int) + assert isinstance(response.columns[output_column_name].completion_tokens, int) + rows = jamai.table.list_table_rows(table_type, TABLE_ID_A) + assert isinstance(rows.items, list) + assert len(rows.items) == 1 + row = rows.items[0] + assert row["photo"]["value"] == upload_response.uri, row["photo"]["value"] + assert row["photo2"]["value"] == upload_response.uri, row["photo"]["value"] + for animal in ["deer", "rabbit"]: + if animal in row["photo"]["value"].split("_")[0]: + assert animal in row["caption"]["value"] + if animal in row["photo2"]["value"].split("_")[0]: + assert "true" in row["question"]["value"].lower() + + +@flaky(max_runs=5, min_passes=1, rerun_filter=_rerun_on_fs_error_with_delay) +@pytest.mark.parametrize("client_cls", CLIENT_CLS) +@pytest.mark.parametrize("table_type", TABLE_TYPES) +@pytest.mark.parametrize("stream", [True, False]) +def test_add_row_map_dtype_file_to_image( + client_cls: Type[JamAI], + table_type: p.TableType, + stream: bool, ): jamai = client_cls() cols = [ p.ColumnSchemaCreate(id="photo", dtype="file"), - p.ColumnSchemaCreate(id="photo2", dtype="file"), + p.ColumnSchemaCreate(id="photo2", dtype="image"), p.ColumnSchemaCreate( id="caption", dtype="str", @@ -772,6 +1101,10 @@ def test_add_row_sequential_image_model_completion( assert animal in row["caption"]["value"] if animal in row["photo2"]["value"].split("_")[0]: assert "true" in row["question"]["value"].lower() + meta = jamai.table.get_table(table_type, TABLE_ID_A) + for col in meta.cols: + if col.id == "photo": + assert col.dtype == "image" # @flaky(max_runs=5, min_passes=1, rerun_filter=_rerun_on_fs_error_with_delay) @@ -785,7 +1118,7 @@ def test_add_row_sequential_image_model_completion( # ): # jamai = client_cls() # cols = [ -# p.ColumnSchemaCreate(id="photo", dtype="file"), +# p.ColumnSchemaCreate(id="photo", dtype="image"), # p.ColumnSchemaCreate(id="question", dtype="str"), # p.ColumnSchemaCreate( # id="captioning", @@ -802,7 +1135,7 @@ def test_add_row_sequential_image_model_completion( # ), # p.ColumnSchemaCreate( # id="compare", -# dtype="file", +# dtype="image", # gen_config=p.LLMGenConfig( # model="", # prompt="Compare ${captioning} and ${answer}.", @@ -820,7 +1153,7 @@ def test_add_row_output_column_referred_image_input_with_chat_model( ): jamai = client_cls() cols = [ - p.ColumnSchemaCreate(id="photo", dtype="file"), + p.ColumnSchemaCreate(id="photo", dtype="image"), p.ColumnSchemaCreate( id="captioning", dtype="str", @@ -959,10 +1292,14 @@ def test_add_row_image_file_type_with_generation( assert all(r.object == "gen_table.completion.chunk" for r in responses) if table_type == p.TableType.chat: assert all( - r.output_column_name in ("summary", "captioning", "AI") for r in responses + r.output_column_name in ("summary", "captioning", "narration", "AI") + for r in responses ) else: - assert all(r.output_column_name in ("summary", "captioning") for r in responses) + assert all( + r.output_column_name in ("summary", "captioning", "narration") + for r in responses + ) assert len("".join(r.text for r in responses)) > 0 else: assert isinstance(response, p.GenTableChatCompletionChunks) @@ -1058,9 +1395,14 @@ def test_add_row_validate_one_image_per_completion( assert all(isinstance(r, p.GenTableStreamChatCompletionChunk) for r in responses) assert all(r.object == "gen_table.completion.chunk" for r in responses) if table_type == p.TableType.chat: - assert all(r.output_column_name in ("summary", "captioning", "AI") for r in responses) + assert all( + r.output_column_name in ("summary", "captioning", "narration", "AI") + for r in responses + ) else: - assert all(r.output_column_name in ("summary", "captioning") for r in responses) + assert all( + r.output_column_name in ("summary", "captioning", "narration") for r in responses + ) assert len("".join(r.text for r in responses)) > 0 rows = jamai.table.list_table_rows(table_type, TABLE_ID_A) @@ -1090,10 +1432,14 @@ def test_add_row_wrong_dtype( assert all(r.object == "gen_table.completion.chunk" for r in responses) if table_type == p.TableType.chat: assert all( - r.output_column_name in ("summary", "captioning", "AI") for r in responses + r.output_column_name in ("summary", "captioning", "narration", "AI") + for r in responses ) else: - assert all(r.output_column_name in ("summary", "captioning") for r in responses) + assert all( + r.output_column_name in ("summary", "captioning", "narration") + for r in responses + ) assert len("".join(r.text for r in responses)) > 0 else: assert isinstance(response, p.GenTableChatCompletionChunks) @@ -1144,10 +1490,14 @@ def test_add_row_missing_columns( assert all(r.object == "gen_table.completion.chunk" for r in responses) if table_type == p.TableType.chat: assert all( - r.output_column_name in ("summary", "captioning", "AI") for r in responses + r.output_column_name in ("summary", "captioning", "narration", "AI") + for r in responses ) else: - assert all(r.output_column_name in ("summary", "captioning") for r in responses) + assert all( + r.output_column_name in ("summary", "captioning", "narration") + for r in responses + ) assert len("".join(r.text for r in responses)) > 0 else: assert isinstance(response, p.GenTableChatCompletionChunks) @@ -1296,7 +1646,12 @@ def test_regen_rows( assert isinstance(table, p.TableMetaResponse) assert all(isinstance(c, p.ColumnSchema) for c in table.cols) - upload_response = jamai.file.upload_file("clients/python/tests/files/jpeg/rabbit.jpeg") + image_upload_response = jamai.file.upload_file( + "clients/python/tests/files/jpeg/rabbit.jpeg" + ) + audio_upload_response = jamai.file.upload_file( + "clients/python/tests/files/mp3/turning-a4-size-magazine.mp3" + ) response = _add_row( jamai, table_type, @@ -1306,7 +1661,8 @@ def test_regen_rows( words=10, stars=9.9, inputs=TEXT, - photo=upload_response.uri, + photo=image_upload_response.uri, + audio=audio_upload_response.uri, ), ) assert isinstance(response, p.GenTableChatCompletionChunks) @@ -1337,10 +1693,14 @@ def test_regen_rows( assert all(r.object == "gen_table.completion.chunk" for r in responses) if table_type == p.TableType.chat: assert all( - r.output_column_name in ("summary", "captioning", "AI") for r in responses + r.output_column_name in ("summary", "captioning", "narration", "AI") + for r in responses ) else: - assert all(r.output_column_name in ("summary", "captioning") for r in responses) + assert all( + r.output_column_name in ("summary", "captioning", "narration") + for r in responses + ) assert len("".join(r.text for r in responses)) > 0 else: assert isinstance(response, p.GenTableRowsChatCompletionChunks) @@ -1353,7 +1713,8 @@ def test_regen_rows( assert row["good"]["value"] is True assert row["words"]["value"] == 10 assert row["stars"]["value"] == 9.9 - assert row["photo"]["value"] == upload_response.uri + assert row["photo"]["value"] == image_upload_response.uri + assert row["audio"]["value"] == audio_upload_response.uri assert row["Updated at"] > original_ts assert "dune" in row["summary"]["value"].lower() @@ -1508,13 +1869,15 @@ def test_get_and_list_rows( "stars", "inputs", "photo", + "audio", "summary", "captioning", + "narration", } if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID"} + expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"} elif table_type == p.TableType.chat: expected_cols |= {"User", "AI"} else: @@ -2238,6 +2601,7 @@ def test_upload_file( assert all(len(r["Title"]["value"]) > 0 for r in rows.items) assert all(isinstance(r["Text"]["value"], str) for r in rows.items) assert all(len(r["Text"]["value"]) > 0 for r in rows.items) + assert all(r["Page"]["value"] > 0 for r in rows.items) @flaky(max_runs=5, min_passes=1, rerun_filter=_rerun_on_fs_error_with_delay) @@ -2356,6 +2720,7 @@ def test_upload_long_file( assert all(len(r["Title"]["value"]) > 0 for r in rows.items) assert all(isinstance(r["Text"]["value"], str) for r in rows.items) assert all(len(r["Text"]["value"]) > 0 for r in rows.items) + assert all(r["Page"]["value"] > 0 for r in rows.items) if __name__ == "__main__": diff --git a/clients/python/tests/oss/gen_table/test_table_ops.py b/clients/python/tests/oss/gen_table/test_table_ops.py index 6e5fea8..a53d587 100644 --- a/clients/python/tests/oss/gen_table/test_table_ops.py +++ b/clients/python/tests/oss/gen_table/test_table_ops.py @@ -20,7 +20,7 @@ "bool": True, "str": '"Arrival" is a 2016 science fiction film. "Arrival" è un film di fantascienza del 2016. 「Arrival」は2016年のSF映画です。', } -KT_FIXED_COLUMN_IDS = ["Title", "Title Embed", "Text", "Text Embed", "File ID"] +KT_FIXED_COLUMN_IDS = ["Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"] CT_FIXED_COLUMN_IDS = ["User"] TABLE_ID_A = "table_a" @@ -103,7 +103,7 @@ def _create_table( p.ColumnSchemaCreate(id="words", dtype="int"), p.ColumnSchemaCreate(id="stars", dtype="float"), p.ColumnSchemaCreate(id="inputs", dtype="str"), - p.ColumnSchemaCreate(id="photo", dtype="file"), + p.ColumnSchemaCreate(id="photo", dtype="image"), p.ColumnSchemaCreate( id="summary", dtype="str", @@ -232,7 +232,7 @@ def _create_table_v2( id=table_id, cols=cols, embedding_model=embedding_model ) ) - expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID"} + expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"} elif table_type == p.TableType.chat: table = jamai.table.create_chat_table( p.ChatTableSchemaCreate(id=table_id, cols=chat_cols + cols) @@ -782,7 +782,7 @@ def test_default_image_model( jamai = client_cls() available_image_models = _get_image_models(jamai) cols = [ - p.ColumnSchemaCreate(id="input0", dtype="file"), + p.ColumnSchemaCreate(id="input0", dtype="image"), p.ColumnSchemaCreate( id="output0", dtype="str", @@ -833,7 +833,7 @@ def test_default_image_model( dtype="str", gen_config=p.LLMGenConfig(prompt="${input0}"), ), - p.ColumnSchemaCreate(id="file_input1", dtype="file"), + p.ColumnSchemaCreate(id="file_input1", dtype="image"), p.ColumnSchemaCreate( id="output3", dtype="str", @@ -890,7 +890,7 @@ def test_invalid_image_model( jamai = client_cls() available_image_models = _get_image_models(jamai) cols = [ - p.ColumnSchemaCreate(id="input0", dtype="file"), + p.ColumnSchemaCreate(id="input0", dtype="image"), p.ColumnSchemaCreate( id="output0", dtype="str", @@ -902,7 +902,7 @@ def test_invalid_image_model( pass cols = [ - p.ColumnSchemaCreate(id="input0", dtype="file"), + p.ColumnSchemaCreate(id="input0", dtype="image"), p.ColumnSchemaCreate( id="output0", dtype="str", @@ -1066,7 +1066,7 @@ def test_default_prompts( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - input_cols |= {"Title", "Text", "File ID"} + input_cols |= {"Title", "Text", "File ID", "Page"} else: input_cols |= {"User"} cols = {c.id: c for c in table.cols} @@ -1116,7 +1116,7 @@ def test_default_prompts( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - input_cols |= {"Title", "Text", "File ID"} + input_cols |= {"Title", "Text", "File ID", "Page"} else: input_cols |= {"User"} cols = {c.id: c for c in table.cols} @@ -1132,7 +1132,7 @@ def test_default_prompts( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - input_cols |= {"Title", "Text", "File ID"} + input_cols |= {"Title", "Text", "File ID", "Page"} else: input_cols |= {"User"} for col_id in ["output3"]: @@ -1201,7 +1201,7 @@ def test_add_drop_columns( table = jamai.table.add_knowledge_columns( p.AddKnowledgeColumnSchema(id=table.id, cols=cols) ) - expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID"} + expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"} elif table_type == p.TableType.chat: expected_cols |= {"User", "AI"} table = jamai.table.add_chat_columns(p.AddChatColumnSchema(id=table.id, cols=cols)) @@ -1254,7 +1254,7 @@ def test_add_drop_columns( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID"} + expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"} elif table_type == p.TableType.chat: expected_cols |= {"User", "AI"} else: @@ -1297,7 +1297,7 @@ def test_add_drop_file_column( # --- COLUMN ADD --- # cols = [ - p.ColumnSchemaCreate(id="add_in_file", dtype="file"), + p.ColumnSchemaCreate(id="add_in_file", dtype="image"), p.ColumnSchemaCreate( id="add_out_str", dtype="str", @@ -1318,7 +1318,7 @@ def test_add_drop_file_column( table = jamai.table.add_knowledge_columns( p.AddKnowledgeColumnSchema(id=table.id, cols=cols) ) - expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID"} + expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"} elif table_type == p.TableType.chat: expected_cols |= {"User", "AI"} table = jamai.table.add_chat_columns(p.AddChatColumnSchema(id=table.id, cols=cols)) @@ -1360,7 +1360,7 @@ def test_add_drop_file_column( cols = [ p.ColumnSchemaCreate( id="add_out_file", - dtype="file", + dtype="image", gen_config=p.LLMGenConfig( model="", system_prompt="", @@ -1393,7 +1393,7 @@ def test_add_drop_file_column( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID"} + expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"} elif table_type == p.TableType.chat: expected_cols |= {"User", "AI"} else: @@ -1475,7 +1475,7 @@ def test_rename_columns( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID"} + expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"} elif table_type == p.TableType.chat: expected_cols |= {"User", "AI"} else: @@ -1500,7 +1500,7 @@ def test_rename_columns( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID"} + expected_cols |= {"Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"} elif table_type == p.TableType.chat: expected_cols |= {"User", "AI"} else: @@ -1599,10 +1599,10 @@ def test_reorder_columns( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - column_names += ["Title", "Title Embed", "Text", "Text Embed", "File ID"] + column_names += ["Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"] expected_order = ( expected_order[:2] - + ["Title", "Title Embed", "Text", "Text Embed", "File ID"] + + ["Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"] + expected_order[2:] ) elif table_type == p.TableType.chat: @@ -1631,7 +1631,7 @@ def test_reorder_columns( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - expected_order += ["Title", "Title Embed", "Text", "Text Embed", "File ID"] + expected_order += ["Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"] elif table_type == p.TableType.chat: expected_order += ["User", "AI"] else: @@ -1690,10 +1690,10 @@ def test_reorder_columns_invalid( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - column_names += ["Title", "Title Embed", "Text", "Text Embed", "File ID"] + column_names += ["Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"] expected_order = ( expected_order[:2] - + ["Title", "Title Embed", "Text", "Text Embed", "File ID"] + + ["Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"] + expected_order[2:] ) elif table_type == p.TableType.chat: @@ -1717,7 +1717,7 @@ def test_reorder_columns_invalid( if table_type == p.TableType.action: pass elif table_type == p.TableType.knowledge: - column_names += ["Title", "Title Embed", "Text", "Text Embed", "File ID"] + column_names += ["Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"] elif table_type == p.TableType.chat: column_names += ["User", "AI"] else: diff --git a/clients/python/tests/oss/test_chat.py b/clients/python/tests/oss/test_chat.py index edd867e..7a0d18a 100644 --- a/clients/python/tests/oss/test_chat.py +++ b/clients/python/tests/oss/test_chat.py @@ -139,10 +139,19 @@ def _get_chat_request(model: str, **kwargs): return request -def _get_models(return_all: bool = False) -> list[str]: - models = JamAI().model_names(capabilities=["chat"]) +def _get_models( + capabilities: list[str] = None, return_all: bool = False, exclude_audio: bool = True +) -> list[str]: + if capabilities is None: + capabilities = ["chat"] + models = JamAI().model_names(capabilities=capabilities) + audio_models = JamAI().model_names(capabilities=["audio"]) if return_all: + if exclude_audio: + return list(set(models) - set(audio_models)) return models + if exclude_audio: + return list(set(models) - set(audio_models)) providers = sorted(set(m.split("/")[0] for m in models)) selected = [] for provider in providers: @@ -190,6 +199,148 @@ async def test_chat_completion( assert response.usage.total_tokens == response.prompt_tokens + response.completion_tokens +TOOLS = { + "get_weather": p.Tool( + type="function", + function=p.Function( + name="get_weather", + description="Get the current weather for a location", + parameters=p.FunctionParameters( + type="object", + properties={ + "location": p.FunctionParameter( + type="string", description="The city and state, e.g. San Francisco, CA" + ) + }, + required=["location"], + additionalProperties=False, + ), + ), + ), + "calculator": p.Tool( + type="function", + function=p.Function( + name="calculator", + description="Perform a basic arithmetic operation", + parameters=p.FunctionParameters( + type="object", + properties={ + "operation": p.FunctionParameter( + type="string", + description="The arithmetic operation to perform", + enum=["add", "subtract", "multiply", "divide"], + ), + "first_number": p.FunctionParameter( + type="number", + description="The first number", + ), + "second_number": p.FunctionParameter( + type="number", + description="The second number", + ), + }, + required=["operation", "first_number", "second_number"], + additionalProperties=False, + ), + ), + ), +} + +TOOL_PROMPTS = [ + { + "tool_choice": "get_weather", + "prompt": "What's the weather like in Paris?", + "response": ['{"location":'], + }, + { + "tool_choice": "calculator", + "prompt": "Divide 5 by 2.", + "response": ['"operation":"divide"', "first_number"], + }, +] + + +@flaky(max_runs=3, min_passes=1) +@pytest.mark.parametrize("client_cls", CLIENT_CLS) +@pytest.mark.parametrize("model", _get_models(capabilities=["tool"], return_all=True)) +@pytest.mark.parametrize("tool_prompt", TOOL_PROMPTS) +@pytest.mark.parametrize("set_multi_tools", [False, True]) +async def test_chat_completion_with_tools( + client_cls: Type[JamAI | JamAIAsync], model: str, tool_prompt: dict, set_multi_tools: bool +): + jamai = client_cls() + + tool_choice = p.ToolChoice( + type="function", + function=p.ToolChoiceFunction( + name=tool_prompt["tool_choice"], + ), + ) + + # Create a chat request with a tool + request = p.ChatRequestWithTools( + id="test", + model=model, + messages=[ + p.ChatEntry.system("You are a concise assistant."), + p.ChatEntry.user(tool_prompt["prompt"]), + ], + tools=[v for _, v in TOOLS.items()] + if set_multi_tools + else [TOOLS[tool_prompt["tool_choice"]]], + tool_choice="auto" if model.startswith("openai/") else tool_choice, + temperature=0.001, + top_p=0.001, + max_tokens=50, + stream=False, + ) + + # Non-streaming + response = await run(jamai.generate_chat_completions, request) + assert isinstance(response, p.ChatCompletionChunk) + assert isinstance(response.text, str) + assert len(response.text) == 0 + tool_calls = response.message.tool_calls + assert isinstance(tool_calls, list) + assert len(tool_calls) == 1 + assert tool_calls[0].function.name == tool_prompt["tool_choice"] + for argument in tool_prompt["response"]: + assert argument in tool_calls[0].function.arguments.replace(" ", "") + assert isinstance(response.usage, p.CompletionUsage) + assert isinstance(response.prompt_tokens, int) + assert isinstance(response.completion_tokens, int) + assert response.references is None + + # Streaming + request.stream = True + responses = await run(jamai.generate_chat_completions, request) + assert len(responses) > 0 + assert all(isinstance(r, p.ChatCompletionChunk) for r in responses) + assert all(isinstance(r.text, str) for r in responses) + assert len("".join(r.text for r in responses)) == 0 + assert all(r.references is None for r in responses) + response = responses[-1] + assert all(isinstance(r.usage, p.CompletionUsage) for r in responses) + assert all(isinstance(r.prompt_tokens, int) for r in responses) + assert all(isinstance(r.completion_tokens, int) for r in responses) + assert response.prompt_tokens > 0 + assert response.completion_tokens > 0 + assert response.usage.total_tokens == response.prompt_tokens + response.completion_tokens + arguments_result = "" + for response in responses: + tool_calls = response.message.tool_calls + assert isinstance(tool_calls, list) or tool_calls is None + if isinstance(tool_calls, list): + assert len(tool_calls) == 1 + arguments_result += tool_calls[0].function.arguments + assert ( + tool_calls[0].function.name == tool_prompt["tool_choice"] + or tool_calls[0].function.name is None + ) + for argument in tool_prompt["response"]: + assert argument in arguments_result.replace(" ", "") + + @flaky(max_runs=3, min_passes=1) @pytest.mark.parametrize("client_cls", CLIENT_CLS) @pytest.mark.parametrize("model", _get_models()) diff --git a/clients/python/tests/oss/test_file.py b/clients/python/tests/oss/test_file.py index 279bedf..d31f90a 100644 --- a/clients/python/tests/oss/test_file.py +++ b/clients/python/tests/oss/test_file.py @@ -16,7 +16,7 @@ GetURLResponse, ) from jamaibase.utils import run -from jamaibase.utils.io import generate_thumbnail +from jamaibase.utils.io import generate_audio_thumbnail, generate_image_thumbnail def read_file_content(file_path): @@ -24,7 +24,7 @@ def read_file_content(file_path): return f.read() -# Define the paths to your test image files +# Define the paths to your test image and audio files IMAGE_FILES = [ "clients/python/tests/files/jpeg/cifar10-deer.jpg", "clients/python/tests/files/png/rabbit.png", @@ -32,12 +32,17 @@ def read_file_content(file_path): "clients/python/tests/files/webp/rabbit_cifar10-deer.webp", ] +AUDIO_FILES = [ + "clients/python/tests/files/wav/turning-a4-size-magazine.wav", + "clients/python/tests/files/mp3/turning-a4-size-magazine.mp3", +] + CLIENT_CLS = [JamAI, JamAIAsync] @pytest.mark.parametrize("client_cls", CLIENT_CLS) @pytest.mark.parametrize("image_file", IMAGE_FILES) -async def test_upload(client_cls: Type[JamAI | JamAIAsync], image_file: str): +async def test_upload_image(client_cls: Type[JamAI | JamAIAsync], image_file: str): # Initialize the client jamai = client_cls() @@ -64,6 +69,35 @@ async def test_upload(client_cls: Type[JamAI | JamAIAsync], image_file: str): print(f"Returned URI matches the expected format: {upload_response.uri}") +@pytest.mark.parametrize("client_cls", CLIENT_CLS) +@pytest.mark.parametrize("audio_file", AUDIO_FILES) +async def test_upload_audio(client_cls: Type[JamAI | JamAIAsync], audio_file: str): + # Initialize the client + jamai = client_cls() + + # Ensure the audio file exists + assert os.path.exists(audio_file), f"Test audio file does not exist: {audio_file}" + # Upload the file + upload_response = await run(jamai.file.upload_file, audio_file) + assert isinstance(upload_response, FileUploadResponse) + assert upload_response.uri.startswith( + ("file://", "s3://") + ), f"Returned URI '{upload_response.uri}' does not start with 'file://' or 's3://'" + + filename = os.path.basename(audio_file) + expected_uri_pattern = re.compile( + r"(file|s3)://[^/]+/raw/default/default/[a-f0-9-]{36}/" + re.escape(filename) + "$" + ) + + # Check if the returned URI matches the expected format + assert expected_uri_pattern.match(upload_response.uri), ( + f"Returned URI '{upload_response.uri}' does not match the expected format: " + f"(file|s3)://file/raw/default/default/{{UUID}}/{filename}" + ) + + print(f"Returned URI matches the expected format: {upload_response.uri}") + + @pytest.mark.parametrize("client_cls", CLIENT_CLS) async def test_upload_large_image_file(client_cls: Type[JamAI | JamAIAsync]): jamai = client_cls() @@ -87,15 +121,15 @@ async def test_get_raw_urls(client_cls: Type[JamAI | JamAIAsync]): jamai = client_cls() # Upload files first uploaded_uris = [] - for file in IMAGE_FILES: + for file in IMAGE_FILES + AUDIO_FILES: response = await run(jamai.file.upload_file, file) uploaded_uris.append(response.uri) # Now test get_raw_urls response = await run(jamai.file.get_raw_urls, uploaded_uris) assert isinstance(response, GetURLResponse) - assert len(response.urls) == len(IMAGE_FILES) - for original_file, url in zip(IMAGE_FILES, response.urls, strict=True): + assert len(response.urls) == len(IMAGE_FILES + AUDIO_FILES) + for original_file, url in zip(IMAGE_FILES + AUDIO_FILES, response.urls, strict=True): if url.startswith(("http://", "https://")): # Handle HTTP/HTTPS URLs HEADERS = {"X-PROJECT-ID": "default"} @@ -129,22 +163,22 @@ async def test_get_thumbnail_urls(client_cls: Type[JamAI | JamAIAsync]): # Upload files first uploaded_uris = [] - for file in IMAGE_FILES: + for file in IMAGE_FILES + AUDIO_FILES: response = await run(jamai.file.upload_file, file) uploaded_uris.append(response.uri) # Now test get_thumbnail_urls response = await run(jamai.file.get_thumbnail_urls, uploaded_uris) assert isinstance(response, GetURLResponse) - assert len(response.urls) == len(IMAGE_FILES) + assert len(response.urls) == len(IMAGE_FILES + AUDIO_FILES) # Generate thumbnails and compare - for original_file, url in zip(IMAGE_FILES, response.urls, strict=True): + for original_file, url in zip(IMAGE_FILES, response.urls[: len(IMAGE_FILES)], strict=True): # Read original file content original_content = read_file_content(original_file) # Generate thumbnail - expected_thumbnail = generate_thumbnail(original_content) + expected_thumbnail = generate_image_thumbnail(original_content) assert expected_thumbnail is not None, f"Failed to generate thumbnail for {original_file}" if url.startswith(("http://", "https://")): @@ -157,6 +191,27 @@ async def test_get_thumbnail_urls(client_cls: Type[JamAI | JamAIAsync]): expected_thumbnail == downloaded_thumbnail ), f"Thumbnail mismatch for file: {original_file}" + # Generate audio thumbnails and compare + for original_file, url in zip(AUDIO_FILES, response.urls[len(IMAGE_FILES) :], strict=True): + # Read original file content + original_content = read_file_content(original_file) + + # Generate audio thumbnail + expected_thumbnail = generate_audio_thumbnail(original_content) + assert expected_thumbnail is not None, f"Failed to generate thumbnail for {original_file}" + + if url.startswith(("http://", "https://")): + downloaded_thumbnail = httpx.get(url, headers={"X-PROJECT-ID": "default"}).content + else: + downloaded_thumbnail = read_file_content(url) + + # Compare thumbnails + # TODO: debug the starting of thumbnail mismatch + assert ( + expected_thumbnail[-round(len(expected_thumbnail) * 0.9) :] + == downloaded_thumbnail[-round(len(expected_thumbnail) * 0.9) :] + ), f"Thumbnail mismatch for file: {original_file}" + # Check if the returned URIs are valid for url in response.urls: parsed_uri = urlparse(url) diff --git a/clients/python/tests/oss/test_gen_executor.py b/clients/python/tests/oss/test_gen_executor.py index dbdce44..1ca3d1d 100644 --- a/clients/python/tests/oss/test_gen_executor.py +++ b/clients/python/tests/oss/test_gen_executor.py @@ -1,17 +1,22 @@ import asyncio +import io import time from contextlib import asynccontextmanager +import httpx import pytest from flaky import flaky +from PIL import Image from jamaibase import JamAI, JamAIAsync from jamaibase.exceptions import ResourceNotFoundError from jamaibase.protocol import ( + CodeGenConfig, ColumnSchemaCreate, GenConfigUpdateRequest, GenTableRowsChatCompletionChunks, GenTableStreamChatCompletionChunk, + GetURLResponse, RegenStrategy, RowAddRequest, RowRegenRequest, @@ -539,5 +544,178 @@ async def test_multicols_regen_invalid_column_id( ) +@pytest.mark.parametrize("client_cls", CLIENT_CLS) +@pytest.mark.parametrize("stream", [True, False], ids=["stream", "non-stream"]) +async def test_code_str(client_cls: JamAI | JamAIAsync, stream: bool): + jamai = client_cls() + cols = [ + ColumnSchemaCreate(id="code_column", dtype="str"), + ColumnSchemaCreate( + id="result_column", dtype="str", gen_config=CodeGenConfig(source_column="code_column") + ), + ] + + async with _create_table(jamai, TableType.action, cols) as table_id: + test_cases = [ + {"code": "print('Hello, World!')", "expected": "Hello, World!"}, + {"code": "result = 2 + 2\nprint(result)", "expected": "4"}, + {"code": "import math\nprint(math.pi)", "expected": "3.141592653589793"}, + {"code": "result = 5 * 5", "expected": "25"}, + {"code": "result = 'Python' + ' ' + 'Programming'", "expected": "Python Programming"}, + {"code": "result = [1, 2, 3, 4, 5]\nresult = sum(result)", "expected": "15"}, + # Define factorial function as globals namespace to be able to executed recursive calls. + # exec() creates a new local scope for the code it's executing, and the recursive calls can't access the function name in this temporary scope. + { + "code": "def factorial(n):\n return 1 if n == 0 else n * factorial(n-1)\nglobals()['factorial'] = factorial\nresult = factorial(5)", + "expected": "120", + }, + { + "code": "result = {x: x**2 for x in range(1, 6)}", + "expected": "{1: 1, 2: 4, 3: 9, 4: 16, 5: 25}", + }, + ] + + for case in test_cases: + row_input_data = {"code_column": case["code"]} + chunks = await run( + jamai.table.add_table_rows, + TableType.action, + RowAddRequest(table_id=table_id, data=[row_input_data], stream=stream), + ) + + if stream: + print(chunks[0]) + assert isinstance(chunks[0], GenTableStreamChatCompletionChunk) + else: + print(chunks) + assert isinstance(chunks, GenTableRowsChatCompletionChunks) + + # Get rows + rows = await run(jamai.table.list_table_rows, TableType.action, table_id) + row_id = rows.items[0]["ID"] + row = await run(jamai.table.get_table_row, TableType.action, table_id, row_id) + assert row["result_column"]["value"].strip() == case["expected"] + + # Test error handling + error_code = "print(undefined_variable)" + row_input_data = {"code_column": error_code} + chunks = await run( + jamai.table.add_table_rows, + TableType.action, + RowAddRequest(table_id=table_id, data=[row_input_data], stream=stream), + ) + rows = await run(jamai.table.list_table_rows, TableType.action, table_id) + row_id = rows.items[0]["ID"] + row = await run(jamai.table.get_table_row, TableType.action, table_id, row_id) + assert "name 'undefined_variable' is not defined" in row["result_column"]["value"] + + +@pytest.mark.parametrize("client_cls", CLIENT_CLS) +@pytest.mark.parametrize("stream", [True, False], ids=["stream", "non-stream"]) +async def test_code_image(client_cls: JamAI | JamAIAsync, stream: bool): + jamai = client_cls() + cols = [ + ColumnSchemaCreate(id="code_column", dtype="str"), + ColumnSchemaCreate( + id="result_column", + dtype="image", + gen_config=CodeGenConfig(source_column="code_column"), + ), + ] + + async with _create_table(jamai, TableType.action, cols) as table_id: + test_cases = [ + { + "code": """ +import matplotlib.pyplot as plt +import io + +plt.figure(figsize=(10, 5)) +plt.plot([1, 2, 3, 4], [1, 4, 2, 3]) +plt.title('Simple Line Plot') +buf = io.BytesIO() +plt.savefig(buf, format='png') +buf.seek(0) +result = buf.getvalue() +""", + "expected_format": "PNG", + }, + { + "code": """ +from PIL import Image, ImageDraw +import io + +img = Image.new('RGB', (200, 200), color='red') +draw = ImageDraw.Draw(img) +draw.ellipse((50, 50, 150, 150), fill='blue') +buf = io.BytesIO() +img.save(buf, format='JPEG') +buf.seek(0) +result = buf.getvalue() +""", + "expected_format": "JPEG", + }, + { + "code": """ +result = b'This is not a valid image file' +""", + "expected_format": None, + }, + ] + + for case in test_cases: + row_input_data = {"code_column": case["code"]} + chunks = await run( + jamai.table.add_table_rows, + TableType.action, + RowAddRequest(table_id=table_id, data=[row_input_data], stream=stream), + ) + + if stream: + print(chunks[0]) + assert isinstance(chunks[0], GenTableStreamChatCompletionChunk) + else: + print(chunks) + assert isinstance(chunks, GenTableRowsChatCompletionChunks) + + # Get rows + rows = await run(jamai.table.list_table_rows, TableType.action, table_id) + row_id = rows.items[0]["ID"] + row = await run(jamai.table.get_table_row, TableType.action, table_id, row_id) + file_uri = row["result_column"]["value"] + + if case["expected_format"] is None: + assert file_uri is None + else: + assert file_uri.startswith(("file://", "s3://")) + + response = await run(jamai.file.get_raw_urls, [file_uri]) + assert isinstance(response, GetURLResponse) + for url in response.urls: + if url.startswith(("http://", "https://")): + # Handle HTTP/HTTPS URLs + HEADERS = {"X-PROJECT-ID": "default"} + with httpx.Client() as client: + downloaded_content = client.get(url, headers=HEADERS).content + + image = Image.open(io.BytesIO(downloaded_content)) + assert image.format == case["expected_format"] + + # Test error handling + error_code = "result = 1 / 0" + row_input_data = {"code_column": error_code} + chunks = await run( + jamai.table.add_table_rows, + TableType.action, + RowAddRequest(table_id=table_id, data=[row_input_data], stream=stream), + ) + + rows = await run(jamai.table.list_table_rows, TableType.action, table_id) + row_id = rows.items[0]["ID"] + row = await run(jamai.table.get_table_row, TableType.action, table_id, row_id) + + assert row["result_column"]["value"] is None + + if __name__ == "__main__": asyncio.run(test_multicols_regen_invalid_column_id(CLIENT_CLS[-1], REGEN_STRATEGY[1], True)) diff --git a/clients/typescript/__tests__/gentable.test.ts b/clients/typescript/__tests__/gentable.test.ts index 9632938..56936bc 100644 --- a/clients/typescript/__tests__/gentable.test.ts +++ b/clients/typescript/__tests__/gentable.test.ts @@ -178,7 +178,7 @@ describe("APIClient Gentable", () => { model: llmModel, prompt: "Suggest a followup questions on ${question}.", temperature: 1, - max_tokens: 100, + max_tokens: 30, top_p: 0.1 } }, @@ -189,7 +189,7 @@ describe("APIClient Gentable", () => { model: llmModel, temperature: 1, - max_tokens: 100, + max_tokens: 30, top_p: 0.1 } } @@ -312,7 +312,7 @@ describe("APIClient Gentable", () => { model: llmModel, prompt: "Suggest a followup questions on ${question}.", temperature: 1, - max_tokens: 100, + max_tokens: 30, top_p: 0.1 } } diff --git a/clients/typescript/src/resources/files/index.ts b/clients/typescript/src/resources/files/index.ts index 728bdbb..1cfe7d6 100644 --- a/clients/typescript/src/resources/files/index.ts +++ b/clients/typescript/src/resources/files/index.ts @@ -14,7 +14,7 @@ import { export class Files extends Base { public async uploadFile(params: IUploadFileRequest): Promise { - const apiURL = `/api/v1/files/upload/`; + const apiURL = `/api/v1/files/upload`; const parsedParams = UploadFileRequestSchema.parse(params); diff --git a/clients/typescript/src/resources/gen_tables/tables.ts b/clients/typescript/src/resources/gen_tables/tables.ts index b86bb01..5cb2de2 100644 --- a/clients/typescript/src/resources/gen_tables/tables.ts +++ b/clients/typescript/src/resources/gen_tables/tables.ts @@ -17,9 +17,9 @@ export const TableTypesSchema = z.enum(["action", "knowledge", "chat"]); export const IdSchema = z.string().regex(/^[A-Za-z0-9]([A-Za-z0-9 _-]{0,98}[A-Za-z0-9])?$/, "Invalid Id"); export const TableIdSchema = z.string().regex(/^[A-Za-z0-9]([A-Za-z0-9._-]{0,98}[A-Za-z0-9])?$/, "Invalid Table Id"); -const DtypeCreateEnumSchema = z.enum(["int", "float", "str", "bool", "file"]); +const DtypeCreateEnumSchema = z.enum(["int", "float", "str", "bool", "image"]); -const DtypeEnumSchema = z.enum(["int", "int8", "float", "float64", "float32", "float16", "bool", "str", "date-time", "file", "bytes"]); +const DtypeEnumSchema = z.enum(["int", "int8", "float", "float64", "float32", "float16", "bool", "str", "date-time", "image", "bytes"]); export const EmbedGenConfigSchema = z.object({ object: z.literal("gen_config.embed").default("gen_config.embed"), diff --git a/clients/typescript/src/resources/llm/model.ts b/clients/typescript/src/resources/llm/model.ts index e43437b..36c6f47 100644 --- a/clients/typescript/src/resources/llm/model.ts +++ b/clients/typescript/src/resources/llm/model.ts @@ -3,7 +3,7 @@ import { z } from "zod"; export const ModelInfoRequestSchema = z.object({ model: z.string().optional(), capabilities: z - .array(z.enum(["completion", "chat", "image", "embed", "rerank"])) + .array(z.enum(["completion", "chat", "image", "audio", "tool", "embed", "rerank"])) .nullable() .optional() }); @@ -14,7 +14,7 @@ export const ModelInfoSchema = z.object({ name: z.string(), context_length: z.number().default(16384), languages: z.array(z.string()), - capabilities: z.array(z.enum(["completion", "chat", "image", "embed", "rerank"])).default(["chat"]), + capabilities: z.array(z.enum(["completion", "chat", "image", "audio", "tool", "embed", "rerank"])).default(["chat"]), owned_by: z.string() }); @@ -26,7 +26,7 @@ export const ModelInfoResponseSchema = z.object({ export const ModelNamesRequestSchema = z.object({ prefer: z.string().optional(), capabilities: z - .array(z.enum(["completion", "chat", "image", "embed", "rerank"])) + .array(z.enum(["completion", "chat", "image", "audio", "tool", "embed", "rerank"])) .nullable() .optional() }); diff --git a/docker/Dockerfile.owl b/docker/Dockerfile.owl index 6c797d9..a6ed08f 100644 --- a/docker/Dockerfile.owl +++ b/docker/Dockerfile.owl @@ -1,6 +1,7 @@ FROM python:3.12 RUN pip install --no-cache-dir --upgrade setuptools +RUN apt-get update -qq && apt-get install ffmpeg libavcodec-extra -y WORKDIR /app diff --git a/docker/compose.cpu.yml b/docker/compose.cpu.yml index a7a1e29..6ff3f25 100644 --- a/docker/compose.cpu.yml +++ b/docker/compose.cpu.yml @@ -177,5 +177,20 @@ services: networks: - jamai + # By default, kopi service is not enabled, and only used for testing. use --profile kopi along docker compose up if kopi is needed. + kopi: + profiles: ["kopi"] + image: hoipangg/kopi + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5569/health')"] + interval: 10s + timeout: 2s + retries: 20 + start_period: 10s + ports: + - "5569:5569" + networks: + - jamai + networks: jamai: diff --git a/docker/compose.mac.yml b/docker/compose.mac.yml deleted file mode 100644 index 6eaa8eb..0000000 --- a/docker/compose.mac.yml +++ /dev/null @@ -1,182 +0,0 @@ -services: - infinity: - image: michaelf34/infinity:0.0.70 - container_name: jamai_infinity - command: ["v2", "--engine", "torch", "--port", "6909", "--model-warmup", "--model-id", "${EMBEDDING_MODEL}", "--model-id", "${RERANKER_MODEL}"] - healthcheck: - test: ["CMD-SHELL", "curl --fail http://localhost:6909/health"] - interval: 10s - timeout: 2s - retries: 20 - start_period: 10s - restart: unless-stopped - env_file: - - ../.env - volumes: - - ${PWD}/infinity_cache:/app/.cache - networks: - - jamai - - unstructuredio: - image: downloads.unstructured.io/unstructured-io/unstructured-api:latest - platform: linux/amd64 - entrypoint: ["/usr/bin/env", "bash", "-c", "uvicorn prepline_general.api.app:app --log-config logger_config.yaml --port 6989 --host 0.0.0.0"] - healthcheck: - test: ["CMD-SHELL", "wget http://localhost:6989/healthcheck -O /dev/null || exit 1"] - interval: 10s - timeout: 2s - retries: 20 - start_period: 10s - restart: unless-stopped - networks: - - jamai - - docio: - build: - context: .. - dockerfile: docker/Dockerfile.docio - image: jamai/docio - pull_policy: build - command: ["python", "-m", "docio.entrypoints.api"] - healthcheck: - test: ["CMD-SHELL", "curl --fail http://localhost:6979/health || exit 1"] - interval: 10s - timeout: 2s - retries: 20 - start_period: 10s - restart: unless-stopped - env_file: - - ../.env - networks: - - jamai - - dragonfly: - image: "docker.dragonflydb.io/dragonflydb/dragonfly" - ulimits: - memlock: -1 - healthcheck: - test: ["CMD-SHELL", "nc -z localhost 6379 || exit 1"] - interval: 10s - timeout: 2s - retries: 20 - start_period: 10s - # For better performance, consider `host` mode instead `port` to avoid docker NAT. - # `host` mode is NOT currently supported in Swarm Mode. - # https://docs.docker.com/compose/compose-file/compose-file-v3/#network_mode - # network_mode: "host" - # volumes: - # - ${PWD}/dragonflydata:/data - networks: - - jamai - - owl: - build: - context: .. - dockerfile: docker/Dockerfile.owl - image: jamai/owl - pull_policy: build - command: ["python", "-m", "owl.entrypoints.api"] - depends_on: - infinity: - condition: service_healthy - unstructuredio: - condition: service_healthy - docio: - condition: service_healthy - dragonfly: - condition: service_healthy - healthcheck: - test: ["CMD-SHELL", "curl --fail localhost:6969/api/health || exit 1"] - interval: 10s - timeout: 2s - retries: 20 - start_period: 10s - restart: unless-stopped - env_file: - - ../.env - volumes: - - ${PWD}/db:/app/api/db - - ${PWD}/logs:/app/api/logs - - ${PWD}/file:/app/api/file - ports: - - "${API_PORT:-6969}:6969" - networks: - - jamai - - starling: - extends: - service: owl - entrypoint: - - /bin/bash - - -c - - | - celery -A owl.entrypoints.starling worker --loglevel=info --max-memory-per-child 65536 --autoscale=2,4 & \ - celery -A owl.entrypoints.starling beat --loglevel=info & \ - FLOWER_UNAUTHENTICATED_API=1 celery -A owl.entrypoints.starling flower --loglevel=info - command: !reset [] - depends_on: - owl: - condition: service_healthy - healthcheck: - test: ["CMD-SHELL", "curl --fail http://localhost:5555/api/workers || exit 1"] - interval: 10s - timeout: 2s - retries: 20 - start_period: 10s - ports: !override - - "${STARLING_PORT:-5555}:5555" - - frontend: - build: - context: .. - dockerfile: docker/Dockerfile.frontend - args: - JAMAI_URL: ${JAMAI_URL} - PUBLIC_JAMAI_URL: ${PUBLIC_JAMAI_URL} - PUBLIC_IS_SPA: ${PUBLIC_IS_SPA} - CHECK_ORIGIN: ${CHECK_ORIGIN} - image: jamai/frontend - pull_policy: build - command: ["node", "server"] - depends_on: - owl: - condition: service_healthy - healthcheck: - test: ["CMD-SHELL", "curl --fail localhost:4000 || exit 1"] - interval: 10s - timeout: 2s - retries: 20 - start_period: 10s - restart: unless-stopped - environment: - - NODE_ENV=production - - BODY_SIZE_LIMIT=Infinity - env_file: - - ../.env - ports: - - "${FRONTEND_PORT:-4000}:4000" - networks: - - jamai - - # By default, minio service is not enabled, and only used for testing. use --profile minio along docker compose up if minio is needed. - minio: - profiles: ["minio"] - image: minio/minio - entrypoint: /bin/sh -c " minio server /data --console-address ':9001' & until (mc config host add myminio http://localhost:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD}) do echo '...waiting...' && sleep 1; done; mc mb myminio/file; wait " - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 10s - timeout: 2s - retries: 20 - start_period: 10s - ports: - - "9000:9000" - - "9001:9001" - networks: - - jamai - -networks: - jamai: diff --git a/scripts/migration_v040.py b/scripts/migration_v040.py new file mode 100644 index 0000000..2d09e11 --- /dev/null +++ b/scripts/migration_v040.py @@ -0,0 +1,234 @@ +import os +import shutil +import sqlite3 +from datetime import datetime, timezone +from glob import glob +from os.path import basename, dirname, join + +import lancedb +import orjson +from loguru import logger +from pydantic_settings import BaseSettings, SettingsConfigDict + +from jamaibase.protocol import ColumnSchema + + +class EnvConfig(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="ignore", cli_parse_args=False + ) + owl_db_dir: str = "db" + + +ENV_CONFIG = EnvConfig() +NOW = datetime.now(tz=timezone.utc).isoformat() + + +def backup_db(db_path: str, backup_dir: str): + """Backup SQLite database.""" + db_path_components = db_path.split(os.sep) + if db_path_components[-1] == "main.db": + bak_db_path = join(backup_dir, db_path_components[-1]) + else: + bak_db_path = join(backup_dir, *db_path_components[-3:]) + os.makedirs(dirname(bak_db_path), exist_ok=True) + with sqlite3.connect(db_path) as src, sqlite3.connect(bak_db_path) as dst: + src.backup(dst) + print(f"└─ Backed up SQLite database: {db_path} to {bak_db_path}") + + +def backup_lance_db(lance_dir: str, backup_dir: str): + """Backup LanceDB directory.""" + lance_dir_components = lance_dir.split(os.sep) + bak_lance_dir = join(backup_dir, *lance_dir_components[-3:]) + os.makedirs(dirname(bak_lance_dir), exist_ok=True) + + # Copy the .lance directory + shutil.copytree(lance_dir, bak_lance_dir, ignore=shutil.ignore_patterns("*.lock")) + print(f"└─ Backed up LanceDB directory: {lance_dir} to {bak_lance_dir}") + + +def find_sqlite_files(directory): + """Find all SQLite files in the directory.""" + sqlite_files = [] + for root, dirs, filenames in os.walk(directory, topdown=True): + # Don't visit Lance directories + lance_dirs = [d for d in dirs if d.endswith(".lance")] + for d in lance_dirs: + dirs.remove(d) + for filename in filenames: + if filename.endswith(".lock"): + continue + if filename.endswith(".db"): + sqlite_files.append(join(root, filename)) + return sqlite_files + + +def find_lance_dirs(directory, table_type): + """Find all LanceDB directories in the directory.""" + lance_dirs = [] + for root, dirs, _ in os.walk(directory, topdown=True): + for dir_name in dirs: + if dir_name.endswith(".lance"): + dir_components = dir_name.split(os.sep) + if root.split(os.sep)[-1] == table_type: + lance_dirs.append(join(root, *dir_components[:-1])) + return list(set(lance_dirs)) + + +def reset_column_dtype_from_file_to_image(db_path: str): + """Reset column dtype from 'file' to 'image' in SQLite tables.""" + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + # Fetch all TableMeta records + cursor.execute("SELECT id, cols FROM TableMeta") + records = cursor.fetchall() + + for i, record in enumerate(records): + table_id = record[0] + cols = orjson.loads(record[1]) + + updated_cols = [] + print(f"└─ (Table {i + 1:,d}/{len(records):,d}) Modifying table: {table_id}") + for col in cols: + col = ColumnSchema.model_validate(col) + if col.dtype == "file": + col.dtype = "image" + col = col.model_dump() + updated_cols.append(col) + + # Update the TableMeta record with the new cols + updated_cols_json = orjson.dumps(updated_cols).decode("utf-8") + cursor.execute( + "UPDATE TableMeta SET cols = ? WHERE id = ?", + (updated_cols_json, table_id), + ) + conn.commit() + print( + f"└─ (Table {i + 1:,d}/{len(records):,d}) Updated 'file' dtype to 'image' in table: {table_id}" + ) + # Checking + cursor.execute("SELECT id, cols FROM TableMeta") + records = cursor.fetchall() + for i, record in enumerate(records): + table_id = record[0] + cols = orjson.loads(record[1]) + print(f"└─ (Table {i + 1:,d}/{len(records):,d}) Checking table: {table_id}") + print( + f"\t└─ Current (column, dtype) pairs: {[(col['id'], col['dtype']) for col in cols]}" + ) + cursor.close() + conn.close() + except Exception as e: + logger.exception(f"└─ Error updating GenTable column due to {e}: {record}") + + +def add_page_column(db_path: str): + """Add 'Page' column to SQLite tables.""" + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + # Fetch all TableMeta records + cursor.execute("SELECT id, cols FROM TableMeta") + records = cursor.fetchall() + PAGE_COLUMN_ID = "Page" + for i, record in enumerate(records): + table_id = record[0] + print(f"└─ (Table {i + 1:,d}/{len(records):,d}) Modifying table: {table_id}") + cols = orjson.loads(record[1]) + has_page_column = False + for col in cols: + col = ColumnSchema.model_validate(col) + if col.id == PAGE_COLUMN_ID: + has_page_column = True + break + if not has_page_column: + cols.append( + ColumnSchema( + id=PAGE_COLUMN_ID, + dtype="int", + ).model_dump() + ) + cols.append( + ColumnSchema( + id=f"{PAGE_COLUMN_ID}_", + dtype="str", + ).model_dump() + ) + updated_cols_json = orjson.dumps(cols).decode("utf-8") + cursor.execute( + "UPDATE TableMeta SET cols = ? WHERE id = ?", + (updated_cols_json, table_id), + ) + conn.commit() + print( + f"└─ (Table {i + 1:,d}/{len(records):,d}) Added '{PAGE_COLUMN_ID}' column to table: {table_id}" + ) + else: + print( + f"└─ (Table {i + 1:,d}/{len(records):,d}) Table: {table_id} already has '{PAGE_COLUMN_ID}' column" + ) + # Checking + cursor.execute("SELECT id, cols FROM TableMeta") + records = cursor.fetchall() + for i, record in enumerate(records): + table_id = record[0] + cols = orjson.loads(record[1]) + print(f"└─ (Table {i + 1:,d}/{len(records):,d}) Checking table: {table_id}") + print(f"\t└─ Current columns: {[col['id'] for col in cols]}") + cursor.close() + conn.close() + except Exception as e: + logger.exception(f"└─ Error adding columns due to {e}") + + +def add_page_column_to_lance_table(lance_dir: str): + """Add 'Page' column to LanceDB tables.""" + try: + db = lancedb.connect(lance_dir) + table_names = [ + basename(table_dir).replace(".lance", "") + for table_dir in glob(join(lance_dir, "*.lance")) + ] + for i, table_name in enumerate(table_names): + print( + f"└─ (Table {i + 1:,d}/{len(table_names):,d}) Modifying LanceDB table: {table_name}" + ) + tbl = db.open_table(table_name) + if "Page" not in tbl.schema.names: + tbl.add_columns( + { + "Page": "cast(NULL as bigint)", + "Page_": "cast('{\"is_null\": true}' as string)", + } + ) + print(f"\t└─ Added 'Page' column to LanceDB table: {table_name}") + else: + print(f"\t└─ LanceDB table: {table_name} already has 'Page' column") + print(f"\t└─ Current columns: {tbl.schema.names}") + except Exception as e: + logger.exception(f"└─ Error adding columns to LanceDB table due to {e}") + + +if __name__ == "__main__": + backup_dir = f"{ENV_CONFIG.owl_db_dir}_BAK_{NOW}" + os.makedirs(backup_dir, exist_ok=False) + + # Backup SQLite files + sqlite_files = find_sqlite_files(ENV_CONFIG.owl_db_dir) + for j, db_file in enumerate(sqlite_files): + print(f"(DB {j + 1:,d}/{len(sqlite_files):,d}): Processing: {db_file}") + backup_db(db_file, backup_dir) + if not db_file.endswith("main.db"): + reset_column_dtype_from_file_to_image(db_file) + if db_file.endswith("knowledge.db"): + add_page_column(db_file) + + # Backup and process knowledge table LanceDB files + kt_lance_dirs = find_lance_dirs(ENV_CONFIG.owl_db_dir, "knowledge") + for k, kt_lance_dir in enumerate(kt_lance_dirs): + print(f"(LanceDB {k + 1:,d}/{len(kt_lance_dirs):,d}): Processing: {kt_lance_dir}") + backup_lance_db(kt_lance_dir, backup_dir) + add_page_column_to_lance_table(kt_lance_dir) diff --git a/services/api/pyproject.toml b/services/api/pyproject.toml index 61ef746..1f45a29 100644 --- a/services/api/pyproject.toml +++ b/services/api/pyproject.toml @@ -25,7 +25,7 @@ filterwarnings = [ [tool.ruff] line-length = 99 indent-width = 4 -target-version = "py310" +target-version = "py312" extend-include = [".pyi?$", ".ipynb"] extend-exclude = ["archive/*"] respect-gitignore = true @@ -84,7 +84,7 @@ description = "Owl: API server for JamAI Base." readme = "README.md" requires-python = "~=3.10" # keywords = ["one", "two"] -license = { text = "Proprietary" } +license = { text = "Apache 2.0" } classifiers = [ # https://pypi.org/classifiers/ "Development Status :: 3 - Alpha", "Programming Language :: Python :: 3 :: Only", @@ -110,7 +110,7 @@ dependencies = [ "lancedb==0.12.0", "langchain-community~=0.2.12", "langchain~=0.2.14", - "litellm~=1.48.17", + "litellm~=1.50.0", "loguru~=0.7.2", "natsort[fast]>=8.4.0", "numpy>=1.26.4", @@ -123,6 +123,7 @@ dependencies = [ "pycryptodomex~=3.20.0", "pydantic-settings~=2.4.0", "pydantic[email,timezone]~=2.8.2", + "pydub~=0.25.1", "pyjwt~=2.9.0", # pylance 0.13.0 has issues with row deletion "pylance==0.16.0", diff --git a/services/api/src/owl/configs/manager.py b/services/api/src/owl/configs/manager.py index 30f3677..a434edc 100644 --- a/services/api/src/owl/configs/manager.py +++ b/services/api/src/owl/configs/manager.py @@ -15,9 +15,9 @@ from redis.retry import Retry from owl.protocol import ( - EXAMPLE_CHAT_MODEL, - EXAMPLE_EMBEDDING_MODEL, - EXAMPLE_RERANKING_MODEL, + EXAMPLE_CHAT_MODEL_IDS, + EXAMPLE_EMBEDDING_MODEL_IDS, + EXAMPLE_RERANKING_MODEL_IDS, ModelListConfig, ) @@ -26,7 +26,11 @@ class EnvConfig(BaseSettings): model_config = SettingsConfigDict( - env_file=".env", env_file_encoding="utf-8", extra="ignore", cli_parse_args=False + # env_prefix="owl_", # TODO: Enable this + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + cli_parse_args=False, ) # API configs owl_is_prod: bool = False @@ -45,7 +49,9 @@ class EnvConfig(BaseSettings): owl_redis_port: int = 6379 owl_internal_org_id: str = "org_82d01c923f25d5939b9d4188" # Configs - owl_file_upload_max_bytes: int = 20 * 1024 * 1024 # 20MB in bytes + owl_embed_file_upload_max_bytes: int = 200 * 1024 * 1024 # 200MB in bytes + owl_image_file_upload_max_bytes: int = 20 * 1024 * 1024 # 20MB in bytes + owl_audio_file_upload_max_bytes: int = 120 * 1024 * 1024 # 120MB in bytes owl_compute_storage_period_min: float = 1 owl_models_config: str = "models.json" owl_pricing_config: str = "cloud_pricing.json" @@ -63,6 +69,8 @@ class EnvConfig(BaseSettings): owl_concurrent_rows_batch_size: int = 3 owl_concurrent_cols_batch_size: int = 5 owl_max_write_batch_size: int = 1000 + # Code Executor configs + code_executor_endpoint: str = "http://kopi:5569" # Loader configs docio_url: str = "http://docio:6979/api/docio" unstructuredio_url: str = "http://unstructuredio:6989" @@ -98,6 +106,7 @@ class EnvConfig(BaseSettings): hyperbolic_api_key: SecretStr = "" cerebras_api_key: SecretStr = "" sambanova_api_key: SecretStr = "" + deepseek_api_key: SecretStr = "" @model_validator(mode="after") def make_paths_absolute(self): @@ -203,6 +212,10 @@ def cerebras_api_key_plain(self): def sambanova_api_key_plain(self): return self.sambanova_api_key.get_secret_value() + @property + def deepseek_api_key_plain(self): + return self.deepseek_api_key.get_secret_value() + MODEL_CONFIG_KEY = " models" PRICES_KEY = " prices" @@ -341,7 +354,11 @@ class _ModelPrice(BaseModel): 'Unique identifier in the form of "{provider}/{model_id}". ' "Users will specify this to select a model." ), - examples=[EXAMPLE_CHAT_MODEL, EXAMPLE_EMBEDDING_MODEL, EXAMPLE_RERANKING_MODEL], + examples=[ + EXAMPLE_CHAT_MODEL_IDS[0], + EXAMPLE_EMBEDDING_MODEL_IDS[0], + EXAMPLE_RERANKING_MODEL_IDS[0], + ], ) name: str = Field( description="Name of the model.", @@ -482,7 +499,6 @@ def get_model_json(self) -> str: if model_json is None: model_json = self._load_model_config_from_file().model_dump_json() self[MODEL_CONFIG_KEY] = model_json - logger.warning(f"Model config set to: {model_json}") return model_json def get_model_config(self) -> ModelListConfig: diff --git a/services/api/src/owl/configs/models_ci.json b/services/api/src/owl/configs/models_ci.json index d37b069..fcc62a0 100644 --- a/services/api/src/owl/configs/models_ci.json +++ b/services/api/src/owl/configs/models_ci.json @@ -5,7 +5,7 @@ "name": "OpenAI GPT-4o Mini", "context_length": 128000, "languages": ["mul"], - "capabilities": ["chat", "image"], + "capabilities": ["chat", "image", "tool"], "deployments": [ { "litellm_id": "", @@ -19,7 +19,7 @@ "name": "Anthropic Claude 3 Haiku", "context_length": 200000, "languages": ["mul"], - "capabilities": ["chat"], + "capabilities": ["chat", "tool"], "deployments": [ { "litellm_id": "", @@ -29,16 +29,31 @@ ] }, { - "id": "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "name": "Together AI Meta Llama 3.1 (8B)", - "context_length": 130000, + "id": "meta/Llama3.2-3b-instruct", + "name": "Meta Llama 3.2 (3B)", + "context_length": 128000, "languages": ["mul"], "capabilities": ["chat"], "deployments": [ { - "litellm_id": "", - "api_base": "", - "provider": "together_ai" + "litellm_id": "openai/meta/Llama3.2-3b-instruct", + "api_base": "https://llmci.embeddedllm.com/chat/v1", + "provider": "custom" + } + ] + }, + { + "id": "ellm/Qwen/Qwen-2-Audio-7B", + "object": "model", + "name": "Qwen 2 Audio 7B (Audio, internal)", + "context_length": 128000, + "languages": ["mul"], + "capabilities": ["chat", "audio"], + "deployments": [ + { + "litellm_id": "openai/Qwen/Qwen-2-Audio-7B", + "api_base": "https://llmci.embeddedllm.com/audio/v1", + "provider": "custom" } ] } diff --git a/services/api/src/owl/db/gen_executor.py b/services/api/src/owl/db/gen_executor.py index 466ebda..493ce07 100644 --- a/services/api/src/owl/db/gen_executor.py +++ b/services/api/src/owl/db/gen_executor.py @@ -21,6 +21,7 @@ ChatCompletionChunk, ChatEntry, ChatRequest, + CodeGenConfig, EmbedGenConfig, ExternalKeys, GenTableChatCompletionChunks, @@ -36,14 +37,15 @@ TableMeta, ) from owl.utils import mask_string, uuid7_draft2_str +from owl.utils.code import code_executor from owl.utils.io import open_uri_async @dataclass(slots=True) class Task: - type: Literal["embed", "chat"] + type: Literal["embed", "chat", "code"] output_column_name: str - body: ChatRequest | EmbedGenConfig + body: ChatRequest | EmbedGenConfig | CodeGenConfig dtype: str @@ -290,9 +292,12 @@ def __init__( self.error_columns = [] self.tag_regen_columns = [] self.skip_regen_columns = [] - self.file_columns = [] - self.img_column_dict = {} - self.doc_column_dict = {} + self.image_columns = [] + self.audio_columns = [] + self.audio_gen_columns = [] + self.image_column_dict = {} + self.document_column_dict = {} + self.audio_column_dict = {} def _log_exception(self, exc: Exception, error_message: str): if not isinstance(exc, (JamaiException, RequestValidationError)): @@ -302,6 +307,52 @@ async def _get_file_binary(self, uri: str) -> bytes: async with open_uri_async(uri) as file_handle: return await file_handle.read() + # TODO: resolve duplicated code + async def _convert_uri_to_base64(self, uri: str, col_id: str) -> tuple[dict, bool]: + """ + Converts a URI to a base64-encoded string with the appropriate prefix and determines the file type. + + Args: + uri (str): The URI of the file. + col_id (str): The column ID for error context. + + Returns: + tuple: A tuple containing: + - dict: A dictionary with the base64-encoded data and its prefix. + - bool: A boolean indicating whether the file is audio. + + Raises: + ValueError: If the file format is unsupported. + """ + if not uri.startswith(("file://", "s3://")): + raise ValueError( + f"Invalid URI format for column {col_id}. URI must start with 'file://' or 's3://'" + ) + + # uri -> file binary -> base64 + file_binary = await self._get_file_binary(uri) + base64_data = self._binary_to_base64(file_binary) + + # uri -> file extension -> prefix + extension = splitext(uri)[1].lower() + + if extension in [".mp3", ".wav"]: + prefix = f"data:audio/{"mpeg" if extension == ".mp3" else "x-wav"};base64," + return { + "data": base64_data, + "format": extension[1:], + "url": prefix + base64_data, + }, True + elif extension in [".jpeg", ".jpg", ".png", ".gif", ".webp"]: + extension = ".jpeg" if extension == ".jpg" else extension + prefix = f"data:image/{extension[1:]};base64," + return {"url": prefix + base64_data}, False + else: + raise ValueError( + "Unsupported file type. Supported formats are: " + "['jpeg/jpg', 'png', 'gif', 'webp'] for images and ['mp3', 'wav'] for audio." + ) + async def gen_row(self) -> Any | tuple[GenTableChatCompletionChunks, dict]: cols = self.meta.cols_schema col_ids = set(c.id for c in cols) @@ -354,32 +405,49 @@ async def gen_row(self) -> Any | tuple[GenTableChatCompletionChunks, dict]: gen_config = ChatRequest( id=self.request.state.id, messages=messages, **col.gen_config.model_dump() ) + if gen_config.model != "": + model_config = self.request.state.all_models.get_llm_model_info( + gen_config.model + ) + if ( + "audio" in model_config.capabilities + and model_config.deployments[0].provider == "openai" + ): + self.audio_gen_columns.append(col.id) + elif isinstance(col.gen_config, CodeGenConfig): + task_type = "code" + gen_config = col.gen_config else: raise ValueError(f'Unexpected "gen_config" type: {type(col.gen_config)}') self.tasks.append( Task(type=task_type, output_column_name=col.id, body=gen_config, dtype=col.dtype) ) - self.file_columns = [col.id for col in cols if col.dtype == "file"] - for col_id in self.file_columns: + self.image_columns = [col.id for col in cols if col.dtype == "image"] + self.audio_columns = [col.id for col in cols if col.dtype == "audio"] + for col_id in self.image_columns + self.audio_columns: if self.column_dict.get(col_id, None) is not None: uri = self.column_dict[col_id] - # uri -> file binary -> base64 - file_binary = await self._get_file_binary(uri) - base64 = self._binary_to_base64(file_binary) - - # uri -> file extension -> prefix - extension = splitext(uri)[1].lower() - if extension in [".jpeg", ".jpg", ".png", ".gif", ".webp"]: - extension = ".jpeg" if extension == ".jpg" else extension - prefix = f"data:image/{extension[1:]};base64," - # url = prefix + base64 - self.img_column_dict[col_id] = prefix + base64 - else: - raise ValueError( - "Unsupported image, make sure the image belongs to " - "one of the following formats: ['jpeg/jpg', 'png', 'gif', 'webp']." + b64, is_audio = await self._convert_uri_to_base64(uri, col_id) + + if is_audio: + if col_id not in self.audio_columns: + raise ValueError( + f"Column {col_id} is not marked as an audio column but contains audio data." + ) + self.audio_column_dict[col_id] = ( + { + "data": b64["data"], + "format": b64["format"], + }, # for audio gen model + {"url": b64["url"]}, # for audio model ) + else: + if col_id not in self.image_columns: + raise ValueError( + f"Column {col_id} is not marked as a file column but contains image data." + ) + self.image_column_dict[col_id] = b64 column_dict_keys = set(self.column_dict.keys()) if len(column_dict_keys - col_ids) > 0: @@ -422,7 +490,7 @@ def _extract_upstream_image_columns(self, text: str) -> list[str]: def _binary_to_base64(self, binary_data: bytes) -> str: return base64.b64encode(binary_data).decode("utf-8") - def _interpolate_column(self, prompt: str) -> str | dict[str, Any]: + def _interpolate_column(self, prompt: str, base_column_name: str) -> str | dict[str, Any]: """ Replaces / interpolates column references in the prompt with their contents. @@ -434,15 +502,22 @@ def _interpolate_column(self, prompt: str) -> str | dict[str, Any]: """ image_column_names = [] + audio_column_names = [] def replace_match(match): column_name = match.group(1) # Extract the column_name from the match try: - if column_name in self.img_column_dict: + if column_name in self.image_column_dict: image_column_names.append(column_name) return "" - elif column_name in self.doc_column_dict: - return self.doc_column_dict[column_name] + elif column_name in self.audio_column_dict: + audio_column_names.append(column_name) + if base_column_name in self.audio_gen_columns: + return "" # follow the content type + else: + return "" + elif column_name in self.document_column_dict: + return self.document_column_dict[column_name] return str(self.column_dict[column_name]) # Data can be non-string except KeyError as e: raise BadInputError(f"Requested column '{column_name}' is not found.") from e @@ -450,6 +525,9 @@ def replace_match(match): content_ = re.sub(GEN_CONFIG_VAR_PATTERN, replace_match, prompt) content = [{"type": "text", "text": content_}] + if len(image_column_names) > 0 and len(audio_column_names) > 0: + raise BadInputError("Either image or audio is supported per completion.") + if len(image_column_names) > 0: if len(image_column_names) > 1: raise BadInputError("Only one image is supported per completion.") @@ -457,10 +535,29 @@ def replace_match(match): content.append( { "type": "image_url", - "image_url": {"url": self.img_column_dict[image_column_names[0]]}, + "image_url": self.image_column_dict[image_column_names[0]], } ) return content + elif len(audio_column_names) > 0: + if len(audio_column_names) > 1: + raise BadInputError("Only one audio is supported per completion.") + + if base_column_name in self.audio_gen_columns: + content.append( + { + "type": "input_audio", + "input_audio": self.audio_column_dict[audio_column_names[0]][0], + } + ) + else: + content.append( + { + "type": "audio_url", + "audio_url": self.audio_column_dict[audio_column_names[0]][1], + } + ) + return content else: return content_ @@ -469,6 +566,53 @@ def _check_upstream_error_chunk(self, content: str) -> None: if any([match in self.error_columns for match in matches]): raise Exception + def _validate_model(self, body: LLMGenConfig, output_column_name: str): + for input_column_name in self.dependencies[output_column_name]: + if input_column_name in self.image_column_dict: + try: + body.model = self.llm.validate_model_id(body.model, ["image"]) + break + except ResourceNotFoundError as e: + raise BadInputError( + f'Column "{output_column_name}" referred to image file input but using a chat model ' + f'"{self.llm.get_model_name(body.model) if self.llm.is_browser else body.model}", ' + "select image model instead.", + ) from e + if input_column_name in self.audio_column_dict: + try: + body.model = self.llm.validate_model_id(body.model, ["audio"]) + break + except ResourceNotFoundError as e: + raise BadInputError( + f'Column "{output_column_name}" referred to audio file input but using a chat model ' + f'"{self.llm.get_model_name(body.model) if self.llm.is_browser else body.model}", ' + "select audio model instead.", + ) from e + + async def _execute_code(self, task: Task) -> str: + output_column_name = task.output_column_name + body: CodeGenConfig = task.body + dtype = task.dtype + source_code = self.column_dict[body.source_column] + + try: + new_column_value = await code_executor(source_code, dtype, self.request) + except Exception as e: + new_column_value = f"[ERROR] {str(e)}" + self._log_exception(e, f'Error executing code for column "{output_column_name}": {e}') + + if dtype == "image" and new_column_value is not None: + try: + ( + self.image_column_dict[output_column_name], + _, + ) = await self._convert_uri_to_base64(new_column_value, output_column_name) + except ValueError as e: + self._log_exception(e, f"Invalid file path for column '{output_column_name}'") + new_column_value = None + + return new_column_value + async def _execute_task_stream(self, task: Task) -> AsyncGenerator[str, None]: """ Executes a single task in a streaming manner, returning an asynchronous generator of chunks. @@ -478,38 +622,21 @@ async def _execute_task_stream(self, task: Task) -> AsyncGenerator[str, None]: try: logger.debug(f"Processing column: {output_column_name}") - self._check_upstream_error_chunk(body.messages[-1].content) - body.messages[-1].content = self._interpolate_column(body.messages[-1].content) - - if isinstance(body.messages[-1].content, list): - for input_column_name in self.dependencies[output_column_name]: - if input_column_name in self.img_column_dict: - try: - body.model = self.llm.validate_model_id(body.model, ["image"]) - break - except ResourceNotFoundError as e: - raise BadInputError( - f'Column "{output_column_name}" referred to image file input but using a chat model ' - f'"{self.llm.get_model_name(body.model) if self.llm.is_browser else body.model}", ' - "select image model instead.", - ) from e if output_column_name in self.skip_regen_columns: new_column_value = self.column_dict[output_column_name] logger.debug( f"Skipped regen for `{output_column_name}`, value: {new_column_value}" ) - elif output_column_name in self.file_columns: - new_column_value = None - logger.info( - f"Identified output column `{output_column_name}` as file type, set value to {new_column_value}" - ) + elif isinstance(body, CodeGenConfig): + new_column_value = await self._execute_code(task) + logger.info(f"Executed Code Execution Column: '{output_column_name}'") chunk = GenTableStreamChatCompletionChunk( id=self.request.state.id, object="gen_table.completion.chunk", created=int(time()), - model="", + model="code_execution", usage=None, choices=[ ChatCompletionChoiceDelta( @@ -522,36 +649,62 @@ async def _execute_task_stream(self, task: Task) -> AsyncGenerator[str, None]: ) yield f"data: {chunk.model_dump_json()}\n\n" - else: - new_column_value = "" - kwargs = body.model_dump() - messages, references = await self.llm.retrieve_references( - messages=kwargs.pop("messages"), - rag_params=kwargs.pop("rag_params", None), - **kwargs, + elif isinstance(body, ChatRequest): + self._check_upstream_error_chunk(body.messages[-1].content) + body.messages[-1].content = self._interpolate_column( + body.messages[-1].content, output_column_name ) - if references is not None: - ref = GenTableStreamReferences( - **references.model_dump(exclude=["object"]), - output_column_name=output_column_name, + + if isinstance(body.messages[-1].content, list): + self._validate_model(body, output_column_name) + + if output_column_name in self.image_columns + self.audio_columns: + new_column_value = None + logger.info( + f"Identified output column `{output_column_name}` as image / audio type, set value to {new_column_value}" ) - yield f"data: {ref.model_dump_json()}\n\n" - async for chunk in self.llm.generate_stream(messages=messages, **kwargs): - new_column_value += chunk.text chunk = GenTableStreamChatCompletionChunk( - **chunk.model_dump(exclude=["object"]), + id=self.request.state.id, + object="gen_table.completion.chunk", + created=int(time()), + model="", + usage=None, + choices=[ + ChatCompletionChoiceDelta( + message=ChatEntry.assistant(new_column_value), + index=0, + ) + ], output_column_name=output_column_name, row_id=self.row_id, ) yield f"data: {chunk.model_dump_json()}\n\n" - if chunk.finish_reason == "error": - self.error_columns.append(output_column_name) - logger.info( - ( - f"{self.request.state.id} - Streamed completion for " - f"{output_column_name}: <{mask_string(new_column_value)}>" + else: + new_column_value = "" + kwargs = body.model_dump() + messages, references = await self.llm.retrieve_references( + messages=kwargs.pop("messages"), + rag_params=kwargs.pop("rag_params", None), + **kwargs, ) - ) + if references is not None: + ref = GenTableStreamReferences( + **references.model_dump(exclude=["object"]), + output_column_name=output_column_name, + ) + yield f"data: {ref.model_dump_json()}\n\n" + async for chunk in self.llm.generate_stream(messages=messages, **kwargs): + new_column_value += chunk.text + chunk = GenTableStreamChatCompletionChunk( + **chunk.model_dump(exclude=["object"]), + output_column_name=output_column_name, + row_id=self.row_id, + ) + yield f"data: {chunk.model_dump_json()}\n\n" + if chunk.finish_reason == "error": + self.error_columns.append(output_column_name) + else: + raise ValueError(f"Unsupported task type: {type(body)}") except Exception as e: error_chunk = GenTableStreamChatCompletionChunk( @@ -580,6 +733,10 @@ async def _execute_task_stream(self, task: Task) -> AsyncGenerator[str, None]: # Append new column data for subsequent tasks self.column_dict[output_column_name] = new_column_value self.regen_column_dict[output_column_name] = new_column_value + logger.info( + f"{self.request.state.id} - Streamed completion for " + f"{output_column_name}: <{mask_string(new_column_value)}>" + ) async def _execute_task_nonstream(self, task: Task): """ @@ -589,15 +746,6 @@ async def _execute_task_nonstream(self, task: Task): body: ChatRequest = task.body try: - body.messages[-1].content = self._interpolate_column(body.messages[-1].content) - except IndexError: - pass - try: - if isinstance(body.messages[-1].content, list): - for input_column_name in self.dependencies[output_column_name]: - if input_column_name in self.img_column_dict: - body.model = self.llm.validate_model_id(body.model, ["image"]) - break if output_column_name in self.skip_regen_columns: new_column_value = self.column_dict[output_column_name] response = ChatCompletionChunk( @@ -616,27 +764,60 @@ async def _execute_task_nonstream(self, task: Task): logger.debug( f"Skipped regen for `{output_column_name}`, value: {new_column_value}" ) - elif output_column_name in self.file_columns: - new_column_value = None + + elif isinstance(body, CodeGenConfig): + new_column_value = await self._execute_code(task) response = ChatCompletionChunk( id=self.request.state.id, object="chat.completion.chunk", created=int(time()), - model="", + model="code_execution", usage=None, choices=[ ChatCompletionChoiceDelta( - message=ChatEntry.assistant(new_column_value), index=0, + message=ChatEntry.assistant(new_column_value), ) ], ) - logger.info( - f"Identified output column `{output_column_name}` as file type, set value to {new_column_value}" + logger.debug( + f"Identified as Code Execution Column: {task.output_column_name}, executing code." ) + elif isinstance(body, ChatRequest): + self._check_upstream_error_chunk(body.messages[-1].content) + try: + body.messages[-1].content = self._interpolate_column( + body.messages[-1].content, output_column_name + ) + except IndexError: + pass + + if isinstance(body.messages[-1].content, list): + self._validate_model(body, output_column_name) + + if output_column_name in self.image_columns + self.audio_columns: + new_column_value = None + response = ChatCompletionChunk( + id=self.request.state.id, + object="chat.completion.chunk", + created=int(time()), + model="", + usage=None, + choices=[ + ChatCompletionChoiceDelta( + message=ChatEntry.assistant(new_column_value), + index=0, + ) + ], + ) + logger.debug( + f"Identified output column `{output_column_name}` as image / audio type, set value to {new_column_value}" + ) + else: + response = await self.llm.rag(**body.model_dump()) + new_column_value = response.text else: - response = await self.llm.rag(**body.model_dump()) - new_column_value = response.text + raise ValueError(f"Unsupported task type: {type(body)}") # append new column data for subsequence tasks self.column_dict[output_column_name] = new_column_value @@ -705,14 +886,25 @@ def _setup_dependencies(self) -> None: self.llm_tasks = { task.output_column_name: task for task in self.tasks if task.type == "chat" } + self.code_tasks = { + task.output_column_name: task for task in self.tasks if task.type == "code" + } self.dependencies = { task.output_column_name: self._extract_upstream_columns(task.body.messages[-1].content) for task in self.llm_tasks.values() } + self.dependencies.update( + { + task.output_column_name: [task.body.source_column] + for task in self.code_tasks.values() + } + ) logger.debug(f"Initial dependencies: {self.dependencies}") self.input_column_names = [ - key for key in self.column_dict.keys() if key not in self.llm_tasks.keys() + key + for key in self.column_dict.keys() + if key not in self.llm_tasks.keys() and key not in self.code_tasks.keys() ] def _mark_regen_columns(self) -> None: @@ -722,8 +914,12 @@ def _mark_regen_columns(self) -> None: if self.is_row_add: return + # Get the current column order from the table metadata + cols = self.meta.cols_schema + col_ids = [col.id for col in cols] + if self.body.regen_strategy == RegenStrategy.RUN_ALL: - self.tag_regen_columns = self.llm_tasks.keys() + self.tag_regen_columns = set(self.llm_tasks.keys()).union(self.code_tasks.keys()) elif self.body.regen_strategy == RegenStrategy.RUN_SELECTED: self.tag_regen_columns.append(self.body.output_column_id) @@ -733,13 +929,13 @@ def _mark_regen_columns(self) -> None: RegenStrategy.RUN_AFTER, ): if self.body.regen_strategy == RegenStrategy.RUN_BEFORE: - for column_name in self.column_dict.keys(): + for column_name in col_ids: self.tag_regen_columns.append(column_name) if column_name == self.body.output_column_id: break else: # RegenStrategy.RUN_AFTER reached_column = False - for column_name in self.column_dict.keys(): + for column_name in col_ids: if column_name == self.body.output_column_id: reached_column = True if reached_column: @@ -749,9 +945,7 @@ def _mark_regen_columns(self) -> None: raise ValueError(f"Invalid regeneration strategy: {self.body.regen_strategy}") self.skip_regen_columns = [ - column_name - for column_name in self.column_dict.keys() - if column_name not in self.tag_regen_columns + column_name for column_name in col_ids if column_name not in self.tag_regen_columns ] async def _nonstream_concurrent_execution(self) -> tuple[GenTableChatCompletionChunks, dict]: @@ -766,7 +960,11 @@ async def _nonstream_concurrent_execution(self) -> tuple[GenTableChatCompletionC responses = {} async def execute_task(task_name): - task = self.llm_tasks[task_name] + try: + task = self.llm_tasks[task_name] + except Exception: + task = self.code_tasks[task_name] + try: responses[task_name] = await self._execute_task_nonstream(task) except Exception as e: @@ -775,7 +973,9 @@ async def execute_task(task_name): completed.add(task_name) tasks_in_progress.remove(task_name) - while len(completed) < (len(self.llm_tasks) + len(self.input_column_names)): + while len(completed) < ( + len(self.llm_tasks) + len(self.code_tasks) + len(self.input_column_names) + ): ready_tasks = [ task_name for task_name, deps in self.dependencies.items() @@ -812,8 +1012,20 @@ async def _stream_concurrent_execution(self) -> AsyncGenerator[str, None]: queue = asyncio.Queue() tasks_in_progress = set() + ready_tasks = [ + task_name + for task_name, deps in self.dependencies.items() + if all(dep in completed for dep in deps) + and task_name not in completed + and task_name not in tasks_in_progress + ] + async def execute_task(task_name): - task = self.llm_tasks[task_name] + try: + task = self.llm_tasks[task_name] + except Exception: + task = self.code_tasks[task_name] + try: async for chunk in self._execute_task_stream(task): await queue.put((task_name, chunk)) @@ -824,7 +1036,9 @@ async def execute_task(task_name): await queue.put((task_name, None)) tasks_in_progress.remove(task_name) - while len(completed) < (len(self.llm_tasks) + len(self.input_column_names)): + while len(completed) < ( + len(self.llm_tasks) + len(self.code_tasks) + len(self.input_column_names) + ): ready_tasks = [ task_name for task_name, deps in self.dependencies.items() diff --git a/services/api/src/owl/db/gen_table.py b/services/api/src/owl/db/gen_table.py index abf1c0d..bcf09ab 100644 --- a/services/api/src/owl/db/gen_table.py +++ b/services/api/src/owl/db/gen_table.py @@ -75,7 +75,8 @@ "float16": 0.0, "bool": False, "str": "''", - "file": "''", + "image": "''", + "audio": "''", } @@ -1101,8 +1102,10 @@ def _interpolate_column( def replace_match(match): col_id = match.group(1) try: - if column_dtypes[col_id] == "file": - return "" + if column_dtypes[col_id] == "image": + return "" + elif column_dtypes[col_id] == "audio": + return "" return str(column_contents[col_id]) except KeyError as e: raise KeyError(f'Referenced column "{col_id}" is not found.') from e @@ -1222,7 +1225,7 @@ def dump_parquet( # Convert into Arrow Table pa_table = table._dataset.to_table(offset=None, limit=None) # Add file data into Arrow Table - file_col_ids = [col.id for col in meta.cols_schema if col.dtype == "file"] + file_col_ids = [col.id for col in meta.cols_schema if col.dtype in ["image", "audio"]] for col_id in file_col_ids: file_bytes = [] for uri in pa_table.column(col_id).to_pylist(): @@ -1277,7 +1280,7 @@ async def import_parquet( if session.get(TableMeta, table_id_dst) is not None: raise ResourceExistsError(f'Table "{table_id_dst}" already exists.') # Upload files - file_col_ids = [col.id for col in meta.cols_schema if col.dtype == "file"] + file_col_ids = [col.id for col in meta.cols_schema if col.dtype in ["image", "audio"]] for col_id in file_col_ids: new_uris = [] for old_uri, content in zip( @@ -1789,7 +1792,7 @@ class ActionTable(GenerativeTable): class KnowledgeTable(GenerativeTable): - FIXED_COLUMN_IDS = ["Title", "Title Embed", "Text", "Text Embed", "File ID"] + FIXED_COLUMN_IDS = ["Title", "Title Embed", "Text", "Text Embed", "File ID", "Page"] @override def create_table( @@ -1831,6 +1834,7 @@ def create_table( ), ), ColumnSchema(id="File ID", dtype=ColumnDtype.STR), + ColumnSchema(id="Page", dtype=ColumnDtype.INT), ] + schema.cols, ) diff --git a/services/api/src/owl/entrypoints/api.py b/services/api/src/owl/entrypoints/api.py index 72b3e0e..d9f2678 100644 --- a/services/api/src/owl/entrypoints/api.py +++ b/services/api/src/owl/entrypoints/api.py @@ -5,7 +5,7 @@ import os from typing import Any -from fastapi import FastAPI, Request, status +from fastapi import BackgroundTasks, FastAPI, Request, status from fastapi.exceptions import RequestValidationError, ResponseValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import ORJSONResponse @@ -195,9 +195,18 @@ async def log_request(request: Request, call_next): ): return await call_next(request) - # --- Call request --- # + # Call request response = await call_next(request) logger.info(make_request_log_str(request, response.status_code)) + + # Add egress events + request.state.billing.create_egress_events( + float(response.headers.get("content-length", 0)) / (1024**3) + ) + # Process billing (this will run AFTER streaming responses are sent) + tasks = BackgroundTasks() + tasks.add_task(request.state.billing.process_all) + response.background = tasks return response diff --git a/services/api/src/owl/llm.py b/services/api/src/owl/llm.py index c8458fd..96a78e7 100644 --- a/services/api/src/owl/llm.py +++ b/services/api/src/owl/llm.py @@ -27,12 +27,14 @@ from owl.models import CloudEmbedder, CloudReranker from owl.protocol import ( ChatCompletionChoiceDelta, + ChatCompletionChoiceOutput, ChatCompletionChunk, ChatEntry, ChatRole, Chunk, CompletionUsage, ExternalKeys, + LLMModelConfig, ModelInfo, ModelInfoResponse, ModelListConfig, @@ -77,7 +79,7 @@ def _get_llm_router(model_json: str, external_api_keys: str): retry_after=5.0, timeout=ENV_CONFIG.owl_llm_timeout_sec, allowed_fails=3, - cooldown_time=0.0, + cooldown_time=5.5, debug_level="DEBUG", redis_host=ENV_CONFIG.owl_redis_host, redis_port=ENV_CONFIG.owl_redis_port, @@ -288,36 +290,57 @@ async def generate_stream( **hyperparams, ) -> AsyncGenerator[ChatCompletionChunk, None]: api_key = "" + usage = None try: model = model.strip() + # check audio model type + is_audio_gen_model = False + if model != "": + model_config: LLMModelConfig = self.request.state.all_models.get_llm_model_info( + model + ) + if ( + "audio" in model_config.capabilities + and model_config.deployments[0].provider == "openai" + ): + is_audio_gen_model = True hyperparams = self._prepare_hyperparams(model, hyperparams, stream=True) messages = self._prepare_messages(messages) + # omit system prompt for audio input with audio gen + if is_audio_gen_model and messages[0].role in (ChatRole.SYSTEM.value, ChatRole.SYSTEM): + messages = messages[1:] messages = [m.model_dump(mode="json", exclude_none=True) for m in messages] model = self.validate_model_id( model=model, capabilities=capabilities, ) self._log_completion_masked(model, messages, **hyperparams) - response = await self.router.acompletion( - model=model, - messages=messages, - # Fixes discrepancy between stream and non-stream token usage - stream_options={"include_usage": True}, - **hyperparams, - ) - chunks = [] - completion = None + if is_audio_gen_model: + response = await self.router.acompletion( + model=model, + modalities=["text", "audio"], + audio={"voice": "alloy", "format": "pcm16"}, + messages=messages, + # Fixes discrepancy between stream and non-stream token usage + stream_options={"include_usage": True}, + **hyperparams, + ) + else: + response = await self.router.acompletion( + model=model, + messages=messages, + # Fixes discrepancy between stream and non-stream token usage + stream_options={"include_usage": True}, + **hyperparams, + ) output_text = "" usage = CompletionUsage() async for chunk in response: - chunks.append(chunk) - content = chunk.choices[0].delta.content if hasattr(chunk, "usage"): - completion = chunk usage = CompletionUsage( - prompt_tokens=completion.usage.prompt_tokens, - completion_tokens=completion.usage.completion_tokens, - total_tokens=completion.usage.total_tokens, + prompt_tokens=chunk.usage.prompt_tokens, + completion_tokens=chunk.usage.completion_tokens, + total_tokens=chunk.usage.total_tokens, ) yield ChatCompletionChunk( id=self.id, @@ -327,7 +350,16 @@ async def generate_stream( usage=usage, choices=[ ChatCompletionChoiceDelta( - message=ChatEntry.assistant(choice.delta.content), + message=ChatEntry.assistant(choice.delta.audio.get("transcript", "")) + if is_audio_gen_model and choice.delta.audio is not None + else ChatCompletionChoiceOutput.assistant( + choice.delta.content, + tool_calls=[ + tool_call.model_dump() for tool_call in choice.delta.tool_calls + ] + if isinstance(chunk.choices[0].delta.tool_calls, list) + else None, + ), index=choice.index, finish_reason=choice.get( "finish_reason", chunk.get("finish_reason", None) @@ -336,16 +368,17 @@ async def generate_stream( for choice in chunk.choices ], ) - output_text += content if content else "" + if is_audio_gen_model and chunk.choices[0].delta.audio is not None: + output_text += chunk.choices[0].delta.audio.get("transcript", "") + else: + content = chunk.choices[0].delta.content + output_text += content if content else "" logger.info(f"{self.id} - Streamed completion: <{mask_string(output_text)}>") - if completion is None: - logger.warning("`completion` should not be None !!!") - return self._billing.create_llm_events( model=model, - input_tokens=completion.usage.prompt_tokens, - output_tokens=completion.usage.completion_tokens, + input_tokens=usage.prompt_tokens, + output_tokens=usage.completion_tokens, ) except Exception as e: self._map_and_log_exception(e, model, messages, api_key, **hyperparams) @@ -354,7 +387,7 @@ async def generate_stream( object="chat.completion.chunk", created=int(time()), model=model, - usage=None, + usage=usage, choices=[ ChatCompletionChoiceDelta( message=ChatEntry.assistant(f"[ERROR] {e!r}"), @@ -374,31 +407,59 @@ async def generate( api_key = "" try: model = model.strip() + # check audio model type + is_audio_gen_model = False + if model != "": + model_config: LLMModelConfig = self.request.state.all_models.get_llm_model_info( + model + ) + if ( + "audio" in model_config.capabilities + and model_config.deployments[0].provider == "openai" + ): + is_audio_gen_model = True hyperparams = self._prepare_hyperparams(model, hyperparams, stream=False) messages = self._prepare_messages(messages) + # omit system prompt for audio input with audio gen + if is_audio_gen_model and messages[0].role in (ChatRole.SYSTEM.value, ChatRole.SYSTEM): + messages = messages[1:] messages = [m.model_dump(mode="json", exclude_none=True) for m in messages] model = self.validate_model_id( model=model, capabilities=capabilities, ) self._log_completion_masked(model, messages, **hyperparams) - completion = await self.router.acompletion( - model=model, - messages=messages, - **hyperparams, - ) + if is_audio_gen_model: + completion = await self.router.acompletion( + model=model, + modalities=["text", "audio"], + audio={"voice": "alloy", "format": "pcm16"}, + messages=messages, + **hyperparams, + ) + else: + completion = await self.router.acompletion( + model=model, + messages=messages, + **hyperparams, + ) self._billing.create_llm_events( model=model, input_tokens=completion.usage.prompt_tokens, output_tokens=completion.usage.completion_tokens, ) + choices = [] + for choice in completion.choices: + if is_audio_gen_model and choice.message.audio.transcript is not None: + choice.message.content = choice.message.audio.transcript + choices.append(choice.model_dump()) completion = ChatCompletionChunk( id=self.id, object="chat.completion", created=completion.created, model=model, usage=completion.usage.model_dump(), - choices=[choice.model_dump() for choice in completion.choices], + choices=choices, ) logger.info(f"{self.id} - Generated completion: <{mask_string(completion.text)}>") return completion diff --git a/services/api/src/owl/loaders.py b/services/api/src/owl/loaders.py index 2ddb7ba..f53a4c0 100644 --- a/services/api/src/owl/loaders.py +++ b/services/api/src/owl/loaders.py @@ -29,7 +29,10 @@ def make_printable(s: str) -> str: return s.translate(NOPRINT_TRANS_TABLE) -def format_chunks(documents: list[Document], file_name: str) -> list[Chunk]: +def format_chunks(documents: list[Document], file_name: str, page: int = None) -> list[Chunk]: + if page is not None: + for d in documents: + d.metadata["page"] = page chunks = [ # TODO: Probably can use regex for this # Replace vertical tabs, form feed, Unicode replacement character @@ -84,7 +87,7 @@ async def load_file( loader = DocIOAPIFileLoader(tmp_path, ENV_CONFIG.docio_url) documents = loader.load() logger.debug('File "{file_name}" loaded: {docs}', file_name=file_name, docs=documents) - chunks = format_chunks(documents, file_name) + chunks = format_chunks(documents, file_name, page=1) if ext == ".json": chunks = split_chunks( SplitChunksRequest( diff --git a/services/api/src/owl/models.py b/services/api/src/owl/models.py index 6efd67f..f609dc4 100644 --- a/services/api/src/owl/models.py +++ b/services/api/src/owl/models.py @@ -67,7 +67,7 @@ def _get_embedding_router(model_json: str, external_api_keys: str): retry_after=5.0, timeout=ENV_CONFIG.owl_embed_timeout_sec, allowed_fails=3, - cooldown_time=0.0, + cooldown_time=5.5, ) diff --git a/services/api/src/owl/protocol.py b/services/api/src/owl/protocol.py index ea5b232..622d23d 100644 --- a/services/api/src/owl/protocol.py +++ b/services/api/src/owl/protocol.py @@ -63,22 +63,27 @@ def sanitise_document_id_list(v: list[str]) -> list[str]: DocumentID = Annotated[str, AfterValidator(sanitise_document_id)] DocumentIDList = Annotated[list[str], AfterValidator(sanitise_document_id_list)] -EXAMPLE_CHAT_MODEL = "openai/gpt-4o-mini" - +EXAMPLE_CHAT_MODEL_IDS = ["openai/gpt-4o-mini"] # for openai embedding models doc: https://platform.openai.com/docs/guides/embeddings # for cohere embedding models doc: https://docs.cohere.com/reference/embed # for jina embedding models doc: https://jina.ai/embeddings/ # for voyage embedding models doc: https://docs.voyageai.com/docs/embeddings # for hf embedding models doc: check the respective hf model page, name should be ellm/{org}/{model} -EXAMPLE_EMBEDDING_MODEL = "openai/text-embedding-3-small-512" - +EXAMPLE_EMBEDDING_MODEL_IDS = [ + "openai/text-embedding-3-small-512", + "ellm/sentence-transformers/all-MiniLM-L6-v2", +] # for cohere reranking models doc: https://docs.cohere.com/reference/rerank-1 # for jina reranking models doc: https://jina.ai/reranker # for colbert reranking models doc: https://docs.voyageai.com/docs/reranker # for hf embedding models doc: check the respective hf model page, name should be ellm/{org}/{model} -EXAMPLE_RERANKING_MODEL = "cohere/rerank-multilingual-v3.0" +EXAMPLE_RERANKING_MODEL_IDS = [ + "cohere/rerank-multilingual-v3.0", + "ellm/cross-encoder/ms-marco-TinyBERT-L-2", +] IMAGE_FILE_EXTENSIONS = [".jpeg", ".jpg", ".png", ".gif", ".webp"] +AUDIO_FILE_EXTENSIONS = [".mp3", ".wav"] DOCUMENT_FILE_EXTENSIONS = [ ".pdf", ".txt", @@ -240,6 +245,7 @@ class ExternalKeys(BaseModel): hyperbolic: str = "" cerebras: str = "" sambanova: str = "" + deepseek: str = "" class OkResponse(BaseModel): @@ -298,6 +304,8 @@ class ModelCapability(str, Enum): COMPLETION = "completion" CHAT = "chat" IMAGE = "image" + AUDIO = "audio" + TOOL = "tool" EMBED = "embed" RERANK = "rerank" @@ -311,7 +319,7 @@ class ModelInfo(BaseModel): 'Unique identifier in the form of "{provider}/{model_id}". ' "Users will specify this to select a model." ), - examples=[EXAMPLE_CHAT_MODEL], + examples=EXAMPLE_CHAT_MODEL_IDS, ) object: str = Field( default="model", @@ -366,7 +374,7 @@ class ModelDeploymentConfig(BaseModel): 'For example, you can map "openai/gpt-4o" calls to "openai/gpt-4o-2024-08-06". ' 'For vLLM with OpenAI compatible server, use "openai/".' ), - examples=[EXAMPLE_CHAT_MODEL], + examples=EXAMPLE_CHAT_MODEL_IDS, ) api_base: str = Field( default="", @@ -397,7 +405,7 @@ class ModelConfig(ModelInfo): 'For example, you can map "openai/gpt-4o" calls to "openai/gpt-4o-2024-08-06". ' 'For vLLM with OpenAI compatible server, use "openai/".' ), - examples=[EXAMPLE_CHAT_MODEL], + examples=EXAMPLE_CHAT_MODEL_IDS, ) api_base: str = Field( default="", @@ -451,7 +459,7 @@ class EmbeddingModelConfig(ModelConfig): 'For self-hosted models with Infinity, use "ellm/{org}/{model}". ' "Users will specify this to select a model." ), - examples=["ellm/sentence-transformers/all-MiniLM-L6-v2", EXAMPLE_EMBEDDING_MODEL], + examples=EXAMPLE_EMBEDDING_MODEL_IDS, ) embedding_size: int = Field( description="Embedding size of the model", @@ -491,7 +499,7 @@ class RerankingModelConfig(ModelConfig): 'For self-hosted models with Infinity, use "ellm/{org}/{model}". ' "Users will specify this to select a model." ), - examples=["ellm/cross-encoder/ms-marco-TinyBERT-L-2", EXAMPLE_RERANKING_MODEL], + examples=EXAMPLE_RERANKING_MODEL_IDS, ) capabilities: list[ModelCapability] = Field( default=[ModelCapability.RERANK], @@ -555,6 +563,9 @@ def get_default_model(self, capabilities: list[str] | None = None) -> str: if capabilities is not None: for capability in capabilities: models = [m for m in models if capability in m.capabilities] + # if `capabilities`` is chat only, filter out audio model + if capabilities == ["chat"]: + models = [m for m in models if "audio" not in m.capabilities] if len(models) == 0: raise ResourceNotFoundError(f"No model found with capabilities: {capabilities}") model = natsorted(models, key=self._sort_key_with_priority)[0] @@ -659,7 +670,7 @@ class RAGParams(BaseModel): reranking_model: str | None = Field( default=None, description="Reranking model to use for hybrid search.", - examples=[EXAMPLE_RERANKING_MODEL, None], + examples=[EXAMPLE_RERANKING_MODEL_IDS[0], None], ) search_query: str = Field( default="", @@ -758,6 +769,17 @@ def sanitise_name(v: str) -> str: MessageName = Annotated[str, AfterValidator(sanitise_name)] +class MessageToolCallFunction(BaseModel): + arguments: str + name: str | None + + +class MessageToolCall(BaseModel): + id: str | None + function: MessageToolCallFunction + type: str + + class ChatEntry(BaseModel): """Represents a message in the chat context.""" @@ -799,6 +821,11 @@ def coerce_input(cls, value: Any) -> str | list[dict[str, str | dict[str, str]]] return str(value) +class ChatCompletionChoiceOutput(ChatEntry): + tool_calls: list[MessageToolCall] | None = None + """List of tool calls if the message includes tool call responses.""" + + class ChatThread(BaseModel): object: str = Field( default="chat.thread", @@ -828,7 +855,9 @@ class CompletionUsage(BaseModel): class ChatCompletionChoice(BaseModel): - message: ChatEntry = Field(description="A chat completion message generated by the model.") + message: ChatEntry | ChatCompletionChoiceOutput = Field( + description="A chat completion message generated by the model." + ) index: int = Field(description="The index of the choice in the list of choices.") finish_reason: str | None = Field( default=None, @@ -848,7 +877,7 @@ def text(self) -> str: class ChatCompletionChoiceDelta(ChatCompletionChoice): @computed_field @property - def delta(self) -> ChatEntry: + def delta(self) -> ChatEntry | ChatCompletionChoiceOutput: return self.message @@ -928,7 +957,7 @@ class ChatCompletionChunk(BaseModel): ) @property - def message(self) -> ChatEntry | None: + def message(self) -> ChatEntry | ChatCompletionChoiceOutput | None: return self.choices[0].message if len(self.choices) > 0 else None @property @@ -987,6 +1016,49 @@ class GenTableStreamChatCompletionChunk(ChatCompletionChunk): row_id: str +class FunctionParameter(BaseModel): + type: str = Field( + default="", description="The type of the parameter, e.g., 'string', 'number'." + ) + description: str = Field(default="", description="A description of the parameter.") + enum: list[str] = Field( + default=[], description="An optional list of allowed values for the parameter." + ) + + +class FunctionParameters(BaseModel): + type: str = Field( + default="object", description="The type of the parameters object, usually 'object'." + ) + properties: dict[str, FunctionParameter] = Field( + description="The properties of the parameters object." + ) + required: list[str] = Field(description="A list of required parameter names.") + additionalProperties: bool = Field( + default=False, description="Whether additional properties are allowed." + ) + + +class Function(BaseModel): + name: str = Field(default="", description="The name of the function.") + description: str = Field(default="", description="A description of what the function does.") + parameters: FunctionParameters = Field(description="The parameters for the function.") + + +class Tool(BaseModel): + type: str = Field(default="function", description="The type of the tool, e.g., 'function'.") + function: Function = Field(description="The function details of the tool.") + + +class ToolChoiceFunction(BaseModel): + name: str = Field(default="", description="The name of the function.") + + +class ToolChoice(BaseModel): + type: str = Field(default="function", description="The type of the tool, e.g., 'function'.") + function: ToolChoiceFunction = Field(description="Select a tool for the chat model to use.") + + class ChatRequest(BaseModel): id: str = Field( default="", @@ -1094,6 +1166,48 @@ def convert_stop(cls, v: list[str] | None) -> list[str] | None: return v +class ChatRequestWithTools(ChatRequest): + tools: list[Tool] = Field( + description="A list of tools available for the chat model to use.", + min_length=1, + examples=[ + # --- [Tool Function] --- + # def get_delivery_date(order_id: str) -> datetime: + # # Connect to the database + # conn = sqlite3.connect('ecommerce.db') + # cursor = conn.cursor() + # # ... + [ + Tool( + type="function", + function=Function( + name="get_delivery_date", + description="Get the delivery date for a customer's order.", + parameters=FunctionParameters( + type="object", + properties={ + "order_id": FunctionParameter( + type="string", description="The customer's order ID." + ) + }, + required=["order_id"], + additionalProperties=False, + ), + ), + ) + ], + ], + ) + tool_choice: str | ToolChoice = Field( + default="auto", + description="Set `auto` to let chat model pick a tool or select a tool for the chat model to use.", + examples=[ + "auto", + ToolChoice(type="function", function=ToolChoiceFunction(name="get_delivery_date")), + ], + ) + + class EmbeddingRequest(BaseModel): input: str | list[str] = Field( description=( @@ -1108,7 +1222,7 @@ class EmbeddingRequest(BaseModel): "The ID of the model to use. " "You can use the List models API to see all of your available models." ), - examples=[EXAMPLE_EMBEDDING_MODEL], + examples=EXAMPLE_EMBEDDING_MODEL_IDS, ) type: Literal["query", "document"] = Field( default="document", @@ -1248,7 +1362,8 @@ def datetime_str_before_validator(x): "bool": pa.bool_(), "str": pa.utf8(), # Alias for `pa.string()` "chat": pa.utf8(), - "file": pa.utf8(), + "image": pa.utf8(), + "audio": pa.utf8(), } _str_to_py_type = { "int": int, @@ -1261,7 +1376,8 @@ def datetime_str_before_validator(x): "str": str, "date-time": datetime, "chat": str, - "file": str, + "image": str, + "audio": str, } @@ -1299,7 +1415,8 @@ class ColumnDtype(str, Enum, metaclass=MetaEnum): BOOL = "bool" STR = "str" DATE_TIME = "date-time" - FILE = "file" + IMAGE = "image" + AUDIO = "audio" def __str__(self) -> str: return self.value @@ -1310,7 +1427,8 @@ class ColumnDtypeCreate(str, Enum, metaclass=MetaEnum): FLOAT = "float" BOOL = "bool" STR = "str" - FILE = "file" + IMAGE = "image" + AUDIO = "audio" def __str__(self) -> str: return self.value @@ -1463,7 +1581,7 @@ class EmbedGenConfig(BaseModel): ) embedding_model: str = Field( description="The embedding model to use.", - examples=[EXAMPLE_EMBEDDING_MODEL], + examples=EXAMPLE_EMBEDDING_MODEL_IDS, ) source_column: str = Field( description="The source column for embedding.", @@ -1471,6 +1589,10 @@ class EmbedGenConfig(BaseModel): ) +class CodeGenConfig(p.CodeGenConfig): + pass + + def _gen_config_discriminator(x: Any) -> str | None: object_attr = getattr(x, "object", None) if object_attr: @@ -1487,9 +1609,10 @@ def _gen_config_discriminator(x: Any) -> str | None: return None -GenConfig = LLMGenConfig | EmbedGenConfig +GenConfig = LLMGenConfig | EmbedGenConfig | CodeGenConfig DiscriminatedGenConfig = Annotated[ Union[ + Annotated[CodeGenConfig, Tag("gen_config.code")], Annotated[LLMGenConfig, Tag("gen_config.llm")], Annotated[LLMGenConfig, Tag("gen_config.chat")], Annotated[EmbedGenConfig, Tag("gen_config.embed")], @@ -1502,7 +1625,7 @@ class ColumnSchema(BaseModel): id: str = Field(description="Column name.") dtype: ColumnDtype = Field( default=ColumnDtype.STR, - description='Column data type, one of ["int", "int8", "float", "float32", "float16", "bool", "str", "date-time", "file"]', + description='Column data type, one of ["int", "int8", "float", "float32", "float16", "bool", "str", "date-time", "image"]', ) vlen: PositiveInt = Field( # type: ignore default=0, @@ -1537,13 +1660,25 @@ class ColumnSchemaCreate(ColumnSchema): id: ColName = Field(description="Column name.") dtype: ColumnDtypeCreate = Field( default=ColumnDtypeCreate.STR, - description='Column data type, one of ["int", "float", "bool", "str", "file"]', + description='Column data type, one of ["int", "float", "bool", "str", "image", "audio"]', ) + @model_validator(mode="before") + def match_column_dtype_file_to_image(self) -> Self: + if self.get("dtype", "") == "file": + self["dtype"] = ColumnDtype.IMAGE + return self + @model_validator(mode="after") def check_output_column_dtype(self) -> Self: - if self.gen_config is not None and self.vlen == 0 and self.dtype != ColumnDtype.STR: - raise ValueError("Output column must be string column.") + if self.gen_config is not None and self.vlen == 0: + if isinstance(self.gen_config, CodeGenConfig): + if self.dtype not in (ColumnDtype.STR, ColumnDtype.IMAGE): + raise ValueError( + "Output column must be either string or image column when gen_config is CodeGenConfig." + ) + elif self.dtype != ColumnDtype.STR: + raise ValueError("Output column must be string column.") return self @@ -1675,6 +1810,29 @@ def check_gen_configs(self) -> Self: f"Available columns: {col_ids}." ) ) + elif isinstance(gen_config, CodeGenConfig): + source_col = next( + (c for c in available_cols if c.id == gen_config.source_column), None + ) + if source_col is None: + raise ValueError( + ( + f"Table '{self.id}': " + f"Code Execution config of column '{col.id}' referenced " + f"an invalid source column '{gen_config.source_column}'. " + "Make sure you only reference columns on its left. " + f"Available columns: {col_ids}." + ) + ) + if source_col.dtype != ColumnDtype.STR: + raise ValueError( + ( + f"Table '{self.id}': " + f"Code Execution config of column '{col.id}' referenced " + f"a source column '{gen_config.source_column}' with an invalid datatype of '{source_col.dtype}'. " + "Make sure the source column is Str typed." + ) + ) elif isinstance(gen_config, LLMGenConfig): # Insert default prompts if needed system_prompt, user_prompt = self.get_default_prompts( @@ -1734,9 +1892,13 @@ class KnowledgeTableSchemaCreate(TableSchemaCreate): @model_validator(mode="after") def check_cols(self) -> Self: super().check_cols() - num_text_cols = sum(c.id.lower() in ("text", "title", "file id") for c in self.cols) + num_text_cols = sum( + c.id.lower() in ("text", "title", "file id", "page") for c in self.cols + ) if num_text_cols != 0: - raise ValueError("Schema cannot contain column names: 'Text', 'Title', 'File ID'.") + raise ValueError( + "Schema cannot contain column names: 'Text', 'Title', 'File ID', 'Page'." + ) return self @staticmethod @@ -1749,9 +1911,13 @@ class AddKnowledgeColumnSchema(TableSchemaCreate): @model_validator(mode="after") def check_cols(self) -> Self: super().check_cols() - num_text_cols = sum(c.id.lower() in ("text", "title", "file id") for c in self.cols) + num_text_cols = sum( + c.id.lower() in ("text", "title", "file id", "page") for c in self.cols + ) if num_text_cols != 0: - raise ValueError("Schema cannot contain column names: 'Text', 'Title', 'File ID'.") + raise ValueError( + "Schema cannot contain column names: 'Text', 'Title', 'File ID', 'Page'." + ) return self @model_validator(mode="after") @@ -1927,7 +2093,7 @@ def _handle_nulls_and_validate(self, check_missing_cols: bool = True) -> Self: d[k] = 0.0 elif col.dtype == ColumnDtype.BOOL: d[k] = False - elif col.dtype in (ColumnDtype.STR, ColumnDtype.FILE): + elif col.dtype in (ColumnDtype.STR, ColumnDtype.IMAGE): # Store null string as "" # https://github.com/lancedb/lancedb/issues/1160 d[k] = "" @@ -2114,11 +2280,12 @@ def check_data(self) -> Self: value.startswith("s3://") or value.startswith("file://") ): extension = splitext(value)[1].lower() - if extension not in IMAGE_FILE_EXTENSIONS: + if extension not in IMAGE_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS: raise ValueError( "Unsupported file type. Make sure the file belongs to " "one of the following formats: \n" - f"[Image File Types]: \n{IMAGE_FILE_EXTENSIONS}" + f"[Image File Types]: \n{IMAGE_FILE_EXTENSIONS} \n" + f"[Audio File Types]: \n{AUDIO_FILE_EXTENSIONS}" ) return self @@ -2160,11 +2327,12 @@ def check_data(self) -> Self: value.startswith("s3://") or value.startswith("file://") ): extension = splitext(value)[1].lower() - if extension not in IMAGE_FILE_EXTENSIONS: + if extension not in IMAGE_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS: raise ValueError( "Unsupported file type. Make sure the file belongs to " "one of the following formats: \n" - f"[Image File Types]: \n{IMAGE_FILE_EXTENSIONS}" + f"[Image File Types]: \n{IMAGE_FILE_EXTENSIONS} \n" + f"[Audio File Types]: \n{AUDIO_FILE_EXTENSIONS}" ) return self diff --git a/services/api/src/owl/routers/file.py b/services/api/src/owl/routers/file.py index 495ce00..fed6675 100644 --- a/services/api/src/owl/routers/file.py +++ b/services/api/src/owl/routers/file.py @@ -1,5 +1,6 @@ import mimetypes import os +from os.path import splitext from typing import Annotated from urllib.parse import quote, urlparse, urlunparse @@ -14,6 +15,7 @@ from owl.utils.auth import ProjectRead, auth_user_project from owl.utils.exceptions import handle_exception from owl.utils.io import ( + AUDIO_WHITE_LIST_EXT, LOCAL_FILE_DIR, S3_CLIENT, UPLOAD_WHITE_LIST_MIME, @@ -91,7 +93,8 @@ async def proxy_file(request: Request, path: str) -> Response: raise ResourceNotFoundError("Neither S3 nor local file store is configured") -@router.options("/v1/files/upload/") +@router.options("/v1/files/upload") +@router.options("/v1/files/upload/", deprecated=True) @handle_exception async def upload_file_options(): headers = { @@ -103,7 +106,8 @@ async def upload_file_options(): return JSONResponse(content={"accepted_types": list(UPLOAD_WHITE_LIST_MIME)}, headers=headers) -@router.post("/v1/files/upload/") +@router.post("/v1/files/upload") +@router.post("/v1/files/upload/", deprecated=True) @handle_exception async def upload_file( project: Annotated[ProjectRead, Depends(auth_user_project)], @@ -164,9 +168,11 @@ async def get_thumbnail_urls(body: GetURLRequest, request: Request) -> GetURLRes file_url = "" if uri.startswith("s3://"): try: + ext = splitext(uri)[1].lower() bucket_name, key = uri[5:].split("/", 1) + thumb_ext = "mp3" if ext in AUDIO_WHITE_LIST_EXT else "webp" thumb_key = key.replace("raw", "thumb") - thumb_key = f"{os.path.splitext(thumb_key)[0]}.webp" + thumb_key = f"{os.path.splitext(thumb_key)[0]}.{thumb_ext}" file_url = await _generate_presigned_url(aclient, bucket_name, thumb_key) except Exception as e: logger.exception( @@ -179,9 +185,11 @@ async def get_thumbnail_urls(body: GetURLRequest, request: Request) -> GetURLRes file_url = "" if uri.startswith("file://"): try: + ext = splitext(uri)[1].lower() local_path = os.path.abspath(uri[7:]) + thumb_ext = "mp3" if ext in AUDIO_WHITE_LIST_EXT else "webp" thumb_path = local_path.replace("raw", "thumb") - thumb_path = f"{os.path.splitext(thumb_path)[0]}.webp" + thumb_path = f"{os.path.splitext(thumb_path)[0]}.{thumb_ext}" if os.path.exists(thumb_path): relative_path = os.path.relpath(thumb_path, LOCAL_FILE_DIR) file_url = str(request.url_for("proxy_file", path=relative_path)) diff --git a/services/api/src/owl/routers/gen_table.py b/services/api/src/owl/routers/gen_table.py index c7a6296..48f65f8 100644 --- a/services/api/src/owl/routers/gen_table.py +++ b/services/api/src/owl/routers/gen_table.py @@ -47,6 +47,7 @@ ChatEntry, ChatTableSchemaCreate, ChatThread, + CodeGenConfig, ColName, ColumnDropRequest, ColumnDtype, @@ -85,7 +86,8 @@ def _validate_gen_config( gen_config: GenConfig | None, table_type: TableType, column_id: str, - file_column_ids: list[str], + image_column_ids: list[str], + audio_column_ids: list[str], ) -> GenConfig | None: if gen_config is None: return gen_config @@ -98,8 +100,10 @@ def _validate_gen_config( capabilities = ["chat"] for message in (gen_config.system_prompt, gen_config.prompt): for col_id in re.findall(GEN_CONFIG_VAR_PATTERN, message): - if col_id in file_column_ids: + if col_id in image_column_ids: capabilities = ["image"] + if col_id in audio_column_ids: + capabilities = ["audio"] break gen_config.model = llm.validate_model_id( model=gen_config.model, @@ -141,6 +145,8 @@ def _validate_gen_config( raise ResourceNotFoundError( f'Column {column_id} used a reranking model "{reranking_model}" that is not available.' ) from e + elif isinstance(gen_config, CodeGenConfig): + pass elif isinstance(gen_config, EmbedGenConfig): pass return gen_config @@ -155,8 +161,15 @@ def _create_table( ) -> TableMetaResponse: # Validate llm = LLMEngine(request=request) - file_column_ids = [ - col.id for col in schema.cols if col.dtype == ColumnDtype.FILE and not col.id.endswith("_") + image_column_ids = [ + col.id + for col in schema.cols + if col.dtype == ColumnDtype.IMAGE and not col.id.endswith("_") + ] + audio_column_ids = [ + col.id + for col in schema.cols + if col.dtype == ColumnDtype.AUDIO and not col.id.endswith("_") ] for col in schema.cols: col.gen_config = _validate_gen_config( @@ -164,7 +177,8 @@ def _create_table( gen_config=col.gen_config, table_type=table_type, column_id=col.id, - file_column_ids=file_column_ids, + image_column_ids=image_column_ids, + audio_column_ids=audio_column_ids, ) if table_type == TableType.KNOWLEDGE: try: @@ -497,10 +511,15 @@ def update_gen_config( with table.create_session() as session: meta = table.open_meta(session, updates.table_id) llm = LLMEngine(request=request) - file_column_ids = [ + image_column_ids = [ + col["id"] + for col in meta.cols + if col["dtype"] == ColumnDtype.IMAGE and not col["id"].endswith("_") + ] + audio_column_ids = [ col["id"] for col in meta.cols - if col["dtype"] == ColumnDtype.FILE and not col["id"].endswith("_") + if col["dtype"] == ColumnDtype.AUDIO and not col["id"].endswith("_") ] if table_type == TableType.KNOWLEDGE: @@ -519,7 +538,8 @@ def update_gen_config( gen_config=gen_config, table_type=table_type, column_id=col_id, - file_column_ids=file_column_ids, + image_column_ids=image_column_ids, + audio_column_ids=audio_column_ids, ) for col_id, gen_config in updates.column_map.items() } @@ -544,8 +564,11 @@ def _add_columns( cols = TableSchema( id=meta.id, cols=[c.model_dump() for c in meta.cols_schema + schema.cols] ).cols - file_column_ids = [ - col.id for col in cols if col.dtype == ColumnDtype.FILE and not col.id.endswith("_") + image_column_ids = [ + col.id for col in cols if col.dtype == ColumnDtype.IMAGE and not col.id.endswith("_") + ] + audio_column_ids = [ + col.id for col in cols if col.dtype == ColumnDtype.AUDIO and not col.id.endswith("_") ] schema.cols = [col for col in cols if col.id in set(c.id for c in schema.cols)] for col in schema.cols: @@ -554,7 +577,8 @@ def _add_columns( gen_config=col.gen_config, table_type=table_type, column_id=col.id, - file_column_ids=file_column_ids, + image_column_ids=image_column_ids, + audio_column_ids=audio_column_ids, ) # Create _, meta = table.add_columns(session, schema) @@ -1126,6 +1150,7 @@ async def _embed_file( "Title": title, "Title Embed": title_embed, "File ID": file_uri, + "Page": chunk.page, } for chunk, text_embed in zip(chunks, text_embeds, strict=True) ] @@ -1197,6 +1222,10 @@ async def embed_file( file_name = file.filename or file_name if splitext(file_name)[1].lower() == ".jsonl": file_content_type = "application/jsonl" + elif splitext(file_name)[1].lower() == ".md": + file_content_type = "text/markdown" + elif splitext(file_name)[1].lower() == ".tsv": + file_content_type = "text/tab-separated-values" else: file_content_type = file.content_type if file_content_type not in EMBED_WHITE_LIST_MIME: @@ -1303,7 +1332,7 @@ async def import_table_data( if dtype == "str": df[col_id] = df[col_id].apply(lambda x: str(x) if not pd.isna(x) else x) else: - if dtype == ColumnDtype.FILE: + if dtype in [ColumnDtype.IMAGE, ColumnDtype.AUDIO]: dtype = "str" df[col_id] = df[col_id].astype(dtype, errors="raise") except ValueError as e: diff --git a/services/api/src/owl/routers/llm.py b/services/api/src/owl/routers/llm.py index a27bd7d..35d754a 100644 --- a/services/api/src/owl/routers/llm.py +++ b/services/api/src/owl/routers/llm.py @@ -13,8 +13,9 @@ from owl.llm import LLMEngine from owl.models import CloudEmbedder from owl.protocol import ( - EXAMPLE_CHAT_MODEL, + EXAMPLE_CHAT_MODEL_IDS, ChatRequest, + ChatRequestWithTools, EmbeddingRequest, EmbeddingResponse, EmbeddingResponseData, @@ -39,7 +40,7 @@ async def get_model_info( str, Query( description="ID of the requested model.", - examples=[EXAMPLE_CHAT_MODEL], + examples=EXAMPLE_CHAT_MODEL_IDS, ), ] = "", capabilities: Annotated[ @@ -79,7 +80,7 @@ async def get_model_names( str, Query( description="ID of the preferred model.", - examples=[EXAMPLE_CHAT_MODEL], + examples=EXAMPLE_CHAT_MODEL_IDS, ), ] = "", capabilities: Annotated[ @@ -109,7 +110,7 @@ async def get_model_names( description="Given a list of messages comprising a conversation, the model will return a response.", ) @handle_exception -async def generate_completions(request: Request, body: ChatRequest): +async def generate_completions(request: Request, body: ChatRequest | ChatRequestWithTools): # Check quota request.state.billing.check_llm_quota(body.model) request.state.billing.check_egress_quota() diff --git a/services/api/src/owl/utils/auth.py b/services/api/src/owl/utils/auth.py index dc3ab57..06f8e9e 100644 --- a/services/api/src/owl/utils/auth.py +++ b/services/api/src/owl/utils/auth.py @@ -2,7 +2,7 @@ from secrets import compare_digest from typing import Annotated, AsyncGenerator -from fastapi import Header, Request, Response +from fastapi import BackgroundTasks, Header, Request, Response from httpx import RequestError from loguru import logger from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random_exponential @@ -169,6 +169,7 @@ def _get_external_keys(organization: OrganizationRead) -> ExternalKeys: hyperbolic=get_non_empty(ext_keys, "hyperbolic", ENV_CONFIG.hyperbolic_api_key_plain), cerebras=get_non_empty(ext_keys, "cerebras", ENV_CONFIG.cerebras_api_key_plain), sambanova=get_non_empty(ext_keys, "sambanova", ENV_CONFIG.sambanova_api_key_plain), + deepseek=get_non_empty(ext_keys, "deepseek", ENV_CONFIG.deepseek_api_key_plain), ) @@ -270,6 +271,7 @@ def _get_valid_modellistconfig(all_models: str, external_keys: str) -> ModelList "sambanova", "cerebras", "hyperbolic", + "deepseek", ] # remove providers without credentials available_providers = [ @@ -346,6 +348,7 @@ async def auth_user_project_oss( async def auth_user_project_cloud( + bg_tasks: BackgroundTasks, request: Request, response: Response, project_id: Annotated[ @@ -361,7 +364,6 @@ async def auth_user_project_cloud( user_id: Annotated[str, Header(alias="X-USER-ID", description="User ID.")] = "", ) -> AsyncGenerator[ProjectRead, None]: route = request.url.path - user_id = "" project_id = project_id.strip() bearer_token = bearer_token.strip() user_id = user_id.strip() @@ -450,21 +452,20 @@ async def auth_user_project_cloud( yield project - # Add egress events - request.state.billing.create_egress_events( - float(response.headers.get("content-length", 0)) / (1024**3) - ) - # Process all billing events - await request.state.billing.process_all() + # NOTE that billing processing is done in middleware where response headers are available # Set project updated at datetime - if "gen_tables" in route and request.method in WRITE_METHODS: - try: - await CLIENT.admin.organization.set_project_updated_at(project_id) - except Exception as e: - logger.warning( - f'{request.state.id} - Error setting project "{project_id}" last updated time: {e}' - ) + async def _set_project_updated_at() -> None: + if "gen_tables" in route and request.method in WRITE_METHODS: + try: + await CLIENT.admin.organization.set_project_updated_at(project_id) + except Exception as e: + logger.warning( + f'{request.state.id} - Error setting project "{project_id}" last updated time: {e}' + ) + + # This will run AFTER streaming responses are sent + bg_tasks.add_task(_set_project_updated_at) auth_user_project = auth_user_project_oss if ENV_CONFIG.is_oss else auth_user_project_cloud diff --git a/services/api/src/owl/utils/code.py b/services/api/src/owl/utils/code.py new file mode 100644 index 0000000..76a764e --- /dev/null +++ b/services/api/src/owl/utils/code.py @@ -0,0 +1,58 @@ +import base64 +import uuid + +import filetype +import httpx +from fastapi import Request +from loguru import logger + +from owl.configs.manager import ENV_CONFIG +from owl.utils.io import upload_file_to_s3 + + +async def code_executor(source_code: str, dtype: str, request: Request) -> str | None: + response = None + + try: + if dtype == "image": + dtype = "file" # for code execution endpoint usage + async with httpx.AsyncClient() as client: + response = await client.post( + f"{ENV_CONFIG.code_executor_endpoint}/execute", + json={"code": source_code}, + ) + response.raise_for_status() + result = response.json() + + if dtype == "file": + if result["type"].startswith("image"): + image_content = base64.b64decode(result["result"]) + content_type = filetype.guess(image_content) + if content_type is None: + raise ValueError("Unable to determine file type") + filename = f"{uuid.uuid4()}.{content_type.extension}" + + # Upload the file + uri = await upload_file_to_s3( + organization_id=request.state.org_id, + project_id=request.state.project_id, + content=image_content, + content_type=content_type.mime, + filename=filename, + ) + response = uri + else: + logger.warning( + f"Code Executor: {request.state.id} - Unsupported file type: {result['type']}" + ) + response = None + else: + response = str(result["result"]) + + logger.info(f"Code Executor: {request.state.id} - Python code execution completed") + + except Exception as e: + logger.error(f"Code Executor: {request.state.id} - An unexpected error occurred: {e}") + response = None + + return response diff --git a/services/api/src/owl/utils/io.py b/services/api/src/owl/utils/io.py index 7541017..91fef1e 100644 --- a/services/api/src/owl/utils/io.py +++ b/services/api/src/owl/utils/io.py @@ -15,7 +15,7 @@ from loguru import logger from jamaibase.exceptions import BadInputError, ResourceNotFoundError -from jamaibase.utils.io import generate_thumbnail +from jamaibase.utils.io import generate_audio_thumbnail, generate_image_thumbnail from owl.configs.manager import ENV_CONFIG from owl.utils import uuid7_str @@ -61,12 +61,22 @@ "image/gif": [".gif"], "image/webp": [".webp"], } -UPLOAD_WHITE_LIST = {**EMBED_WHITE_LIST, **IMAGE_WHITE_LIST} +AUDIO_WHITE_LIST = { + "audio/mpeg": [".mp3"], + "audio/vnd.wav": [".wav"], + "audio/x-wav": [".wav"], + "audio/x-pn-wav": [".wav"], + "audio/wave": [".wav"], + "audio/vnd.wave": [".wav"], +} +UPLOAD_WHITE_LIST = {**EMBED_WHITE_LIST, **IMAGE_WHITE_LIST, **AUDIO_WHITE_LIST} EMBED_WHITE_LIST_MIME = set(EMBED_WHITE_LIST.keys()) EMBED_WHITE_LIST_EXT = set(ext for exts in EMBED_WHITE_LIST.values() for ext in exts) IMAGE_WHITE_LIST_MIME = set(IMAGE_WHITE_LIST.keys()) IMAGE_WHITE_LIST_EXT = set(ext for exts in IMAGE_WHITE_LIST.values() for ext in exts) +AUDIO_WHITE_LIST_MIME = set(AUDIO_WHITE_LIST.keys()) +AUDIO_WHITE_LIST_EXT = set(ext for exts in AUDIO_WHITE_LIST.values() for ext in exts) UPLOAD_WHITE_LIST_MIME = set(UPLOAD_WHITE_LIST.keys()) UPLOAD_WHITE_LIST_EXT = set(ext for exts in UPLOAD_WHITE_LIST.values() for ext in exts) @@ -253,19 +263,40 @@ async def upload_file_to_s3( raise BadInputError( f"Unsupported file extension: {file_extension}. Allowed types are: {', '.join(UPLOAD_WHITE_LIST_EXT)}" ) - - if len(content) > ENV_CONFIG.owl_file_upload_max_bytes: - raise BadInputError( - f"File size exceeds {ENV_CONFIG.owl_file_upload_max_bytes/1024**2} MB limit: {len(content)/1024**2} MB" - ) + else: + if ( + file_extension in EMBED_WHITE_LIST_EXT + and len(content) > ENV_CONFIG.owl_embed_file_upload_max_bytes + ): + raise BadInputError( + f"File size exceeds {ENV_CONFIG.owl_embed_file_upload_max_bytes / 1024**2} MB limit: {len(content) / 1024**2} MB" + ) + elif ( + file_extension in AUDIO_WHITE_LIST_EXT + and len(content) > ENV_CONFIG.owl_audio_file_upload_max_bytes + ): + raise BadInputError( + f"File size exceeds {ENV_CONFIG.owl_audio_file_upload_max_bytes / 1024**2} MB limit: {len(content) / 1024**2} MB" + ) + elif ( + file_extension in IMAGE_WHITE_LIST_EXT + and len(content) > ENV_CONFIG.owl_image_file_upload_max_bytes + ): + raise BadInputError( + f"File size exceeds {ENV_CONFIG.owl_image_file_upload_max_bytes / 1024**2} MB limit: {len(content) / 1024**2} MB" + ) uuid = uuid7_str() raw_path = os.path.join("raw", organization_id, project_id, uuid, filename) raw_key = os_path_to_s3_key(raw_path) - thumb_filename = f"{os.path.splitext(filename)[0]}.webp" + thumb_ext = "mp3" if file_extension in AUDIO_WHITE_LIST_EXT else "webp" + thumb_filename = f"{os.path.splitext(filename)[0]}.{thumb_ext}" thumb_path = os.path.join("thumb", organization_id, project_id, uuid, thumb_filename) thumb_key = os_path_to_s3_key(thumb_path) - thumbnail_task = asyncio.create_task(asyncio.to_thread(generate_thumbnail, content)) + if file_extension in AUDIO_WHITE_LIST_EXT: + thumbnail_task = asyncio.create_task(asyncio.to_thread(generate_audio_thumbnail, content)) + else: + thumbnail_task = asyncio.create_task(asyncio.to_thread(generate_image_thumbnail, content)) thumbnail = await thumbnail_task if S3_CLIENT: @@ -282,7 +313,7 @@ async def upload_file_to_s3( Body=thumbnail, Bucket=S3_BUCKET_NAME, Key=thumb_key, - ContentType="image/webp", + ContentType=f"{content_type.split('/')[0]}/{"mpeg" if thumb_ext == "mp3" else thumb_ext}", ) logger.info( f"File Uploaded: [{organization_id}/{project_id}] " diff --git a/services/api/src/owl/utils/jwt.py b/services/api/src/owl/utils/jwt.py index 5e2df6e..b443e57 100644 --- a/services/api/src/owl/utils/jwt.py +++ b/services/api/src/owl/utils/jwt.py @@ -2,7 +2,6 @@ from typing import Any import jwt -from fastapi import Request from loguru import logger from jamaibase.exceptions import AuthorizationError @@ -19,7 +18,7 @@ def decode_jwt( token: str, expired_token_message: str, invalid_token_message: str, - request: Request | None = None, + request_id: str | None = None, ) -> dict[str, Any]: try: data = jwt.decode( @@ -33,10 +32,10 @@ def decode_jwt( except jwt.exceptions.PyJWTError as e: raise AuthorizationError(invalid_token_message) from e except Exception as e: - if request is None: + if request_id is None: logger.exception(f'Failed to decode "{token}" due to {e.__class__.__name__}: {e}') else: logger.exception( - f'{request.state.id} - Failed to decode "{token}" due to {e.__class__.__name__}: {e}' + f'{request_id} - Failed to decode "{token}" due to {e.__class__.__name__}: {e}' ) raise AuthorizationError(invalid_token_message) from e diff --git a/services/app/package-lock.json b/services/app/package-lock.json index 4115254..258406c 100644 --- a/services/app/package-lock.json +++ b/services/app/package-lock.json @@ -19,6 +19,7 @@ "chartjs-adapter-moment": "^1.0.1", "clsx": "^2.1.0", "cors": "^2.8.5", + "dexie": "^4.0.10", "dotenv": "^16.4.5", "electron-serve": "^2.0.0", "express": "^4.19.2", @@ -5646,6 +5647,12 @@ "integrity": "sha512-maua5KUiapvEwiEAe+XnlZ3Rh0GD+qI1J/nb9vrJc3muPXvcF/8gXYTWF76+5DAqHyDUtOIImEuo0YKE9mshVw==", "dev": true }, + "node_modules/dexie": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dexie/-/dexie-4.0.10.tgz", + "integrity": "sha512-eM2RzuR3i+M046r2Q0Optl3pS31qTWf8aFuA7H9wnsHTwl8EPvroVLwvQene/6paAs39Tbk6fWZcn2aZaHkc/w==", + "license": "Apache-2.0" + }, "node_modules/didyoumean": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", diff --git a/services/app/package.json b/services/app/package.json index a534027..f89ec85 100644 --- a/services/app/package.json +++ b/services/app/package.json @@ -80,6 +80,7 @@ "chartjs-adapter-moment": "^1.0.1", "clsx": "^2.1.0", "cors": "^2.8.5", + "dexie": "^4.0.10", "dotenv": "^16.4.5", "electron-serve": "^2.0.0", "express": "^4.19.2", diff --git a/services/app/src/hooks.server.ts b/services/app/src/hooks.server.ts index 399d9b7..ca9c53f 100644 --- a/services/app/src/hooks.server.ts +++ b/services/app/src/hooks.server.ts @@ -1,7 +1,7 @@ import { PUBLIC_IS_LOCAL } from '$env/static/public'; import { JAMAI_URL, JAMAI_SERVICE_KEY } from '$env/static/private'; import { dev } from '$app/environment'; -import { json, type Handle } from '@sveltejs/kit'; +import { json, redirect, type Handle } from '@sveltejs/kit'; import { Agent } from 'undici'; import { getPrices } from '$lib/server/nodeCache'; import logger from '$lib/logger'; @@ -82,25 +82,40 @@ const handleApiProxy: Handle = async ({ event }) => { }; export const handle: Handle = async ({ event, resolve }) => { - if (dev && !event.request.url.includes('/api/v1/files')) - console.log('Connecting', event.request.url); + const { cookies, locals, request, url } = event; + if (dev && !request.url.includes('/api/v1/files')) console.log('Connecting', request.url); if (PUBLIC_IS_LOCAL === 'false') { //? Workaround for event.platform unavailable in development if (dev) { const user = await ( - await fetch(`${event.url.origin}/dev-profile`, { - headers: { cookie: `appSession=${event.cookies.get('appSession')}` } + await fetch(`${url.origin}/dev-profile`, { + headers: { cookie: `appSession=${cookies.get('appSession')}` } }) ).json(); - event.locals.user = Object.keys(user).length ? user : undefined; + locals.user = Object.keys(user).length ? user : undefined; } else { // @ts-expect-error missing type - event.locals.user = event.platform?.req?.res?.locals?.user; + locals.user = event.platform?.req?.res?.locals?.user; } } - if (PROXY_PATHS.some((p) => event.url.pathname.startsWith(p.path))) { + if (PUBLIC_IS_LOCAL === 'false' && !url.pathname.startsWith('/api')) { + if (!locals.user) { + const originalUrl = + url.pathname + (url.searchParams.size > 0 ? `?${url.searchParams.toString()}` : ''); + throw redirect(302, `/login${originalUrl ? `?returnTo=${originalUrl}` : ''}`); + } else { + if (!locals.user.email_verified && !url.pathname.startsWith('/verify-email')) { + throw redirect( + 302, + `/verify-email${url.searchParams.size > 0 ? `?${url.searchParams.toString()}` : ''}` + ); + } + } + } + + if (PROXY_PATHS.some((p) => url.pathname.startsWith(p.path))) { return await handleApiProxy({ event, resolve }); } diff --git a/services/app/src/lib/components/preset/ModelSelect.svelte b/services/app/src/lib/components/preset/ModelSelect.svelte index c430f62..5452464 100644 --- a/services/app/src/lib/components/preset/ModelSelect.svelte +++ b/services/app/src/lib/components/preset/ModelSelect.svelte @@ -119,9 +119,11 @@ {#each $modelsAvailable as { id, name, languages, capabilities, owned_by }} {@const isDisabled = owned_by !== 'ellm' && - $page.data.organizationData?.tier === 'free' && - !$page.data.organizationData?.credit && - !$page.data.organizationData?.external_keys?.[owned_by]} + owned_by !== 'custom' && + $page.data.organizationData && + $page.data.organizationData.credit === 0 && + $page.data.organizationData.credit_grant === 0 && + !$page.data.organizationData.external_keys?.[owned_by]} {#if !capabilityFilter || capabilities.includes(capabilityFilter)} diff --git a/services/app/src/lib/components/tables/(sub)/ColumnDropdown.svelte b/services/app/src/lib/components/tables/(sub)/ColumnDropdown.svelte index b15ce22..c4b7f53 100644 --- a/services/app/src/lib/components/tables/(sub)/ColumnDropdown.svelte +++ b/services/app/src/lib/components/tables/(sub)/ColumnDropdown.svelte @@ -3,9 +3,9 @@ import { page } from '$app/stores'; import toUpper from 'lodash/toUpper'; import Trash2 from 'lucide-svelte/icons/trash-2'; - import { genTableRows } from '../tablesStore'; + import { genTableRows, tableState } from '../tablesStore'; import logger from '$lib/logger'; - import { chatTableStaticCols, knowledgeTableStaticCols } from '$lib/constants'; + import { tableStaticCols } from '$lib/constants'; import type { GenTable, GenTableCol, GenTableStreamEvent } from '$lib/types'; import { CustomToastDesc, toast } from '$lib/components/ui/sonner'; @@ -17,13 +17,8 @@ import StarIcon from '$lib/icons/StarIcon.svelte'; export let tableType: 'action' | 'knowledge' | 'chat'; - export let column: GenTableCol; export let tableData: GenTable | undefined; - export let selectedRows: string[]; - export let streamingRows: Record; - export let isColumnSettingsOpen: { column: any; showMenu: boolean }; - export let isRenamingColumn: string | null; - export let isDeletingColumn: string | null; + export let column: GenTableCol; export let refetchTable: () => Promise; export let readonly; @@ -31,12 +26,12 @@ async function handleRegen(regenStrategy: 'run_before' | 'run_selected' | 'run_after') { if (!tableData || !$genTableRows) return; - if (Object.keys(streamingRows).length !== 0) return; + if (Object.keys($tableState.streamingRows).length !== 0) return; - const toRegenRowIds = selectedRows.filter((i) => !streamingRows[i]); + const toRegenRowIds = $tableState.selectedRows.filter((i) => !$tableState.streamingRows[i]); if (toRegenRowIds.length === 0) return toast.info('Select a row to start generating', { id: 'row-select-req' }); - selectedRows = []; + tableState.setSelectedRows([]); let colsToClear: string[]; switch (regenStrategy) { @@ -58,16 +53,15 @@ } } - streamingRows = { - ...streamingRows, - ...toRegenRowIds.reduce( + tableState.addStreamingRows( + toRegenRowIds.reduce( (acc, curr) => ({ ...acc, [curr]: colsToClear }), {} ) - }; + ); //? Optimistic update, clear row const originalValues = toRegenRowIds.map((toRegenRowId) => ({ @@ -148,12 +142,16 @@ break; } default: { - streamingRows = { - ...streamingRows, - [parsedValue.row_id]: streamingRows[parsedValue.row_id].filter( - (col) => col !== parsedValue.output_column_name - ) - }; + const streamingCols = $tableState.streamingRows[parsedValue.row_id].filter( + (col) => col !== parsedValue.output_column_name + ); + if (streamingCols.length === 0) { + tableState.delStreamingRows([parsedValue.row_id]); + } else { + tableState.addStreamingRows({ + [parsedValue.row_id]: streamingCols + }); + } break; } } @@ -176,12 +174,7 @@ // logger.error(toUpper(`${tableType}TBL_ROW_REGENSTREAM`), err); console.error(err); - //? Below necessary for retry - for (const toRegenRowId of toRegenRowIds) { - delete streamingRows[toRegenRowId]; - } - streamingRows = streamingRows; - + tableState.delStreamingRows(toRegenRowIds); refetchTable(); throw err; @@ -191,11 +184,7 @@ refetchTable(); } - for (const toRegenRowId of toRegenRowIds) { - delete streamingRows[toRegenRowId]; - } - streamingRows = streamingRows; - + tableState.delStreamingRows(toRegenRowIds); refetchTable(); } @@ -219,20 +208,20 @@ > {#if colType === 'output'} - (isColumnSettingsOpen = { column, showMenu: true })}> + tableState.setColumnSettings({ column, isOpen: true })}> Open settings {/if} - {#if colType === 'output' && !readonly && (tableType !== 'chat' || !chatTableStaticCols.includes(column.id)) && (tableType !== 'knowledge' || !knowledgeTableStaticCols.includes(column.id))} + {#if colType === 'output' && !readonly && !tableStaticCols[tableType].includes(column.id)} {/if} - {#if !readonly && (tableType !== 'chat' || !chatTableStaticCols.includes(column.id)) && (tableType !== 'knowledge' || !knowledgeTableStaticCols.includes(column.id))} + {#if !readonly && !tableStaticCols[tableType].includes(column.id)} - {#if selectedRows.length > 0} + {#if colType === 'output' && $tableState.selectedRows.length > 0} @@ -254,7 +243,7 @@ { - isRenamingColumn = column.id; + tableState.setRenamingCol(column.id); //? Tick doesn't work setTimeout(() => document.getElementById('column-id-edit')?.focus(), 100); }} @@ -262,7 +251,10 @@ Rename - (isDeletingColumn = column.id)} class="!text-[#F04438]"> + tableState.setDeletingCol(column.id)} + class="!text-[#F04438]" + > Delete column diff --git a/services/app/src/lib/components/tables/(sub)/ColumnHeader.svelte b/services/app/src/lib/components/tables/(sub)/ColumnHeader.svelte new file mode 100644 index 0000000..a996446 --- /dev/null +++ b/services/app/src/lib/components/tables/(sub)/ColumnHeader.svelte @@ -0,0 +1,425 @@ + + + { + if ($tableState.resizingCol) { + db[`${tableType}_table`].put({ + id: tableData.id, + columns: $tableState.colSizes + }); + $tableState.resizingCol = null; + } + }} +/> + +{#each tableData.cols as column, index (column.id)} + {@const colType = !column.gen_config ? 'input' : 'output'} + {@const isCustomCol = column.id !== 'ID' && column.id !== 'Updated at'} + + +
handleColumnHeaderClick(column)} + on:dragover={(e) => { + if (isCustomCol) { + e.preventDefault(); + hoveredColumnIndex = index; + } + }} + class={cn( + 'relative [&>*]:z-[-5] flex items-center gap-1 [&:not(:last-child)]:border-r border-[#E4E7EC] data-dark:border-[#333] cursor-default', + isCustomCol && !readonly ? 'px-1' : 'pl-2 pr-1', + $tableState.columnSettings.column?.id == column.id && + $tableState.columnSettings.isOpen && + 'bg-[#30A8FF33]', + draggingColumn?.id == column.id && 'opacity-0' + )} + > + {#if isCustomCol} + + {/if} + + {#if isCustomCol && !readonly} + + {/if} + + {#if column.id !== 'ID' && column.id !== 'Updated at'} + {#if !$tableState.colSizes[column.id] || $tableState.colSizes[column.id] >= 150} + + + {colType} + + {#if !$tableState.colSizes[column.id] || $tableState.colSizes[column.id] >= 220} + + {column.dtype} + + {/if} + + {#if column.gen_config?.object === 'gen_config.llm' && column.gen_config.multi_turn} +
+
+ +
+ {/if} + + {/if} + {/if} + + {#if $tableState.renamingCol === column.id} + + { + if (e.key === 'Enter') { + e.preventDefault(); + + handleSaveColumnTitle(e); + } else if (e.key === 'Escape') { + tableState.setRenamingCol(null); + } + }} + on:blur={() => setTimeout(() => tableState.setRenamingCol(null), 100)} + class="w-full bg-transparent border-0 outline outline-1 outline-[#4169e1] data-dark:outline-[#5b7ee5] rounded-[2px]" + /> + {:else} + + {column.id} + + {/if} + + {#if (!tableStaticCols[tableType].includes(column.id) || colType === 'output') && !readonly} + + {/if} +
+{/each} + +{#if dragMouseCoords && draggingColumn} + {@const colType = !draggingColumn.gen_config /* || Object.keys(column.gen_config).length === 0 */ + ? 'input' + : 'output'} + +
+ + + {#if !$tableState.colSizes[draggingColumn.id] || $tableState.colSizes[draggingColumn.id] >= 150} + + + {colType} + + {#if !$tableState.colSizes[draggingColumn.id] || $tableState.colSizes[draggingColumn.id] >= 220} + + {draggingColumn.dtype} + + {/if} + + {#if draggingColumn.gen_config?.object === 'gen_config.llm' && draggingColumn.gen_config.multi_turn} +
+
+ +
+ {/if} + + {/if} + + + {draggingColumn.id} + + + +
+ +{/if} diff --git a/services/app/src/lib/components/tables/(sub)/ColumnSettings.svelte b/services/app/src/lib/components/tables/(sub)/ColumnSettings.svelte index 49250ac..7ce4c18 100644 --- a/services/app/src/lib/components/tables/(sub)/ColumnSettings.svelte +++ b/services/app/src/lib/components/tables/(sub)/ColumnSettings.svelte @@ -1,8 +1,10 @@ + + + +
+ +
+ +
+ + + + +
diff --git a/services/app/src/lib/components/tables/(sub)/Conversations.svelte b/services/app/src/lib/components/tables/(sub)/Conversations.svelte new file mode 100644 index 0000000..9e77283 --- /dev/null +++ b/services/app/src/lib/components/tables/(sub)/Conversations.svelte @@ -0,0 +1,383 @@ + + +Chat history + +
+ { + //@ts-expect-error Generic type + debouncedSearchConv(e.target?.value ?? ''); + }} + bind:value={searchQuery} + type="search" + placeholder="Search" + class="pl-8 h-9 placeholder:not-italic placeholder:text-[#98A2B3] bg-[#F2F4F7] rounded-full" + > + + {#if isLoadingSearch} +
+ +
+ {:else} + + {/if} + + +
+ +
+ { + currentOffset = 0; + moreConversationsFinished = false; + pastConversations = []; + getPastConversations(); + }} + class="h-[20px] w-[30px] [&>[data-switch-thumb]]:h-4 [&>[data-switch-thumb]]:data-[state=checked]:translate-x-2.5" + /> + +
+ +{#if searchResults.length || isNoResults} + {#if isNoResults} +
+ No results found +
+ {:else} + + Search results: + {searchQuery} + + +
+ {/if} +{/if} + + { + autoAnimateController = autoAnimate(e.detail[0].elements().viewport); + }} + class="grow flex flex-col my-3 rounded-md overflow-auto os-dark" +> + {#each !searchResults.length && !isNoResults ? pastConversations : searchResults as conversation, index (conversation.id)} + {#if !searchResults.length && !isNoResults} + {#each timestampKeys as time (time)} + {#if timestamps[time] == index} +
+ + {timestampsDisplayName[time]} + +
+ {/if} + {/each} + {/if} + {#if isEditingTitle === conversation.id} +
+ +
+ + {/if} {/if}
diff --git a/services/app/src/lib/components/tables/(sub)/TablePagination.svelte b/services/app/src/lib/components/tables/(sub)/TablePagination.svelte index 63e972b..291fc77 100644 --- a/services/app/src/lib/components/tables/(sub)/TablePagination.svelte +++ b/services/app/src/lib/components/tables/(sub)/TablePagination.svelte @@ -1,8 +1,9 @@ @@ -320,7 +158,7 @@ const editingCell = document.querySelector('[data-editing="true"]'); //@ts-ignore if (e.target && editingCell && !editingCell.contains(e.target)) { - isEditingCell = null; + tableState.setEditingCell(null); } }} on:keydown={keyboardNavigate} @@ -329,7 +167,7 @@ {#if tableData}
+ >
+ >
{#if !readonly} { if ($genTableRows) { - return $genTableRows.every((row) => selectedRows.includes(row.ID)) - ? (selectedRows = selectedRows.filter( - (i) => !$genTableRows?.some(({ ID }) => ID === i) - )) - : (selectedRows = [ - ...selectedRows.filter((i) => !$genTableRows?.some(({ ID }) => ID === i)), - ...$genTableRows.map(({ ID }) => ID) - ]); + return tableState.selectAllRows($genTableRows); } else return false; }} - checked={($genTableRows ?? []).every((row) => selectedRows.includes(row.ID))} + checked={($genTableRows ?? []).every((row) => + $tableState.selectedRows.includes(row.ID) + )} class="h-4 sm:h-[18px] w-4 sm:w-[18px] [&>svg]:h-3 sm:[&>svg]:h-3.5 [&>svg]:w-3 sm:[&>svg]:w-3.5 [&>svg]:translate-x-[1px]" /> {/if}
- {#each tableData.cols as column, index (column.id)} - {@const colType = !column.gen_config ? 'input' : 'output'} - {@const isCustomCol = column.id !== 'ID' && column.id !== 'Updated at'} - - -
handleColumnHeaderClick(column)} - on:dragover={(e) => { - if (isCustomCol) { - e.preventDefault(); - hoveredColumnIndex = index; - } - }} - class="flex items-center gap-1 {isCustomCol && !readonly - ? 'px-1' - : 'pl-2 pr-1'} cursor-default [&:not(:last-child)]:border-r border-[#E4E7EC] data-dark:border-[#333] {isColumnSettingsOpen - .column?.id == column.id && isColumnSettingsOpen.showMenu - ? 'bg-[#30A8FF33]' - : ''} {draggingColumn?.id == column.id ? 'opacity-0' : ''}" - > - {#if isCustomCol} - {#if !readonly} - - {/if} - - - - {colType} - - - {column.dtype} - - - {#if column.gen_config?.object === 'gen_config.llm' && column.gen_config.multi_turn} -
-
- -
- {/if} - - {/if} - - {#if isRenamingColumn === column.id} - - { - if (e.key === 'Enter') { - e.preventDefault(); - - handleSaveColumnTitle(e); - } else if (e.key === 'Escape') { - isRenamingColumn = null; - } - }} - on:blur={() => setTimeout(() => (isRenamingColumn = null), 100)} - class="w-full bg-transparent border-0 outline outline-1 outline-[#4169e1] data-dark:outline-[#5b7ee5] rounded-[2px]" - /> - {:else} - - {column.id} - - {/if} - - {#if (!actionTableStaticCols.includes(column.id) || colType === 'output') && !readonly} - - {/if} -
- {/each} +
{#if $genTableRows} {#if !readonly} - + {/if} {#each $genTableRows as row (row.ID)}
- {#if streamingRows[row.ID]} + {#if $tableState.streamingRows[row.ID]}
+ >
{/if}
+ class={cn( + 'absolute -z-10 top-0 -left-4 h-full w-[calc(100%_+_16px)]', + $tableState.streamingRows[row.ID] + ? 'bg-[#FDEFF4]' + : 'bg-[#FAFBFC] data-dark:bg-[#1E2024] group-hover:bg-[#ECEDEE]' + )} + >
{#if !readonly} handleSelectRow(e, row)} - checked={!!selectedRows.find((i) => i === row.ID)} + checked={!!$tableState.selectedRows.find((i) => i === row.ID)} class="mt-[1px] h-4 sm:h-[18px] w-4 sm:w-[18px] [&>svg]:h-3 sm:[&>svg]:h-3.5 [&>svg]:w-3 sm:[&>svg]:w-3.5 [&>svg]:translate-x-[1px]" /> {/if}
{#each tableData.cols as column} {@const editMode = - isEditingCell && - isEditingCell.rowID === row.ID && - isEditingCell.columnID === column.id} + $tableState.editingCell && + $tableState.editingCell.rowID === row.ID && + $tableState.editingCell.columnID === column.id} {@const isValidFileUri = isValidUri(row[column.id]?.value)}
{ if (column.id === 'ID' || column.id === 'Updated at') return; - if (column.dtype === 'file' && row[column.id]?.value && isValidFileUri) return; + if ( + (column.dtype === 'file' || column.dtype === 'audio') && + row[column.id]?.value && + isValidFileUri + ) + return; if (uploadController) return; - if (streamingRows[row.ID] || isEditingCell) return; + if ($tableState.streamingRows[row.ID] || $tableState.editingCell) return; if (e.detail > 1) { e.preventDefault(); @@ -602,45 +326,59 @@ if (readonly) return; if (column.id === 'ID' || column.id === 'Updated at') return; - if (column.dtype === 'file' && row[column.id]?.value && isValidFileUri) return; + if ( + (column.dtype === 'file' || column.dtype === 'audio') && + row[column.id]?.value && + isValidFileUri + ) + return; if (uploadController) return; - if (!streamingRows[row.ID]) { - isEditingCell = { rowID: row.ID, columnID: column.id }; + if (!$tableState.streamingRows[row.ID]) { + tableState.setEditingCell({ rowID: row.ID, columnID: column.id }); } }} on:keydown={(e) => { if (readonly) return; if (column.id === 'ID' || column.id === 'Updated at') return; - if (column.dtype === 'file' && row[column.id]?.value && isValidFileUri) return; + if ( + (column.dtype === 'file' || column.dtype === 'audio') && + row[column.id]?.value && + isValidFileUri + ) + return; if (uploadController) return; - if (!editMode && e.key == 'Enter' && !streamingRows[row.ID]) { - isEditingCell = { rowID: row.ID, columnID: column.id }; + if (!editMode && e.key == 'Enter' && !$tableState.streamingRows[row.ID]) { + tableState.setEditingCell({ rowID: row.ID, columnID: column.id }); } }} - style={isColumnSettingsOpen.column?.id == column.id && isColumnSettingsOpen.showMenu + style={$tableState.columnSettings.column?.id == column.id && + $tableState.columnSettings.isOpen ? 'background-color: #30A8FF17;' : ''} - class="flex flex-col justify-start gap-1 {editMode - ? 'p-0 bg-black/5 data-dark:bg-white/5' - : 'p-2 overflow-auto whitespace-pre-line'} h-full max-h-[99px] sm:max-h-[149px] w-full break-words {streamingRows[ - row.ID - ] - ? 'bg-[#FDEFF4]' - : 'group-hover:bg-[#ECEDEE] data-dark:group-hover:bg-white/5'} [&:not(:last-child)]:border-r border-[#E4E7EC] data-dark:border-[#333]" + class={cn( + 'flex flex-col justify-start gap-1 h-full max-h-[99px] sm:max-h-[149px] w-full break-words [&:not(:last-child)]:border-r border-[#E4E7EC] data-dark:border-[#333]', + editMode + ? 'p-0 bg-black/5 data-dark:bg-white/5' + : 'p-2 overflow-auto whitespace-pre-line', + $tableState.streamingRows[row.ID] + ? 'bg-[#FDEFF4]' + : 'group-hover:bg-[#ECEDEE] data-dark:group-hover:bg-white/5' + )} > - {#if streamingRows[row.ID]?.includes(column.id) && !editMode && column.id !== 'ID' && column.id !== 'Updated at' && column.gen_config} + {#if $tableState.streamingRows[row.ID]?.includes(column.id) && !editMode && column.id !== 'ID' && column.id !== 'Updated at' && column.gen_config} {/if} {#if editMode} - {#if column.dtype === 'file'} + {#if column.dtype === 'file' || column.dtype === 'audio'} {:else} @@ -656,9 +394,9 @@ } }} class="min-h-[100px] sm:min-h-[150px] h-full w-full p-2 bg-transparent outline outline-secondary resize-none" - /> + > {/if} - {:else if column.dtype === 'file'} + {:else if column.dtype === 'file' || column.dtype === 'audio'} {/if} @@ -715,52 +453,7 @@
{/if} -{#if dragMouseCoords && draggingColumn} - {@const colType = !draggingColumn.gen_config /* || Object.keys(column.gen_config).length === 0 */ - ? 'input' - : 'output'} - -
- - - - - {colType} - - - {draggingColumn.dtype} - - - - - {draggingColumn.id} - - - -
- -{/if} - - + { diff --git a/services/app/src/lib/components/tables/ChatTable.svelte b/services/app/src/lib/components/tables/ChatTable.svelte index f937898..a9b3795 100644 --- a/services/app/src/lib/components/tables/ChatTable.svelte +++ b/services/app/src/lib/components/tables/ChatTable.svelte @@ -2,15 +2,13 @@ import { PUBLIC_JAMAI_URL } from '$env/static/public'; import { onDestroy } from 'svelte'; import { page } from '$app/stores'; - import GripVertical from 'lucide-svelte/icons/grip-vertical'; - import { genTableRows } from '$lib/components/tables/tablesStore'; - import { isValidUri } from '$lib/utils'; - import { chatTableStaticCols } from '$lib/constants'; + import { genTableRows, tableState } from '$lib/components/tables/tablesStore'; + import { cn, isValidUri } from '$lib/utils'; import logger from '$lib/logger'; - import type { GenTable, GenTableCol, GenTableRow, UserRead } from '$lib/types'; + import type { GenTable, GenTableRow, UserRead } from '$lib/types'; import { - ColumnDropdown, + ColumnHeader, DeleteFileDialog, FileColumnView, FileSelect, @@ -18,34 +16,14 @@ NewRow } from '$lib/components/tables/(sub)'; import Checkbox from '$lib/components/Checkbox.svelte'; - import Portal from '$lib/components/Portal.svelte'; import FoundProjectOrgSwitcher from '$lib/components/preset/FoundProjectOrgSwitcher.svelte'; import RowStreamIndicator from '$lib/components/preset/RowStreamIndicator.svelte'; import { toast, CustomToastDesc } from '$lib/components/ui/sonner'; - import { Button } from '$lib/components/ui/button'; import LoadingSpinner from '$lib/icons/LoadingSpinner.svelte'; - import MoreVertIcon from '$lib/icons/MoreVertIcon.svelte'; - import MultiturnChatIcon from '$lib/icons/MultiturnChatIcon.svelte'; export let userData: UserRead | undefined; - export let table: Promise< - | { - error: number; - message: any; - data?: undefined; - } - | { - data: GenTable; - error?: undefined; - message?: undefined; - } - >; export let tableData: GenTable | undefined; - export let tableError: { error: number; message: Awaited['message'] } | undefined; - export let selectedRows: string[]; - export let streamingRows: Record; - export let isColumnSettingsOpen: { column: any; showMenu: boolean }; - export let isDeletingColumn: string | null; + export let tableError: { error: number; message?: any } | undefined; export let readonly = false; export let refetchTable: (hideColumnSettings?: boolean) => Promise; @@ -56,150 +34,16 @@ //? Expanding ID and Updated at columns let focusedCol: string | null = null; - //? Column header click handler - let isRenamingColumn: string | null = null; - let dblClickTimer: NodeJS.Timeout | null = null; - function handleColumnHeaderClick(column: GenTableCol) { - if (!tableData) return; - if (isRenamingColumn) return; - - if (dblClickTimer) { - clearTimeout(dblClickTimer); - dblClickTimer = null; - if (!readonly && !chatTableStaticCols.includes(column.id)) { - isRenamingColumn = column.id; - } - } else { - dblClickTimer = setTimeout(() => { - if (column.id !== 'ID' && column.id !== 'Updated at' && column.gen_config) { - isColumnSettingsOpen = { column, showMenu: true }; - } - dblClickTimer = null; - }, 200); - } - } - - async function handleSaveColumnTitle( - e: KeyboardEvent & { - currentTarget: EventTarget & HTMLInputElement; - } - ) { - if (!tableData || !$genTableRows) return; - if (!isRenamingColumn) return; - - const response = await fetch(`${PUBLIC_JAMAI_URL}/api/v1/gen_tables/chat/columns/rename`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-project-id': $page.params.project_id - }, - body: JSON.stringify({ - table_id: $page.params.table_id, - column_map: { - [isRenamingColumn]: e.currentTarget.value - } - }) - }); - - if (response.ok) { - refetchTable(); - tableData = { - ...tableData, - cols: tableData.cols.map((col) => - col.id === isRenamingColumn ? { ...col, id: e.currentTarget.value } : col - ) - }; - isRenamingColumn = null; - } else { - const responseBody = await response.json(); - logger.error('CHATTBL_COLUMN_RENAME', responseBody); - toast.error('Failed to rename column', { - id: responseBody.message || JSON.stringify(responseBody), - description: CustomToastDesc, - componentProps: { - description: responseBody.message || JSON.stringify(responseBody), - requestID: responseBody.request_id - } - }); - } - } - - //? Reorder columns - let isReorderLoading = false; - let dragMouseCoords: { - x: number; - y: number; - startX: number; - startY: number; - width: number; - } | null = null; - let draggingColumn: GenTable['cols'][number] | null = null; - let draggingColumnIndex: number | null = null; - let hoveredColumnIndex: number | null = null; - - $: if ( - tableData && - draggingColumnIndex != null && - hoveredColumnIndex != null && - draggingColumnIndex != hoveredColumnIndex - ) { - [tableData.cols[draggingColumnIndex], tableData.cols[hoveredColumnIndex]] = [ - tableData.cols[hoveredColumnIndex], - tableData.cols[draggingColumnIndex] - ]; - - draggingColumnIndex = hoveredColumnIndex; - } - - async function handleSaveOrder() { - if (!tableData || !$genTableRows) return; - if (isReorderLoading) return; - isReorderLoading = true; - - const response = await fetch(`${PUBLIC_JAMAI_URL}/api/v1/gen_tables/chat/columns/reorder`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-project-id': $page.params.project_id - }, - body: JSON.stringify({ - table_id: tableData.id, - column_names: tableData.cols.flatMap(({ id }) => - id === 'ID' || id === 'Updated at' ? [] : id - ) - }) - }); - - if (!response.ok) { - const responseBody = await response.json(); - logger.error('CHATTBL_TBL_REORDER', responseBody); - toast.error('Failed to reorder columns', { - id: responseBody.message || JSON.stringify(responseBody), - description: CustomToastDesc, - componentProps: { - description: responseBody.message || JSON.stringify(responseBody), - requestID: responseBody.request_id - } - }); - tableData = (await table)?.data; - } else { - refetchTable(); - } - - isReorderLoading = false; - } - - let isEditingCell: { rowID: string; columnID: string } | null = null; async function handleSaveEdit( e: KeyboardEvent & { currentTarget: EventTarget & HTMLTextAreaElement; } ) { if (!tableData || !$genTableRows) return; - if (!isEditingCell) return; + if (!$tableState.editingCell) return; const editedValue = e.currentTarget.value; - const cellToUpdate = isEditingCell; + const cellToUpdate = $tableState.editingCell; await saveEditCell(cellToUpdate, editedValue); } @@ -246,7 +90,7 @@ //? Revert back to original value genTableRows.setCell(cellToUpdate, originalValue); } else { - isEditingCell = null; + tableState.setEditingCell(null); refetchTable(); } } @@ -260,32 +104,24 @@ if (!tableData || !$genTableRows) return; //? Select multiple rows with shift key const rowIndex = $genTableRows.findIndex(({ ID }) => ID === row.ID); - if (e.detail.event.shiftKey && selectedRows.length && shiftOrigin != null) { + if (e.detail.event.shiftKey && $tableState.selectedRows.length && shiftOrigin != null) { if (shiftOrigin < rowIndex) { - selectedRows = [ - ...selectedRows.filter((i) => !$genTableRows?.some(({ ID }) => ID === i)), + tableState.setSelectedRows([ + ...$tableState.selectedRows.filter((i) => !$genTableRows?.some(({ ID }) => ID === i)), ...$genTableRows.slice(shiftOrigin, rowIndex + 1).map(({ ID }) => ID) - ]; + ]); } else if (shiftOrigin > rowIndex) { - selectedRows = [ - ...selectedRows.filter((i) => !$genTableRows?.some(({ ID }) => ID === i)), + tableState.setSelectedRows([ + ...$tableState.selectedRows.filter((i) => !$genTableRows?.some(({ ID }) => ID === i)), ...$genTableRows.slice(rowIndex, shiftOrigin + 1).map(({ ID }) => ID) - ]; + ]); } else { - selectOne(); + tableState.toggleRowSelection(row.ID); } } else { - selectOne(); + tableState.toggleRowSelection(row.ID); shiftOrigin = rowIndex; } - - function selectOne() { - if (selectedRows.find((i) => i === row.ID)) { - selectedRows = selectedRows.filter((i) => i !== row.ID); - } else { - selectedRows = [...selectedRows, row.ID]; - } - } } function keyboardNavigate(e: KeyboardEvent) { @@ -296,21 +132,22 @@ if (isCtrl && e.key === 'a' && !isInputActive) { e.preventDefault(); - if (Object.keys(streamingRows).length !== 0) return; + if (Object.keys($tableState.streamingRows).length !== 0) return; - selectedRows = [ - ...selectedRows.filter((i) => !$genTableRows?.some(({ ID }) => ID === i)), + tableState.setSelectedRows([ + ...$tableState.selectedRows.filter((i) => !$genTableRows?.some(({ ID }) => ID === i)), ...$genTableRows.map(({ ID }) => ID) - ]; + ]); } if (e.key === 'Escape') { - isEditingCell = null; + tableState.setEditingCell(null); } } onDestroy(() => { $genTableRows = undefined; + tableState.reset(); }); @@ -319,7 +156,7 @@ const editingCell = document.querySelector('[data-editing="true"]'); //@ts-ignore if (e.target && editingCell && !editingCell.contains(e.target)) { - isEditingCell = null; + tableState.setEditingCell(null); } }} on:keydown={keyboardNavigate} @@ -328,7 +165,7 @@ {#if tableData}
{ if ($genTableRows) { - return $genTableRows.every((row) => selectedRows.includes(row.ID)) - ? (selectedRows = selectedRows.filter( - (i) => !$genTableRows?.some(({ ID }) => ID === i) - )) - : (selectedRows = [ - ...selectedRows.filter((i) => !$genTableRows?.some(({ ID }) => ID === i)), - ...$genTableRows.map(({ ID }) => ID) - ]); + return tableState.selectAllRows($genTableRows); } else return false; }} - checked={($genTableRows ?? []).every((row) => selectedRows.includes(row.ID))} + checked={($genTableRows ?? []).every((row) => + $tableState.selectedRows.includes(row.ID) + )} class="h-4 sm:h-[18px] w-4 sm:w-[18px] [&>svg]:h-3 sm:[&>svg]:h-3.5 [&>svg]:w-3 sm:[&>svg]:w-3.5 [&>svg]:translate-x-[1px]" /> {/if}
- {#each tableData.cols as column, index (column.id)} - {@const colType = !column.gen_config ? 'input' : 'output'} - {@const isCustomCol = column.id !== 'ID' && column.id !== 'Updated at'} - - -
handleColumnHeaderClick(column)} - on:dragover={(e) => { - if (isCustomCol) { - e.preventDefault(); - hoveredColumnIndex = index; - } - }} - class="flex items-center gap-1 {isCustomCol && !readonly - ? 'px-1' - : 'pl-2 pr-1'} cursor-default [&:not(:last-child)]:border-r border-[#E4E7EC] data-dark:border-[#333] {isColumnSettingsOpen - .column?.id == column.id && isColumnSettingsOpen.showMenu - ? 'bg-[#30A8FF33]' - : ''} {draggingColumn?.id == column.id ? 'opacity-0' : ''}" - > - {#if isCustomCol && !readonly} - - {/if} - - {#if column.id !== 'ID' && column.id !== 'Updated at'} - - - {colType} - - - {column.dtype} - - - {#if column.gen_config?.object === 'gen_config.llm' && column.gen_config.multi_turn} -
-
- -
- {/if} - - {/if} - - {#if isRenamingColumn === column.id} - - { - if (e.key === 'Enter') { - e.preventDefault(); - - handleSaveColumnTitle(e); - } else if (e.key === 'Escape') { - isRenamingColumn = null; - } - }} - on:blur={() => setTimeout(() => (isRenamingColumn = null), 100)} - class="w-full bg-transparent border-0 outline outline-1 outline-[#4169e1] data-dark:outline-[#5b7ee5] rounded-[2px]" - /> - {:else} - - {column.id} - - {/if} - - {#if (!chatTableStaticCols.includes(column.id) || colType === 'output') && !readonly} - - {/if} -
- {/each} +
{#if $genTableRows} {#if !readonly} - + {/if}