From 55ac8c0e24e745cbdcdfb91c25047d9e05fbcafb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 17:53:42 +1300 Subject: [PATCH 01/12] Add pool adapter --- composer.json | 3 +- composer.lock | 94 ++++++-- src/Database/Adapter/Pool.php | 416 ++++++++++++++++++++++++++++++++++ 3 files changed, 491 insertions(+), 22 deletions(-) create mode 100644 src/Database/Adapter/Pool.php diff --git a/composer.json b/composer.json index 1be5215cd..2b3ed503a 100755 --- a/composer.json +++ b/composer.json @@ -37,7 +37,8 @@ "ext-pdo": "*", "ext-mbstring": "*", "utopia-php/framework": "0.33.*", - "utopia-php/cache": "0.12.*" + "utopia-php/cache": "0.12.*", + "utopia-php/pools": "0.8.*" }, "require-dev": { "fakerphp/faker": "1.23.*", diff --git a/composer.lock b/composer.lock index ec648d2fc..b92c6fea0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d1e1cb1921161014c22372feccc945f9", + "content-hash": "5d1e7adf0910bdd2df5324f9cc531a01", "packages": [ { "name": "brick/math", @@ -1082,16 +1082,16 @@ }, { "name": "ramsey/collection", - "version": "2.1.0", + "version": "2.1.1", "source": { "type": "git", "url": "https://github.com/ramsey/collection.git", - "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109" + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/collection/zipball/3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", - "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", + "url": "https://api.github.com/repos/ramsey/collection/zipball/344572933ad0181accbf4ba763e85a0306a8c5e2", + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2", "shasum": "" }, "require": { @@ -1152,9 +1152,9 @@ ], "support": { "issues": "https://github.com/ramsey/collection/issues", - "source": "https://github.com/ramsey/collection/tree/2.1.0" + "source": "https://github.com/ramsey/collection/tree/2.1.1" }, - "time": "2025-03-02T04:48:29+00:00" + "time": "2025-03-22T05:38:12+00:00" }, { "name": "ramsey/uuid", @@ -1923,18 +1923,70 @@ }, "time": "2025-03-06T11:37:49+00:00" }, + { + "name": "utopia-php/pools", + "version": "0.8.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/pools.git", + "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/60733929dc328e7ea47e800579c8bbf0d49df5ba", + "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba", + "shasum": "" + }, + "require": { + "php": ">=8.3", + "utopia-php/telemetry": "0.1.*" + }, + "require-dev": { + "laravel/pint": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "11.*" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Pools\\": "src/Pools" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], + "description": "A simple library to manage connection pools", + "keywords": [ + "framework", + "php", + "pools", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/pools/issues", + "source": "https://github.com/utopia-php/pools/tree/0.8.0" + }, + "time": "2025-03-19T10:22:03+00:00" + }, { "name": "utopia-php/telemetry", - "version": "0.1.0", + "version": "0.1.1", "source": { "type": "git", "url": "https://github.com/utopia-php/telemetry.git", - "reference": "d35f2f0632f4ee0be63fb7ace6a94a6adda71a80" + "reference": "437f0021777f0e575dfb9e8a1a081b3aed75e33f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/d35f2f0632f4ee0be63fb7ace6a94a6adda71a80", - "reference": "d35f2f0632f4ee0be63fb7ace6a94a6adda71a80", + "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/437f0021777f0e575dfb9e8a1a081b3aed75e33f", + "reference": "437f0021777f0e575dfb9e8a1a081b3aed75e33f", "shasum": "" }, "require": { @@ -1955,7 +2007,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\": "src/" + "Utopia\\Telemetry\\": "src/Telemetry" } }, "notification-url": "https://packagist.org/downloads/", @@ -1969,9 +2021,9 @@ ], "support": { "issues": "https://github.com/utopia-php/telemetry/issues", - "source": "https://github.com/utopia-php/telemetry/tree/0.1.0" + "source": "https://github.com/utopia-php/telemetry/tree/0.1.1" }, - "time": "2024-11-13T10:29:53+00:00" + "time": "2025-03-17T11:57:52+00:00" } ], "packages-dev": [ @@ -2444,16 +2496,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.21", + "version": "1.12.23", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "14276fdef70575106a3392a4ed553c06a984df28" + "reference": "29201e7a743a6ab36f91394eab51889a82631428" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/14276fdef70575106a3392a4ed553c06a984df28", - "reference": "14276fdef70575106a3392a4ed553c06a984df28", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/29201e7a743a6ab36f91394eab51889a82631428", + "reference": "29201e7a743a6ab36f91394eab51889a82631428", "shasum": "" }, "require": { @@ -2498,7 +2550,7 @@ "type": "github" } ], - "time": "2025-03-09T09:24:50+00:00" + "time": "2025-03-23T14:57:32+00:00" }, { "name": "phpunit/php-code-coverage", @@ -4069,7 +4121,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -4077,6 +4129,6 @@ "ext-pdo": "*", "ext-mbstring": "*" }, - "platform-dev": [], + "platform-dev": {}, "plugin-api-version": "2.6.0" } diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php new file mode 100644 index 000000000..506310720 --- /dev/null +++ b/src/Database/Adapter/Pool.php @@ -0,0 +1,416 @@ +pool = $pool; + + $this->pool->use(function (mixed $resource) { + if (!($resource instanceof Adapter)) { + throw new DatabaseException('Pool must contain instances of Utopia\Database\Adapter'); + } + }); + } + + /** + * Forward method calls to the internal adapter instance via the pool. + * + * Required because __call() can't be used to implement abstract methods. + * + * @template T + * @param string $method + * @param array $args + * @return T + */ + public function delegate(string $method, array $args): mixed + { + return $this->pool->use(function (Adapter $adapter) use ($method, $args) { + return $adapter->{$method}(...$args); + }); + } + + public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): static + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function startTransaction(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function commitTransaction(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function rollbackTransaction(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function ping(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function reconnect(): void + { + $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function create(string $name): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function exists(string $database, ?string $collection = null): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function list(): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function delete(string $name): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function createCollection(string $name, array $attributes = [], array $indexes = []): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function deleteCollection(string $id): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function analyzeCollection(string $collection): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function createAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function deleteAttribute(string $collection, string $id): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function renameAttribute(string $collection, string $old, string $new): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function createRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay = false, string $id = '', string $twoWayKey = ''): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function updateRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, string $side, ?string $newKey = null, ?string $newTwoWayKey = null): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function deleteRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, string $side): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function renameIndex(string $collection, string $old, string $new): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function deleteIndex(string $collection, string $id): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function createDocument(string $collection, Document $document): Document + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function createDocuments(string $collection, array $documents): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function updateDocuments(string $collection, Document $updates, array $documents): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function createOrUpdateDocuments(string $collection, string $attribute, array $documents): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function deleteDocument(string $collection, string $id): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function deleteDocuments(string $collection, array $ids): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null): float|int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function count(string $collection, array $queries = [], ?int $max = null): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSizeOfCollection(string $collection): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSizeOfCollectionOnDisk(string $collection): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getLimitForString(): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getLimitForInt(): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getLimitForAttributes(): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getLimitForIndexes(): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getMaxIndexLength(): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getMinDateTime(): \DateTime + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForSchemas(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForAttributes(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForSchemaAttributes(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForIndex(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForUniqueIndex(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForFulltextIndex(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForFulltextWildcardIndex(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForCasting(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForQueryContains(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForTimeouts(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForRelationships(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForUpdateLock(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForBatchOperations(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForAttributeResizing(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForGetConnectionId(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForCastIndexArray(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForUpserts(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForCacheSkipOnFailure(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSupportForReconnection(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getCountOfAttributes(Document $collection): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getCountOfIndexes(Document $collection): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getCountOfDefaultAttributes(): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getCountOfDefaultIndexes(): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getDocumentSizeLimit(): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getAttributeWidth(Document $collection): int + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getKeywords(): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + protected function getAttributeProjection(array $selections, string $prefix = ''): mixed + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function increaseDocumentAttribute(string $collection, string $id, string $attribute, float|int $value, string $updatedAt, float|int|null $min = null, float|int|null $max = null): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getConnectionId(): string + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getInternalIndexesKeys(): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getSchemaAttributes(string $collection): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function getTenantQuery(string $collection, string $parentAlias = ''): string + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } +} From 442367e8129df0e6d90131672c612b2337b181a2 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 17:54:13 +1300 Subject: [PATCH 02/12] Fix static usage --- src/Database/Adapter.php | 13 ++++++------- src/Database/Adapter/SQL.php | 10 +++++----- tests/e2e/Adapter/Base.php | 8 ++++---- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 694af73f9..4261ecb63 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -245,12 +245,12 @@ public function resetMetadata(): static * and an appropriate error or exception will be raised to handle the timeout condition. * * @param int $milliseconds The timeout value in milliseconds for database queries. - * @param string $event The event the timeout should fire fore - * @return void + * @param string $event The event the timeout should fire for + * @return $this * * @throws Exception The provided timeout value must be greater than or equal to 0. */ - abstract public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void; + abstract public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): static; /** * Clears a global timeout for database queries. @@ -301,7 +301,6 @@ abstract public function rollbackTransaction(): bool; * Check if a transaction is active. * * @return bool - * @throws DatabaseException */ public function inTransaction(): bool { @@ -932,14 +931,14 @@ abstract public function getCountOfIndexes(Document $collection): int; * * @return int */ - abstract public static function getCountOfDefaultAttributes(): int; + abstract public function getCountOfDefaultAttributes(): int; /** * Returns number of indexes used by default. * * @return int */ - abstract public static function getCountOfDefaultIndexes(): int; + abstract public function getCountOfDefaultIndexes(): int; /** * Get maximum width, in bytes, allowed for a SQL row @@ -947,7 +946,7 @@ abstract public static function getCountOfDefaultIndexes(): int; * * @return int */ - abstract public static function getDocumentSizeLimit(): int; + abstract public function getDocumentSizeLimit(): int; /** * Estimate maximum number of bytes required to store a document in $collection. diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index eafa917d6..8c1020f55 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -427,7 +427,7 @@ public function getCountOfAttributes(Document $collection): int { $attributes = \count($collection->getAttribute('attributes') ?? []); - return $attributes + static::getCountOfDefaultAttributes(); + return $attributes + $this->getCountOfDefaultAttributes(); } /** @@ -439,7 +439,7 @@ public function getCountOfAttributes(Document $collection): int public function getCountOfIndexes(Document $collection): int { $indexes = \count($collection->getAttribute('indexes') ?? []); - return $indexes + static::getCountOfDefaultIndexes(); + return $indexes + $this->getCountOfDefaultIndexes(); } /** @@ -447,7 +447,7 @@ public function getCountOfIndexes(Document $collection): int * * @return int */ - public static function getCountOfDefaultAttributes(): int + public function getCountOfDefaultAttributes(): int { return \count(Database::INTERNAL_ATTRIBUTES); } @@ -457,7 +457,7 @@ public static function getCountOfDefaultAttributes(): int * * @return int */ - public static function getCountOfDefaultIndexes(): int + public function getCountOfDefaultIndexes(): int { return \count(Database::INTERNAL_INDEXES); } @@ -468,7 +468,7 @@ public static function getCountOfDefaultIndexes(): int * * @return int */ - public static function getDocumentSizeLimit(): int + public function getDocumentSizeLimit(): int { return 65535; } diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 54b38f7c0..7939ec975 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -5802,7 +5802,7 @@ public function testNoChangeUpdateDocumentWithRelationWithoutPermission(): void public function testWidthLimit(): void { - if (static::getDatabase()->getAdapter()::getDocumentSizeLimit() === 0) { + if (static::getDatabase()->getAdapter()->getDocumentSizeLimit() === 0) { $this->expectNotToPerformAssertions(); return; } @@ -5885,7 +5885,7 @@ public function testExceptionAttributeLimit(): void return; } - $limit = static::getDatabase()->getAdapter()->getLimitForAttributes() - static::getDatabase()->getAdapter()::getCountOfDefaultAttributes(); + $limit = static::getDatabase()->getAdapter()->getLimitForAttributes() - static::getDatabase()->getAdapter()->getCountOfDefaultAttributes(); $attributes = []; @@ -16504,12 +16504,12 @@ public function testDeleteBulkDocuments(): void 'required' => true, ]) ], - documentSecurity: false, permissions: [ Permission::create(Role::any()), Permission::read(Role::any()), Permission::delete(Role::any()) - ] + ], + documentSecurity: false ); $this->propagateBulkDocuments('bulk_delete'); From 19f9a3f2abe7b319bc1abe78c28c1d520b4863df Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 18:51:20 +1300 Subject: [PATCH 03/12] Fix timeout refs --- src/Database/Adapter.php | 2 +- src/Database/Adapter/Pool.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 4261ecb63..72105e3d6 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -250,7 +250,7 @@ public function resetMetadata(): static * * @throws Exception The provided timeout value must be greater than or equal to 0. */ - abstract public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): static; + abstract public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void; /** * Clears a global timeout for database queries. diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 506310720..10c4fc914 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -44,9 +44,9 @@ public function delegate(string $method, array $args): mixed }); } - public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): static + public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void { - return $this->delegate(__FUNCTION__, \func_get_args()); + $this->delegate(__FUNCTION__, \func_get_args()); } public function startTransaction(): bool From 6fa9f365ac2d9d2707a69ac7c09c191103ecac97 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 18:51:37 +1300 Subject: [PATCH 04/12] Remove redundant code --- tests/e2e/Adapter/Base.php | 7 +------ tests/e2e/Adapter/MariaDBTest.php | 11 ----------- tests/e2e/Adapter/MirrorTest.php | 5 ----- tests/e2e/Adapter/MySQLTest.php | 17 ++++++----------- tests/e2e/Adapter/PostgresTest.php | 10 ---------- tests/e2e/Adapter/SQLiteTest.php | 11 ----------- 6 files changed, 7 insertions(+), 54 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 7939ec975..d22751376 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -32,7 +32,7 @@ use Utopia\Database\Validator\Structure; use Utopia\Validator\Range; -ini_set('memory_limit', '2048M'); +\ini_set('memory_limit', '2048M'); abstract class Base extends TestCase { @@ -59,11 +59,6 @@ abstract protected static function deleteColumn(string $collection, string $colu */ abstract protected static function deleteIndex(string $collection, string $index): bool; - /** - * @return string - */ - abstract protected static function getAdapterName(): string; - public function setUp(): void { Authorization::setRole('any'); diff --git a/tests/e2e/Adapter/MariaDBTest.php b/tests/e2e/Adapter/MariaDBTest.php index 6d3bf835d..8a4893af3 100644 --- a/tests/e2e/Adapter/MariaDBTest.php +++ b/tests/e2e/Adapter/MariaDBTest.php @@ -15,17 +15,6 @@ class MariaDBTest extends Base protected static ?PDO $pdo = null; protected static string $namespace; - // Remove once all methods are implemented - /** - * Return name of adapter - * - * @return string - */ - public static function getAdapterName(): string - { - return "mariadb"; - } - /** * @return Database */ diff --git a/tests/e2e/Adapter/MirrorTest.php b/tests/e2e/Adapter/MirrorTest.php index 25af331de..71793b881 100644 --- a/tests/e2e/Adapter/MirrorTest.php +++ b/tests/e2e/Adapter/MirrorTest.php @@ -104,11 +104,6 @@ protected static function getDatabase(bool $fresh = false): Mirror return self::$database = $database; } - protected static function getAdapterName(): string - { - return "Mirror"; - } - /** * @throws Exception * @throws \RedisException diff --git a/tests/e2e/Adapter/MySQLTest.php b/tests/e2e/Adapter/MySQLTest.php index 36841202a..31cb6b828 100644 --- a/tests/e2e/Adapter/MySQLTest.php +++ b/tests/e2e/Adapter/MySQLTest.php @@ -7,6 +7,9 @@ use Utopia\Cache\Cache; use Utopia\Database\Adapter\MySQL; use Utopia\Database\Database; +use Utopia\Database\Exception; +use Utopia\Database\Exception\Duplicate; +use Utopia\Database\Exception\Limit; use Utopia\Database\PDO; class MySQLTest extends Base @@ -15,19 +18,11 @@ class MySQLTest extends Base protected static ?PDO $pdo = null; protected static string $namespace; - // Remove once all methods are implemented - /** - * Return name of adapter - * - * @return string - */ - public static function getAdapterName(): string - { - return "mysql"; - } - /** * @return Database + * @throws Duplicate + * @throws Exception + * @throws Limit */ public static function getDatabase(): Database { diff --git a/tests/e2e/Adapter/PostgresTest.php b/tests/e2e/Adapter/PostgresTest.php index a8e93e4a0..4e63eea81 100644 --- a/tests/e2e/Adapter/PostgresTest.php +++ b/tests/e2e/Adapter/PostgresTest.php @@ -15,16 +15,6 @@ class PostgresTest extends Base protected static ?PDO $pdo = null; protected static string $namespace; - /** - * Return name of adapter - * - * @return string - */ - public static function getAdapterName(): string - { - return "postgres"; - } - /** * @reture Adapter */ diff --git a/tests/e2e/Adapter/SQLiteTest.php b/tests/e2e/Adapter/SQLiteTest.php index 651e3eafd..b40c1259f 100644 --- a/tests/e2e/Adapter/SQLiteTest.php +++ b/tests/e2e/Adapter/SQLiteTest.php @@ -15,17 +15,6 @@ class SQLiteTest extends Base protected static ?PDO $pdo = null; protected static string $namespace; - // Remove once all methods are implemented - /** - * Return name of adapter - * - * @return string - */ - public static function getAdapterName(): string - { - return "sqlite"; - } - /** * @return Database */ From da0e1bcf284c646be51e2eec2093f164379b8877 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 19:09:47 +1300 Subject: [PATCH 05/12] Fix stan --- src/Database/Adapter.php | 2 +- src/Database/Adapter/Pool.php | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 72105e3d6..ffdd172d1 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -246,7 +246,7 @@ public function resetMetadata(): static * * @param int $milliseconds The timeout value in milliseconds for database queries. * @param string $event The event the timeout should fire for - * @return $this + * @return void * * @throws Exception The provided timeout value must be greater than or equal to 0. */ diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 10c4fc914..52197643d 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -32,10 +32,9 @@ public function __construct(UtopiaPool $pool) * * Required because __call() can't be used to implement abstract methods. * - * @template T * @param string $method - * @param array $args - * @return T + * @param array $args + * @return mixed */ public function delegate(string $method, array $args): mixed { From 417701ba75c5f676f3885feefe2a5cb210e33d4f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 19:10:25 +1300 Subject: [PATCH 06/12] Fix debug --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 22ecf2532..76bbdfc82 100755 --- a/Dockerfile +++ b/Dockerfile @@ -12,12 +12,11 @@ RUN composer install \ --no-scripts \ --prefer-dist -FROM php:8.3.10-cli-alpine3.20 AS compile +FROM php:8.3.19-cli-alpine3.21 AS compile ENV PHP_REDIS_VERSION="6.0.2" \ PHP_SWOOLE_VERSION="v5.1.3" \ - PHP_MONGO_VERSION="1.16.1" \ - PHP_XDEBUG_VERSION="3.3.2" + PHP_XDEBUG_VERSION="3.4.2" RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone From a619c1de8eb0c21424181f0f4dc702c226c9ba73 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 21:22:12 +1300 Subject: [PATCH 07/12] Fix setter delegation --- src/Database/Adapter.php | 7 +++++++ src/Database/Adapter/MariaDB.php | 2 ++ src/Database/Adapter/MySQL.php | 3 +++ src/Database/Adapter/Pool.php | 16 ++++++++++++++++ src/Database/Adapter/Postgres.php | 3 +++ 5 files changed, 31 insertions(+) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index ffdd172d1..5aabf6775 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -18,6 +18,8 @@ abstract class Adapter protected ?int $tenant = null; + protected int $timeout = 0; + protected int $inTransaction = 0; /** @@ -252,6 +254,11 @@ public function resetMetadata(): static */ abstract public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void; + public function getTimeout(): int + { + return $this->timeout; + } + /** * Clears a global timeout for database queries. * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index ecfe4e3d5..f4e2b4e08 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -2655,6 +2655,8 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL throw new DatabaseException('Timeout must be greater than 0'); } + $this->timeout = $milliseconds; + $seconds = $milliseconds / 1000; $this->before($event, 'timeout', function ($sql) use ($seconds) { diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 60cd66ab9..b803dd74b 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -25,6 +25,9 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL if ($milliseconds <= 0) { throw new DatabaseException('Timeout must be greater than 0'); } + + $this->timeout = $milliseconds; + $this->before($event, 'timeout', function ($sql) use ($milliseconds) { return \preg_replace( pattern: '/SELECT/', diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 52197643d..4eb422991 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -35,10 +35,26 @@ public function __construct(UtopiaPool $pool) * @param string $method * @param array $args * @return mixed + * @throws DatabaseException */ public function delegate(string $method, array $args): mixed { return $this->pool->use(function (Adapter $adapter) use ($method, $args) { + $adapter->setDatabase($this->getDatabase()); + $adapter->setNamespace($this->getNamespace()); + $adapter->setSharedTables($this->getSharedTables()); + $adapter->setTenant($this->getTenant()); + + if ($this->getTimeout() > 0) { + $adapter->setTimeout($this->getTimeout()); + } + foreach ($this->getDebug() as $key => $value) { + $adapter->setDebug($key, $value); + } + foreach($this->getMetadata() as $key => $value) { + $adapter->setMetadata($key, $value); + } + return $adapter->{$method}(...$args); }); } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 2ef065d7b..9f37519ca 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -95,6 +95,9 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL if ($milliseconds <= 0) { throw new DatabaseException('Timeout must be greater than 0'); } + + $this->timeout = $milliseconds; + $this->before($event, 'timeout', function ($sql) use ($milliseconds) { return " SET statement_timeout = {$milliseconds}; From 58295ed93e63d5beaed9b7b25fb6343d09890f97 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 21:22:38 +1300 Subject: [PATCH 08/12] Only expose withTransaction to avoid transactions on different connections --- src/Database/Database.php | 42 +-------------------------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 068a68b44..1d110f09b 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1010,48 +1010,8 @@ public function getAdapter(): Adapter } /** - * Start a new transaction. + * Run a callback inside a transaction. * - * If a transaction is already active, this will only increment the transaction count and return true. - * - * @return bool - * @throws DatabaseException - */ - public function startTransaction(): bool - { - return $this->adapter->startTransaction(); - } - - /** - * Commit a transaction. - * - * If no transaction is active, this will be a no-op and will return false. - * If there is more than one active transaction, this decrement the transaction count and return true. - * If the transaction count is 1, it will be commited, the transaction count will be reset to 0, and return true. - * - * @return bool - * @throws DatabaseException - */ - public function commitTransaction(): bool - { - return $this->adapter->startTransaction(); - } - - /** - * Rollback a transaction. - * - * If no transaction is active, this will be a no-op and will return false. - * If 1 or more transactions are active, this will roll back all transactions, reset the count to 0, and return true. - * - * @return bool - * @throws DatabaseException - */ - public function rollbackTransaction(): bool - { - return $this->adapter->rollbackTransaction(); - } - - /** * @template T * @param callable(): T $callback * @return T From 09f02e21954e3a98a6b0240b5c062975dfe76c58 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 22:14:17 +1300 Subject: [PATCH 09/12] Add pool test --- .github/workflows/tests.yml | 1 + tests/e2e/Adapter/PoolTest.php | 106 +++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 tests/e2e/Adapter/PoolTest.php diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d6a4c8033..eea0e7842 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,6 +77,7 @@ jobs: Postgres, SQLite, Mirror, + Pool, SharedTables/MariaDB, SharedTables/MySQL, SharedTables/Postgres, diff --git a/tests/e2e/Adapter/PoolTest.php b/tests/e2e/Adapter/PoolTest.php new file mode 100644 index 000000000..0211e6d8a --- /dev/null +++ b/tests/e2e/Adapter/PoolTest.php @@ -0,0 +1,106 @@ +connect('redis', 6379); + $redis->flushAll(); + $cache = new Cache(new RedisAdapter($redis)); + + $pool = new UtopiaPool('mysql', 10, function () { + $dbHost = 'mysql'; + $dbPort = '3307'; + $dbUser = 'root'; + $dbPass = 'password'; + + return new MySQL(new PDO( + dsn: "mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", + username: $dbUser, + password: $dbPass, + config: MySQL::getPDOAttributes(), + )); + }); + + $database = new Database(new Pool($pool), $cache); + + $database + ->setDatabase('utopiaTests') + ->setNamespace(static::$namespace = 'myapp_' . uniqid()); + + if ($database->exists()) { + $database->delete(); + } + + $database->create(); + + self::$pool = $pool; + + return self::$database = $database; + } + + protected static function deleteColumn(string $collection, string $column): bool + { + $sqlTable = "`" . self::getDatabase()->getDatabase() . "`.`" . self::getDatabase()->getNamespace() . "_" . $collection . "`"; + $sql = "ALTER TABLE {$sqlTable} DROP COLUMN `{$column}`"; + + self::$pool->use(function (Adapter $adapter) use ($sql) { + // Hack to get adapter PDO + $class = new ReflectionClass($adapter); + $property = $class->getProperty('pdo'); + $property->setAccessible(true); + $pdo = $property->getValue($adapter); + $pdo->exec($sql); + }); + + return true; + } + + protected static function deleteIndex(string $collection, string $index): bool + { + $sqlTable = "`" . self::getDatabase()->getDatabase() . "`.`" . self::getDatabase()->getNamespace() . "_" . $collection . "`"; + $sql = "DROP INDEX `{$index}` ON {$sqlTable}"; + + self::$pool->use(function (Adapter $adapter) use ($sql) { + // Hack to get adapter PDO + $class = new ReflectionClass($adapter); + $property = $class->getProperty('pdo'); + $property->setAccessible(true); + $pdo = $property->getValue($adapter); + $pdo->exec($sql); + }); + + return true; + } +} From 4d48d1350fa9fb3c499fc60f04edfd41b0c72423 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 23:34:38 +1300 Subject: [PATCH 10/12] Fix triggers --- Dockerfile | 12 ------------ README.md | 3 +-- docker-compose.yml | 11 ----------- src/Database/Adapter/Pool.php | 12 ++++++++++++ 4 files changed, 13 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index 76bbdfc82..429d4499d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -57,16 +57,6 @@ RUN \ && ./configure --enable-http2 \ && make && make install -## MongoDB Extension -FROM compile AS mongodb -RUN \ - git clone --depth 1 --branch $PHP_MONGO_VERSION https://github.com/mongodb/mongo-php-driver.git \ - && cd mongo-php-driver \ - && git submodule update --init \ - && phpize \ - && ./configure \ - && make && make install - ## PCOV Extension FROM compile AS pcov RUN \ @@ -96,7 +86,6 @@ WORKDIR /usr/src/code RUN echo extension=redis.so >> /usr/local/etc/php/conf.d/redis.ini RUN echo extension=swoole.so >> /usr/local/etc/php/conf.d/swoole.ini -RUN echo extension=mongodb.so >> /usr/local/etc/php/conf.d/mongodb.ini RUN echo extension=pcov.so >> /usr/local/etc/php/conf.d/pcov.ini RUN echo extension=xdebug.so >> /usr/local/etc/php/conf.d/xdebug.ini @@ -109,7 +98,6 @@ RUN echo "memory_limit=1024M" >> $PHP_INI_DIR/php.ini COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor COPY --from=swoole /usr/local/lib/php/extensions/no-debug-non-zts-20230831/swoole.so /usr/local/lib/php/extensions/no-debug-non-zts-20230831/ COPY --from=redis /usr/local/lib/php/extensions/no-debug-non-zts-20230831/redis.so /usr/local/lib/php/extensions/no-debug-non-zts-20230831/ -COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20230831/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20230831/ COPY --from=pcov /usr/local/lib/php/extensions/no-debug-non-zts-20230831/pcov.so /usr/local/lib/php/extensions/no-debug-non-zts-20230831/ COPY --from=xdebug /usr/local/lib/php/extensions/no-debug-non-zts-20230831/xdebug.so /usr/local/lib/php/extensions/no-debug-non-zts-20230831/ diff --git a/README.md b/README.md index db7ea5866..835bee0ee 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,10 @@ The database document interface only supports primitives types (`strings`, `inte Below is a list of supported databases, and their compatibly tested versions alongside a list of supported features and relevant limits. | Adapter | Status | Version | -| -------- | ------ | ------- | +|----------|--------|---------| | MariaDB | ✅ | 10.5 | | MySQL | ✅ | 8.0 | | Postgres | ✅ | 13.0 | -| MongoDB | ✅ | 5.0 | | SQLite | ✅ | 3.38 | ` ✅ - supported ` diff --git a/docker-compose.yml b/docker-compose.yml index c49290aaf..a560cc9cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,17 +70,6 @@ services: - "8704:3306" environment: - MYSQL_ROOT_PASSWORD=password - - mongo: - image: mongo:5.0 - container_name: utopia-mongo - networks: - - database - ports: - - "8705:27017" - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: example mysql: image: mysql:8.0.33 diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 4eb422991..a320a0c00 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -59,6 +59,18 @@ public function delegate(string $method, array $args): mixed }); } + public function before(string $event, string $name = '', ?callable $callback = null): static + { + $this->delegate(__FUNCTION__, \func_get_args()); + + return $this; + } + + protected function trigger(string $event, mixed $query): mixed + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void { $this->delegate(__FUNCTION__, \func_get_args()); From d0e97266dcf830b7e010267fd64d193d35c71d1e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 23:50:46 +1300 Subject: [PATCH 11/12] Format --- src/Database/Adapter/Pool.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index a320a0c00..e3b5bcb06 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -51,7 +51,7 @@ public function delegate(string $method, array $args): mixed foreach ($this->getDebug() as $key => $value) { $adapter->setDebug($key, $value); } - foreach($this->getMetadata() as $key => $value) { + foreach ($this->getMetadata() as $key => $value) { $adapter->setMetadata($key, $value); } From b84273e06ee43b5a6f10198a3a0973de66221ca5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 27 Mar 2025 20:09:35 +1300 Subject: [PATCH 12/12] Fix merge --- composer.lock | 12 ++++++------ src/Database/Adapter/Pool.php | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/composer.lock b/composer.lock index b92c6fea0..e85d8e38c 100644 --- a/composer.lock +++ b/composer.lock @@ -149,16 +149,16 @@ }, { "name": "google/protobuf", - "version": "v4.30.1", + "version": "v4.30.2", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24" + "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24", - "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/a4c4d8565b40b9f76debc9dfeb221412eacb8ced", + "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced", "shasum": "" }, "require": { @@ -187,9 +187,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.2" }, - "time": "2025-03-13T21:08:17+00:00" + "time": "2025-03-26T18:01:50+00:00" }, { "name": "nyholm/psr7", diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index e3b5bcb06..b043b7e77 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -221,7 +221,7 @@ public function deleteDocument(string $collection, string $id): bool return $this->delegate(__FUNCTION__, \func_get_args()); } - public function deleteDocuments(string $collection, array $ids): int + public function deleteDocuments(string $collection, array $internalIds, array $permissionIds): int { return $this->delegate(__FUNCTION__, \func_get_args()); }