From 8e14b9b337367e7ccbe90574801b5fbd518df800 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 6 Feb 2025 13:53:23 +0200 Subject: [PATCH 01/31] internal ids chunks --- src/Database/Adapter/MariaDB.php | 39 ++++++++++++++++++++------------ src/Database/Database.php | 10 ++++++++ 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 1615274d2..63823c7c5 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -981,7 +981,7 @@ public function createDocuments(string $collection, array $documents, int $batch try { $name = $this->filter($collection); $batches = \array_chunk($documents, \max(1, $batchSize)); - $documentIds = \array_map(fn ($document) => $document->getId(), $documents); + $documentIds = []; foreach ($batches as $batch) { $bindIndex = 0; @@ -990,6 +990,9 @@ public function createDocuments(string $collection, array $documents, int $batch $permissions = []; foreach ($batch as $document) { + /** + * @var Document $document + */ $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); $attributes['_createdAt'] = $document->getCreatedAt(); @@ -997,8 +1000,9 @@ public function createDocuments(string $collection, array $documents, int $batch $attributes['_permissions'] = \json_encode($document->getPermissions()); if (!empty($document->getInternalId())) { - $internalIds[$document->getId()] = true; $attributes['_id'] = $document->getInternalId(); + } else { + $documentIds[] = $document->getId(); } if ($this->sharedTables) { @@ -1077,27 +1081,32 @@ public function createDocuments(string $collection, array $documents, int $batch } } - // Get internal IDs - $sql = " + $internalIds = []; + + foreach (\array_chunk($documentIds, 3) as $documentIdsChunk) { + // Get internal IDs + $sql = " SELECT _uid, _id FROM {$this->getSQLTable($collection)} - WHERE _uid IN (" . implode(',', array_map(fn ($index) => ":_key_{$index}", array_keys($documentIds))) . ") + WHERE _uid IN (" . implode(',', array_map(fn ($index) => ":_key_{$index}", array_keys($documentIdsChunk))) . ") {$this->getTenantQuery($collection)} "; + $stmt = $this->getPDO()->prepare($sql); - $stmt = $this->getPDO()->prepare($sql); + foreach ($documentIdsChunk as $index => $id) { + $stmt->bindValue(":_key_{$index}", $id); + } - foreach ($documentIds as $index => $id) { - $stmt->bindValue(":_key_{$index}", $id); - } + if ($this->sharedTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } - if ($this->sharedTables) { - $stmt->bindValue(':_tenant', $this->tenant); - } + $stmt->execute(); + $results = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] + $stmt->closeCursor(); - $stmt->execute(); - $internalIds = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] - $stmt->closeCursor(); + $internalIds = array_merge($internalIds, $results); + } foreach ($documents as $document) { if (isset($internalIds[$document->getId()])) { diff --git a/src/Database/Database.php b/src/Database/Database.php index 9ef753f13..239e8c6ed 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3418,6 +3418,16 @@ public function createDocuments( $collection = $this->silent(fn () => $this->getCollection($collection)); + /** + * Check collection exist + */ + if ($collection->getId() !== self::METADATA) { + $authorization = new Authorization(self::PERMISSION_CREATE); + if (!$authorization->isValid($collection->getCreate())) { + throw new AuthorizationException($authorization->getDescription()); + } + } + $time = DateTime::now(); foreach ($documents as $key => $document) { From d8110bfe5c70d9719d9a5cbb8d9dac7a0d415612 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 6 Feb 2025 14:08:16 +0200 Subject: [PATCH 02/31] lock --- composer.lock | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/composer.lock b/composer.lock index 1bfbc9925..54c85f47c 100644 --- a/composer.lock +++ b/composer.lock @@ -465,16 +465,16 @@ }, { "name": "open-telemetry/api", - "version": "1.2.1", + "version": "1.2.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0" + "reference": "8b925df3047628968bc5be722468db1b98b82d51" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/74b1a03263be8c5acb578f41da054b4bac3af4a0", - "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/8b925df3047628968bc5be722468db1b98b82d51", + "reference": "8b925df3047628968bc5be722468db1b98b82d51", "shasum": "" }, "require": { @@ -531,7 +531,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-20T23:35:16+00:00" + "time": "2025-02-03T21:49:11+00:00" }, { "name": "open-telemetry/context", @@ -721,16 +721,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.1", + "version": "1.2.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1" + "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1", - "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/37eec0fe47ddd627911f318f29b6cd48196be0c0", + "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0", "shasum": "" }, "require": { @@ -807,24 +807,24 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-09T23:17:14+00:00" + "time": "2025-01-29T21:40:28+00:00" }, { "name": "open-telemetry/sem-conv", - "version": "1.27.1", + "version": "1.30.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sem-conv.git", - "reference": "1dba705fea74bc0718d04be26090e3697e56f4e6" + "reference": "4178c9f390da8e4dbca9b181a9d1efd50cf7ee0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/1dba705fea74bc0718d04be26090e3697e56f4e6", - "reference": "1dba705fea74bc0718d04be26090e3697e56f4e6", + "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/4178c9f390da8e4dbca9b181a9d1efd50cf7ee0a", + "reference": "4178c9f390da8e4dbca9b181a9d1efd50cf7ee0a", "shasum": "" }, "require": { - "php": "^8.1" + "php": "^8.0" }, "type": "library", "extra": { @@ -864,7 +864,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-08-28T09:20:31+00:00" + "time": "2025-02-06T00:21:48+00:00" }, { "name": "php-http/discovery", From c989b3b417e2cddbb4e38f228aa463ba67782391 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 6 Feb 2025 15:10:05 +0200 Subject: [PATCH 03/31] INSERT_BATCH_SIZE limit --- src/Database/Database.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 239e8c6ed..b0702d7f3 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -141,8 +141,8 @@ class Database public const EVENT_INDEX_CREATE = 'index_create'; public const EVENT_INDEX_DELETE = 'index_delete'; - public const INSERT_BATCH_SIZE = 10_000; - public const DELETE_BATCH_SIZE = 10_000; + public const INSERT_BATCH_SIZE = 3_000; + public const DELETE_BATCH_SIZE = 3_000; /** * List of Internal attributes From 0a12deebaad8c2ab6452c5585f7b3f588e03865e Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 6 Feb 2025 15:12:58 +0200 Subject: [PATCH 04/31] Chunk document ids --- src/Database/Adapter/MariaDB.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 63823c7c5..4160828db 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1083,7 +1083,10 @@ public function createDocuments(string $collection, array $documents, int $batch $internalIds = []; - foreach (\array_chunk($documentIds, 3) as $documentIdsChunk) { + /** + * UID, _tenant bottleneck is ~ 5000 rows since we use _uid IN query + */ + foreach (\array_chunk($documentIds, 3000) as $documentIdsChunk) { // Get internal IDs $sql = " SELECT _uid, _id From a63f8eb3655a7b878b2385f60a5d7462169fe704 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 9 Feb 2025 15:14:28 +0200 Subject: [PATCH 05/31] translate document ids --- src/Database/Adapter/MariaDB.php | 74 +++++++++++++++++++------------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 4160828db..8d690beec 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1081,35 +1081,7 @@ public function createDocuments(string $collection, array $documents, int $batch } } - $internalIds = []; - - /** - * UID, _tenant bottleneck is ~ 5000 rows since we use _uid IN query - */ - foreach (\array_chunk($documentIds, 3000) as $documentIdsChunk) { - // Get internal IDs - $sql = " - SELECT _uid, _id - FROM {$this->getSQLTable($collection)} - WHERE _uid IN (" . implode(',', array_map(fn ($index) => ":_key_{$index}", array_keys($documentIdsChunk))) . ") - {$this->getTenantQuery($collection)} - "; - $stmt = $this->getPDO()->prepare($sql); - - foreach ($documentIdsChunk as $index => $id) { - $stmt->bindValue(":_key_{$index}", $id); - } - - if ($this->sharedTables) { - $stmt->bindValue(':_tenant', $this->tenant); - } - - $stmt->execute(); - $results = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] - $stmt->closeCursor(); - - $internalIds = array_merge($internalIds, $results); - } + $internalIds = $this->translateUidsToInternalIds($collection, $documentIds); foreach ($documents as $document) { if (isset($internalIds[$document->getId()])) { @@ -1123,6 +1095,48 @@ public function createDocuments(string $collection, array $documents, int $batch return $documents; } + /** + * Translate Uid's to internal id's + * + * @param string $collection + * @param array $documentIds + * @return array + * @throws DatabaseException + */ + private function translateUidsToInternalIds(string $collection, array $documentIds): array + { + $internalIds = []; + + /** + * UID, _tenant bottleneck is ~ 5000 rows since we use _uid IN query + */ + foreach (\array_chunk($documentIds, 1000) as $documentIdsChunk) { + // Get internal IDs + $sql = " + SELECT _uid, _id + FROM {$this->getSQLTable($collection)} + WHERE _uid IN (" . implode(',', array_map(fn ($index) => ":_key_{$index}", array_keys($documentIdsChunk))) . ") + {$this->getTenantQuery($collection)} + "; + $stmt = $this->getPDO()->prepare($sql); + + foreach ($documentIdsChunk as $index => $id) { + $stmt->bindValue(":_key_{$index}", $id); + } + + if ($this->sharedTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + + $stmt->execute(); + $results = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] + $stmt->closeCursor(); + + $internalIds = array_merge($internalIds, $results); + } + + return $internalIds; + } /** * Update Document * @@ -1625,11 +1639,13 @@ public function createOrUpdateDocuments( $name = $this->filter($collection); $attribute = $this->filter($attribute); $batches = \array_chunk($documents, \max(1, $batchSize)); + $documentIds = []; foreach ($batches as $batch) { $bindIndex = 0; $batchKeys = []; $bindValues = []; + $attributes = []; $documentIds = array_map(fn ($doc) => $doc->getId(), $batch); From 6cb0e4b0fb23a71e0e29faf3fb7d066ca7939d57 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 9 Feb 2025 15:43:56 +0200 Subject: [PATCH 06/31] INSERT_BATCH_SIZE --- src/Database/Adapter/MariaDB.php | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 8d690beec..8812fd80a 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -978,9 +978,12 @@ public function createDocuments(string $collection, array $documents, int $batch return $documents; } + $batchSize = \max(1, $batchSize); + $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); + try { $name = $this->filter($collection); - $batches = \array_chunk($documents, \max(1, $batchSize)); + $batches = \array_chunk($documents, $batchSize); $documentIds = []; foreach ($batches as $batch) { @@ -1635,10 +1638,13 @@ public function createOrUpdateDocuments( return $documents; } + $batchSize = \max(1, $batchSize); + $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); + try { $name = $this->filter($collection); $attribute = $this->filter($attribute); - $batches = \array_chunk($documents, \max(1, $batchSize)); + $batches = \array_chunk($documents, $batchSize); $documentIds = []; foreach ($batches as $batch) { @@ -1831,27 +1837,7 @@ public function createOrUpdateDocuments( } } - // Get internal IDs - $sql = " - SELECT _uid, _id - FROM {$this->getSQLTable($collection)} - WHERE _uid IN (" . \implode(',', \array_map(fn ($index) => ":_key_{$index}", \array_keys($documentIds))) . ") - {$this->getTenantQuery($collection)} - "; - - $stmt = $this->getPDO()->prepare($sql); - - foreach ($documentIds as $index => $id) { - $stmt->bindValue(":_key_{$index}", $id); - } - - if ($this->sharedTables) { - $stmt->bindValue(':_tenant', $this->tenant); - } - - $stmt->execute(); - $internalIds = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] - $stmt->closeCursor(); + $internalIds = $this->translateUidsToInternalIds($collection, $documentIds); foreach ($documents as $document) { if (isset($internalIds[$document->getId()])) { From ee79c00d0d671caddd0b2c5be98ca3578964fa9c Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 9 Feb 2025 18:53:50 +0200 Subject: [PATCH 07/31] return type --- src/Database/Adapter/MariaDB.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 8812fd80a..27212dc18 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1103,7 +1103,7 @@ public function createDocuments(string $collection, array $documents, int $batch * * @param string $collection * @param array $documentIds - * @return array + * @return array * @throws DatabaseException */ private function translateUidsToInternalIds(string $collection, array $documentIds): array From 6fabc266256c3b5a03e3e47063e2ec6cd0986809 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 11:55:51 +0200 Subject: [PATCH 08/31] Create documents chunking from outside --- src/Database/Adapter.php | 3 +- src/Database/Adapter/MariaDB.php | 157 ++++++++++++++---------------- src/Database/Adapter/Mongo.php | 3 +- src/Database/Adapter/Postgres.php | 120 +++++++++++------------ src/Database/Database.php | 29 ++++-- 5 files changed, 155 insertions(+), 157 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 12b74513b..b206ee286 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -604,13 +604,12 @@ abstract public function createDocument(string $collection, Document $document): * * @param string $collection * @param array $documents - * @param int $batchSize * * @return array * * @throws DatabaseException */ - abstract public function createDocuments(string $collection, array $documents, int $batchSize): array; + abstract public function createDocuments(string $collection, array $documents): array; /** * Update Document diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 27212dc18..63c6ff8e2 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -965,123 +965,116 @@ public function createDocument(string $collection, Document $document): Document * * @param string $collection * @param array $documents - * @param int $batchSize * * @return array * * @throws DuplicateException * @throws \Throwable */ - public function createDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array + public function createDocuments(string $collection, array $documents): array { if (empty($documents)) { return $documents; } - $batchSize = \max(1, $batchSize); - $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); - try { $name = $this->filter($collection); - $batches = \array_chunk($documents, $batchSize); - $documentIds = []; - - foreach ($batches as $batch) { - $bindIndex = 0; - $batchKeys = []; - $bindValues = []; - $permissions = []; - foreach ($batch as $document) { - /** - * @var Document $document - */ - $attributes = $document->getAttributes(); - $attributes['_uid'] = $document->getId(); - $attributes['_createdAt'] = $document->getCreatedAt(); - $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = \json_encode($document->getPermissions()); + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + $permissions = []; + $documentIds = []; - if (!empty($document->getInternalId())) { - $attributes['_id'] = $document->getInternalId(); - } else { - $documentIds[] = $document->getId(); - } + foreach ($documents as $document) { + /** + * @var Document $document + */ + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = \json_encode($document->getPermissions()); + + if (! empty($document->getInternalId())) { + $attributes['_id'] = $document->getInternalId(); + } else { + $documentIds[] = $document->getId(); + } - if ($this->sharedTables) { - $attributes['_tenant'] = $this->tenant; - } + if ($this->sharedTables) { + $attributes['_tenant'] = $this->tenant; + } - $columns = []; - foreach (\array_keys($attributes) as $key => $attribute) { - $columns[$key] = "`{$this->filter($attribute)}`"; - } + $columns = []; + foreach (\array_keys($attributes) as $key => $attribute) { + $columns[$key] = "`{$this->filter($attribute)}`"; + } - $columns = '(' . \implode(', ', $columns) . ')'; + $columns = '(' . \implode(', ', $columns) . ')'; - $bindKeys = []; + $bindKeys = []; - foreach ($attributes as $value) { - if (\is_array($value)) { - $value = \json_encode($value); - } - $value = (\is_bool($value)) ? (int)$value : $value; - $bindKey = 'key_' . $bindIndex; - $bindKeys[] = ':' . $bindKey; - $bindValues[$bindKey] = $value; - $bindIndex++; + foreach ($attributes as $value) { + if (\is_array($value)) { + $value = \json_encode($value); } + $value = (\is_bool($value)) ? (int)$value : $value; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $value; + $bindIndex++; + } - $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; - foreach (Database::PERMISSIONS as $type) { - foreach ($document->getPermissionsByType($type) as $permission) { - $permission = \str_replace('"', '', $permission); - $permission = "('{$type}', '{$permission}', '{$document->getId()}'"; - - if ($this->sharedTables) { - $permission .= ", :_tenant)"; - } else { - $permission .= ")"; - } + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; + foreach (Database::PERMISSIONS as $type) { + foreach ($document->getPermissionsByType($type) as $permission) { + $permission = \str_replace('"', '', $permission); + $permission = "('{$type}', '{$permission}', '{$document->getId()}'"; - $permissions[] = $permission; + if ($this->sharedTables) { + $permission .= ", :_tenant)"; + } else { + $permission .= ")"; } + + $permissions[] = $permission; } } + } - $stmt = $this->getPDO()->prepare( - " - INSERT INTO {$this->getSQLTable($name)} {$columns} - VALUES " . \implode(', ', $batchKeys) - ); - - foreach ($bindValues as $key => $value) { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } + $stmt = $this->getPDO()->prepare( + " + INSERT INTO {$this->getSQLTable($name)} {$columns} + VALUES " . \implode(', ', $batchKeys) + ); - $stmt->execute(); + foreach ($bindValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } - if (!empty($permissions)) { - $sqlPermissions = " - INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document - "; + $stmt->execute(); - if ($this->sharedTables) { - $sqlPermissions .= ', _tenant)'; - } else { - $sqlPermissions .= ")"; - } + if (!empty($permissions)) { + $sqlPermissions = " + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document + "; - $sqlPermissions .= " VALUES " . \implode(', ', $permissions); + if ($this->sharedTables) { + $sqlPermissions .= ', _tenant)'; + } else { + $sqlPermissions .= ")"; + } - $stmtPermissions = $this->getPDO()->prepare($sqlPermissions); + $sqlPermissions .= " VALUES " . \implode(', ', $permissions); - if ($this->sharedTables) { - $stmtPermissions->bindValue(':_tenant', $this->tenant); - } + $stmtPermissions = $this->getPDO()->prepare($sqlPermissions); - $stmtPermissions?->execute(); + if ($this->sharedTables) { + $stmtPermissions->bindValue(':_tenant', $this->tenant); } + + $stmtPermissions?->execute(); } $internalIds = $this->translateUidsToInternalIds($collection, $documentIds); diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index b35add565..5daf424ce 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -747,13 +747,12 @@ public function createDocument(string $collection, Document $document): Document * * @param string $collection * @param array $documents - * @param int $batchSize * * @return array * * @throws Duplicate */ - public function createDocuments(string $collection, array $documents, int $batchSize): array + public function createDocuments(string $collection, array $documents): array { $name = $this->getNamespace() . '_' . $this->filter($collection); diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index c2b886776..e1000a0ba 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1025,13 +1025,12 @@ public function createDocument(string $collection, Document $document): Document * * @param string $collection * @param array $documents - * @param int $batchSize * * @return array * * @throws DuplicateException */ - public function createDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array + public function createDocuments(string $collection, array $documents): array { if (empty($documents)) { return $documents; @@ -1039,81 +1038,78 @@ public function createDocuments(string $collection, array $documents, int $batch try { $name = $this->filter($collection); - $batches = \array_chunk($documents, max(1, $batchSize)); $internalIds = []; - foreach ($batches as $batch) { - $bindIndex = 0; - $batchKeys = []; - $bindValues = []; - $permissions = []; - - foreach ($batch as $document) { - $attributes = $document->getAttributes(); - $attributes['_uid'] = $document->getId(); - $attributes['_createdAt'] = $document->getCreatedAt(); - $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = \json_encode($document->getPermissions()); - - if (!empty($document->getInternalId())) { - $internalIds[$document->getId()] = true; - $attributes['_id'] = $document->getInternalId(); - } + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + $permissions = []; + + foreach ($documents as $document) { + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = \json_encode($document->getPermissions()); + + if (!empty($document->getInternalId())) { + $internalIds[$document->getId()] = true; + $attributes['_id'] = $document->getInternalId(); + } - if ($this->sharedTables) { - $attributes['_tenant'] = $this->tenant; - } + if ($this->sharedTables) { + $attributes['_tenant'] = $this->tenant; + } - $columns = []; - foreach (\array_keys($attributes) as $key => $attribute) { - $columns[$key] = "\"{$this->filter($attribute)}\""; - } + $columns = []; + foreach (\array_keys($attributes) as $key => $attribute) { + $columns[$key] = "\"{$this->filter($attribute)}\""; + } - $columns = '(' . \implode(', ', $columns) . ')'; + $columns = '(' . \implode(', ', $columns) . ')'; - $bindKeys = []; + $bindKeys = []; - foreach ($attributes as $value) { - if (\is_array($value)) { - $value = \json_encode($value); - } - $value = (\is_bool($value)) ? (int)$value : $value; - $bindKey = 'key_' . $bindIndex; - $bindKeys[] = ':' . $bindKey; - $bindValues[$bindKey] = $value; - $bindIndex++; + foreach ($attributes as $value) { + if (\is_array($value)) { + $value = \json_encode($value); } + $value = (\is_bool($value)) ? (int)$value : $value; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $value; + $bindIndex++; + } - $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; - foreach (Database::PERMISSIONS as $type) { - foreach ($document->getPermissionsByType($type) as $permission) { - $permission = \str_replace('"', '', $permission); - $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', :_tenant)"; - } + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; + foreach (Database::PERMISSIONS as $type) { + foreach ($document->getPermissionsByType($type) as $permission) { + $permission = \str_replace('"', '', $permission); + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', :_tenant)"; } } + } - $stmt = $this->getPDO()->prepare( - " - INSERT INTO {$this->getSQLTable($name)} {$columns} - VALUES " . \implode(', ', $batchKeys) - ); + $stmt = $this->getPDO()->prepare( + " + INSERT INTO {$this->getSQLTable($name)} {$columns} + VALUES " . \implode(', ', $batchKeys) + ); - foreach ($bindValues as $key => $value) { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } + foreach ($bindValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } - $stmt->execute(); + $stmt->execute(); - if (!empty($permissions)) { - $stmtPermissions = $this->getPDO()->prepare( - " - INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document, _tenant) - VALUES " . \implode(', ', $permissions) - ); - $stmtPermissions->bindValue(':_tenant', $this->tenant); - $stmtPermissions?->execute(); - } + if (!empty($permissions)) { + $stmtPermissions = $this->getPDO()->prepare( + " + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document, _tenant) + VALUES " . \implode(', ', $permissions) + ); + $stmtPermissions->bindValue(':_tenant', $this->tenant); + $stmtPermissions?->execute(); } } catch (PDOException $e) { throw $this->processException($e); diff --git a/src/Database/Database.php b/src/Database/Database.php index b0702d7f3..6854bb40b 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -141,8 +141,8 @@ class Database public const EVENT_INDEX_CREATE = 'index_create'; public const EVENT_INDEX_DELETE = 'index_delete'; - public const INSERT_BATCH_SIZE = 3_000; - public const DELETE_BATCH_SIZE = 3_000; + public const INSERT_BATCH_SIZE = 1_000; + public const DELETE_BATCH_SIZE = 1_000; /** * List of Internal attributes @@ -3418,6 +3418,8 @@ public function createDocuments( $collection = $this->silent(fn () => $this->getCollection($collection)); + $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + /** * Check collection exist */ @@ -3459,11 +3461,13 @@ public function createDocuments( } $documents = $this->withTransaction(function () use ($collection, $documents, $batchSize) { - return $this->adapter->createDocuments( - $collection->getId(), - $documents, - $batchSize, - ); + $stack = []; + + foreach (\array_chunk($documents, $batchSize) as $chunk) { + $stack = array_merge($stack, $this->adapter->createDocuments($collection->getId(), $chunk)); + } + + return $stack; }); foreach ($documents as $key => $document) { @@ -4080,7 +4084,6 @@ public function updateDocuments(string $collection, Document $updates, array $qu } $documents = $this->withTransaction(function () use ($collection, $queries, $batchSize, $updates, $limit, $cursor) { - $lastDocument = null; $documents = []; $documentSecurity = $collection->getAttribute('documentSecurity', false); @@ -5330,6 +5333,9 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } + $batchSize = \max(1, $batchSize); + $batchSize = \min(Database::DELETE_BATCH_SIZE, $batchSize); + $collection = $this->silent(fn () => $this->getCollection($collection)); if ($collection->isEmpty()) { @@ -5435,7 +5441,12 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba 'modified' => count($documents) ])); - $this->adapter->deleteDocuments($collection->getId(), array_map(fn ($document) => $document->getId(), $documents)); + foreach (\array_chunk($documents, $batchSize) as $chunk) { + $this->adapter->deleteDocuments( + $collection->getId(), + array_map(fn ($document) => $document->getId(), $chunk) + ); + } return $documents; }); From f085ca0c0e82fa36e57248ab4ef08bc59291b3ff Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 12:24:46 +0200 Subject: [PATCH 09/31] updateDocuments batch limit --- src/Database/Database.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 67518681e..ddb224d01 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4032,6 +4032,8 @@ public function updateDocuments(string $collection, Document $updates, array $qu return []; } + $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $collection = $this->silent(fn () => $this->getCollection($collection)); if ($collection->isEmpty()) { @@ -5334,8 +5336,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = \max(1, $batchSize); - $batchSize = \min(Database::DELETE_BATCH_SIZE, $batchSize); + $batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); From a28fa1e6cd561112a597e3dd9cc9e72aa77249d7 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 12:56:37 +0200 Subject: [PATCH 10/31] Chunk createOrUpdateDocuments --- src/Database/Adapter.php | 4 +- src/Database/Adapter/MariaDB.php | 320 +++++++++++++++--------------- src/Database/Adapter/Mongo.php | 3 +- src/Database/Adapter/Postgres.php | 7 +- src/Database/Database.php | 15 +- 5 files changed, 173 insertions(+), 176 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index b206ee286..c91ad2ef0 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -644,14 +644,12 @@ abstract public function updateDocuments(string $collection, Document $updates, * @param string $collection * @param string $attribute * @param array $documents - * @param int $batchSize * @return array */ abstract public function createOrUpdateDocuments( string $collection, string $attribute, - array $documents, - int $batchSize + array $documents ): array; /** diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 63c6ff8e2..e28d12013 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1617,217 +1617,207 @@ public function updateDocuments(string $collection, Document $updates, array $do * @param string $collection * @param string $attribute * @param array $documents - * @param int $batchSize * @return array * @throws DatabaseException */ public function createOrUpdateDocuments( string $collection, string $attribute, - array $documents, - int $batchSize + array $documents ): array { if (empty($documents)) { return $documents; } - $batchSize = \max(1, $batchSize); - $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); - try { $name = $this->filter($collection); $attribute = $this->filter($attribute); - $batches = \array_chunk($documents, $batchSize); - $documentIds = []; - foreach ($batches as $batch) { - $bindIndex = 0; - $batchKeys = []; - $bindValues = []; - $attributes = []; - - $documentIds = array_map(fn ($doc) => $doc->getId(), $batch); - - foreach ($batch as $document) { - /** - * @var array $attributes - */ - $attributes = $document->getAttributes(); - $attributes['_uid'] = $document->getId(); - $attributes['_createdAt'] = $document->getCreatedAt(); - $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = \json_encode($document->getPermissions()); - - if (!empty($document->getInternalId())) { - $attributes['_id'] = $document->getInternalId(); - } + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + $attributes = []; + $documentIds = []; - if ($this->sharedTables) { - $attributes['_tenant'] = $this->tenant; - } + foreach ($documents as $document) { + /** + * @var array $attributes + */ + $attributes = $document->getAttributes(); + $documentIds[] = $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = \json_encode($document->getPermissions()); - $columns = []; - foreach (\array_keys($attributes) as $key => $attr) { - $columns[$key] = "`{$this->filter($attr)}`"; - } + if (!empty($document->getInternalId())) { + $attributes['_id'] = $document->getInternalId(); + } - $columns = '(' . \implode(', ', $columns) . ')'; + if ($this->sharedTables) { + $attributes['_tenant'] = $this->tenant; + } - $bindKeys = []; + $columns = []; + foreach (\array_keys($attributes) as $key => $attr) { + $columns[$key] = "`{$this->filter($attr)}`"; + } - foreach ($attributes as $attrValue) { - if (\is_array($attrValue)) { - $attrValue = \json_encode($attrValue); - } - $attrValue = (\is_bool($attrValue)) ? (int)$attrValue : $attrValue; - $bindKey = 'key_' . $bindIndex; - $bindKeys[] = ':' . $bindKey; - $bindValues[$bindKey] = $attrValue; - $bindIndex++; - } + $columns = '(' . \implode(', ', $columns) . ')'; - $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; - } + $bindKeys = []; - if (!empty($attribute)) { - // Increment specific column by its new value in place - $updateColumns = [ - "`{$attribute}` = `{$attribute}` + VALUES(`{$attribute}`)", - "`_updatedAt` = VALUES(`_updatedAt`)" - ]; - } else { - // Update all columns - $updateColumns = []; - foreach (\array_keys($attributes) as $attr) { - $updateColumns[] = "`{$this->filter($attr)}` = VALUES(`{$this->filter($attr)}`)"; + foreach ($attributes as $attrValue) { + if (\is_array($attrValue)) { + $attrValue = \json_encode($attrValue); } + $attrValue = (\is_bool($attrValue)) ? (int)$attrValue : $attrValue; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $attrValue; + $bindIndex++; } - $stmt = $this->getPDO()->prepare( - " - INSERT INTO {$this->getSQLTable($name)} {$columns} - VALUES " . \implode(', ', $batchKeys) . " - ON DUPLICATE KEY UPDATE - " . \implode(', ', $updateColumns) - ); + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; + } - foreach ($bindValues as $key => $binding) { - $stmt->bindValue($key, $binding, $this->getPDOType($binding)); + if (!empty($attribute)) { + // Increment specific column by its new value in place + $updateColumns = [ + "`{$attribute}` = `{$attribute}` + VALUES(`{$attribute}`)", + "`_updatedAt` = VALUES(`_updatedAt`)" + ]; + } else { + // Update all columns + $updateColumns = []; + foreach (\array_keys($attributes) as $attr) { + $updateColumns[] = "`{$this->filter($attr)}` = VALUES(`{$this->filter($attr)}`)"; } + } - $stmt->execute(); + $stmt = $this->getPDO()->prepare( + " + INSERT INTO {$this->getSQLTable($name)} {$columns} + VALUES " . \implode(', ', $batchKeys) . " + ON DUPLICATE KEY UPDATE + " . \implode(', ', $updateColumns) + ); - // Fetch existing permissions in bulk after data updates - $sql = " - SELECT _document, _type, _permission - FROM {$this->getSQLTable($name . '_perms')} - WHERE _document IN (" . \implode(',', \array_map(fn ($index) => ":_key_{$index}", \array_keys($documentIds))) . ") - {$this->getTenantQuery($collection)} - "; + foreach ($bindValues as $key => $binding) { + $stmt->bindValue($key, $binding, $this->getPDOType($binding)); + } - $stmt = $this->getPDO()->prepare($sql); + $stmt->execute(); - foreach ($documentIds as $index => $id) { - $stmt->bindValue(":_key_{$index}", $id); - } + // Fetch existing permissions in bulk after data updates + $sql = " + SELECT _document, _type, _permission + FROM {$this->getSQLTable($name . '_perms')} + WHERE _document IN (" . \implode(',', \array_map(fn ($index) => ":_key_{$index}", \array_keys($documentIds))) . ") + {$this->getTenantQuery($collection)} + "; - if ($this->sharedTables) { - $stmt->bindValue(':_tenant', $this->tenant); - } + $stmt = $this->getPDO()->prepare($sql); - $stmt->execute(); - $existing = $stmt->fetchAll(); - $stmt->closeCursor(); + foreach ($documentIds as $index => $id) { + $stmt->bindValue(":_key_{$index}", $id); + } - // Group permissions by document - $permissionsByDocument = []; - foreach ($existing as $row) { - $permissionsByDocument[$row['_document']][$row['_type']][] = $row['_permission']; - } + if ($this->sharedTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } - foreach ($documentIds as $id) { - foreach (Database::PERMISSIONS as $type) { - $permissionsByDocument[$id][$type] = $permissionsByDocument[$id][$type] ?? []; - } - } + $stmt->execute(); + $existing = $stmt->fetchAll(); + $stmt->closeCursor(); - $removeQueries = []; - $removeBindValues = []; - $addQueries = []; - $addBindValues = []; - - foreach ($batch as $index => $document) { - $currentPermissions = $permissionsByDocument[$document->getId()] ?? []; - - // Calculate removals - foreach (Database::PERMISSIONS as $type) { - $toRemove = \array_diff($currentPermissions[$type], $document->getPermissionsByType($type)); - if (!empty($toRemove)) { - $removeQueries[] = "( - _document = :uid_{$index} - {$this->getTenantQuery($collection)} - AND _type = '{$type}' - AND _permission IN (" . \implode(',', \array_map(fn ($i) => ":remove_{$type}_{$index}_{$i}", \array_keys($toRemove))) . ") - )"; - $removeBindValues[":uid_{$index}"] = $document->getId(); - foreach ($toRemove as $i => $perm) { - $removeBindValues[":remove_{$type}_{$index}_{$i}"] = $perm; - } - } - } + // Group permissions by document + $permissionsByDocument = []; + foreach ($existing as $row) { + $permissionsByDocument[$row['_document']][$row['_type']][] = $row['_permission']; + } - // Calculate additions - foreach (Database::PERMISSIONS as $type) { - $toAdd = \array_diff($document->getPermissionsByType($type), $currentPermissions[$type]); - foreach ($toAdd as $i => $permission) { - $addQuery = "(:uid_{$index}, '{$type}', :add_{$type}_{$index}_{$i}"; + foreach ($documentIds as $id) { + foreach (Database::PERMISSIONS as $type) { + $permissionsByDocument[$id][$type] = $permissionsByDocument[$id][$type] ?? []; + } + } - if ($this->sharedTables) { - $addQuery .= ", :_tenant)"; - } else { - $addQuery .= ")"; - } + $removeQueries = []; + $removeBindValues = []; + $addQueries = []; + $addBindValues = []; + + foreach ($documents as $index => $document) { + $currentPermissions = $permissionsByDocument[$document->getId()] ?? []; - $addQueries[] = $addQuery; - $addBindValues[":uid_{$index}"] = $document->getId(); - $addBindValues[":add_{$type}_{$index}_{$i}"] = $permission; + // Calculate removals + foreach (Database::PERMISSIONS as $type) { + $toRemove = \array_diff($currentPermissions[$type], $document->getPermissionsByType($type)); + if (!empty($toRemove)) { + $removeQueries[] = "( + _document = :uid_{$index} + {$this->getTenantQuery($collection)} + AND _type = '{$type}' + AND _permission IN (" . \implode(',', \array_map(fn ($i) => ":remove_{$type}_{$index}_{$i}", \array_keys($toRemove))) . ") + )"; + $removeBindValues[":uid_{$index}"] = $document->getId(); + foreach ($toRemove as $i => $perm) { + $removeBindValues[":remove_{$type}_{$index}_{$i}"] = $perm; } } } - // Execute permission removals - if (!empty($removeQueries)) { - $removeQuery = \implode(' OR ', $removeQueries); - $stmtRemovePermissions = $this->getPDO()->prepare("DELETE FROM {$this->getSQLTable($name . '_perms')} WHERE {$removeQuery}"); - foreach ($removeBindValues as $key => $value) { - $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); - } - if ($this->sharedTables) { - $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + // Calculate additions + foreach (Database::PERMISSIONS as $type) { + $toAdd = \array_diff($document->getPermissionsByType($type), $currentPermissions[$type]); + foreach ($toAdd as $i => $permission) { + $addQuery = "(:uid_{$index}, '{$type}', :add_{$type}_{$index}_{$i}"; + + if ($this->sharedTables) { + $addQuery .= ", :_tenant)"; + } else { + $addQuery .= ")"; + } + + $addQueries[] = $addQuery; + $addBindValues[":uid_{$index}"] = $document->getId(); + $addBindValues[":add_{$type}_{$index}_{$i}"] = $permission; } - $stmtRemovePermissions->execute(); } + } - // Execute permission additions - if (!empty($addQuery)) { - $sqlAddPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission"; - if ($this->sharedTables) { - $sqlAddPermissions .= ", _tenant)"; - } else { - $sqlAddPermissions .= ")"; - } - $addQuery = \implode(', ', $addQueries); - $sqlAddPermissions .= " VALUES {$addQuery}"; - $stmtAddPermissions = $this->getPDO()->prepare($sqlAddPermissions); - foreach ($addBindValues as $key => $value) { - $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); - } - if ($this->sharedTables) { - $stmtAddPermissions->bindValue(':_tenant', $this->tenant); - } + // Execute permission removals + if (!empty($removeQueries)) { + $removeQuery = \implode(' OR ', $removeQueries); + $stmtRemovePermissions = $this->getPDO()->prepare("DELETE FROM {$this->getSQLTable($name . '_perms')} WHERE {$removeQuery}"); + foreach ($removeBindValues as $key => $value) { + $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); + } + if ($this->sharedTables) { + $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + } + $stmtRemovePermissions->execute(); + } - $stmtAddPermissions->execute(); + // Execute permission additions + if (!empty($addQuery)) { + $sqlAddPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission"; + if ($this->sharedTables) { + $sqlAddPermissions .= ", _tenant)"; + } else { + $sqlAddPermissions .= ")"; + } + $addQuery = \implode(', ', $addQueries); + $sqlAddPermissions .= " VALUES {$addQuery}"; + $stmtAddPermissions = $this->getPDO()->prepare($sqlAddPermissions); + foreach ($addBindValues as $key => $value) { + $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); } + if ($this->sharedTables) { + $stmtAddPermissions->bindValue(':_tenant', $this->tenant); + } + + $stmtAddPermissions->execute(); } $internalIds = $this->translateUidsToInternalIds($collection, $documentIds); @@ -1962,6 +1952,10 @@ public function deleteDocument(string $collection, string $id): bool */ public function deleteDocuments(string $collection, array $ids): int { + if (empty($ids)) { + return 0; + } + try { $name = $this->filter($collection); $where = []; diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 5daf424ce..44df82c67 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -894,10 +894,9 @@ public function updateDocuments(string $collection, Document $updates, array $do * @param string $collection * @param string $attribute * @param array $documents - * @param int $batchSize * @return array */ - public function createOrUpdateDocuments(string $collection, string $attribute, array $documents, int $batchSize): array + public function createOrUpdateDocuments(string $collection, string $attribute, array $documents): array { return $documents; } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index e1000a0ba..b718f13d9 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1605,10 +1605,9 @@ public function updateDocuments(string $collection, Document $updates, array $do * @param string $collection * @param string $attribute * @param array $documents - * @param int $batchSize * @return array */ - public function createOrUpdateDocuments(string $collection, string $attribute, array $documents, int $batchSize): array + public function createOrUpdateDocuments(string $collection, string $attribute, array $documents): array { return $documents; } @@ -1731,6 +1730,10 @@ public function deleteDocument(string $collection, string $id): bool */ public function deleteDocuments(string $collection, array $ids): int { + if (empty($ids)){ + return 0; + } + try { $name = $this->filter($collection); $where = []; diff --git a/src/Database/Database.php b/src/Database/Database.php index ddb224d01..eb90a9556 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4613,6 +4613,8 @@ public function createOrUpdateDocumentsWithIncrease( return []; } + $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $collection = $this->silent(fn () => $this->getCollection($collection)); $time = DateTime::now(); @@ -4660,12 +4662,13 @@ public function createOrUpdateDocumentsWithIncrease( } $documents = $this->withTransaction(function () use ($collection, $attribute, $documents, $batchSize) { - return $this->adapter->createOrUpdateDocuments( - $collection->getId(), - $attribute, - $documents, - $batchSize, - ); + $stack = []; + + foreach (\array_chunk($documents, $batchSize) as $chunk) { + $stack = array_merge($stack, $this->adapter->createOrUpdateDocuments($collection->getId(), $attribute, $chunk)); + } + + return $stack; }); foreach ($documents as $key => $document) { From e47ea693ddb9d21cc1f956e6a0008245f3a3e5de Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 14:42:56 +0200 Subject: [PATCH 11/31] Formatting --- src/Database/Adapter/Postgres.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index b718f13d9..d64d42f53 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1730,7 +1730,7 @@ public function deleteDocument(string $collection, string $id): bool */ public function deleteDocuments(string $collection, array $ids): int { - if (empty($ids)){ + if (empty($ids)) { return 0; } From 3ed992d9af17be30e24139f570226c7b9c1533c7 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 16:27:52 +0200 Subject: [PATCH 12/31] column counts --- src/Database/Database.php | 7 +++++++ tests/e2e/Adapter/Base.php | 21 +++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index eb90a9556..ace4aea6b 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3432,6 +3432,7 @@ public function createDocuments( } $time = DateTime::now(); + $columnCount = null; foreach ($documents as $key => $document) { $createdAt = $document->getCreatedAt(); @@ -3443,6 +3444,12 @@ public function createDocuments( ->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt) ->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt); + if(is_null($columnCount)){ + $columnCount = count($document); + } if ($columnCount != count($document) ){ + throw new StructureException('Insert value list does not match column list'); + } + $document = $this->encode($collection, $document); $validator = new Structure( diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 4bef96a01..fbeba198b 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2352,8 +2352,8 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void $collection = 'testCreateOrUpdateInplace'; static::getDatabase()->createCollection($collection); - static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true); - static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true); + static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, false); + static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, false); $documents = [ new Document([ @@ -2396,6 +2396,23 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void foreach ($documents as $document) { $this->assertEquals(6, $document->getAttribute('integer')); } + + $documents = [ + new Document([ + '$id' => 'with_integer', + 'integer' => 1, + ]), + new Document([ + '$id' => 'without_integer', + ]), + ]; + + try { + static::getDatabase()->createDocuments($collection, $documents); + $this->fail('Failed to throw exception'); + } catch (Exception $e) { + $this->assertInstanceOf(StructureException::class, $e); + } } public function testCreateOrUpdateDocumentsPermissions(): void From d641a4b53ca73204f28821161375932ea65b8486 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 16:28:55 +0200 Subject: [PATCH 13/31] Fix elseif --- src/Database/Database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index ace4aea6b..e678cff4a 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3446,7 +3446,7 @@ public function createDocuments( if(is_null($columnCount)){ $columnCount = count($document); - } if ($columnCount != count($document) ){ + } elseif ($columnCount != count($document) ){ throw new StructureException('Insert value list does not match column list'); } From 314286fdafaf8158208de48b24e6977a47b06508 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 16:31:37 +0200 Subject: [PATCH 14/31] Check message --- tests/e2e/Adapter/Base.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index fbeba198b..cff56ace9 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2412,6 +2412,7 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void $this->fail('Failed to throw exception'); } catch (Exception $e) { $this->assertInstanceOf(StructureException::class, $e); + $this->assertEquals('Insert value list does not match column list', $e->getMessage()); } } From 578d18a267c41ae8396740e4a42cb0675ee7056a Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 16:34:14 +0200 Subject: [PATCH 15/31] lint --- src/Database/Database.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index e678cff4a..dd992d6dc 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3444,9 +3444,9 @@ public function createDocuments( ->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt) ->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt); - if(is_null($columnCount)){ + if (is_null($columnCount)) { $columnCount = count($document); - } elseif ($columnCount != count($document) ){ + } elseif ($columnCount != count($document)) { throw new StructureException('Insert value list does not match column list'); } From 271794ccf89c506e25f86103b6b83542c1a86223 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 17:19:47 +0200 Subject: [PATCH 16/31] Fix analyse --- src/Database/Database.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index dd992d6dc..56cb2af84 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3419,7 +3419,8 @@ public function createDocuments( $collection = $this->silent(fn () => $this->getCollection($collection)); - $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = \max(1, $batchSize); + $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); /** * Check collection exist @@ -4039,7 +4040,8 @@ public function updateDocuments(string $collection, Document $updates, array $qu return []; } - $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = \max(1, $batchSize); + $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -4620,7 +4622,8 @@ public function createOrUpdateDocumentsWithIncrease( return []; } - $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = \max(1, $batchSize); + $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -5346,7 +5349,8 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = \max(1, $batchSize); + $batchSize = \min(Database::DELETE_BATCH_SIZE, $batchSize); $collection = $this->silent(fn () => $this->getCollection($collection)); From 519cbb4ba246d284ed5101115420ac81dcfb752a Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 17:28:40 +0200 Subject: [PATCH 17/31] Fix Code QL --- src/Database/Database.php | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 56cb2af84..a31ac7a5a 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3419,8 +3419,7 @@ public function createDocuments( $collection = $this->silent(fn () => $this->getCollection($collection)); - $batchSize = \max(1, $batchSize); - $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); + $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); /** * Check collection exist @@ -4040,8 +4039,7 @@ public function updateDocuments(string $collection, Document $updates, array $qu return []; } - $batchSize = \max(1, $batchSize); - $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); + $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -4622,8 +4620,7 @@ public function createOrUpdateDocumentsWithIncrease( return []; } - $batchSize = \max(1, $batchSize); - $batchSize = \min(Database::INSERT_BATCH_SIZE, $batchSize); + $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -5349,8 +5346,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = \max(1, $batchSize); - $batchSize = \min(Database::DELETE_BATCH_SIZE, $batchSize); + $batchSize = (int)\min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); From 748f601b0310b84bc6cd4549c382c0d5d2172020 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 17:31:19 +0200 Subject: [PATCH 18/31] Fix Code QL intval --- src/Database/Database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index a31ac7a5a..da9185390 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5346,7 +5346,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = (int)\min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = \intval(\min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize))); $collection = $this->silent(fn () => $this->getCollection($collection)); From 1d85d1198a5d7966e6856dd5c8d0cd499dfdef78 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 17:32:32 +0200 Subject: [PATCH 19/31] Fix Code QL casting --- src/Database/Database.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index da9185390..efb3dcf5c 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3419,7 +3419,7 @@ public function createDocuments( $collection = $this->silent(fn () => $this->getCollection($collection)); - $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, (int)\max(1, $batchSize)); /** * Check collection exist @@ -4039,7 +4039,7 @@ public function updateDocuments(string $collection, Document $updates, array $qu return []; } - $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, (int)\max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -4620,7 +4620,7 @@ public function createOrUpdateDocumentsWithIncrease( return []; } - $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, (int)\max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); From d0be423eb5bf0c488128494441c95fc412543d29 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 17:35:22 +0200 Subject: [PATCH 20/31] Fix Code QL casting --- src/Database/Database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index efb3dcf5c..c9c2eb89f 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5346,7 +5346,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = \intval(\min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize))); + $batchSize = (int)\min(Database::DELETE_BATCH_SIZE, (int)\max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); From 23b293721e43b5243440952ddc27ddc5abd9502a Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 17:50:25 +0200 Subject: [PATCH 21/31] Split 2 lines --- src/Database/Database.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index c9c2eb89f..848023220 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5346,7 +5346,8 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = (int)\min(Database::DELETE_BATCH_SIZE, (int)\max(1, $batchSize)); + $batchSize = \max(1, $batchSize); + $batchSize = \min(Database::DELETE_BATCH_SIZE, $batchSize); $collection = $this->silent(fn () => $this->getCollection($collection)); From 735bc7eb9e311fdfe9ff244c612e5d81b1d1f95c Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 18:02:51 +0200 Subject: [PATCH 22/31] Another try --- src/Database/Database.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 848023220..cdbef1e69 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5346,8 +5346,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = \max(1, $batchSize); - $batchSize = \min(Database::DELETE_BATCH_SIZE, $batchSize); + $batchSize = \max(1, \min(Database::DELETE_BATCH_SIZE, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); From 7ad36d625277c041f0dfdeeb6b8d5b88cca62312 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 18:11:57 +0200 Subject: [PATCH 23/31] Add var --- src/Database/Database.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index cdbef1e69..ecc531aff 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3419,7 +3419,7 @@ public function createDocuments( $collection = $this->silent(fn () => $this->getCollection($collection)); - $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, (int)\max(1, $batchSize)); + $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); /** * Check collection exist @@ -4039,7 +4039,7 @@ public function updateDocuments(string $collection, Document $updates, array $qu return []; } - $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, (int)\max(1, $batchSize)); + $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -4620,7 +4620,7 @@ public function createOrUpdateDocumentsWithIncrease( return []; } - $batchSize = (int)\min(Database::INSERT_BATCH_SIZE, (int)\max(1, $batchSize)); + $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -5346,7 +5346,8 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = \max(1, \min(Database::DELETE_BATCH_SIZE, $batchSize)); + /** @var int<1, max> $batchSize */ + $batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); From de899184930e8b08fefef94879a6facdcaa0f085 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 18:15:31 +0200 Subject: [PATCH 24/31] Another try --- src/Database/Database.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index ecc531aff..b322840fc 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5346,8 +5346,8 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - /** @var int<1, max> $batchSize */ - $batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = \max($batchSize, 1); + $batchSize = \min(Database::DELETE_BATCH_SIZE, $batchSize); $collection = $this->silent(fn () => $this->getCollection($collection)); From a3fac0ffe887d83a8b97610942f12c5f8a513140 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 18:21:05 +0200 Subject: [PATCH 25/31] Another try --- src/Database/Database.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index b322840fc..cfa577a36 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5346,8 +5346,8 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = \max($batchSize, 1); - $batchSize = \min(Database::DELETE_BATCH_SIZE, $batchSize); + /** @var int<1, 999> $batchSize */ + $batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); From 9ae826f487d2e6a2495c050bfb141902b9e6644d Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 18:25:07 +0200 Subject: [PATCH 26/31] Another try --- src/Database/Database.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index cfa577a36..6693bd4bd 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5346,8 +5346,10 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - /** @var int<1, 999> $batchSize */ - $batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = $batchSize < 1 ? 1: $batchSize; + $batchSize = $batchSize > Database::DELETE_BATCH_SIZE ? Database::DELETE_BATCH_SIZE : $batchSize; + + //$batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); From 07832ba5a29ed4a7317e0c6c1ab10aec535d2056 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 18:27:53 +0200 Subject: [PATCH 27/31] Another abs --- src/Database/Database.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 6693bd4bd..7b5dcafda 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5346,10 +5346,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $batchSize = $batchSize < 1 ? 1: $batchSize; - $batchSize = $batchSize > Database::DELETE_BATCH_SIZE ? Database::DELETE_BATCH_SIZE : $batchSize; - - //$batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); + $batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -5456,7 +5453,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba 'modified' => count($documents) ])); - foreach (\array_chunk($documents, $batchSize) as $chunk) { + foreach (\array_chunk($documents, abs($batchSize)) as $chunk) { $this->adapter->deleteDocuments( $collection->getId(), array_map(fn ($document) => $document->getId(), $chunk) From 6db3d717bc4aaeb86779a3a1439cb376e9974542 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 18:33:32 +0200 Subject: [PATCH 28/31] Avoid negative --- src/Database/Database.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 7b5dcafda..dbd7bbf9b 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5393,7 +5393,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba $lastDocument = $cursor; while (true) { - if ($limit && $limit < $batchSize) { + if ($limit && $limit < $batchSize && $limit > 0) { $batchSize = $limit; } elseif (!empty($limit)) { $limit -= $batchSize; @@ -5453,7 +5453,10 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba 'modified' => count($documents) ])); - foreach (\array_chunk($documents, abs($batchSize)) as $chunk) { + /** + * abs is for phpstan + */ + foreach (\array_chunk($documents, $batchSize) as $chunk) { $this->adapter->deleteDocuments( $collection->getId(), array_map(fn ($document) => $document->getId(), $chunk) From e0f2d5bccd484bd2de28a16ca9ee511c25fc072c Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 11 Feb 2025 18:35:45 +0200 Subject: [PATCH 29/31] Remove comment --- src/Database/Database.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index dbd7bbf9b..6d59fe730 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5453,9 +5453,6 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba 'modified' => count($documents) ])); - /** - * abs is for phpstan - */ foreach (\array_chunk($documents, $batchSize) as $chunk) { $this->adapter->deleteDocuments( $collection->getId(), From 77f42161ef4c0ad827865b0354d8ab87d0890955 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Feb 2025 19:15:21 +1300 Subject: [PATCH 30/31] Apply suggestions from code review --- src/Database/Adapter/MariaDB.php | 6 +++--- src/Database/Database.php | 7 ------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index e28d12013..ebf0d18a7 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1077,7 +1077,7 @@ public function createDocuments(string $collection, array $documents): array $stmtPermissions?->execute(); } - $internalIds = $this->translateUidsToInternalIds($collection, $documentIds); + $internalIds = $this->getInternalIds($collection, $documentIds); foreach ($documents as $document) { if (isset($internalIds[$document->getId()])) { @@ -1092,14 +1092,14 @@ public function createDocuments(string $collection, array $documents): array } /** - * Translate Uid's to internal id's + * Get internal IDs for the given documents * * @param string $collection * @param array $documentIds * @return array * @throws DatabaseException */ - private function translateUidsToInternalIds(string $collection, array $documentIds): array + private function getInternalIds(string $collection, array $documentIds): array { $internalIds = []; diff --git a/src/Database/Database.php b/src/Database/Database.php index 6d59fe730..2298e11fd 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3432,7 +3432,6 @@ public function createDocuments( } $time = DateTime::now(); - $columnCount = null; foreach ($documents as $key => $document) { $createdAt = $document->getCreatedAt(); @@ -3444,12 +3443,6 @@ public function createDocuments( ->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt) ->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt); - if (is_null($columnCount)) { - $columnCount = count($document); - } elseif ($columnCount != count($document)) { - throw new StructureException('Insert value list does not match column list'); - } - $document = $this->encode($collection, $document); $validator = new Structure( From 735337f51ff73fae31978e229873d74f15ed2aeb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Feb 2025 19:16:54 +1300 Subject: [PATCH 31/31] Update src/Database/Adapter/MariaDB.php --- src/Database/Adapter/MariaDB.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index ebf0d18a7..3d6064f50 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1820,7 +1820,7 @@ public function createOrUpdateDocuments( $stmtAddPermissions->execute(); } - $internalIds = $this->translateUidsToInternalIds($collection, $documentIds); + $internalIds = $this->getInternalIds($collection, $documentIds); foreach ($documents as $document) { if (isset($internalIds[$document->getId()])) {