diff --git a/composer.lock b/composer.lock index 1bfbc9925..54c85f47c 100644 --- a/composer.lock +++ b/composer.lock @@ -465,16 +465,16 @@ }, { "name": "open-telemetry/api", - "version": "1.2.1", + "version": "1.2.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0" + "reference": "8b925df3047628968bc5be722468db1b98b82d51" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/74b1a03263be8c5acb578f41da054b4bac3af4a0", - "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/8b925df3047628968bc5be722468db1b98b82d51", + "reference": "8b925df3047628968bc5be722468db1b98b82d51", "shasum": "" }, "require": { @@ -531,7 +531,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-20T23:35:16+00:00" + "time": "2025-02-03T21:49:11+00:00" }, { "name": "open-telemetry/context", @@ -721,16 +721,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.1", + "version": "1.2.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1" + "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1", - "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/37eec0fe47ddd627911f318f29b6cd48196be0c0", + "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0", "shasum": "" }, "require": { @@ -807,24 +807,24 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-09T23:17:14+00:00" + "time": "2025-01-29T21:40:28+00:00" }, { "name": "open-telemetry/sem-conv", - "version": "1.27.1", + "version": "1.30.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sem-conv.git", - "reference": "1dba705fea74bc0718d04be26090e3697e56f4e6" + "reference": "4178c9f390da8e4dbca9b181a9d1efd50cf7ee0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/1dba705fea74bc0718d04be26090e3697e56f4e6", - "reference": "1dba705fea74bc0718d04be26090e3697e56f4e6", + "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/4178c9f390da8e4dbca9b181a9d1efd50cf7ee0a", + "reference": "4178c9f390da8e4dbca9b181a9d1efd50cf7ee0a", "shasum": "" }, "require": { - "php": "^8.1" + "php": "^8.0" }, "type": "library", "extra": { @@ -864,7 +864,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-08-28T09:20:31+00:00" + "time": "2025-02-06T00:21:48+00:00" }, { "name": "php-http/discovery", diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 12b74513b..c91ad2ef0 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -604,13 +604,12 @@ abstract public function createDocument(string $collection, Document $document): * * @param string $collection * @param array $documents - * @param int $batchSize * * @return array * * @throws DatabaseException */ - abstract public function createDocuments(string $collection, array $documents, int $batchSize): array; + abstract public function createDocuments(string $collection, array $documents): array; /** * Update Document @@ -645,14 +644,12 @@ abstract public function updateDocuments(string $collection, Document $updates, * @param string $collection * @param string $attribute * @param array $documents - * @param int $batchSize * @return array */ abstract public function createOrUpdateDocuments( string $collection, string $attribute, - array $documents, - int $batchSize + array $documents ): array; /** diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 1615274d2..3d6064f50 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -965,14 +965,13 @@ public function createDocument(string $collection, Document $document): Document * * @param string $collection * @param array $documents - * @param int $batchSize * * @return array * * @throws DuplicateException * @throws \Throwable */ - public function createDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array + public function createDocuments(string $collection, array $documents): array { if (empty($documents)) { return $documents; @@ -980,114 +979,144 @@ public function createDocuments(string $collection, array $documents, int $batch try { $name = $this->filter($collection); - $batches = \array_chunk($documents, \max(1, $batchSize)); - $documentIds = \array_map(fn ($document) => $document->getId(), $documents); - - foreach ($batches as $batch) { - $bindIndex = 0; - $batchKeys = []; - $bindValues = []; - $permissions = []; - - foreach ($batch as $document) { - $attributes = $document->getAttributes(); - $attributes['_uid'] = $document->getId(); - $attributes['_createdAt'] = $document->getCreatedAt(); - $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = \json_encode($document->getPermissions()); - - if (!empty($document->getInternalId())) { - $internalIds[$document->getId()] = true; - $attributes['_id'] = $document->getInternalId(); - } - if ($this->sharedTables) { - $attributes['_tenant'] = $this->tenant; - } + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + $permissions = []; + $documentIds = []; - $columns = []; - foreach (\array_keys($attributes) as $key => $attribute) { - $columns[$key] = "`{$this->filter($attribute)}`"; - } + foreach ($documents as $document) { + /** + * @var Document $document + */ + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = \json_encode($document->getPermissions()); + + if (! empty($document->getInternalId())) { + $attributes['_id'] = $document->getInternalId(); + } else { + $documentIds[] = $document->getId(); + } - $columns = '(' . \implode(', ', $columns) . ')'; + if ($this->sharedTables) { + $attributes['_tenant'] = $this->tenant; + } - $bindKeys = []; + $columns = []; + foreach (\array_keys($attributes) as $key => $attribute) { + $columns[$key] = "`{$this->filter($attribute)}`"; + } - foreach ($attributes as $value) { - if (\is_array($value)) { - $value = \json_encode($value); - } - $value = (\is_bool($value)) ? (int)$value : $value; - $bindKey = 'key_' . $bindIndex; - $bindKeys[] = ':' . $bindKey; - $bindValues[$bindKey] = $value; - $bindIndex++; - } + $columns = '(' . \implode(', ', $columns) . ')'; - $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; - foreach (Database::PERMISSIONS as $type) { - foreach ($document->getPermissionsByType($type) as $permission) { - $permission = \str_replace('"', '', $permission); - $permission = "('{$type}', '{$permission}', '{$document->getId()}'"; + $bindKeys = []; - if ($this->sharedTables) { - $permission .= ", :_tenant)"; - } else { - $permission .= ")"; - } + foreach ($attributes as $value) { + if (\is_array($value)) { + $value = \json_encode($value); + } + $value = (\is_bool($value)) ? (int)$value : $value; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $value; + $bindIndex++; + } - $permissions[] = $permission; + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; + foreach (Database::PERMISSIONS as $type) { + foreach ($document->getPermissionsByType($type) as $permission) { + $permission = \str_replace('"', '', $permission); + $permission = "('{$type}', '{$permission}', '{$document->getId()}'"; + + if ($this->sharedTables) { + $permission .= ", :_tenant)"; + } else { + $permission .= ")"; } + + $permissions[] = $permission; } } + } - $stmt = $this->getPDO()->prepare( - " - INSERT INTO {$this->getSQLTable($name)} {$columns} - VALUES " . \implode(', ', $batchKeys) - ); + $stmt = $this->getPDO()->prepare( + " + INSERT INTO {$this->getSQLTable($name)} {$columns} + VALUES " . \implode(', ', $batchKeys) + ); - foreach ($bindValues as $key => $value) { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } + foreach ($bindValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } - $stmt->execute(); + $stmt->execute(); - if (!empty($permissions)) { - $sqlPermissions = " - INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document - "; + if (!empty($permissions)) { + $sqlPermissions = " + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document + "; - if ($this->sharedTables) { - $sqlPermissions .= ', _tenant)'; - } else { - $sqlPermissions .= ")"; - } + if ($this->sharedTables) { + $sqlPermissions .= ', _tenant)'; + } else { + $sqlPermissions .= ")"; + } - $sqlPermissions .= " VALUES " . \implode(', ', $permissions); + $sqlPermissions .= " VALUES " . \implode(', ', $permissions); - $stmtPermissions = $this->getPDO()->prepare($sqlPermissions); + $stmtPermissions = $this->getPDO()->prepare($sqlPermissions); - if ($this->sharedTables) { - $stmtPermissions->bindValue(':_tenant', $this->tenant); - } + if ($this->sharedTables) { + $stmtPermissions->bindValue(':_tenant', $this->tenant); + } + + $stmtPermissions?->execute(); + } + + $internalIds = $this->getInternalIds($collection, $documentIds); - $stmtPermissions?->execute(); + foreach ($documents as $document) { + if (isset($internalIds[$document->getId()])) { + $document['$internalId'] = $internalIds[$document->getId()]; } } + } catch (PDOException $e) { + throw $this->processException($e); + } + return $documents; + } + + /** + * Get internal IDs for the given documents + * + * @param string $collection + * @param array $documentIds + * @return array + * @throws DatabaseException + */ + private function getInternalIds(string $collection, array $documentIds): array + { + $internalIds = []; + + /** + * UID, _tenant bottleneck is ~ 5000 rows since we use _uid IN query + */ + foreach (\array_chunk($documentIds, 1000) as $documentIdsChunk) { // Get internal IDs $sql = " SELECT _uid, _id FROM {$this->getSQLTable($collection)} - WHERE _uid IN (" . implode(',', array_map(fn ($index) => ":_key_{$index}", array_keys($documentIds))) . ") + WHERE _uid IN (" . implode(',', array_map(fn ($index) => ":_key_{$index}", array_keys($documentIdsChunk))) . ") {$this->getTenantQuery($collection)} "; - $stmt = $this->getPDO()->prepare($sql); - foreach ($documentIds as $index => $id) { + foreach ($documentIdsChunk as $index => $id) { $stmt->bindValue(":_key_{$index}", $id); } @@ -1096,21 +1125,14 @@ public function createDocuments(string $collection, array $documents, int $batch } $stmt->execute(); - $internalIds = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] + $results = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] $stmt->closeCursor(); - foreach ($documents as $document) { - if (isset($internalIds[$document->getId()])) { - $document['$internalId'] = $internalIds[$document->getId()]; - } - } - } catch (PDOException $e) { - throw $this->processException($e); + $internalIds = array_merge($internalIds, $results); } - return $documents; + return $internalIds; } - /** * Update Document * @@ -1595,15 +1617,13 @@ public function updateDocuments(string $collection, Document $updates, array $do * @param string $collection * @param string $attribute * @param array $documents - * @param int $batchSize * @return array * @throws DatabaseException */ public function createOrUpdateDocuments( string $collection, string $attribute, - array $documents, - int $batchSize + array $documents ): array { if (empty($documents)) { return $documents; @@ -1612,218 +1632,195 @@ public function createOrUpdateDocuments( try { $name = $this->filter($collection); $attribute = $this->filter($attribute); - $batches = \array_chunk($documents, \max(1, $batchSize)); - - foreach ($batches as $batch) { - $bindIndex = 0; - $batchKeys = []; - $bindValues = []; - - $documentIds = array_map(fn ($doc) => $doc->getId(), $batch); - - foreach ($batch as $document) { - /** - * @var array $attributes - */ - $attributes = $document->getAttributes(); - $attributes['_uid'] = $document->getId(); - $attributes['_createdAt'] = $document->getCreatedAt(); - $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = \json_encode($document->getPermissions()); - - if (!empty($document->getInternalId())) { - $attributes['_id'] = $document->getInternalId(); - } - if ($this->sharedTables) { - $attributes['_tenant'] = $this->tenant; - } + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + $attributes = []; + $documentIds = []; - $columns = []; - foreach (\array_keys($attributes) as $key => $attr) { - $columns[$key] = "`{$this->filter($attr)}`"; - } + foreach ($documents as $document) { + /** + * @var array $attributes + */ + $attributes = $document->getAttributes(); + $documentIds[] = $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = \json_encode($document->getPermissions()); + + if (!empty($document->getInternalId())) { + $attributes['_id'] = $document->getInternalId(); + } - $columns = '(' . \implode(', ', $columns) . ')'; + if ($this->sharedTables) { + $attributes['_tenant'] = $this->tenant; + } - $bindKeys = []; + $columns = []; + foreach (\array_keys($attributes) as $key => $attr) { + $columns[$key] = "`{$this->filter($attr)}`"; + } - foreach ($attributes as $attrValue) { - if (\is_array($attrValue)) { - $attrValue = \json_encode($attrValue); - } - $attrValue = (\is_bool($attrValue)) ? (int)$attrValue : $attrValue; - $bindKey = 'key_' . $bindIndex; - $bindKeys[] = ':' . $bindKey; - $bindValues[$bindKey] = $attrValue; - $bindIndex++; - } + $columns = '(' . \implode(', ', $columns) . ')'; - $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; - } + $bindKeys = []; - if (!empty($attribute)) { - // Increment specific column by its new value in place - $updateColumns = [ - "`{$attribute}` = `{$attribute}` + VALUES(`{$attribute}`)", - "`_updatedAt` = VALUES(`_updatedAt`)" - ]; - } else { - // Update all columns - $updateColumns = []; - foreach (\array_keys($attributes) as $attr) { - $updateColumns[] = "`{$this->filter($attr)}` = VALUES(`{$this->filter($attr)}`)"; + foreach ($attributes as $attrValue) { + if (\is_array($attrValue)) { + $attrValue = \json_encode($attrValue); } + $attrValue = (\is_bool($attrValue)) ? (int)$attrValue : $attrValue; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $attrValue; + $bindIndex++; } - $stmt = $this->getPDO()->prepare( - " - INSERT INTO {$this->getSQLTable($name)} {$columns} - VALUES " . \implode(', ', $batchKeys) . " - ON DUPLICATE KEY UPDATE - " . \implode(', ', $updateColumns) - ); + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; + } - foreach ($bindValues as $key => $binding) { - $stmt->bindValue($key, $binding, $this->getPDOType($binding)); + if (!empty($attribute)) { + // Increment specific column by its new value in place + $updateColumns = [ + "`{$attribute}` = `{$attribute}` + VALUES(`{$attribute}`)", + "`_updatedAt` = VALUES(`_updatedAt`)" + ]; + } else { + // Update all columns + $updateColumns = []; + foreach (\array_keys($attributes) as $attr) { + $updateColumns[] = "`{$this->filter($attr)}` = VALUES(`{$this->filter($attr)}`)"; } + } - $stmt->execute(); + $stmt = $this->getPDO()->prepare( + " + INSERT INTO {$this->getSQLTable($name)} {$columns} + VALUES " . \implode(', ', $batchKeys) . " + ON DUPLICATE KEY UPDATE + " . \implode(', ', $updateColumns) + ); - // Fetch existing permissions in bulk after data updates - $sql = " - SELECT _document, _type, _permission - FROM {$this->getSQLTable($name . '_perms')} - WHERE _document IN (" . \implode(',', \array_map(fn ($index) => ":_key_{$index}", \array_keys($documentIds))) . ") - {$this->getTenantQuery($collection)} - "; + foreach ($bindValues as $key => $binding) { + $stmt->bindValue($key, $binding, $this->getPDOType($binding)); + } - $stmt = $this->getPDO()->prepare($sql); + $stmt->execute(); - foreach ($documentIds as $index => $id) { - $stmt->bindValue(":_key_{$index}", $id); - } + // Fetch existing permissions in bulk after data updates + $sql = " + SELECT _document, _type, _permission + FROM {$this->getSQLTable($name . '_perms')} + WHERE _document IN (" . \implode(',', \array_map(fn ($index) => ":_key_{$index}", \array_keys($documentIds))) . ") + {$this->getTenantQuery($collection)} + "; - if ($this->sharedTables) { - $stmt->bindValue(':_tenant', $this->tenant); - } + $stmt = $this->getPDO()->prepare($sql); - $stmt->execute(); - $existing = $stmt->fetchAll(); - $stmt->closeCursor(); + foreach ($documentIds as $index => $id) { + $stmt->bindValue(":_key_{$index}", $id); + } - // Group permissions by document - $permissionsByDocument = []; - foreach ($existing as $row) { - $permissionsByDocument[$row['_document']][$row['_type']][] = $row['_permission']; - } + if ($this->sharedTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } - foreach ($documentIds as $id) { - foreach (Database::PERMISSIONS as $type) { - $permissionsByDocument[$id][$type] = $permissionsByDocument[$id][$type] ?? []; - } - } + $stmt->execute(); + $existing = $stmt->fetchAll(); + $stmt->closeCursor(); - $removeQueries = []; - $removeBindValues = []; - $addQueries = []; - $addBindValues = []; - - foreach ($batch as $index => $document) { - $currentPermissions = $permissionsByDocument[$document->getId()] ?? []; - - // Calculate removals - foreach (Database::PERMISSIONS as $type) { - $toRemove = \array_diff($currentPermissions[$type], $document->getPermissionsByType($type)); - if (!empty($toRemove)) { - $removeQueries[] = "( - _document = :uid_{$index} - {$this->getTenantQuery($collection)} - AND _type = '{$type}' - AND _permission IN (" . \implode(',', \array_map(fn ($i) => ":remove_{$type}_{$index}_{$i}", \array_keys($toRemove))) . ") - )"; - $removeBindValues[":uid_{$index}"] = $document->getId(); - foreach ($toRemove as $i => $perm) { - $removeBindValues[":remove_{$type}_{$index}_{$i}"] = $perm; - } - } - } + // Group permissions by document + $permissionsByDocument = []; + foreach ($existing as $row) { + $permissionsByDocument[$row['_document']][$row['_type']][] = $row['_permission']; + } - // Calculate additions - foreach (Database::PERMISSIONS as $type) { - $toAdd = \array_diff($document->getPermissionsByType($type), $currentPermissions[$type]); - foreach ($toAdd as $i => $permission) { - $addQuery = "(:uid_{$index}, '{$type}', :add_{$type}_{$index}_{$i}"; + foreach ($documentIds as $id) { + foreach (Database::PERMISSIONS as $type) { + $permissionsByDocument[$id][$type] = $permissionsByDocument[$id][$type] ?? []; + } + } - if ($this->sharedTables) { - $addQuery .= ", :_tenant)"; - } else { - $addQuery .= ")"; - } + $removeQueries = []; + $removeBindValues = []; + $addQueries = []; + $addBindValues = []; + + foreach ($documents as $index => $document) { + $currentPermissions = $permissionsByDocument[$document->getId()] ?? []; - $addQueries[] = $addQuery; - $addBindValues[":uid_{$index}"] = $document->getId(); - $addBindValues[":add_{$type}_{$index}_{$i}"] = $permission; + // Calculate removals + foreach (Database::PERMISSIONS as $type) { + $toRemove = \array_diff($currentPermissions[$type], $document->getPermissionsByType($type)); + if (!empty($toRemove)) { + $removeQueries[] = "( + _document = :uid_{$index} + {$this->getTenantQuery($collection)} + AND _type = '{$type}' + AND _permission IN (" . \implode(',', \array_map(fn ($i) => ":remove_{$type}_{$index}_{$i}", \array_keys($toRemove))) . ") + )"; + $removeBindValues[":uid_{$index}"] = $document->getId(); + foreach ($toRemove as $i => $perm) { + $removeBindValues[":remove_{$type}_{$index}_{$i}"] = $perm; } } } - // Execute permission removals - if (!empty($removeQueries)) { - $removeQuery = \implode(' OR ', $removeQueries); - $stmtRemovePermissions = $this->getPDO()->prepare("DELETE FROM {$this->getSQLTable($name . '_perms')} WHERE {$removeQuery}"); - foreach ($removeBindValues as $key => $value) { - $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); - } - if ($this->sharedTables) { - $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); - } - $stmtRemovePermissions->execute(); - } + // Calculate additions + foreach (Database::PERMISSIONS as $type) { + $toAdd = \array_diff($document->getPermissionsByType($type), $currentPermissions[$type]); + foreach ($toAdd as $i => $permission) { + $addQuery = "(:uid_{$index}, '{$type}', :add_{$type}_{$index}_{$i}"; - // Execute permission additions - if (!empty($addQuery)) { - $sqlAddPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission"; - if ($this->sharedTables) { - $sqlAddPermissions .= ", _tenant)"; - } else { - $sqlAddPermissions .= ")"; - } - $addQuery = \implode(', ', $addQueries); - $sqlAddPermissions .= " VALUES {$addQuery}"; - $stmtAddPermissions = $this->getPDO()->prepare($sqlAddPermissions); - foreach ($addBindValues as $key => $value) { - $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); - } - if ($this->sharedTables) { - $stmtAddPermissions->bindValue(':_tenant', $this->tenant); - } + if ($this->sharedTables) { + $addQuery .= ", :_tenant)"; + } else { + $addQuery .= ")"; + } - $stmtAddPermissions->execute(); + $addQueries[] = $addQuery; + $addBindValues[":uid_{$index}"] = $document->getId(); + $addBindValues[":add_{$type}_{$index}_{$i}"] = $permission; + } } } - // Get internal IDs - $sql = " - SELECT _uid, _id - FROM {$this->getSQLTable($collection)} - WHERE _uid IN (" . \implode(',', \array_map(fn ($index) => ":_key_{$index}", \array_keys($documentIds))) . ") - {$this->getTenantQuery($collection)} - "; - - $stmt = $this->getPDO()->prepare($sql); - - foreach ($documentIds as $index => $id) { - $stmt->bindValue(":_key_{$index}", $id); + // Execute permission removals + if (!empty($removeQueries)) { + $removeQuery = \implode(' OR ', $removeQueries); + $stmtRemovePermissions = $this->getPDO()->prepare("DELETE FROM {$this->getSQLTable($name . '_perms')} WHERE {$removeQuery}"); + foreach ($removeBindValues as $key => $value) { + $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); + } + if ($this->sharedTables) { + $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + } + $stmtRemovePermissions->execute(); } - if ($this->sharedTables) { - $stmt->bindValue(':_tenant', $this->tenant); + // Execute permission additions + if (!empty($addQuery)) { + $sqlAddPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission"; + if ($this->sharedTables) { + $sqlAddPermissions .= ", _tenant)"; + } else { + $sqlAddPermissions .= ")"; + } + $addQuery = \implode(', ', $addQueries); + $sqlAddPermissions .= " VALUES {$addQuery}"; + $stmtAddPermissions = $this->getPDO()->prepare($sqlAddPermissions); + foreach ($addBindValues as $key => $value) { + $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); + } + if ($this->sharedTables) { + $stmtAddPermissions->bindValue(':_tenant', $this->tenant); + } + + $stmtAddPermissions->execute(); } - $stmt->execute(); - $internalIds = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] - $stmt->closeCursor(); + $internalIds = $this->getInternalIds($collection, $documentIds); foreach ($documents as $document) { if (isset($internalIds[$document->getId()])) { @@ -1955,6 +1952,10 @@ public function deleteDocument(string $collection, string $id): bool */ public function deleteDocuments(string $collection, array $ids): int { + if (empty($ids)) { + return 0; + } + try { $name = $this->filter($collection); $where = []; diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index b35add565..44df82c67 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -747,13 +747,12 @@ public function createDocument(string $collection, Document $document): Document * * @param string $collection * @param array $documents - * @param int $batchSize * * @return array * * @throws Duplicate */ - public function createDocuments(string $collection, array $documents, int $batchSize): array + public function createDocuments(string $collection, array $documents): array { $name = $this->getNamespace() . '_' . $this->filter($collection); @@ -895,10 +894,9 @@ public function updateDocuments(string $collection, Document $updates, array $do * @param string $collection * @param string $attribute * @param array $documents - * @param int $batchSize * @return array */ - public function createOrUpdateDocuments(string $collection, string $attribute, array $documents, int $batchSize): array + public function createOrUpdateDocuments(string $collection, string $attribute, array $documents): array { return $documents; } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index c2b886776..d64d42f53 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1025,13 +1025,12 @@ public function createDocument(string $collection, Document $document): Document * * @param string $collection * @param array $documents - * @param int $batchSize * * @return array * * @throws DuplicateException */ - public function createDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array + public function createDocuments(string $collection, array $documents): array { if (empty($documents)) { return $documents; @@ -1039,81 +1038,78 @@ public function createDocuments(string $collection, array $documents, int $batch try { $name = $this->filter($collection); - $batches = \array_chunk($documents, max(1, $batchSize)); $internalIds = []; - foreach ($batches as $batch) { - $bindIndex = 0; - $batchKeys = []; - $bindValues = []; - $permissions = []; - - foreach ($batch as $document) { - $attributes = $document->getAttributes(); - $attributes['_uid'] = $document->getId(); - $attributes['_createdAt'] = $document->getCreatedAt(); - $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = \json_encode($document->getPermissions()); - - if (!empty($document->getInternalId())) { - $internalIds[$document->getId()] = true; - $attributes['_id'] = $document->getInternalId(); - } + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + $permissions = []; + + foreach ($documents as $document) { + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = \json_encode($document->getPermissions()); + + if (!empty($document->getInternalId())) { + $internalIds[$document->getId()] = true; + $attributes['_id'] = $document->getInternalId(); + } - if ($this->sharedTables) { - $attributes['_tenant'] = $this->tenant; - } + if ($this->sharedTables) { + $attributes['_tenant'] = $this->tenant; + } - $columns = []; - foreach (\array_keys($attributes) as $key => $attribute) { - $columns[$key] = "\"{$this->filter($attribute)}\""; - } + $columns = []; + foreach (\array_keys($attributes) as $key => $attribute) { + $columns[$key] = "\"{$this->filter($attribute)}\""; + } - $columns = '(' . \implode(', ', $columns) . ')'; + $columns = '(' . \implode(', ', $columns) . ')'; - $bindKeys = []; + $bindKeys = []; - foreach ($attributes as $value) { - if (\is_array($value)) { - $value = \json_encode($value); - } - $value = (\is_bool($value)) ? (int)$value : $value; - $bindKey = 'key_' . $bindIndex; - $bindKeys[] = ':' . $bindKey; - $bindValues[$bindKey] = $value; - $bindIndex++; + foreach ($attributes as $value) { + if (\is_array($value)) { + $value = \json_encode($value); } + $value = (\is_bool($value)) ? (int)$value : $value; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $value; + $bindIndex++; + } - $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; - foreach (Database::PERMISSIONS as $type) { - foreach ($document->getPermissionsByType($type) as $permission) { - $permission = \str_replace('"', '', $permission); - $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', :_tenant)"; - } + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; + foreach (Database::PERMISSIONS as $type) { + foreach ($document->getPermissionsByType($type) as $permission) { + $permission = \str_replace('"', '', $permission); + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', :_tenant)"; } } + } - $stmt = $this->getPDO()->prepare( - " - INSERT INTO {$this->getSQLTable($name)} {$columns} - VALUES " . \implode(', ', $batchKeys) - ); + $stmt = $this->getPDO()->prepare( + " + INSERT INTO {$this->getSQLTable($name)} {$columns} + VALUES " . \implode(', ', $batchKeys) + ); - foreach ($bindValues as $key => $value) { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } + foreach ($bindValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } - $stmt->execute(); + $stmt->execute(); - if (!empty($permissions)) { - $stmtPermissions = $this->getPDO()->prepare( - " - INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document, _tenant) - VALUES " . \implode(', ', $permissions) - ); - $stmtPermissions->bindValue(':_tenant', $this->tenant); - $stmtPermissions?->execute(); - } + if (!empty($permissions)) { + $stmtPermissions = $this->getPDO()->prepare( + " + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document, _tenant) + VALUES " . \implode(', ', $permissions) + ); + $stmtPermissions->bindValue(':_tenant', $this->tenant); + $stmtPermissions?->execute(); } } catch (PDOException $e) { throw $this->processException($e); @@ -1609,10 +1605,9 @@ public function updateDocuments(string $collection, Document $updates, array $do * @param string $collection * @param string $attribute * @param array $documents - * @param int $batchSize * @return array */ - public function createOrUpdateDocuments(string $collection, string $attribute, array $documents, int $batchSize): array + public function createOrUpdateDocuments(string $collection, string $attribute, array $documents): array { return $documents; } @@ -1735,6 +1730,10 @@ public function deleteDocument(string $collection, string $id): bool */ public function deleteDocuments(string $collection, array $ids): int { + if (empty($ids)) { + return 0; + } + try { $name = $this->filter($collection); $where = []; diff --git a/src/Database/Database.php b/src/Database/Database.php index 6e0c20b77..2298e11fd 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -142,8 +142,8 @@ class Database public const EVENT_INDEX_CREATE = 'index_create'; public const EVENT_INDEX_DELETE = 'index_delete'; - public const INSERT_BATCH_SIZE = 10_000; - public const DELETE_BATCH_SIZE = 10_000; + public const INSERT_BATCH_SIZE = 1_000; + public const DELETE_BATCH_SIZE = 1_000; /** * List of Internal attributes @@ -3419,6 +3419,18 @@ public function createDocuments( $collection = $this->silent(fn () => $this->getCollection($collection)); + $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + + /** + * Check collection exist + */ + if ($collection->getId() !== self::METADATA) { + $authorization = new Authorization(self::PERMISSION_CREATE); + if (!$authorization->isValid($collection->getCreate())) { + throw new AuthorizationException($authorization->getDescription()); + } + } + $time = DateTime::now(); foreach ($documents as $key => $document) { @@ -3450,11 +3462,13 @@ public function createDocuments( } $documents = $this->withTransaction(function () use ($collection, $documents, $batchSize) { - return $this->adapter->createDocuments( - $collection->getId(), - $documents, - $batchSize, - ); + $stack = []; + + foreach (\array_chunk($documents, $batchSize) as $chunk) { + $stack = array_merge($stack, $this->adapter->createDocuments($collection->getId(), $chunk)); + } + + return $stack; }); foreach ($documents as $key => $document) { @@ -4018,6 +4032,8 @@ public function updateDocuments(string $collection, Document $updates, array $qu return []; } + $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $collection = $this->silent(fn () => $this->getCollection($collection)); if ($collection->isEmpty()) { @@ -4071,7 +4087,6 @@ public function updateDocuments(string $collection, Document $updates, array $qu } $documents = $this->withTransaction(function () use ($collection, $queries, $batchSize, $updates, $limit, $cursor) { - $lastDocument = null; $documents = []; $documentSecurity = $collection->getAttribute('documentSecurity', false); @@ -4598,6 +4613,8 @@ public function createOrUpdateDocumentsWithIncrease( return []; } + $batchSize = \min(Database::INSERT_BATCH_SIZE, \max(1, $batchSize)); + $collection = $this->silent(fn () => $this->getCollection($collection)); $time = DateTime::now(); @@ -4645,12 +4662,13 @@ public function createOrUpdateDocumentsWithIncrease( } $documents = $this->withTransaction(function () use ($collection, $attribute, $documents, $batchSize) { - return $this->adapter->createOrUpdateDocuments( - $collection->getId(), - $attribute, - $documents, - $batchSize, - ); + $stack = []; + + foreach (\array_chunk($documents, $batchSize) as $chunk) { + $stack = array_merge($stack, $this->adapter->createOrUpdateDocuments($collection->getId(), $attribute, $chunk)); + } + + return $stack; }); foreach ($documents as $key => $document) { @@ -5321,6 +5339,8 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } + $batchSize = \min(Database::DELETE_BATCH_SIZE, \max(1, $batchSize)); + $collection = $this->silent(fn () => $this->getCollection($collection)); if ($collection->isEmpty()) { @@ -5366,7 +5386,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba $lastDocument = $cursor; while (true) { - if ($limit && $limit < $batchSize) { + if ($limit && $limit < $batchSize && $limit > 0) { $batchSize = $limit; } elseif (!empty($limit)) { $limit -= $batchSize; @@ -5426,7 +5446,12 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba 'modified' => count($documents) ])); - $this->adapter->deleteDocuments($collection->getId(), array_map(fn ($document) => $document->getId(), $documents)); + foreach (\array_chunk($documents, $batchSize) as $chunk) { + $this->adapter->deleteDocuments( + $collection->getId(), + array_map(fn ($document) => $document->getId(), $chunk) + ); + } return $documents; }); diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 78ff7773c..6f38b0ff4 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2352,8 +2352,8 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void $collection = 'testCreateOrUpdateInplace'; static::getDatabase()->createCollection($collection); - static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true); - static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true); + static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, false); + static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, false); $documents = [ new Document([