[1.0.0] Optimize server side sort on sqlite / mysql

resources/schema/sqlite/baseline.sql

@@ -17,7 +17,8 @@ CREATE TABLE IF NOT EXISTS entry_attribute_values (
 
 CREATE INDEX IF NOT EXISTS idx_eav_attr_value ON entry_attribute_values (attr_name_lower, value_lower);
 
-CREATE INDEX IF NOT EXISTS idx_eav_entry ON entry_attribute_values (entry_lc_dn);
+-- Covering (entry_lc_dn leads): entry-scoped lookups plus index-only sort MIN() and correlated EXISTS.
+CREATE INDEX IF NOT EXISTS idx_eav_entry ON entry_attribute_values (entry_lc_dn, attr_name_lower, value_lower);
 
 CREATE TABLE IF NOT EXISTS ldap_change_journal (
     seq          INTEGER NOT NULL PRIMARY KEY,

src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/Dialect/PdoDialectTrait.php

@@ -139,24 +139,48 @@ public function querySidecarInsertPrefix(): string
         return 'INSERT INTO entry_attribute_values (entry_lc_dn, attr_name_lower, value_lower, value_original) VALUES ';
     }
 
-    public function sortKeyClause(
-        string $attributeLower,
-        string $direction,
-    ): SortClause {
-        $expr = <<<SQL
-            (SELECT MIN(eav.value_lower)
-             FROM entry_attribute_values eav
-             WHERE eav.entry_lc_dn = LOWER(dn)
-               AND eav.attr_name_lower = ?)
+    public function sortedQuery(
+        string $baseSql,
+        array $baseParams,
+        array $sortKeys,
+    ): SortedQuery {
+        $projections = [];
+        $orderTerms = [];
+        $sortParams = [];
+
+        // MySQL/MariaDB lack NULLS FIRST/LAST and would re-run the correlated subquery per ORDER BY term; project the
+        // key once into a derived table, then order by the materialised column (single evaluation per candidate).
+        foreach ($sortKeys as $index => $sortKey) {
+            $alias = '__sk' . $index;
+            $projections[] = <<<SQL
+                (SELECT MIN(eav.value_lower)
+                 FROM entry_attribute_values eav
+                 WHERE eav.entry_lc_dn = LOWER(__base.dn)
+                   AND eav.attr_name_lower = ?) AS {$alias}
+                SQL;
+            $orderTerms[] = "{$alias} IS NULL {$sortKey->direction}, {$alias} {$sortKey->direction}";
+            $sortParams[] = $sortKey->attributeLower;
+        }
+
+        $projection = implode(",\n", $projections);
+        $order = implode(', ', $orderTerms);
+        $sql = <<<SQL
+            SELECT dn, attributes FROM (
+                SELECT __base.dn, __base.attributes,
+                {$projection}
+                FROM ({$baseSql}) __base
+            ) __keyed
+            ORDER BY {$order}
             SQL;
 
-        // MySQL/MariaDB lack NULLS FIRST/LAST...emulate via an "IS NULL" ordering prefix.
-        return new SortClause(
-            $expr . ' IS NULL ' . $direction . ', ' . $expr . ' ' . $direction,
-            [
-                $attributeLower,
-                $attributeLower,
-            ],
+        // The projected subqueries precede the nested base query textually.
+        // So their params bind first.
+        return new SortedQuery(
+            $sql,
+            array_merge(
+                $sortParams,
+                $baseParams,
+            ),
         );
     }
 }

src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/Dialect/PdoEntryDialectInterface.php

@@ -109,10 +109,14 @@ public function querySidecarInsertPrefix(): string;
     public function maxDnLength(): ?int;
 
     /**
-     * Returns the ORDER BY term and bound params for one sort key, with NULL/missing values ordered per RFC 2891 §2.2.
-     */
-    public function sortKeyClause(
-        string $attributeLower,
-        string $direction,
-    ): SortClause;
+     * Rewrites the base entry query with an ORDER BY for the sort keys, NULL/missing values ordered per RFC 2891 §2.2.
+     *
+     * @param list<string> $baseParams
+     * @param list<SortKeySpec> $sortKeys
+     */
+    public function sortedQuery(
+        string $baseSql,
+        array $baseParams,
+        array $sortKeys,
+    ): SortedQuery;
 }

src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/Dialect/SortKeySpec.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * This file is part of the FreeDSx LDAP package.
+ *
+ * (c) Chad Sikorra <Chad.Sikorra@gmail.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace FreeDSx\Ldap\Server\Backend\Storage\Adapter\Dialect;
+
+/**
+ * One resolved sort key: the lowercased attribute and its SQL direction.
+ *
+ * @author Chad Sikorra <Chad.Sikorra@gmail.com>
+ */
+final readonly class SortKeySpec
+{
+    public function __construct(
+        public string $attributeLower,
+        public string $direction,
+    ) {}
+}

src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/Dialect/SortClause.php → src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/Dialect/SortedQuery.php

@@ -14,11 +14,11 @@
 namespace FreeDSx\Ldap\Server\Backend\Storage\Adapter\Dialect;
 
 /**
- * ORDER BY term for one sort key plus its bound parameters.
+ * A base entry query rewritten with an ORDER BY, plus its parameters in bind order.
  *
  * @author Chad Sikorra <Chad.Sikorra@gmail.com>
  */
-final readonly class SortClause
+final readonly class SortedQuery
 {
     /**
      * @param list<string> $params

src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/Dialect/SqliteDialect.php

@@ -74,23 +74,35 @@ public function maxDnLength(): ?int
         return null;
     }
 
-    public function sortKeyClause(
-        string $attributeLower,
-        string $direction,
-    ): SortClause {
-        // RFC 2891 §2.2: NULL is the largest value, so missing entries sort last (ASC) or first (DESC).
-        $nulls = $direction === 'ASC'
-            ? 'NULLS LAST'
-            : 'NULLS FIRST';
+    public function sortedQuery(
+        string $baseSql,
+        array $baseParams,
+        array $sortKeys,
+    ): SortedQuery {
+        $terms = [];
+        $sortParams = [];
 
-        return new SortClause(
-            <<<SQL
+        // RFC 2891 §2.2: NULL is the largest value, so missing entries sort last (ASC) or first (DESC). Native
+        // NULLS ordering evaluates the correlated subquery once, so the base query needs no wrapping.
+        foreach ($sortKeys as $sortKey) {
+            $nulls = $sortKey->direction === 'ASC'
+                ? 'NULLS LAST'
+                : 'NULLS FIRST';
+            $terms[] = <<<SQL
                 (SELECT MIN(eav.value_lower)
                  FROM entry_attribute_values eav
                  WHERE eav.entry_lc_dn = LOWER(dn)
-                   AND eav.attr_name_lower = ?) $direction $nulls
-            SQL,
-            [$attributeLower],
+                   AND eav.attr_name_lower = ?) {$sortKey->direction} {$nulls}
+                SQL;
+            $sortParams[] = $sortKey->attributeLower;
+        }
+
+        return new SortedQuery(
+            $baseSql . ' ORDER BY ' . implode(', ', $terms),
+            array_merge(
+                $baseParams,
+                $sortParams,
+            ),
         );
     }
 

src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/Pdo/PdoListQueryBuilder.php

@@ -15,6 +15,7 @@
 
 use FreeDSx\Ldap\Control\Sorting\SortKey;
 use FreeDSx\Ldap\Server\Backend\Storage\Adapter\Dialect\PdoDialectInterface;
+use FreeDSx\Ldap\Server\Backend\Storage\Adapter\Dialect\SortKeySpec;
 use FreeDSx\Ldap\Server\Backend\Storage\Adapter\SqlFilter\SqlFilterResult;
 use FreeDSx\Ldap\Server\Backend\Storage\Adapter\SqlFilter\SqlFilterUtility;
 
@@ -58,7 +59,7 @@ public function build(
         };
 
         if ($sortKeys !== []) {
-            $query = $this->appendSortClause(
+            $query = $this->applySort(
                 $query,
                 $sortKeys,
             );
@@ -218,28 +219,27 @@ private function buildFilteredSubtreeQuery(
     /**
      * @param SortKey[] $sortKeys
      */
-    private function appendSortClause(
+    private function applySort(
         SqlQuery $query,
         array $sortKeys,
     ): SqlQuery {
-        $clauses = [];
-        $params = [];
-
-        foreach ($sortKeys as $sortKey) {
-            $clause = $this->dialect->sortKeyClause(
+        $specs = array_values(array_map(
+            static fn(SortKey $sortKey): SortKeySpec => new SortKeySpec(
                 strtolower($sortKey->getAttribute()),
                 $sortKey->getUseReverseOrder() ? 'DESC' : 'ASC',
-            );
-            $clauses[] = $clause->sql;
-            $params = array_merge(
-                $params,
-                $clause->params,
-            );
-        }
+            ),
+            $sortKeys,
+        ));
 
-        return $query->appending(
-            ' ORDER BY ' . implode(', ', $clauses),
-            $params,
+        $sorted = $this->dialect->sortedQuery(
+            $query->sql,
+            $query->params,
+            $specs,
+        );
+
+        return new SqlQuery(
+            $sorted->sql,
+            $sorted->params,
         );
     }
 }

tests/performance/Config.php

@@ -54,7 +54,7 @@ final class Config
         'json',
     ];
 
-    public const DEFAULT_MIX = 'bind=5,search-read=50,search-eq=25,search-sub=10,search-list=5,add=2,modify=2,delete=1';
+    public const DEFAULT_MIX = 'bind=5,search-read=50,search-eq=25,search-sub=10,search-list=5,search-sort=3,add=2,modify=2,delete=1';
 
     public const DEFAULT_BIND_DN = 'cn=user,dc=foo,dc=bar';
 
@@ -66,6 +66,11 @@ final class Config
 
     public const DEFAULT_SEARCH_SUB_SIZE_LIMIT = 500;
 
+    /**
+     * A sorted search models a bounded display page typically, so give it a realistic limit.
+     */
+    public const DEFAULT_SEARCH_SORT_SIZE_LIMIT = 50;
+
     /**
      * Mirrors the realistic ServerOptions default so load tests run bounded rather than with lookthrough disabled.
      */
@@ -91,6 +96,7 @@ public function __construct(
         public readonly string $writeBase = self::DEFAULT_WRITE_BASE,
         public readonly bool $jit = true,
         public readonly int $searchSubSizeLimit = self::DEFAULT_SEARCH_SUB_SIZE_LIMIT,
+        public readonly int $searchSortSizeLimit = self::DEFAULT_SEARCH_SORT_SIZE_LIMIT,
         public readonly bool $monitor = true,
         public readonly ?string $searchAttributes = null,
         public readonly bool $attributesOnly = false,
@@ -106,6 +112,7 @@ public function __construct(
         $this->assertNonNegative('warmup', $warmup);
         $this->assertNonNegative('seed-entries', $seedEntries);
         $this->assertNonNegative('search-sub-size-limit', $searchSubSizeLimit);
+        $this->assertNonNegative('search-sort-size-limit', $searchSortSizeLimit);
         $this->assertNonNegative('seed-attributes', $seedAttributes);
         $this->assertNonNegative('max-search-lookthrough', $maxSearchLookthrough);
 

tests/performance/LoadTestCommand.php

@@ -214,6 +214,13 @@ protected function configure(): void
                 'Per-request size limit applied to search-sub ops (0 = unlimited)',
                 (string) Config::DEFAULT_SEARCH_SUB_SIZE_LIMIT,
             )
+            ->addOption(
+                'search-sort-size-limit',
+                null,
+                InputOption::VALUE_REQUIRED,
+                'Per-request size limit applied to search-sort ops (0 = unlimited)',
+                (string) Config::DEFAULT_SEARCH_SORT_SIZE_LIMIT,
+            )
             ->addOption(
                 'search-attributes',
                 null,
@@ -445,6 +452,7 @@ private function buildConfig(InputInterface $input): Config
             // Swoole keeps it. --no-jit forces it off either way.
             jit: !(bool) $input->getOption('no-jit') && $runner !== 'pcntl',
             searchSubSizeLimit: $this->requireInt($input, 'search-sub-size-limit'),
+            searchSortSizeLimit: $this->requireInt($input, 'search-sort-size-limit'),
             monitor: (bool) $input->getOption('monitor'),
             searchAttributes: $this->optionalString($input, 'search-attributes'),
             attributesOnly: (bool) $input->getOption('attributes-only'),

tests/performance/Threshold/CiThresholds.php

@@ -65,11 +65,14 @@ public static function forProfile(string $key): ThresholdSet
                 maxErrors: 0,
                 minThroughput: 720.0,
                 maxP99Ms: 200.0,
+                // Server-sides sort file-sorts + materializes per query, so it's going to be slower.
+                perOpMaxP99Ms: ['search-sort' => 250.0],
             ),
             'mysql:swoole' => new ThresholdSet(
                 maxErrors: 0,
                 minThroughput: 720.0,
                 maxP99Ms: 200.0,
+                perOpMaxP99Ms: ['search-sort' => 250.0],
             ),
             default => throw new InvalidArgumentException(sprintf(
                 'Unknown CI profile "%s". Known: %s.',

tests/performance/Threshold/ThresholdEvaluator.php

@@ -47,10 +47,11 @@ public function evaluate(
             );
         }
 
-        if ($thresholds->maxP99Ms !== null) {
+        if ($thresholds->maxP99Ms !== null || $thresholds->perOpMaxP99Ms !== []) {
             $gates[] = $this->evaluateMaxP99(
                 $snapshot,
                 $thresholds->maxP99Ms,
+                $thresholds->perOpMaxP99Ms,
             );
         }
 
@@ -122,32 +123,48 @@ private function evaluateMinThroughput(
     }
 
     /**
+     * Gates each op against its own ceiling (per-op override, else the global), reporting the op nearest its limit.
+     *
+     * @param array<string, float> $perOpMaxMs
+     *
      * @return array{gate: string, passed: bool, expected: string, actual: string}
      */
     private function evaluateMaxP99(
         StatsSnapshot $snapshot,
-        float $maxMs,
+        ?float $globalMaxMs,
+        array $perOpMaxMs,
     ): array {
         $worstOp = null;
-        $worstP99Ns = 0;
+        $worstRatio = 0.0;
+        $worstP99Ms = 0.0;
+        $worstCeilingMs = 0.0;
 
         foreach ($snapshot->operations() as $op) {
-            $p99 = $snapshot->latencyStats($op)['p99'];
+            $ceilingMs = $perOpMaxMs[$op] ?? $globalMaxMs;
 
-            if ($p99 <= $worstP99Ns) {
+            if ($ceilingMs === null) {
                 continue;
             }
 
-            $worstP99Ns = $p99;
+            $p99Ms = $snapshot->latencyStats($op)['p99'] / 1_000_000;
+            $ratio = $p99Ms / $ceilingMs;
+
+            if ($ratio <= $worstRatio) {
+                continue;
+            }
+
+            $worstRatio = $ratio;
             $worstOp = $op;
+            $worstP99Ms = $p99Ms;
+            $worstCeilingMs = $ceilingMs;
         }
 
-        $worstP99Ms = $worstP99Ns / 1_000_000;
-
         return [
             'gate' => 'max-p99-ms',
-            'passed' => $worstP99Ms <= $maxMs,
-            'expected' => sprintf('<= %.2f ms', $maxMs),
+            'passed' => $worstOp === null || $worstP99Ms <= $worstCeilingMs,
+            'expected' => $worstOp !== null
+                ? sprintf('<= %.2f ms (%s)', $worstCeilingMs, $worstOp)
+                : 'no gated ops',
             'actual' => $worstOp !== null
                 ? sprintf('%.2f ms (%s)', $worstP99Ms, $worstOp)
                 : 'no data',