Skip to content

feat(redis): improve pagination and filtering logic in getRowsFromTable method #1442

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Artuomka merged 1 commit into from
Nov 20, 2025
Merged

feat(redis): improve pagination and filtering logic in getRowsFromTable method #1442

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 86 additions & 61 deletions shared-code/src/data-access-layer/data-access-objects/data-access-object-redis.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,12 +161,14 @@ export class DataAccessObjectRedis extends BasicDataAccessObject implements IDat
): Promise<FoundRowsDS> {
const redisClient = await this.getClient();

const safeSettings = settings || ({} as TableSettingsDS);

page = page > 0 ? page : DAO_CONSTANTS.DEFAULT_PAGINATION.page;
perPage =
perPage > 0
? perPage
: settings.list_per_page > 0
? settings.list_per_page
: safeSettings.list_per_page > 0
? safeSettings.list_per_page
: DAO_CONSTANTS.DEFAULT_PAGINATION.perPage;

if (autocompleteFields?.value && autocompleteFields.fields?.length > 0) {
Expand All @@ -176,13 +178,11 @@ export class DataAccessObjectRedis extends BasicDataAccessObject implements IDat
const pattern = `${tableName}:*`;
const allRows: Array<Record<string, unknown>> = [];
let cursor = '0';
let processedCount = 0;
const targetRows = page * perPage;

do {
const scanResult = await redisClient.scan(cursor, {
MATCH: pattern,
COUNT: 100,
COUNT: 1000,
});

cursor = scanResult.cursor;
Expand All @@ -193,37 +193,49 @@ export class DataAccessObjectRedis extends BasicDataAccessObject implements IDat
keys.forEach((key) => pipeline.get(key));
const results = await pipeline.exec();

for (let i = 0; i < keys.length; i++) {
const key = keys[i];
const data = results[i] as unknown as string;

if (data) {
try {
const parsedData = JSON.parse(data);
const keyPart = key.split(':').slice(1).join(':');
const rowData = { key: keyPart, ...parsedData };
const passesFilter = await this.passesFilters(rowData, filteringFields, searchedFieldValue, settings);
if (passesFilter) {
allRows.push(rowData);
processedCount++;
if (results) {
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
const result = results[i];
let data: string | null = null;
if (Array.isArray(result)) {
const [error, value] = result;
if (!error && value !== null) {
data = value as string;
}
} catch (_error) {
continue;
} else {
data = result as unknown as string;
}
}
}

if (processedCount >= targetRows + perPage && cursor === '0') {
break;
if (data) {
try {
const parsedData = JSON.parse(data);
const keyPart = key.split(':').slice(1).join(':');
const rowData = { key: keyPart, ...parsedData };
const passesFilter = await this.passesFilters(
rowData,
filteringFields,
searchedFieldValue,
safeSettings,
tableName,
);
if (passesFilter) {
allRows.push(rowData);
}
} catch (_error) {
continue;
}
}
}
}
}

Copilot AI Nov 20, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The removal of early exit logic means the code now always scans the entire Redis keyspace, even when only a small page of results is needed. This can significantly impact performance for large datasets.

Consider re-implementing early exit logic, but fix the original bug where the condition cursor === '0' prevented it from working correctly. A better approach would be to break when allRows.length >= (page * perPage) + perPage, which ensures we have enough rows to satisfy the current page request.

Suggested change
}
}
// Early exit: break if we have enough rows for the requested page
if (allRows.length >= (page * perPage) + perPage) {
break;
}

Copilot uses AI. Check for mistakes.
} while (cursor !== '0');

if (settings.ordering_field && settings.ordering) {
if (safeSettings.ordering_field && safeSettings.ordering) {
allRows.sort((a, b) => {
const aVal = a[settings.ordering_field];
const bVal = b[settings.ordering_field];
const modifier = settings.ordering === 'ASC' ? 1 : -1;
const aVal = a[safeSettings.ordering_field];
const bVal = b[safeSettings.ordering_field];
const modifier = safeSettings.ordering === 'ASC' ? 1 : -1;

if (aVal < bVal) return -1 * modifier;
if (aVal > bVal) return 1 * modifier;
Expand All @@ -239,7 +251,7 @@ export class DataAccessObjectRedis extends BasicDataAccessObject implements IDat
tableStructure = await this.getTableStructure(tableName);
}

const availableFields = this.findAvailableFields(settings, tableStructure);
const availableFields = this.findAvailableFields(safeSettings, tableStructure);

const finalRows = paginatedRows.map((row) => {
if (availableFields.length > 0) {
Expand Down Expand Up @@ -567,40 +579,52 @@ export class DataAccessObjectRedis extends BasicDataAccessObject implements IDat
keys.forEach((key) => pipeline.get(key));
const results = await pipeline.exec();

for (let i = 0; i < keys.length; i++) {
const key = keys[i];
const data = results[i] as unknown as string;

if (data) {
try {
const parsedData = JSON.parse(data);
const keyPart = key.split(':').slice(1).join(':');
const rowData = { key: keyPart, ...parsedData };

if (
value === '*' ||
fields.some((field) =>
String(rowData[field] || '')
.toLowerCase()
.includes(String(value).toLowerCase()),
)
) {
const autocompleteRow: Record<string, unknown> = {};
fields.forEach((field) => {
if (rowData[field] !== undefined) {
autocompleteRow[field] = rowData[field];
}
});
rows.push(autocompleteRow);
processedCount++;
if (results) {
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
const result = results[i];

let data: string | null = null;
if (Array.isArray(result)) {
const [error, value] = result;
if (!error && value !== null) {
data = value as string;
}
} else {
data = result as unknown as string;
}

if (data) {
try {
const parsedData = JSON.parse(data);
const keyPart = key.split(':').slice(1).join(':');
const rowData = { key: keyPart, ...parsedData };

if (
value === '*' ||
fields.some((field) =>
String(rowData[field] || '')
.toLowerCase()
.includes(String(value).toLowerCase()),
)
) {
const autocompleteRow: Record<string, unknown> = {};
fields.forEach((field) => {
if (rowData[field] !== undefined) {
autocompleteRow[field] = rowData[field];
}
});
rows.push(autocompleteRow);
processedCount++;
}
} catch (_error) {
continue;
}
} catch (_error) {
continue;
}
}

if (processedCount >= DAO_CONSTANTS.AUTOCOMPLETE_ROW_LIMIT) {
break;
if (processedCount >= DAO_CONSTANTS.AUTOCOMPLETE_ROW_LIMIT) {
break;
}
}
}
}
Expand All @@ -622,6 +646,7 @@ export class DataAccessObjectRedis extends BasicDataAccessObject implements IDat
filteringFields: Array<FilteringFieldsDS>,
searchedFieldValue: string,
settings: TableSettingsDS,
tableName: string,
): Promise<boolean> {
if (filteringFields?.length > 0) {
const filterResults: boolean[] = [];
Expand Down Expand Up @@ -681,7 +706,7 @@ export class DataAccessObjectRedis extends BasicDataAccessObject implements IDat
const searchFields =
settings.search_fields?.length > 0
? settings.search_fields
: await this.getTableStructure(row.key as string).then((structure) => structure.map((s) => s.column_name));
: await this.getTableStructure(tableName).then((structure) => structure.map((s) => s.column_name));

const passesSearch = searchFields.some((field) =>
String(row[field] || '')
Expand Down
Loading