From eb5fea4d7d884da387dabd3e2c0c15363b1070f7 Mon Sep 17 00:00:00 2001 From: Michal Turek Date: Wed, 8 Jul 2026 09:08:40 +0200 Subject: [PATCH 1/3] feat: Compress requests using brotli algo - https://github.com/apify/apify-core/pull/28971 added support for brotli compression to BE. - Pros: higher compression, cons: more CPU intensive. - The code could be running on too old Node.js, so brotli compression is applied only if possible; the code otherwise falls back to gzip. Too small payloads and unsupported types are still not compressed - no change. --- src/interceptors.ts | 14 ++++---- src/utils.ts | 67 +++++++++++++++++++++++++++-------- test/datasets.test.ts | 2 +- test/key_value_stores.test.ts | 4 +-- test/utils.test.ts | 30 +++++++++++++++- 5 files changed, 91 insertions(+), 26 deletions(-) diff --git a/src/interceptors.ts b/src/interceptors.ts index 4229ff3e0..dcbebe64e 100644 --- a/src/interceptors.ts +++ b/src/interceptors.ts @@ -5,7 +5,7 @@ import type { JsonObject } from 'type-fest'; import { maybeParseBody } from './body_parser'; import type { ApifyRequestConfig, ApifyResponse } from './http_client'; -import { isNode, maybeGzipValue } from './utils'; +import { isNode, maybeCompressValue } from './utils'; /** * This error exists for the quite common situation, where only a partial JSON response is received and @@ -78,14 +78,14 @@ function stringifyWithFunctions(obj: JsonObject) { }); } -async function maybeGzipRequest(config: ApifyRequestConfig): Promise { +async function maybeCompressRequest(config: ApifyRequestConfig): Promise { if (config.headers?.['content-encoding']) return config; - const maybeZippedData = await maybeGzipValue(config.data); - if (maybeZippedData) { + const maybeCompressed = await maybeCompressValue(config.data); + if (maybeCompressed) { config.headers ??= {}; - config.headers['content-encoding'] = 'gzip'; - config.data = maybeZippedData; + config.headers['content-encoding'] = maybeCompressed.encoding; + config.data = maybeCompressed.data; } return config; @@ -121,7 +121,7 @@ export type RequestInterceptorFunction = Parameters['use']>[0]; export const requestInterceptors: RequestInterceptorFunction[] = [ - maybeGzipRequest, + maybeCompressRequest, serializeRequest, ensureHeadersPrototype, ]; diff --git a/src/utils.ts b/src/utils.ts index 4f28b8075..f8ef3a6c8 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -13,7 +13,7 @@ import type { WebhookUpdateData } from './resource_clients/webhook'; const NOT_FOUND_STATUS_CODE = 404; const RECORD_NOT_FOUND_TYPE = 'record-not-found'; const RECORD_OR_TOKEN_NOT_FOUND_TYPE = 'record-or-token-not-found'; -const MIN_GZIP_BYTES = 1024; +const MIN_COMPRESS_BYTES = 1024; /** * Generic interface for objects that may contain a data property. @@ -112,28 +112,65 @@ export function stringifyWebhooksToBase64(webhooks: WebhookUpdateData[]): string let gzipPromisified: ((arg: string | Buffer) => Promise) | undefined; /** - * Gzip provided value, otherwise returns undefined. + * Gzip-compress the provided value. */ -export async function maybeGzipValue(value: unknown): Promise { - if (!isNode()) return; - if (typeof value !== 'string' && !Buffer.isBuffer(value)) return; +async function gzipValue(value: string | Buffer): Promise { + if (!gzipPromisified) { + const { promisify } = await import('node:util'); + const { gzip } = await import('node:zlib'); + gzipPromisified = promisify(gzip); + } - // Request compression is not that important so let's - // skip it instead of throwing for unsupported types. - const areDataLargeEnough = Buffer.byteLength(value as string) >= MIN_GZIP_BYTES; - if (areDataLargeEnough) { - if (!gzipPromisified) { - const { promisify } = await import('node:util'); - const { gzip } = await import('node:zlib'); - gzipPromisified = promisify(gzip); - } + return gzipPromisified(value); +} + +// null = confirmed unavailable; undefined = not yet checked +let brotliCompressPromisified: ((arg: string | Buffer) => Promise) | null | undefined; - return gzipPromisified(value); +/** + * Brotli-compress the provided value, or return undefined if brotli is unavailable + * (Node.js < v10.16.0), this is a strict defensive guard. + */ +async function maybeBrotliValue(value: string | Buffer): Promise { + if (brotliCompressPromisified === undefined) { + const { promisify } = await import('node:util'); + const { brotliCompress } = await import('node:zlib'); + brotliCompressPromisified = typeof brotliCompress === 'function' ? promisify(brotliCompress) : null; + } + + if (brotliCompressPromisified !== null) { + return brotliCompressPromisified(value); } return undefined; } +export interface CompressedValue { + data: Buffer; + encoding: 'br' | 'gzip'; +} + +/** + * Compress the passed value using brotli if available or using gzip as a fallback. Returns undefined + * if the data is too small / wrong type. + */ +export async function maybeCompressValue(value: unknown): Promise { + if (!isNode()) return undefined; + + // Request compression is not that important so let's + // skip it instead of throwing for unsupported types. + if (typeof value !== 'string' && !Buffer.isBuffer(value)) return undefined; + + const areDataLargeEnough = Buffer.byteLength(value) >= MIN_COMPRESS_BYTES; + if (!areDataLargeEnough) return undefined; + + const brotli = await maybeBrotliValue(value); + if (brotli) return { data: brotli, encoding: 'br' }; + + const gzipped = await gzipValue(value); + return { data: gzipped, encoding: 'gzip' }; +} + /** * Helper function slice the items from array to fit the max byte length. */ diff --git a/test/datasets.test.ts b/test/datasets.test.ts index c27a9a143..5c23b2d55 100644 --- a/test/datasets.test.ts +++ b/test/datasets.test.ts @@ -369,7 +369,7 @@ describe('Dataset methods', () => { const expectedHeaders = { 'content-type': 'application/json; charset=utf-8', - 'content-encoding': 'gzip', + 'content-encoding': 'br', }; const res = await client.dataset(datasetId).pushItems(data); diff --git a/test/key_value_stores.test.ts b/test/key_value_stores.test.ts index dd34757b9..077b9d224 100644 --- a/test/key_value_stores.test.ts +++ b/test/key_value_stores.test.ts @@ -557,7 +557,7 @@ describe('Key-Value Store methods', () => { validateRequest({ params: { storeId, key }, body: JSON.parse(value), additionalHeaders: expectedHeaders }); }); - test('setRecord() uploads gzipped buffer in node context', async () => { + test('setRecord() uploads compressed buffer in node context', async () => { const key = 'some-key'; const storeId = 'some-id'; const value = []; @@ -573,7 +573,7 @@ describe('Key-Value Store methods', () => { body: value, additionalHeaders: { 'content-type': contentType, - 'content-encoding': 'gzip', + 'content-encoding': 'br', }, }); diff --git a/test/utils.test.ts b/test/utils.test.ts index 9b3a3683f..1af4e5612 100644 --- a/test/utils.test.ts +++ b/test/utils.test.ts @@ -1,6 +1,6 @@ import type { WebhookUpdateData } from 'apify-client'; import { ApifyApiError } from 'apify-client'; -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; import * as utils from '../src/utils'; @@ -124,6 +124,34 @@ describe('utils.parseDateFields()', () => { }); }); +describe('utils.maybeCompressValue()', () => { + test('returns undefined for small values', async () => { + expect(await utils.maybeCompressValue('small')).toBeUndefined(); + }); + + test('returns undefined for non-string non-Buffer values', async () => { + expect(await utils.maybeCompressValue({ foo: 'bar' })).toBeUndefined(); + }); + + test('compresses large string using brotli in Node.js', async () => { + const largeValue = 'x'.repeat(2048); + const result = await utils.maybeCompressValue(largeValue); + expect(result).not.toBeUndefined(); + expect(result!.encoding).toBe('br'); + expect(result!.data).toBeInstanceOf(Buffer); + expect(result!.data.length).toBeLessThan(Buffer.byteLength(largeValue)); + }); + + test('compresses large Buffer using brotli in Node.js', async () => { + const largeValue = Buffer.alloc(2048, 'a'); + const result = await utils.maybeCompressValue(largeValue); + expect(result).not.toBeUndefined(); + expect(result!.encoding).toBe('br'); + expect(result!.data).toBeInstanceOf(Buffer); + expect(result!.data.length).toBeLessThan(largeValue.length); + }); +}); + describe('utils.stringifyWebhooksToBase64()', () => { test('works', () => { const webhooks: WebhookUpdateData[] = [ From 90b2c64c363f3ed041544461280577d6b5ed8445 Mon Sep 17 00:00:00 2001 From: Michal Turek Date: Wed, 8 Jul 2026 09:27:48 +0200 Subject: [PATCH 2/3] feat: Fix broken link ``` Errors in website/build/reference/next/class/ActorClient.html [404] https://docs.apify.com/api/v2/act-input-validate-post | Rejected status code: 404 Not Found (configurable with "accept" option) ``` --- src/resource_clients/actor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index c3c85d2e2..90e9bdd6e 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -269,7 +269,7 @@ export class ActorClient extends ResourceClient { * (e.g., `'latest'` or `'1.2.345'`). If not provided, uses the default build. * @param options.contentType - Content type of the input. If specified, input must be a string or Buffer. * @returns `true` if the input is valid. Invalid input causes the underlying API call to throw an `ApifyApiError`. - * @see https://docs.apify.com/api/v2/act-input-validate-post + * @see https://docs.apify.com/api/v2/act-validate-input-post * * @example * ```javascript From 72d9bf0daacd623d60f1a8258b92e8229c2f8254 Mon Sep 17 00:00:00 2001 From: Michal Turek Date: Wed, 8 Jul 2026 17:34:45 +0200 Subject: [PATCH 3/3] feat: Use brotli quality 6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - https://github.com/apify/apify-client-python/pull/927#discussion_r3543818071 - The default was 11 (max). Now it uses quality 6 - roughly 2–4× faster with only a modest increase in compressed size. --- src/utils.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index f8ef3a6c8..7fbc62424 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -134,8 +134,14 @@ let brotliCompressPromisified: ((arg: string | Buffer) => Promi async function maybeBrotliValue(value: string | Buffer): Promise { if (brotliCompressPromisified === undefined) { const { promisify } = await import('node:util'); - const { brotliCompress } = await import('node:zlib'); - brotliCompressPromisified = typeof brotliCompress === 'function' ? promisify(brotliCompress) : null; + const { brotliCompress, constants } = await import('node:zlib'); + if (typeof brotliCompress === 'function') { + const compress = promisify(brotliCompress); + brotliCompressPromisified = async (value) => + compress(value, { params: { [constants.BROTLI_PARAM_QUALITY]: 6 } }); + } else { + brotliCompressPromisified = null; + } } if (brotliCompressPromisified !== null) {