diff --git a/src/interceptors.ts b/src/interceptors.ts index 4229ff3e..dcbebe64 100644 --- a/src/interceptors.ts +++ b/src/interceptors.ts @@ -5,7 +5,7 @@ import type { JsonObject } from 'type-fest'; import { maybeParseBody } from './body_parser'; import type { ApifyRequestConfig, ApifyResponse } from './http_client'; -import { isNode, maybeGzipValue } from './utils'; +import { isNode, maybeCompressValue } from './utils'; /** * This error exists for the quite common situation, where only a partial JSON response is received and @@ -78,14 +78,14 @@ function stringifyWithFunctions(obj: JsonObject) { }); } -async function maybeGzipRequest(config: ApifyRequestConfig): Promise { +async function maybeCompressRequest(config: ApifyRequestConfig): Promise { if (config.headers?.['content-encoding']) return config; - const maybeZippedData = await maybeGzipValue(config.data); - if (maybeZippedData) { + const maybeCompressed = await maybeCompressValue(config.data); + if (maybeCompressed) { config.headers ??= {}; - config.headers['content-encoding'] = 'gzip'; - config.data = maybeZippedData; + config.headers['content-encoding'] = maybeCompressed.encoding; + config.data = maybeCompressed.data; } return config; @@ -121,7 +121,7 @@ export type RequestInterceptorFunction = Parameters['use']>[0]; export const requestInterceptors: RequestInterceptorFunction[] = [ - maybeGzipRequest, + maybeCompressRequest, serializeRequest, ensureHeadersPrototype, ]; diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index c3c85d2e..90e9bdd6 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -269,7 +269,7 @@ export class ActorClient extends ResourceClient { * (e.g., `'latest'` or `'1.2.345'`). If not provided, uses the default build. * @param options.contentType - Content type of the input. If specified, input must be a string or Buffer. * @returns `true` if the input is valid. Invalid input causes the underlying API call to throw an `ApifyApiError`. - * @see https://docs.apify.com/api/v2/act-input-validate-post + * @see https://docs.apify.com/api/v2/act-validate-input-post * * @example * ```javascript diff --git a/src/utils.ts b/src/utils.ts index 4f28b807..7fbc6242 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -13,7 +13,7 @@ import type { WebhookUpdateData } from './resource_clients/webhook'; const NOT_FOUND_STATUS_CODE = 404; const RECORD_NOT_FOUND_TYPE = 'record-not-found'; const RECORD_OR_TOKEN_NOT_FOUND_TYPE = 'record-or-token-not-found'; -const MIN_GZIP_BYTES = 1024; +const MIN_COMPRESS_BYTES = 1024; /** * Generic interface for objects that may contain a data property. @@ -112,28 +112,71 @@ export function stringifyWebhooksToBase64(webhooks: WebhookUpdateData[]): string let gzipPromisified: ((arg: string | Buffer) => Promise) | undefined; /** - * Gzip provided value, otherwise returns undefined. + * Gzip-compress the provided value. */ -export async function maybeGzipValue(value: unknown): Promise { - if (!isNode()) return; - if (typeof value !== 'string' && !Buffer.isBuffer(value)) return; +async function gzipValue(value: string | Buffer): Promise { + if (!gzipPromisified) { + const { promisify } = await import('node:util'); + const { gzip } = await import('node:zlib'); + gzipPromisified = promisify(gzip); + } - // Request compression is not that important so let's - // skip it instead of throwing for unsupported types. - const areDataLargeEnough = Buffer.byteLength(value as string) >= MIN_GZIP_BYTES; - if (areDataLargeEnough) { - if (!gzipPromisified) { - const { promisify } = await import('node:util'); - const { gzip } = await import('node:zlib'); - gzipPromisified = promisify(gzip); + return gzipPromisified(value); +} + +// null = confirmed unavailable; undefined = not yet checked +let brotliCompressPromisified: ((arg: string | Buffer) => Promise) | null | undefined; + +/** + * Brotli-compress the provided value, or return undefined if brotli is unavailable + * (Node.js < v10.16.0), this is a strict defensive guard. + */ +async function maybeBrotliValue(value: string | Buffer): Promise { + if (brotliCompressPromisified === undefined) { + const { promisify } = await import('node:util'); + const { brotliCompress, constants } = await import('node:zlib'); + if (typeof brotliCompress === 'function') { + const compress = promisify(brotliCompress); + brotliCompressPromisified = async (value) => + compress(value, { params: { [constants.BROTLI_PARAM_QUALITY]: 6 } }); + } else { + brotliCompressPromisified = null; } + } - return gzipPromisified(value); + if (brotliCompressPromisified !== null) { + return brotliCompressPromisified(value); } return undefined; } +export interface CompressedValue { + data: Buffer; + encoding: 'br' | 'gzip'; +} + +/** + * Compress the passed value using brotli if available or using gzip as a fallback. Returns undefined + * if the data is too small / wrong type. + */ +export async function maybeCompressValue(value: unknown): Promise { + if (!isNode()) return undefined; + + // Request compression is not that important so let's + // skip it instead of throwing for unsupported types. + if (typeof value !== 'string' && !Buffer.isBuffer(value)) return undefined; + + const areDataLargeEnough = Buffer.byteLength(value) >= MIN_COMPRESS_BYTES; + if (!areDataLargeEnough) return undefined; + + const brotli = await maybeBrotliValue(value); + if (brotli) return { data: brotli, encoding: 'br' }; + + const gzipped = await gzipValue(value); + return { data: gzipped, encoding: 'gzip' }; +} + /** * Helper function slice the items from array to fit the max byte length. */ diff --git a/test/datasets.test.ts b/test/datasets.test.ts index c27a9a14..5c23b2d5 100644 --- a/test/datasets.test.ts +++ b/test/datasets.test.ts @@ -369,7 +369,7 @@ describe('Dataset methods', () => { const expectedHeaders = { 'content-type': 'application/json; charset=utf-8', - 'content-encoding': 'gzip', + 'content-encoding': 'br', }; const res = await client.dataset(datasetId).pushItems(data); diff --git a/test/key_value_stores.test.ts b/test/key_value_stores.test.ts index dd34757b..077b9d22 100644 --- a/test/key_value_stores.test.ts +++ b/test/key_value_stores.test.ts @@ -557,7 +557,7 @@ describe('Key-Value Store methods', () => { validateRequest({ params: { storeId, key }, body: JSON.parse(value), additionalHeaders: expectedHeaders }); }); - test('setRecord() uploads gzipped buffer in node context', async () => { + test('setRecord() uploads compressed buffer in node context', async () => { const key = 'some-key'; const storeId = 'some-id'; const value = []; @@ -573,7 +573,7 @@ describe('Key-Value Store methods', () => { body: value, additionalHeaders: { 'content-type': contentType, - 'content-encoding': 'gzip', + 'content-encoding': 'br', }, }); diff --git a/test/utils.test.ts b/test/utils.test.ts index 9b3a3683..1af4e561 100644 --- a/test/utils.test.ts +++ b/test/utils.test.ts @@ -1,6 +1,6 @@ import type { WebhookUpdateData } from 'apify-client'; import { ApifyApiError } from 'apify-client'; -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; import * as utils from '../src/utils'; @@ -124,6 +124,34 @@ describe('utils.parseDateFields()', () => { }); }); +describe('utils.maybeCompressValue()', () => { + test('returns undefined for small values', async () => { + expect(await utils.maybeCompressValue('small')).toBeUndefined(); + }); + + test('returns undefined for non-string non-Buffer values', async () => { + expect(await utils.maybeCompressValue({ foo: 'bar' })).toBeUndefined(); + }); + + test('compresses large string using brotli in Node.js', async () => { + const largeValue = 'x'.repeat(2048); + const result = await utils.maybeCompressValue(largeValue); + expect(result).not.toBeUndefined(); + expect(result!.encoding).toBe('br'); + expect(result!.data).toBeInstanceOf(Buffer); + expect(result!.data.length).toBeLessThan(Buffer.byteLength(largeValue)); + }); + + test('compresses large Buffer using brotli in Node.js', async () => { + const largeValue = Buffer.alloc(2048, 'a'); + const result = await utils.maybeCompressValue(largeValue); + expect(result).not.toBeUndefined(); + expect(result!.encoding).toBe('br'); + expect(result!.data).toBeInstanceOf(Buffer); + expect(result!.data.length).toBeLessThan(largeValue.length); + }); +}); + describe('utils.stringifyWebhooksToBase64()', () => { test('works', () => { const webhooks: WebhookUpdateData[] = [