feat: Compress requests using brotli algo

src/interceptors.ts

@@ -5,7 +5,7 @@ import type { JsonObject } from 'type-fest';
 
 import { maybeParseBody } from './body_parser';
 import type { ApifyRequestConfig, ApifyResponse } from './http_client';
-import { isNode, maybeGzipValue } from './utils';
+import { isNode, maybeCompressValue } from './utils';
 
 /**
  * This error exists for the quite common situation, where only a partial JSON response is received and
@@ -78,14 +78,14 @@ function stringifyWithFunctions(obj: JsonObject) {
     });
 }
 
-async function maybeGzipRequest(config: ApifyRequestConfig): Promise<ApifyRequestConfig> {
+async function maybeCompressRequest(config: ApifyRequestConfig): Promise<ApifyRequestConfig> {
     if (config.headers?.['content-encoding']) return config;
 
-    const maybeZippedData = await maybeGzipValue(config.data);
-    if (maybeZippedData) {
+    const maybeCompressed = await maybeCompressValue(config.data);
+    if (maybeCompressed) {
         config.headers ??= {};
-        config.headers['content-encoding'] = 'gzip';
-        config.data = maybeZippedData;
+        config.headers['content-encoding'] = maybeCompressed.encoding;
+        config.data = maybeCompressed.data;
     }
 
     return config;
@@ -121,7 +121,7 @@ export type RequestInterceptorFunction = Parameters<AxiosInterceptorManager<Apif
 export type ResponseInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyResponse>['use']>[0];
 
 export const requestInterceptors: RequestInterceptorFunction[] = [
-    maybeGzipRequest,
+    maybeCompressRequest,
     serializeRequest,
     ensureHeadersPrototype,
 ];

src/resource_clients/actor.ts

@@ -269,7 +269,7 @@ export class ActorClient extends ResourceClient {
      *                         (e.g., `'latest'` or `'1.2.345'`). If not provided, uses the default build.
      * @param options.contentType - Content type of the input. If specified, input must be a string or Buffer.
      * @returns `true` if the input is valid. Invalid input causes the underlying API call to throw an `ApifyApiError`.
-     * @see https://docs.apify.com/api/v2/act-input-validate-post
+     * @see https://docs.apify.com/api/v2/act-validate-input-post
      *
      * @example
      * ```javascript

src/utils.ts

@@ -13,7 +13,7 @@ import type { WebhookUpdateData } from './resource_clients/webhook';
 const NOT_FOUND_STATUS_CODE = 404;
 const RECORD_NOT_FOUND_TYPE = 'record-not-found';
 const RECORD_OR_TOKEN_NOT_FOUND_TYPE = 'record-or-token-not-found';
-const MIN_GZIP_BYTES = 1024;
+const MIN_COMPRESS_BYTES = 1024;
 
 /**
  * Generic interface for objects that may contain a data property.
@@ -112,28 +112,71 @@ export function stringifyWebhooksToBase64(webhooks: WebhookUpdateData[]): string
 let gzipPromisified: ((arg: string | Buffer<ArrayBufferLike>) => Promise<Buffer>) | undefined;
 
 /**
- * Gzip provided value, otherwise returns undefined.
+ * Gzip-compress the provided value.
  */
-export async function maybeGzipValue(value: unknown): Promise<Buffer | undefined> {
-    if (!isNode()) return;
-    if (typeof value !== 'string' && !Buffer.isBuffer(value)) return;
+async function gzipValue(value: string | Buffer<ArrayBufferLike>): Promise<Buffer> {
+    if (!gzipPromisified) {
+        const { promisify } = await import('node:util');
+        const { gzip } = await import('node:zlib');
+        gzipPromisified = promisify(gzip);
+    }
 
-    // Request compression is not that important so let's
-    // skip it instead of throwing for unsupported types.
-    const areDataLargeEnough = Buffer.byteLength(value as string) >= MIN_GZIP_BYTES;
-    if (areDataLargeEnough) {
-        if (!gzipPromisified) {
-            const { promisify } = await import('node:util');
-            const { gzip } = await import('node:zlib');
-            gzipPromisified = promisify(gzip);
+    return gzipPromisified(value);
+}
+
+// null = confirmed unavailable; undefined = not yet checked
+let brotliCompressPromisified: ((arg: string | Buffer<ArrayBufferLike>) => Promise<Buffer>) | null | undefined;
+
+/**
+ * Brotli-compress the provided value, or return undefined if brotli is unavailable
+ * (Node.js < v10.16.0), this is a strict defensive guard.
+ */
+async function maybeBrotliValue(value: string | Buffer<ArrayBufferLike>): Promise<Buffer | undefined> {
+    if (brotliCompressPromisified === undefined) {
+        const { promisify } = await import('node:util');
+        const { brotliCompress, constants } = await import('node:zlib');
+        if (typeof brotliCompress === 'function') {
+            const compress = promisify(brotliCompress);
+            brotliCompressPromisified = async (value) =>
+                compress(value, { params: { [constants.BROTLI_PARAM_QUALITY]: 6 } });
+        } else {
+            brotliCompressPromisified = null;
         }
+    }
 
-        return gzipPromisified(value);
+    if (brotliCompressPromisified !== null) {
+        return brotliCompressPromisified(value);
     }
 
     return undefined;
 }
 
+export interface CompressedValue {
+    data: Buffer;
+    encoding: 'br' | 'gzip';
+}
+
+/**
+ * Compress the passed value using brotli if available or using gzip as a fallback. Returns undefined
+ * if the data is too small / wrong type.
+ */
+export async function maybeCompressValue(value: unknown): Promise<CompressedValue | undefined> {
+    if (!isNode()) return undefined;
+
+    // Request compression is not that important so let's
+    // skip it instead of throwing for unsupported types.
+    if (typeof value !== 'string' && !Buffer.isBuffer(value)) return undefined;
+
+    const areDataLargeEnough = Buffer.byteLength(value) >= MIN_COMPRESS_BYTES;
+    if (!areDataLargeEnough) return undefined;
+
+    const brotli = await maybeBrotliValue(value);
+    if (brotli) return { data: brotli, encoding: 'br' };
+
+    const gzipped = await gzipValue(value);
+    return { data: gzipped, encoding: 'gzip' };
+}
+
 /**
  * Helper function slice the items from array to fit the max byte length.
  */

test/datasets.test.ts

@@ -369,7 +369,7 @@ describe('Dataset methods', () => {
 
             const expectedHeaders = {
                 'content-type': 'application/json; charset=utf-8',
-                'content-encoding': 'gzip',
+                'content-encoding': 'br',
             };
 
             const res = await client.dataset(datasetId).pushItems(data);

test/key_value_stores.test.ts

@@ -557,7 +557,7 @@ describe('Key-Value Store methods', () => {
             validateRequest({ params: { storeId, key }, body: JSON.parse(value), additionalHeaders: expectedHeaders });
         });
 
-        test('setRecord() uploads gzipped buffer in node context', async () => {
+        test('setRecord() uploads compressed buffer in node context', async () => {
             const key = 'some-key';
             const storeId = 'some-id';
             const value = [];
@@ -573,7 +573,7 @@ describe('Key-Value Store methods', () => {
                 body: value,
                 additionalHeaders: {
                     'content-type': contentType,
-                    'content-encoding': 'gzip',
+                    'content-encoding': 'br',
                 },
             });
 

test/utils.test.ts

@@ -1,6 +1,6 @@
 import type { WebhookUpdateData } from 'apify-client';
 import { ApifyApiError } from 'apify-client';
-import { describe, expect, test } from 'vitest';
+import { describe, expect, test, vi } from 'vitest';
 
 import * as utils from '../src/utils';
 
@@ -124,6 +124,34 @@ describe('utils.parseDateFields()', () => {
     });
 });
 
+describe('utils.maybeCompressValue()', () => {
+    test('returns undefined for small values', async () => {
+        expect(await utils.maybeCompressValue('small')).toBeUndefined();
+    });
+
+    test('returns undefined for non-string non-Buffer values', async () => {
+        expect(await utils.maybeCompressValue({ foo: 'bar' })).toBeUndefined();
+    });
+
+    test('compresses large string using brotli in Node.js', async () => {
+        const largeValue = 'x'.repeat(2048);
+        const result = await utils.maybeCompressValue(largeValue);
+        expect(result).not.toBeUndefined();
+        expect(result!.encoding).toBe('br');
+        expect(result!.data).toBeInstanceOf(Buffer);
+        expect(result!.data.length).toBeLessThan(Buffer.byteLength(largeValue));
+    });
+
+    test('compresses large Buffer using brotli in Node.js', async () => {
+        const largeValue = Buffer.alloc(2048, 'a');
+        const result = await utils.maybeCompressValue(largeValue);
+        expect(result).not.toBeUndefined();
+        expect(result!.encoding).toBe('br');
+        expect(result!.data).toBeInstanceOf(Buffer);
+        expect(result!.data.length).toBeLessThan(largeValue.length);
+    });
+});
+
 describe('utils.stringifyWebhooksToBase64()', () => {
     test('works', () => {
         const webhooks: WebhookUpdateData[] = [