From 152c3c6c203595c508cae2a1e9ae2f17e2438b1d Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Thu, 9 Apr 2026 12:00:48 -0700 Subject: [PATCH 01/14] docs: add go docs for payload storage --- .../data-handling/data-conversion.mdx | 66 ++++++ .../data-encryption.mdx} | 79 +------ .../go/best-practices/data-handling/index.mdx | 35 +++ .../data-handling/large-payload-storage.mdx | 202 ++++++++++++++++++ docs/develop/go/best-practices/index.mdx | 2 +- docs/develop/go/index.mdx | 2 +- .../data-conversion/external-storage.mdx | 10 +- .../data-encryption.mdx | 2 +- sidebars.js | 15 +- 9 files changed, 335 insertions(+), 78 deletions(-) create mode 100644 docs/develop/go/best-practices/data-handling/data-conversion.mdx rename docs/develop/go/best-practices/{converters-and-encryption.mdx => data-handling/data-encryption.mdx} (60%) create mode 100644 docs/develop/go/best-practices/data-handling/index.mdx create mode 100644 docs/develop/go/best-practices/data-handling/large-payload-storage.mdx diff --git a/docs/develop/go/best-practices/data-handling/data-conversion.mdx b/docs/develop/go/best-practices/data-handling/data-conversion.mdx new file mode 100644 index 0000000000..746ac3628b --- /dev/null +++ b/docs/develop/go/best-practices/data-handling/data-conversion.mdx @@ -0,0 +1,66 @@ +--- +id: data-conversion +title: Payload conversion - Go SDK +sidebar_label: Payload conversion +slug: /develop/go/data-handling/data-conversion +toc_max_heading_level: 2 +tags: + - Data Converters + - Go SDK + - Temporal SDKs +description: Customize how Temporal serializes application objects using Payload Converters in the Go SDK, including composite data converters and custom type examples. +--- + +import { CaptionedImage } from '@site/src/components'; + +Temporal SDKs provide a default [Payload Converter](/payload-converter) that can be customized to convert a custom data type to [Payload](/dataconversion#payload) and back. + +The order in which your encoding Payload Converters are applied depend on the order given to the Data Converter. +You can set multiple encoding Payload Converters to run your conversions. +When the Data Converter receives a value for conversion, it passes through each Payload Converter in sequence until the converter that handles the data type does the conversion. + +Payload Converters can be customized independently of a Payload Codec. +Temporal's Converter architecture looks like this: + + + +## Use a custom Payload Converter {#custom-payload-converter} + +Use a [Composite Data Converter](https://pkg.go.dev/go.temporal.io/sdk/converter#CompositeDataConverter) to apply custom, type-specific Payload Converters in a specified order. +Defining a new Composite Data Converter is not always necessary to implement custom data handling. +You can override the default Converter with a custom Codec, but a Composite Data Converter may be necessary for complex Workflow logic. + +`NewCompositeDataConverter` creates a new instance of `CompositeDataConverter` from an ordered list of type-specific Payload Converters. +The following type-specific Payload Converters are available in the Go SDK, listed in the order that they are applied by the default Data Converter: + +- [NewNilPayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#NilPayloadConverter.ToString) +- [NewByteSlicePayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#ByteSlicePayloadConverter) +- [NewProtoJSONPayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#ProtoJSONPayloadConverter) +- [NewProtoPayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#ProtoPayloadConverter) +- [NewJSONPayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#JSONPayloadConverter) + +The order in which the Payload Converters are applied is important because during serialization the Data Converter tries the Payload Converters in that specific order until a Payload Converter returns a non-nil Payload. + +To set your custom Payload Converter, use [`NewCompositeDataConverter`](https://pkg.go.dev/go.temporal.io/sdk/converter#NewCompositeDataConverter) and set it as the Data Converter in the Client options. + +- To replace the default Data Converter with a custom `NewCompositeDataConverter`, use the following. + + ```go + dataConverter := converter.NewCompositeDataConverter(YourCustomPayloadConverter()) + ``` + +- To add your custom type conversion to the default Data Converter, use the following to keep the defaults but set yours just before the default JSON fall through. + + ```go + dataConverter := converter.NewCompositeDataConverter( + converter.NewNilPayloadConverter(), + converter.NewByteSlicePayloadConverter(), + converter.NewProtoJSONPayloadConverter(), + converter.NewProtoPayloadConverter(), + YourCustomPayloadConverter(), + converter.NewJSONPayloadConverter(), + ) + ``` diff --git a/docs/develop/go/best-practices/converters-and-encryption.mdx b/docs/develop/go/best-practices/data-handling/data-encryption.mdx similarity index 60% rename from docs/develop/go/best-practices/converters-and-encryption.mdx rename to docs/develop/go/best-practices/data-handling/data-encryption.mdx index ecf3af6e13..201a6e1059 100644 --- a/docs/develop/go/best-practices/converters-and-encryption.mdx +++ b/docs/develop/go/best-practices/data-handling/data-encryption.mdx @@ -1,29 +1,22 @@ --- -id: converters-and-encryption -title: Converters and encryption - Go SDK -sidebar_label: Converters and encryption -description: Use a custom Payload Codec and Payload Converter in Go. Create custom PayloadCodec implementations, set Data Converters, and apply transformations effectively using the Temporal SDK. -toc_max_heading_level: 4 -keywords: - - go sdk - - data converter - - payload conversion - - payload converter +id: data-encryption +title: Payload encryption - Go SDK +sidebar_label: Payload encryption +slug: /develop/go/data-handling/data-encryption +toc_max_heading_level: 2 tags: - Security - - Codec Server - - Data Converters - Encryption + - Codec Server - Go SDK - Temporal SDKs +description: Encrypt data sent to and from the Temporal Service using a custom Payload Codec in the Go SDK. --- -import { CaptionedImage } from '@site/src/components'; - Temporal's security model is designed around client-side encryption of Payloads. A client may encrypt Payloads before sending them to the server, and decrypt them after receiving them from the server. This provides a high degree of confidentiality because the Temporal Server itself has absolutely no knowledge of the actual data. -It also gives implementers more power and more freedom regarding which client is able to read which data -- they can control access with keys, algorithms, or other security measures. +It also gives implementers more power and more freedom regarding which client is able to read which data. Implementers can control access with keys, algorithms, or other security measures. A Temporal developer adds client-side encryption of Payloads by providing a Custom Payload Codec to its Client. Depending on business needs, a complete implementation of Payload Encryption may involve selecting appropriate encryption algorithms, managing encryption keys, restricting a subset of their users from viewing payload output, or a combination of these. @@ -32,7 +25,7 @@ The server itself never adds encryption over Payloads. Therefore, unless client-side encryption is implemented, Payload data will be persisted in non-encrypted form to the data store, and any Client that can make requests to a Temporal namespace (including the Temporal UI and CLI) will be able to read Payloads contained in Workflows. When working with sensitive data, you should always implement Payload encryption. -## Use a custom Payload Codec in Go {#custom-payload-codec} +## Use a custom Payload Codec {#custom-payload-codec} **Step 1: Create a custom Payload Codec** @@ -138,57 +131,3 @@ The Temporal CLI and the Web UI in turn provide built-in hooks to call the Codec Refer to the [Codec Server](/production-deployment/data-encryption) documentation for information on how to design and deploy a Codec Server. For reference, see the [Codec server](https://github.com/temporalio/samples-go/tree/main/codec-server) sample. - -## Use custom Payload conversion {#custom-payload-conversion} - -Temporal SDKs provide a default [Payload Converter](/payload-converter) that can be customized to convert a custom data type to [Payload](/dataconversion#payload) and back. - -The order in which your encoding Payload Converters are applied depend on the order given to the Data Converter. -You can set multiple encoding Payload Converters to run your conversions. -When the Data Converter receives a value for conversion, it passes through each Payload Converter in sequence until the converter that handles the data type does the conversion. - -Payload Converters can be customized independently of a Payload Codec. -Temporal's Converter architecture looks like this: - - - -## How to use a custom Payload Converter in Go {#custom-payload-converter} - -Use a [Composite Data Converter](https://pkg.go.dev/go.temporal.io/sdk/converter#CompositeDataConverter) to apply custom, type-specific Payload Converters in a specified order. -Defining a new Composite Data Converter is not always necessary to implement custom data handling. -You can override the default Converter with a custom Codec, but a Composite Data Converter may be necessary for complex Workflow logic. - -`NewCompositeDataConverter` creates a new instance of `CompositeDataConverter` from an ordered list of type-specific Payload Converters. -The following type-specific Payload Converters are available in the Go SDK, listed in the order that they are applied by the default Data Converter: - -- [NewNilPayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#NilPayloadConverter.ToString) -- [NewByteSlicePayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#ByteSlicePayloadConverter) -- [NewProtoJSONPayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#ProtoJSONPayloadConverter) -- [NewProtoPayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#ProtoPayloadConverter) -- [NewJSONPayloadConverter()](https://pkg.go.dev/go.temporal.io/sdk/converter#JSONPayloadConverter) - -The order in which the Payload Converters are applied is important because during serialization the Data Converter tries the Payload Converters in that specific order until a Payload Converter returns a non-nil Payload. - -To set your custom Payload Converter, use [`NewCompositeDataConverter`](https://pkg.go.dev/go.temporal.io/sdk/converter#NewCompositeDataConverter) and set it as the Data Converter in the Client options. - -- To replace the default Data Converter with a custom `NewCompositeDataConverter`, use the following. - - ```go - dataConverter := converter.NewCompositeDataConverter(YourCustomPayloadConverter()) - ``` - -- To add your custom type conversion to the default Data Converter, use the following to keep the defaults but set yours just before the default JSON fall through. - - ```go - dataConverter := converter.NewCompositeDataConverter( - converter.NewNilPayloadConverter(), - converter.NewByteSlicePayloadConverter(), - converter.NewProtoJSONPayloadConverter(), - converter.NewProtoPayloadConverter(), - YourCustomPayloadConverter(), - converter.NewJSONPayloadConverter(), - ) - ``` diff --git a/docs/develop/go/best-practices/data-handling/index.mdx b/docs/develop/go/best-practices/data-handling/index.mdx new file mode 100644 index 0000000000..4d47ef5717 --- /dev/null +++ b/docs/develop/go/best-practices/data-handling/index.mdx @@ -0,0 +1,35 @@ +--- +id: data-handling +title: Data handling - Go SDK +sidebar_label: Data handling +slug: /develop/go/data-handling +description: + Learn how Temporal handles data through the Data Converter, including payload conversion, encryption, and large + payload storage. +toc_max_heading_level: 3 +tags: + - Go SDK + - Temporal SDKs + - Data Converters +--- + +import { CaptionedImage } from '@site/src/components'; + +All data sent to and from the Temporal Service passes through the **Data Converter**. The Data Converter has three +layers that handle different concerns: + + + +Of these three layers, only the PayloadConverter is required. Temporal uses a default PayloadConverter that handles JSON +serialization. The PayloadCodec and ExternalStorage layers are optional. You only need to customize these layers when +your application requires non-JSON types, encryption, or payload offloading. + +| | [PayloadConverter](/develop/go/data-handling/data-conversion) | [PayloadCodec](/develop/go/data-handling/data-encryption) | [ExternalStorage](/develop/go/data-handling/large-payload-storage) | +| ------------------------- | ------------------------------------------------------------- | --------------------------------------------------------- | ------------------------------------------------------------------ | +| **Purpose** | Serialize application data to bytes | Transform encoded payloads (encrypt, compress) | Offload large payloads to external store | +| **Default** | JSON serialization | None (passthrough) | None (all payloads are stored in Workflow History) | + +For a deeper conceptual explanation, see the [Data Conversion encyclopedia](/dataconversion) and [External Storage](/external-storage). diff --git a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx new file mode 100644 index 0000000000..01496f1481 --- /dev/null +++ b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx @@ -0,0 +1,202 @@ +--- +id: large-payload-storage +title: External Storage - Go SDK +sidebar_label: External Storage +slug: /develop/go/data-handling/large-payload-storage +toc_max_heading_level: 3 +tags: + - Go SDK + - Temporal SDKs + - Data Converters +description: Offload large payloads to external storage using the claim check pattern in the Go SDK. +--- + +:::info Release, stability, and dependency info + +External Storage is in [Pre-Release](/evaluate/development-production-features/release-stages#pre-release). APIs and +configuration may change before the stable release. Join the +[#large-payloads Slack channel](https://temporalio.slack.com/archives/C09VA2DE15Y) to provide feedback or ask for help. + +::: + +The Temporal Service enforces a 2 MB per-payload limit by default. This limit is configurable on self-hosted +deployments. When your Workflows or Activities handle data larger than the limit, you can offload payloads to external +storage, such as Amazon S3, and pass a small reference token through the Event History instead. This page shows you how +to set up External Storage with Amazon S3 and how to implement a custom storage driver. + +For a conceptual overview of External Storage and its use cases, see [External Storage](/external-storage). + +## Store and retrieve large payloads with Amazon S3 + +The Go SDK includes an S3 storage driver. Follow these steps to set it up: + +### Prerequisites + +- An Amazon S3 bucket that you have read and write access to. Refer to [lifecycle management](/external-storage#lifecycle) + to ensure that your payloads remain available for the entire lifetime of the Workflow. +- Install the S3 driver module: `go get go.temporal.io/sdk/contrib/aws/s3driver` + +### Procedure + +1. Load your AWS configuration and create the S3 storage driver. The driver uses your standard AWS + credentials from the environment (environment variables, IAM role, or AWS config file): + + + + ```go + import ( + "github.com/aws/aws-sdk-go-v2/config" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "go.temporal.io/sdk/contrib/aws/s3driver" + "go.temporal.io/sdk/contrib/aws/s3driver/awssdkv2" + ) + + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion("us-east-2"), + ) + if err != nil { + log.Fatalf("load AWS config: %v", err) + } + + driver, err := s3driver.NewDriver(s3driver.Options{ + Client: awssdkv2.NewClient(awss3.NewFromConfig(cfg)), + Bucket: s3driver.StaticBucket("my-temporal-payloads"), + }) + if err != nil { + log.Fatalf("create S3 driver: %v", err) + } + ``` + + + +2. Configure the driver on `ExternalStorage` and pass it in your Client options: + + + + ```go + import ( + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/converter" + "go.temporal.io/sdk/worker" + ) + + c, err := client.Dial(client.Options{ + HostPort: "localhost:7233", + ExternalStorage: converter.ExternalStorage{ + Drivers: []converter.StorageDriver{driver}, + }, + }) + if err != nil { + log.Fatalf("connect to Temporal: %v", err) + } + defer c.Close() + + w := worker.New(c, "my-task-queue", worker.Options{}) + ``` + + + + By default, payloads larger than 256 KiB are offloaded to external storage. You can adjust this with the + `PayloadSizeThreshold` option, even setting it to 0 to externalize all payloads regardless of size. Refer to + [Configure payload size threshold](#configure-payload-size-threshold) for more information. + + All Workflows and Activities running on the Worker use the storage driver automatically without changes to your + business logic. The driver uploads and downloads payloads concurrently and validates payload integrity on retrieve. + +## Implement a custom storage driver + +If you need a storage backend other than what the built-in drivers allow, you can implement your own storage driver. +Store payloads durably so that they survive process crashes and remain available for debugging and auditing after the +Workflow completes. Refer to [lifecycle management](/external-storage#lifecycle) for retention requirements. + +A custom driver implements the `converter.StorageDriver` interface, which has four methods: + +- `Name()` returns a unique string that identifies the driver. The SDK stores this name in the claim check reference so + it can route retrieval requests to the correct driver. Changing the name after payloads have been stored breaks retrieval. +- `Store()` receives a slice of payloads and returns one `StorageDriverClaim` per payload. A claim is a set of string + key-value pairs that the driver uses to locate the payload later. +- `Retrieve()` receives the claims that `Store()` produced and returns the original payloads. +- `Options()` returns the driver's configuration options. + +### Store payloads + +In `Store()`, marshal each Payload protobuf message to bytes with `proto.Marshal(payload)` and write the bytes to +your storage system. The application data has already been serialized by the [Payload Converter](/develop/go/data-handling/data-conversion) +and [Payload Codec](/develop/go/data-handling/data-encryption) before it reaches the driver. +See the [data conversion pipeline](/external-storage#data-pipeline) for more details. + +Return a `StorageDriverClaim` for each payload with enough information to retrieve it later. The `context` parameter +provides identity information (namespace, Workflow ID) depending on the operation. Consider structuring +your storage keys to include this information so that you can identify which Workflow owns each payload. + +### Retrieve payloads + +In `Retrieve()`, download the bytes using the claim data, then reconstruct the Payload protobuf message with +`proto.Unmarshal(data, payload)`. The Payload Converter handles deserializing the application data after the driver +returns the payload. + +### Configure the Client + +Pass an `ExternalStorage` struct with your driver in the Client options: + +```go +c, err := client.Dial(client.Options{ + ExternalStorage: converter.ExternalStorage{ + Drivers: []converter.StorageDriver{myCustomDriver}, + }, +}) +``` + +## Configure payload size threshold + +You can configure the payload size threshold that triggers external storage. By default, payloads larger than 256 KiB +are offloaded to external storage. You can adjust this with the `PayloadSizeThreshold` option, or set it to 0 to +externalize all payloads regardless of size. + + + +```go +import "go.temporal.io/sdk/converter" + +c, err := client.Dial(client.Options{ + ExternalStorage: converter.ExternalStorage{ + Drivers: []converter.StorageDriver{driver}, + PayloadSizeThreshold: 0, + }, +}) +``` + + + +## Use multiple storage drivers + +When you register multiple drivers, you must provide a `DriverSelector` function that chooses which driver stores each +payload. Any driver in the list that is not selected for storing is still available for retrieval, which is useful when +migrating between storage backends. Return `nil` from the selector to keep a specific payload inline in Event History. + +Multiple drivers are useful in scenarios such as: + +- Driver migration. Your Worker needs to retrieve payloads created by clients that use a different driver than the + one you prefer. Register both drivers and use the selector to always pick your preferred driver for new payloads. + The old driver remains available for retrieving existing claims. +- Latency vs. durability tradeoffs. Some Workflow types may benefit from a faster storage backend + at the cost of durability, while others require a durable backend like S3. Use the selector to route based on + Workflow context. + +The following example registers two drivers but always selects `preferredDriver` for new payloads. The `legacyDriver` +is only registered so the Worker can retrieve payloads that were previously stored with it: + + + +```go +import "go.temporal.io/sdk/converter" + +converter.ExternalStorage{ + Drivers: []converter.StorageDriver{preferredDriver, legacyDriver}, + DriverSelector: func(ctx converter.StorageDriverStoreContext, payload *commonpb.Payload) converter.StorageDriver { + return preferredDriver + }, +} +``` + + diff --git a/docs/develop/go/best-practices/index.mdx b/docs/develop/go/best-practices/index.mdx index 0d6c322d2d..9bf37acbcd 100644 --- a/docs/develop/go/best-practices/index.mdx +++ b/docs/develop/go/best-practices/index.mdx @@ -21,4 +21,4 @@ import * as Components from '@site/src/components'; - [Error handling](/develop/go/best-practices/error-handling) - [Debugging](/develop/go/best-practices/debugging) - [Testing](/develop/go/best-practices/testing-suite) -- [Converters and encryption](/develop/go/best-practices/converters-and-encryption) \ No newline at end of file +- [Data handling](/develop/go/data-handling) \ No newline at end of file diff --git a/docs/develop/go/index.mdx b/docs/develop/go/index.mdx index 901f3e4ab0..422ddc32f1 100644 --- a/docs/develop/go/index.mdx +++ b/docs/develop/go/index.mdx @@ -85,7 +85,7 @@ From there, you can dive deeper into any of the Temporal primitives to start bui - [Error handling](/develop/go/best-practices/error-handling) - [Debugging](/develop/go/best-practices/debugging) - [Testing](/develop/go/best-practices/testing-suite) -- [Converters and encryption](/develop/go/best-practices/converters-and-encryption) +- [Data handling](/develop/go/data-handling) ## Temporal Go Technical Resources diff --git a/docs/encyclopedia/data-conversion/external-storage.mdx b/docs/encyclopedia/data-conversion/external-storage.mdx index db6fe00e2b..1f7b65727b 100644 --- a/docs/encyclopedia/data-conversion/external-storage.mdx +++ b/docs/encyclopedia/data-conversion/external-storage.mdx @@ -34,6 +34,7 @@ the Event History instead. This is called the [claim check pattern](https://en.w For SDK-specific usage guides, see: +- [Go SDK: Large payload storage](/develop/go/data-handling/large-payload-storage) - [Python SDK: Large payload storage](/develop/python/data-handling/large-payload-storage) ## Why use External Storage @@ -63,6 +64,7 @@ External Storage addresses several common scenarios: For SDK-specific usage guides, see: +- [Go SDK: Large payload storage](/develop/go/data-handling/large-payload-storage) - [Python SDK: Large payload storage](/develop/python/data-handling/large-payload-storage) ## How External Storage fits in the data conversion pipeline {#data-pipeline} @@ -104,10 +106,10 @@ as hot and cold storage tiers. ### Custom storage drivers -If the built-in drivers don't support your storage backend, you can implement a custom driver by extending the -`StorageDriver` abstract class. For an example, see -[Implement a custom storage driver](/develop/python/data-handling/large-payload-storage#implement-a-custom-storage-driver) -in the Python SDK guide. +If the built-in drivers don't support your storage backend, you can implement a custom driver. For SDK-specific examples, see: + +- [Go SDK: Implement a custom storage driver](/develop/go/data-handling/large-payload-storage#implement-a-custom-storage-driver) +- [Python SDK: Implement a custom storage driver](/develop/python/data-handling/large-payload-storage#implement-a-custom-storage-driver) ## Key configuration settings diff --git a/docs/evaluate/development-production-features/data-encryption.mdx b/docs/evaluate/development-production-features/data-encryption.mdx index 0b221b0655..57dd6e75cf 100644 --- a/docs/evaluate/development-production-features/data-encryption.mdx +++ b/docs/evaluate/development-production-features/data-encryption.mdx @@ -29,7 +29,7 @@ Temporal's data encryption capabilities ensure the security and confidentiality Jump straight to a Temporal SDK feature guide. - + diff --git a/sidebars.js b/sidebars.js index 2de9f83ba9..cc9986a84d 100644 --- a/sidebars.js +++ b/sidebars.js @@ -205,7 +205,20 @@ module.exports = { 'develop/go/best-practices/error-handling', 'develop/go/best-practices/debugging', 'develop/go/best-practices/testing-suite', - 'develop/go/best-practices/converters-and-encryption', + { + type: 'category', + label: 'Data handling', + collapsed: true, + link: { + type: 'doc', + id: 'develop/go/best-practices/data-handling/data-handling', + }, + items: [ + 'develop/go/best-practices/data-handling/data-conversion', + 'develop/go/best-practices/data-handling/data-encryption', + 'develop/go/best-practices/data-handling/large-payload-storage', + ], + }, ], }, ], From 10bcdbbd257a51e1b35fbc84e9beb752d43f510c Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Thu, 9 Apr 2026 12:26:49 -0700 Subject: [PATCH 02/14] custom driver addition --- .../data-handling/large-payload-storage.mdx | 142 ++++++++++++++++-- 1 file changed, 126 insertions(+), 16 deletions(-) diff --git a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx index 01496f1481..9214919287 100644 --- a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx +++ b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx @@ -109,40 +109,136 @@ If you need a storage backend other than what the built-in drivers allow, you ca Store payloads durably so that they survive process crashes and remain available for debugging and auditing after the Workflow completes. Refer to [lifecycle management](/external-storage#lifecycle) for retention requirements. -A custom driver implements the `converter.StorageDriver` interface, which has four methods: +The following example shows a custom driver that uses local disk as the backing store. This example is for local +development and testing only. In production, use a durable storage system that is accessible to all Workers: + + + +```go +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/google/uuid" + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/sdk/converter" + "google.golang.org/protobuf/proto" +) + +type LocalDiskStorageDriver struct { + storeDir string +} + +func NewLocalDiskStorageDriver(storeDir string) *LocalDiskStorageDriver { + return &LocalDiskStorageDriver{storeDir: storeDir} +} + +func (d *LocalDiskStorageDriver) Name() string { + return "local-disk" +} + +func (d *LocalDiskStorageDriver) Type() string { + return "local-disk" +} + +func (d *LocalDiskStorageDriver) Store( + ctx converter.StorageDriverStoreContext, + payloads []*commonpb.Payload, +) ([]converter.StorageDriverClaim, error) { + dir := d.storeDir + if info, ok := ctx.Target.(converter.StorageDriverWorkflowInfo); ok && info.WorkflowID != "" { + dir = filepath.Join(d.storeDir, info.Namespace, info.WorkflowID) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create store directory: %w", err) + } + + claims := make([]converter.StorageDriverClaim, len(payloads)) + for i, payload := range payloads { + key := uuid.NewString() + ".bin" + filePath := filepath.Join(dir, key) + + data, err := proto.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("marshal payload: %w", err) + } + if err := os.WriteFile(filePath, data, 0o644); err != nil { + return nil, fmt.Errorf("write payload: %w", err) + } + + claims[i] = converter.StorageDriverClaim{ + ClaimData: map[string]string{"path": filePath}, + } + } + return claims, nil +} + +func (d *LocalDiskStorageDriver) Retrieve( + ctx converter.StorageDriverRetrieveContext, + claims []converter.StorageDriverClaim, +) ([]*commonpb.Payload, error) { + payloads := make([]*commonpb.Payload, len(claims)) + for i, claim := range claims { + filePath := claim.ClaimData["path"] + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("read payload: %w", err) + } + payload := &commonpb.Payload{} + if err := proto.Unmarshal(data, payload); err != nil { + return nil, fmt.Errorf("unmarshal payload: %w", err) + } + payloads[i] = payload + } + return payloads, nil +} +``` + + + +The following sections walk through the key parts of the driver implementation. + +### 1. Implement the StorageDriver interface + +A custom driver implements the `converter.StorageDriver` interface with four methods: - `Name()` returns a unique string that identifies the driver. The SDK stores this name in the claim check reference so it can route retrieval requests to the correct driver. Changing the name after payloads have been stored breaks retrieval. +- `Type()` returns a string that identifies the driver implementation. Unlike `Name()`, this must be the same across all + instances of the same driver type regardless of configuration. - `Store()` receives a slice of payloads and returns one `StorageDriverClaim` per payload. A claim is a set of string key-value pairs that the driver uses to locate the payload later. - `Retrieve()` receives the claims that `Store()` produced and returns the original payloads. -- `Options()` returns the driver's configuration options. -### Store payloads +### 2. Store payloads In `Store()`, marshal each Payload protobuf message to bytes with `proto.Marshal(payload)` and write the bytes to your storage system. The application data has already been serialized by the [Payload Converter](/develop/go/data-handling/data-conversion) and [Payload Codec](/develop/go/data-handling/data-encryption) before it reaches the driver. See the [data conversion pipeline](/external-storage#data-pipeline) for more details. -Return a `StorageDriverClaim` for each payload with enough information to retrieve it later. The `context` parameter -provides identity information (namespace, Workflow ID) depending on the operation. Consider structuring +Return a `StorageDriverClaim` for each payload with enough information to retrieve it later. The `ctx.Target` +provides identity information (namespace, Workflow ID) depending on the operation. Use a type switch on +`StorageDriverWorkflowInfo` and `StorageDriverActivityInfo` to access the concrete values. Consider structuring your storage keys to include this information so that you can identify which Workflow owns each payload. -### Retrieve payloads +### 3. Retrieve payloads In `Retrieve()`, download the bytes using the claim data, then reconstruct the Payload protobuf message with `proto.Unmarshal(data, payload)`. The Payload Converter handles deserializing the application data after the driver returns the payload. -### Configure the Client +### 4. Configure the Client Pass an `ExternalStorage` struct with your driver in the Client options: ```go c, err := client.Dial(client.Options{ ExternalStorage: converter.ExternalStorage{ - Drivers: []converter.StorageDriver{myCustomDriver}, + Drivers: []converter.StorageDriver{NewLocalDiskStorageDriver("/tmp/temporal-payload-store")}, }, }) ``` @@ -170,9 +266,10 @@ c, err := client.Dial(client.Options{ ## Use multiple storage drivers -When you register multiple drivers, you must provide a `DriverSelector` function that chooses which driver stores each -payload. Any driver in the list that is not selected for storing is still available for retrieval, which is useful when -migrating between storage backends. Return `nil` from the selector to keep a specific payload inline in Event History. +When you register multiple drivers, you must provide a `DriverSelector` that implements the `StorageDriverSelector` +interface. The selector chooses which driver stores each payload. Any driver in the list that is not selected for +storing is still available for retrieval, which is useful when migrating between storage backends. Return `nil` from +the selector to keep a specific payload inline in Event History. Multiple drivers are useful in scenarios such as: @@ -189,13 +286,26 @@ is only registered so the Worker can retrieve payloads that were previously stor ```go -import "go.temporal.io/sdk/converter" +import ( + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/sdk/converter" +) + +type PreferredSelector struct { + preferred converter.StorageDriver +} +func (s *PreferredSelector) SelectDriver( + ctx converter.StorageDriverStoreContext, + payload *commonpb.Payload, +) (converter.StorageDriver, error) { + return s.preferred, nil +} + +// Usage: converter.ExternalStorage{ - Drivers: []converter.StorageDriver{preferredDriver, legacyDriver}, - DriverSelector: func(ctx converter.StorageDriverStoreContext, payload *commonpb.Payload) converter.StorageDriver { - return preferredDriver - }, + Drivers: []converter.StorageDriver{preferredDriver, legacyDriver}, + DriverSelector: &PreferredSelector{preferred: preferredDriver}, } ``` From c73b2444de1e569d57fd21b8a69bf11156455689 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Thu, 9 Apr 2026 13:35:03 -0700 Subject: [PATCH 03/14] custom driver addition --- .../best-practices/data-handling/large-payload-storage.mdx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx index 9214919287..bf02b89126 100644 --- a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx +++ b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx @@ -267,9 +267,10 @@ c, err := client.Dial(client.Options{ ## Use multiple storage drivers When you register multiple drivers, you must provide a `DriverSelector` that implements the `StorageDriverSelector` -interface. The selector chooses which driver stores each payload. Any driver in the list that is not selected for -storing is still available for retrieval, which is useful when migrating between storage backends. Return `nil` from -the selector to keep a specific payload inline in Event History. +interface. The SDK returns an error at client creation if multiple drivers are registered without a selector. The +selector chooses which driver stores each payload. Any driver in the list that is not selected for storing is still +available for retrieval, which is useful when migrating between storage backends. Return `nil` from the selector to +keep a specific payload inline in Event History. Multiple drivers are useful in scenarios such as: From cdd350de988ad3e8c7283f3f1de408c0334290c3 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Thu, 9 Apr 2026 15:10:48 -0700 Subject: [PATCH 04/14] address feedback --- .../data-handling/large-payload-storage.mdx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx index bf02b89126..842e940f75 100644 --- a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx +++ b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx @@ -137,7 +137,7 @@ func NewLocalDiskStorageDriver(storeDir string) *LocalDiskStorageDriver { } func (d *LocalDiskStorageDriver) Name() string { - return "local-disk" + return "my-local-disk" } func (d *LocalDiskStorageDriver) Type() string { @@ -205,10 +205,12 @@ The following sections walk through the key parts of the driver implementation. A custom driver implements the `converter.StorageDriver` interface with four methods: -- `Name()` returns a unique string that identifies the driver. The SDK stores this name in the claim check reference so - it can route retrieval requests to the correct driver. Changing the name after payloads have been stored breaks retrieval. +- `Name()` returns a unique string that identifies the driver instance. The SDK stores this name in the claim check + reference so it can route retrieval requests to the correct driver. Changing the name after payloads have been stored + breaks retrieval. For example, two S3 drivers could be named `"s3-us-east"` and `"s3-eu-west"`. - `Type()` returns a string that identifies the driver implementation. Unlike `Name()`, this must be the same across all - instances of the same driver type regardless of configuration. + instances of the same driver type regardless of configuration. Both S3 drivers in the example above would return + `"aws.s3driver"` as their type. - `Store()` receives a slice of payloads and returns one `StorageDriverClaim` per payload. A claim is a set of string key-value pairs that the driver uses to locate the payload later. - `Retrieve()` receives the claims that `Store()` produced and returns the original payloads. From a3dbb8ccff7f61491d2e2758cdadc64cdc6d1d26 Mon Sep 17 00:00:00 2001 From: Lenny Chen <55669665+lennessyy@users.noreply.github.com> Date: Fri, 10 Apr 2026 09:47:56 -0700 Subject: [PATCH 05/14] Apply suggestion from @lennessyy --- .../go/best-practices/data-handling/large-payload-storage.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx index 842e940f75..b058f62fa6 100644 --- a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx +++ b/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx @@ -1,5 +1,5 @@ --- -id: large-payload-storage +id: external-storage title: External Storage - Go SDK sidebar_label: External Storage slug: /develop/go/data-handling/large-payload-storage From f1045bf026752d271b9a76ef495a34beeeb9b045 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 10 Apr 2026 09:58:28 -0700 Subject: [PATCH 06/14] page name change --- .../{large-payload-storage.mdx => external-storage.mdx} | 2 +- docs/develop/go/best-practices/data-handling/index.mdx | 2 +- docs/encyclopedia/data-conversion/external-storage.mdx | 6 +++--- vercel.json | 5 +++++ 4 files changed, 10 insertions(+), 5 deletions(-) rename docs/develop/go/best-practices/data-handling/{large-payload-storage.mdx => external-storage.mdx} (99%) diff --git a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx b/docs/develop/go/best-practices/data-handling/external-storage.mdx similarity index 99% rename from docs/develop/go/best-practices/data-handling/large-payload-storage.mdx rename to docs/develop/go/best-practices/data-handling/external-storage.mdx index b058f62fa6..4e4543f965 100644 --- a/docs/develop/go/best-practices/data-handling/large-payload-storage.mdx +++ b/docs/develop/go/best-practices/data-handling/external-storage.mdx @@ -2,7 +2,7 @@ id: external-storage title: External Storage - Go SDK sidebar_label: External Storage -slug: /develop/go/data-handling/large-payload-storage +slug: /develop/go/data-handling/external-storage toc_max_heading_level: 3 tags: - Go SDK diff --git a/docs/develop/go/best-practices/data-handling/index.mdx b/docs/develop/go/best-practices/data-handling/index.mdx index 4d47ef5717..5f3ffd0c61 100644 --- a/docs/develop/go/best-practices/data-handling/index.mdx +++ b/docs/develop/go/best-practices/data-handling/index.mdx @@ -27,7 +27,7 @@ Of these three layers, only the PayloadConverter is required. Temporal uses a de serialization. The PayloadCodec and ExternalStorage layers are optional. You only need to customize these layers when your application requires non-JSON types, encryption, or payload offloading. -| | [PayloadConverter](/develop/go/data-handling/data-conversion) | [PayloadCodec](/develop/go/data-handling/data-encryption) | [ExternalStorage](/develop/go/data-handling/large-payload-storage) | +| | [PayloadConverter](/develop/go/data-handling/data-conversion) | [PayloadCodec](/develop/go/data-handling/data-encryption) | [ExternalStorage](/develop/go/data-handling/external-storage) | | ------------------------- | ------------------------------------------------------------- | --------------------------------------------------------- | ------------------------------------------------------------------ | | **Purpose** | Serialize application data to bytes | Transform encoded payloads (encrypt, compress) | Offload large payloads to external store | | **Default** | JSON serialization | None (passthrough) | None (all payloads are stored in Workflow History) | diff --git a/docs/encyclopedia/data-conversion/external-storage.mdx b/docs/encyclopedia/data-conversion/external-storage.mdx index 1f7b65727b..9a46654023 100644 --- a/docs/encyclopedia/data-conversion/external-storage.mdx +++ b/docs/encyclopedia/data-conversion/external-storage.mdx @@ -34,7 +34,7 @@ the Event History instead. This is called the [claim check pattern](https://en.w For SDK-specific usage guides, see: -- [Go SDK: Large payload storage](/develop/go/data-handling/large-payload-storage) +- [Go SDK: Large payload storage](/develop/go/data-handling/external-storage) - [Python SDK: Large payload storage](/develop/python/data-handling/large-payload-storage) ## Why use External Storage @@ -64,7 +64,7 @@ External Storage addresses several common scenarios: For SDK-specific usage guides, see: -- [Go SDK: Large payload storage](/develop/go/data-handling/large-payload-storage) +- [Go SDK: Large payload storage](/develop/go/data-handling/external-storage) - [Python SDK: Large payload storage](/develop/python/data-handling/large-payload-storage) ## How External Storage fits in the data conversion pipeline {#data-pipeline} @@ -108,7 +108,7 @@ as hot and cold storage tiers. If the built-in drivers don't support your storage backend, you can implement a custom driver. For SDK-specific examples, see: -- [Go SDK: Implement a custom storage driver](/develop/go/data-handling/large-payload-storage#implement-a-custom-storage-driver) +- [Go SDK: Implement a custom storage driver](/develop/go/data-handling/external-storage#implement-a-custom-storage-driver) - [Python SDK: Implement a custom storage driver](/develop/python/data-handling/large-payload-storage#implement-a-custom-storage-driver) ## Key configuration settings diff --git a/vercel.json b/vercel.json index ee8335b2e2..682b050ef1 100644 --- a/vercel.json +++ b/vercel.json @@ -5,6 +5,11 @@ "silent": true }, "redirects": [ + { + "source": "/develop/go/data-handling/large-payload-storage", + "destination": "/develop/go/data-handling/external-storage", + "permanent": true + }, { "source": "/cloud/introduction", "destination": "/cloud/overview", From fd978fb4f866ae420e1f7e35ce049d66c69b0f88 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 10 Apr 2026 09:58:52 -0700 Subject: [PATCH 07/14] update page name --- .../{large-payload-storage.mdx => external-storage.mdx} | 4 ++-- docs/develop/python/best-practices/data-handling/index.mdx | 2 +- docs/encyclopedia/data-conversion/external-storage.mdx | 6 +++--- vercel.json | 5 +++++ 4 files changed, 11 insertions(+), 6 deletions(-) rename docs/develop/python/best-practices/data-handling/{large-payload-storage.mdx => external-storage.mdx} (99%) diff --git a/docs/develop/python/best-practices/data-handling/large-payload-storage.mdx b/docs/develop/python/best-practices/data-handling/external-storage.mdx similarity index 99% rename from docs/develop/python/best-practices/data-handling/large-payload-storage.mdx rename to docs/develop/python/best-practices/data-handling/external-storage.mdx index b63ffb6995..2b23de7fde 100644 --- a/docs/develop/python/best-practices/data-handling/large-payload-storage.mdx +++ b/docs/develop/python/best-practices/data-handling/external-storage.mdx @@ -1,8 +1,8 @@ --- -id: large-payload-storage +id: external-storage title: External Storage - Python SDK sidebar_label: External Storage -slug: /develop/python/data-handling/large-payload-storage +slug: /develop/python/data-handling/external-storage toc_max_heading_level: 3 tags: - Python SDK diff --git a/docs/develop/python/best-practices/data-handling/index.mdx b/docs/develop/python/best-practices/data-handling/index.mdx index c53cab272a..3367ee126c 100644 --- a/docs/develop/python/best-practices/data-handling/index.mdx +++ b/docs/develop/python/best-practices/data-handling/index.mdx @@ -27,7 +27,7 @@ Of these three layers, only the PayloadConverter is required. Temporal uses a de serialization. The PayloadCodec and ExternalStorage layers are optional. You only need to customize these layers when your application requires non-JSON types, encryption, or payload offloading. -| | [PayloadConverter](/develop/python/data-handling/data-conversion) | [PayloadCodec](/develop/python/data-handling/data-encryption) | [ExternalStorage](/develop/python/data-handling/large-payload-storage) | +| | [PayloadConverter](/develop/python/data-handling/data-conversion) | [PayloadCodec](/develop/python/data-handling/data-encryption) | [ExternalStorage](/develop/python/data-handling/external-storage) | | ------------------------- | ----------------------------------------------------------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------- | | **Purpose** | Serialize application data to bytes | Transform encoded payloads (encrypt, compress) | Offload large payloads to external store | | **Default** | JSON serialization | None (passthrough) | None (all payloads are stored in Workflow History) | diff --git a/docs/encyclopedia/data-conversion/external-storage.mdx b/docs/encyclopedia/data-conversion/external-storage.mdx index 9a46654023..a54dd66081 100644 --- a/docs/encyclopedia/data-conversion/external-storage.mdx +++ b/docs/encyclopedia/data-conversion/external-storage.mdx @@ -35,7 +35,7 @@ the Event History instead. This is called the [claim check pattern](https://en.w For SDK-specific usage guides, see: - [Go SDK: Large payload storage](/develop/go/data-handling/external-storage) -- [Python SDK: Large payload storage](/develop/python/data-handling/large-payload-storage) +- [Python SDK: Large payload storage](/develop/python/data-handling/external-storage) ## Why use External Storage @@ -65,7 +65,7 @@ External Storage addresses several common scenarios: For SDK-specific usage guides, see: - [Go SDK: Large payload storage](/develop/go/data-handling/external-storage) -- [Python SDK: Large payload storage](/develop/python/data-handling/large-payload-storage) +- [Python SDK: Large payload storage](/develop/python/data-handling/external-storage) ## How External Storage fits in the data conversion pipeline {#data-pipeline} @@ -109,7 +109,7 @@ as hot and cold storage tiers. If the built-in drivers don't support your storage backend, you can implement a custom driver. For SDK-specific examples, see: - [Go SDK: Implement a custom storage driver](/develop/go/data-handling/external-storage#implement-a-custom-storage-driver) -- [Python SDK: Implement a custom storage driver](/develop/python/data-handling/large-payload-storage#implement-a-custom-storage-driver) +- [Python SDK: Implement a custom storage driver](/develop/python/data-handling/external-storage#implement-a-custom-storage-driver) ## Key configuration settings diff --git a/vercel.json b/vercel.json index 682b050ef1..837e4d0ae5 100644 --- a/vercel.json +++ b/vercel.json @@ -10,6 +10,11 @@ "destination": "/develop/go/data-handling/external-storage", "permanent": true }, + { + "source": "/develop/python/data-handling/large-payload-storage", + "destination": "/develop/python/data-handling/external-storage", + "permanent": true + }, { "source": "/cloud/introduction", "destination": "/cloud/overview", From a6662b52ed1ca80cdfb8d5213d42504f4c47bf45 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 10 Apr 2026 10:01:36 -0700 Subject: [PATCH 08/14] sidebar change --- sidebars.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sidebars.js b/sidebars.js index cc9986a84d..7bfedd9f2e 100644 --- a/sidebars.js +++ b/sidebars.js @@ -216,7 +216,7 @@ module.exports = { items: [ 'develop/go/best-practices/data-handling/data-conversion', 'develop/go/best-practices/data-handling/data-encryption', - 'develop/go/best-practices/data-handling/large-payload-storage', + 'develop/go/best-practices/data-handling/external-storage', ], }, ], @@ -569,7 +569,7 @@ module.exports = { items: [ 'develop/python/best-practices/data-handling/data-conversion', 'develop/python/best-practices/data-handling/data-encryption', - 'develop/python/best-practices/data-handling/large-payload-storage', + 'develop/python/best-practices/data-handling/external-storage', ], }, 'develop/python/best-practices/python-sdk-sync-vs-async', From 6edd970a5a4456cc7711cc730129a4edad17f4e3 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 10 Apr 2026 11:37:49 -0700 Subject: [PATCH 09/14] use themed images --- .../go/best-practices/data-handling/index.mdx | 16 +++++++++++----- .../best-practices/data-handling/index.mdx | 16 +++++++++++----- .../data-conversion/external-storage.mdx | 16 +++++++++++----- static/diagrams/data-converter-flow-dark.svg | 1 + 4 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 static/diagrams/data-converter-flow-dark.svg diff --git a/docs/develop/go/best-practices/data-handling/index.mdx b/docs/develop/go/best-practices/data-handling/index.mdx index 5f3ffd0c61..177e23210e 100644 --- a/docs/develop/go/best-practices/data-handling/index.mdx +++ b/docs/develop/go/best-practices/data-handling/index.mdx @@ -13,15 +13,21 @@ tags: - Data Converters --- -import { CaptionedImage } from '@site/src/components'; +import ThemedImage from '@theme/ThemedImage'; All data sent to and from the Temporal Service passes through the **Data Converter**. The Data Converter has three layers that handle different concerns: - +
+ +
The Flow of Data through a Data Converter
+
Of these three layers, only the PayloadConverter is required. Temporal uses a default PayloadConverter that handles JSON serialization. The PayloadCodec and ExternalStorage layers are optional. You only need to customize these layers when diff --git a/docs/develop/python/best-practices/data-handling/index.mdx b/docs/develop/python/best-practices/data-handling/index.mdx index 3367ee126c..0a066047e9 100644 --- a/docs/develop/python/best-practices/data-handling/index.mdx +++ b/docs/develop/python/best-practices/data-handling/index.mdx @@ -13,15 +13,21 @@ tags: - Data Converters --- -import { CaptionedImage } from '@site/src/components'; +import ThemedImage from '@theme/ThemedImage'; All data sent to and from the Temporal Service passes through the **Data Converter**. The Data Converter has three layers that handle different concerns: - +
+ +
The Flow of Data through a Data Converter
+
Of these three layers, only the PayloadConverter is required. Temporal uses a default PayloadConverter that handles JSON serialization. The PayloadCodec and ExternalStorage layers are optional. You only need to customize these layers when diff --git a/docs/encyclopedia/data-conversion/external-storage.mdx b/docs/encyclopedia/data-conversion/external-storage.mdx index bf8346949a..c2715bd48d 100644 --- a/docs/encyclopedia/data-conversion/external-storage.mdx +++ b/docs/encyclopedia/data-conversion/external-storage.mdx @@ -19,7 +19,7 @@ tags: - Data Converters --- -import { CaptionedImage } from '@site/src/components'; +import ThemedImage from '@theme/ThemedImage'; :::info Release, stability, and dependency info @@ -72,10 +72,16 @@ For SDK-specific usage guides, see: During [Data Conversion](/dataconversion), External Storage sits at the end of the pipeline, after both the [Payload Converter](/payload-converter) and the [Payload Codec](/payload-codec): - +
+ +
The Flow of Data through a Data Converter
+
When a Temporal Client sends a payload that exceeds the configured size threshold, the storage driver uploads the payload to your external store and replaces it with a lightweight reference. Payloads below the threshold stay inline in diff --git a/static/diagrams/data-converter-flow-dark.svg b/static/diagrams/data-converter-flow-dark.svg new file mode 100644 index 0000000000..a843063f85 --- /dev/null +++ b/static/diagrams/data-converter-flow-dark.svg @@ -0,0 +1 @@ + \ No newline at end of file From 29a4c3ba7fde0b4a99dab908ce9cc477caa4ed53 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 10 Apr 2026 11:44:13 -0700 Subject: [PATCH 10/14] add border to dark image --- static/diagrams/data-converter-flow-dark.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/diagrams/data-converter-flow-dark.svg b/static/diagrams/data-converter-flow-dark.svg index a843063f85..87cbac8af1 100644 --- a/static/diagrams/data-converter-flow-dark.svg +++ b/static/diagrams/data-converter-flow-dark.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From 4a3a76d8a2c49257b09b67f5304966f72da74eaa Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 10 Apr 2026 12:00:14 -0700 Subject: [PATCH 11/14] address PR feedback --- .../data-handling/external-storage.mdx | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/develop/go/best-practices/data-handling/external-storage.mdx b/docs/develop/go/best-practices/data-handling/external-storage.mdx index 4e4543f965..aa07faa885 100644 --- a/docs/develop/go/best-practices/data-handling/external-storage.mdx +++ b/docs/develop/go/best-practices/data-handling/external-storage.mdx @@ -46,7 +46,7 @@ The Go SDK includes an S3 storage driver. Follow these steps to set it up: ```go import ( "github.com/aws/aws-sdk-go-v2/config" - awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3" "go.temporal.io/sdk/contrib/aws/s3driver" "go.temporal.io/sdk/contrib/aws/s3driver/awssdkv2" ) @@ -59,7 +59,7 @@ The Go SDK includes an S3 storage driver. Follow these steps to set it up: } driver, err := s3driver.NewDriver(s3driver.Options{ - Client: awssdkv2.NewClient(awss3.NewFromConfig(cfg)), + Client: awssdkv2.NewClient(s3.NewFromConfig(cfg)), Bucket: s3driver.StaticBucket("my-temporal-payloads"), }) if err != nil { @@ -97,7 +97,7 @@ The Go SDK includes an S3 storage driver. Follow these steps to set it up: By default, payloads larger than 256 KiB are offloaded to external storage. You can adjust this with the - `PayloadSizeThreshold` option, even setting it to 0 to externalize all payloads regardless of size. Refer to + `PayloadSizeThreshold` option, even setting it to 1 to externalize all payloads regardless of size. Refer to [Configure payload size threshold](#configure-payload-size-threshold) for more information. All Workflows and Activities running on the Worker use the storage driver automatically without changes to your @@ -132,7 +132,7 @@ type LocalDiskStorageDriver struct { storeDir string } -func NewLocalDiskStorageDriver(storeDir string) *LocalDiskStorageDriver { +func NewLocalDiskStorageDriver(storeDir string) converter.StorageDriver { return &LocalDiskStorageDriver{storeDir: storeDir} } @@ -207,10 +207,10 @@ A custom driver implements the `converter.StorageDriver` interface with four met - `Name()` returns a unique string that identifies the driver instance. The SDK stores this name in the claim check reference so it can route retrieval requests to the correct driver. Changing the name after payloads have been stored - breaks retrieval. For example, two S3 drivers could be named `"s3-us-east"` and `"s3-eu-west"`. + breaks retrieval. For example, two S3 drivers could be named `"s3-primary"` and `"s3-archive"`. - `Type()` returns a string that identifies the driver implementation. Unlike `Name()`, this must be the same across all instances of the same driver type regardless of configuration. Both S3 drivers in the example above would return - `"aws.s3driver"` as their type. + `"aws.s3driver"` as their type, while the local disk driver in the custom example returns `"local-disk"`. - `Store()` receives a slice of payloads and returns one `StorageDriverClaim` per payload. A claim is a set of string key-value pairs that the driver uses to locate the payload later. - `Retrieve()` receives the claims that `Store()` produced and returns the original payloads. @@ -248,8 +248,8 @@ c, err := client.Dial(client.Options{ ## Configure payload size threshold You can configure the payload size threshold that triggers external storage. By default, payloads larger than 256 KiB -are offloaded to external storage. You can adjust this with the `PayloadSizeThreshold` option, or set it to 0 to -externalize all payloads regardless of size. +are offloaded to external storage. You can adjust this with the `PayloadSizeThreshold` option, or set it to 1 to +externalize all payloads regardless of size. A value of 0 is interpreted as the default (256 KiB). @@ -259,7 +259,7 @@ import "go.temporal.io/sdk/converter" c, err := client.Dial(client.Options{ ExternalStorage: converter.ExternalStorage{ Drivers: []converter.StorageDriver{driver}, - PayloadSizeThreshold: 0, + PayloadSizeThreshold: 1, }, }) ``` From c1a215ffe892c2fb751e0a26bed8441dff7bce3a Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 10 Apr 2026 12:06:58 -0700 Subject: [PATCH 12/14] clarify wording --- .../go/best-practices/data-handling/external-storage.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/develop/go/best-practices/data-handling/external-storage.mdx b/docs/develop/go/best-practices/data-handling/external-storage.mdx index aa07faa885..e8113e2bda 100644 --- a/docs/develop/go/best-practices/data-handling/external-storage.mdx +++ b/docs/develop/go/best-practices/data-handling/external-storage.mdx @@ -209,8 +209,8 @@ A custom driver implements the `converter.StorageDriver` interface with four met reference so it can route retrieval requests to the correct driver. Changing the name after payloads have been stored breaks retrieval. For example, two S3 drivers could be named `"s3-primary"` and `"s3-archive"`. - `Type()` returns a string that identifies the driver implementation. Unlike `Name()`, this must be the same across all - instances of the same driver type regardless of configuration. Both S3 drivers in the example above would return - `"aws.s3driver"` as their type, while the local disk driver in the custom example returns `"local-disk"`. + instances of the same driver type regardless of configuration. Two S3 drivers named `"s3-primary"` and `"s3-archive"` would both return + `"aws.s3driver"` as their type, while the local disk driver in the custom driver code sample returns `"local-disk"`. - `Store()` receives a slice of payloads and returns one `StorageDriverClaim` per payload. A claim is a set of string key-value pairs that the driver uses to locate the payload later. - `Retrieve()` receives the claims that `Store()` produced and returns the original payloads. From 19c48c4a30f1d48040cfc7c2f9406b6d279a605d Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 10 Apr 2026 12:53:22 -0700 Subject: [PATCH 13/14] add StorageDriverActivityInfo handling to custom driver example Co-Authored-By: Claude Opus 4.6 (1M context) --- .../best-practices/data-handling/external-storage.mdx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/develop/go/best-practices/data-handling/external-storage.mdx b/docs/develop/go/best-practices/data-handling/external-storage.mdx index e8113e2bda..f17697c661 100644 --- a/docs/develop/go/best-practices/data-handling/external-storage.mdx +++ b/docs/develop/go/best-practices/data-handling/external-storage.mdx @@ -149,8 +149,15 @@ func (d *LocalDiskStorageDriver) Store( payloads []*commonpb.Payload, ) ([]converter.StorageDriverClaim, error) { dir := d.storeDir - if info, ok := ctx.Target.(converter.StorageDriverWorkflowInfo); ok && info.WorkflowID != "" { - dir = filepath.Join(d.storeDir, info.Namespace, info.WorkflowID) + switch info := ctx.Target.(type) { + case converter.StorageDriverWorkflowInfo: + if info.WorkflowID != "" { + dir = filepath.Join(d.storeDir, info.Namespace, info.WorkflowID) + } + case converter.StorageDriverActivityInfo: + if info.ActivityID != "" { + dir = filepath.Join(d.storeDir, info.Namespace, info.ActivityID) + } } if err := os.MkdirAll(dir, 0o755); err != nil { return nil, fmt.Errorf("create store directory: %w", err) From a3e537070ddef5d3e661f19cd2836d1be205397d Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 10 Apr 2026 12:59:35 -0700 Subject: [PATCH 14/14] add comment clarifying StorageDriverActivityInfo usage Co-Authored-By: Claude Opus 4.6 (1M context) --- .../go/best-practices/data-handling/external-storage.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/develop/go/best-practices/data-handling/external-storage.mdx b/docs/develop/go/best-practices/data-handling/external-storage.mdx index f17697c661..c658c5a05a 100644 --- a/docs/develop/go/best-practices/data-handling/external-storage.mdx +++ b/docs/develop/go/best-practices/data-handling/external-storage.mdx @@ -155,6 +155,8 @@ func (d *LocalDiskStorageDriver) Store( dir = filepath.Join(d.storeDir, info.Namespace, info.WorkflowID) } case converter.StorageDriverActivityInfo: + // StorageDriverActivityInfo is only used for standalone (non-workflow-bound) + // activities. Activities started by a workflow use StorageDriverWorkflowInfo. if info.ActivityID != "" { dir = filepath.Join(d.storeDir, info.Namespace, info.ActivityID) }