From 3a983cb82a1e8ff9cd655bafb49722a99989cdf5 Mon Sep 17 00:00:00 2001 From: Vishwak Thatikonda Date: Thu, 14 May 2026 11:11:34 -0700 Subject: [PATCH] feat(sync): add --force-upload flag to control S3 upload dedup `sam sync` previously hardcoded `force_upload=True` in three places, which bypassed the SHA-based dedup in `S3Uploader.upload_with_dedup` and caused every sync to re-upload all artifacts even when nothing had changed. Issue #8168 reports a 20%+ (90+ second) speedup from toggling this off in a fork. Expose `--force-upload` on `sam sync` (default False, matching `sam deploy`'s default) and thread it through: - PackageContext and DeployContext, replacing the two hardcoded `True` values in `samcli/commands/sync/command.py`. - The S3Uploader instantiation in ZipFunctionSyncFlow now reads `force_upload` from `self._deploy_context.force_upload` so the flag actually controls the oversized-zip upload path too. Schema regenerated. Unit tests updated to assert the flag flows through PackageContext, DeployContext, and the ZipFunctionSyncFlow S3 upload path for both True and False values. Closes #8168 --- samcli/commands/sync/command.py | 9 +- .../lib/sync/flows/zip_function_sync_flow.py | 2 +- schema/samcli.json | 7 +- .../unit/commands/samconfig/test_samconfig.py | 1 + tests/unit/commands/sync/test_command.py | 83 ++++++++++++++++++- .../sync/flows/test_zip_function_sync_flow.py | 33 ++++++++ 6 files changed, 127 insertions(+), 8 deletions(-) diff --git a/samcli/commands/sync/command.py b/samcli/commands/sync/command.py index a3bcbc324fa..f0c7e492129 100644 --- a/samcli/commands/sync/command.py +++ b/samcli/commands/sync/command.py @@ -26,6 +26,7 @@ build_in_source_option, capabilities_option, container_env_var_file_option, + force_upload_option, image_repositories_option, image_repository_option, kms_key_id_option, @@ -169,6 +170,7 @@ @image_repository_option @image_repositories_option @s3_bucket_option(disable_callback=True) # pylint: disable=E1120 +@force_upload_option @s3_prefix_option @kms_key_id_option @role_arn_option @@ -204,6 +206,7 @@ def cli( image_repository: str, image_repositories: Optional[List[str]], s3_bucket: str, + force_upload: bool, s3_prefix: str, kms_key_id: str, capabilities: Optional[List[str]], @@ -243,6 +246,7 @@ def cli( image_repository, image_repositories, s3_bucket, + force_upload, s3_prefix, kms_key_id, capabilities, @@ -277,6 +281,7 @@ def do_cli( image_repository: str, image_repositories: Optional[List[str]], s3_bucket: str, + force_upload: bool, s3_prefix: str, kms_key_id: str, capabilities: Optional[List[str]], @@ -362,7 +367,7 @@ def do_cli( region=region, profile=profile, use_json=False, - force_upload=True, + force_upload=force_upload, ) as package_context: # 500ms of sleep time between stack checks and describe stack events. DEFAULT_POLL_DELAY = 0.5 @@ -393,7 +398,7 @@ def do_cli( fail_on_empty_changeset=True, confirm_changeset=False, use_changeset=False, - force_upload=True, + force_upload=force_upload, signing_profiles=None, disable_rollback=False, poll_delay=poll_delay, diff --git a/samcli/lib/sync/flows/zip_function_sync_flow.py b/samcli/lib/sync/flows/zip_function_sync_flow.py index 239c0ff3b21..d6e976fe848 100644 --- a/samcli/lib/sync/flows/zip_function_sync_flow.py +++ b/samcli/lib/sync/flows/zip_function_sync_flow.py @@ -165,7 +165,7 @@ def sync(self) -> None: bucket_name=self._deploy_context.s3_bucket, prefix=self._deploy_context.s3_prefix, kms_key_id=self._deploy_context.kms_key_id, - force_upload=True, + force_upload=self._deploy_context.force_upload, no_progressbar=True, ) s3_url = uploader.upload_with_dedup(self._zip_file) diff --git a/schema/samcli.json b/schema/samcli.json index f2c93aa4492..71ae3177f29 100644 --- a/schema/samcli.json +++ b/schema/samcli.json @@ -1744,7 +1744,7 @@ "properties": { "parameters": { "title": "Parameters for the sync command", - "description": "Available parameters for the sync command:\n* template_file:\nAWS SAM template file.\n* code:\nSync ONLY code resources. This includes Lambda Functions, API Gateway, and Step Functions.\n* watch:\nWatch local files and automatically sync with cloud.\n* resource_id:\nSync code for all the resources with the ID. To sync a resource within a nested stack, use the following pattern {ChildStack}/{logicalId}.\n* resource:\nSync code for all resources of the given resource type. Accepted values are ['AWS::Serverless::Function', 'AWS::Lambda::Function', 'AWS::Serverless::LayerVersion', 'AWS::Lambda::LayerVersion', 'AWS::Serverless::Api', 'AWS::ApiGateway::RestApi', 'AWS::Serverless::HttpApi', 'AWS::ApiGatewayV2::Api', 'AWS::Serverless::StateMachine', 'AWS::StepFunctions::StateMachine']\n* dependency_layer:\nSeparate dependencies of individual function into a Lambda layer for improved performance.\n* skip_deploy_sync:\nThis option will skip the initial infrastructure deployment if it is not required by comparing the local template with the template deployed in cloud.\n* container_env_var_file:\nEnvironment variables json file (e.g., env_vars.json) to be passed to containers.\n* watch_exclude:\nExcludes a file or folder from being observed for file changes. Files and folders that are excluded will not trigger a sync workflow. This option can be provided multiple times.\n\nExamples:\n\nHelloWorldFunction=package-lock.json\n\nChildStackA/FunctionName=database.sqlite3\n* stack_name:\nName of the AWS CloudFormation stack.\n* base_dir:\nResolve relative paths to function's source code with respect to this directory. Use this if SAM template and source code are not in same enclosing folder. By default, relative paths are resolved with respect to the SAM template's location.\n* use_container:\nBuild functions within an AWS Lambda-like container.\n* build_in_source:\nOpts in to build project in the source folder. The following workflows support building in source: ['nodejs16.x', 'nodejs18.x', 'nodejs20.x', 'nodejs22.x', 'Makefile', 'esbuild']\n* build_image:\nContainer image URIs for building functions/layers. You can specify for all functions/layers with just the image URI (--build-image public.ecr.aws/sam/build-nodejs18.x:latest). You can specify for each individual function with (--build-image FunctionLogicalID=public.ecr.aws/sam/build-nodejs18.x:latest). A combination of the two can be used. If a function does not have build image specified or an image URI for all functions, the default SAM CLI build images will be used.\n* image_repository:\nAWS ECR repository URI where artifacts referenced in the template are uploaded.\n* image_repositories:\nMapping of Function Logical ID to AWS ECR Repository URI.\n\nExample: Function_Logical_ID=ECR_Repo_Uri\nThis option can be specified multiple times.\n* s3_bucket:\nAWS S3 bucket where artifacts referenced in the template are uploaded.\n* s3_prefix:\nPrefix name that is added to the artifact's name when it is uploaded to the AWS S3 bucket.\n* kms_key_id:\nThe ID of an AWS KMS key that is used to encrypt artifacts that are at rest in the AWS S3 bucket.\n* role_arn:\nARN of an IAM role that AWS Cloudformation assumes when executing a deployment change set.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* metadata:\nMap of metadata to attach to ALL the artifacts that are referenced in the template.\n* notification_arns:\nARNs of SNS topics that AWS Cloudformation associates with the stack.\n* tags:\nList of tags to associate with the stack.\n* capabilities:\nList of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html\n* save_params:\nSave the parameters provided via the command line to the configuration file.", + "description": "Available parameters for the sync command:\n* template_file:\nAWS SAM template file.\n* code:\nSync ONLY code resources. This includes Lambda Functions, API Gateway, and Step Functions.\n* watch:\nWatch local files and automatically sync with cloud.\n* resource_id:\nSync code for all the resources with the ID. To sync a resource within a nested stack, use the following pattern {ChildStack}/{logicalId}.\n* resource:\nSync code for all resources of the given resource type. Accepted values are ['AWS::Serverless::Function', 'AWS::Lambda::Function', 'AWS::Serverless::LayerVersion', 'AWS::Lambda::LayerVersion', 'AWS::Serverless::Api', 'AWS::ApiGateway::RestApi', 'AWS::Serverless::HttpApi', 'AWS::ApiGatewayV2::Api', 'AWS::Serverless::StateMachine', 'AWS::StepFunctions::StateMachine']\n* dependency_layer:\nSeparate dependencies of individual function into a Lambda layer for improved performance.\n* skip_deploy_sync:\nThis option will skip the initial infrastructure deployment if it is not required by comparing the local template with the template deployed in cloud.\n* container_env_var_file:\nEnvironment variables json file (e.g., env_vars.json) to be passed to containers.\n* watch_exclude:\nExcludes a file or folder from being observed for file changes. Files and folders that are excluded will not trigger a sync workflow. This option can be provided multiple times.\n\nExamples:\n\nHelloWorldFunction=package-lock.json\n\nChildStackA/FunctionName=database.sqlite3\n* stack_name:\nName of the AWS CloudFormation stack.\n* base_dir:\nResolve relative paths to function's source code with respect to this directory. Use this if SAM template and source code are not in same enclosing folder. By default, relative paths are resolved with respect to the SAM template's location.\n* use_container:\nBuild functions within an AWS Lambda-like container.\n* build_in_source:\nOpts in to build project in the source folder. The following workflows support building in source: ['nodejs16.x', 'nodejs18.x', 'nodejs20.x', 'nodejs22.x', 'Makefile', 'esbuild']\n* build_image:\nContainer image URIs for building functions/layers. You can specify for all functions/layers with just the image URI (--build-image public.ecr.aws/sam/build-nodejs18.x:latest). You can specify for each individual function with (--build-image FunctionLogicalID=public.ecr.aws/sam/build-nodejs18.x:latest). A combination of the two can be used. If a function does not have build image specified or an image URI for all functions, the default SAM CLI build images will be used.\n* image_repository:\nAWS ECR repository URI where artifacts referenced in the template are uploaded.\n* image_repositories:\nMapping of Function Logical ID to AWS ECR Repository URI.\n\nExample: Function_Logical_ID=ECR_Repo_Uri\nThis option can be specified multiple times.\n* s3_bucket:\nAWS S3 bucket where artifacts referenced in the template are uploaded.\n* force_upload:\nIndicates whether to override existing files in the S3 bucket. Specify this flag to upload artifacts even if they match existing artifacts in the S3 bucket.\n* s3_prefix:\nPrefix name that is added to the artifact's name when it is uploaded to the AWS S3 bucket.\n* kms_key_id:\nThe ID of an AWS KMS key that is used to encrypt artifacts that are at rest in the AWS S3 bucket.\n* role_arn:\nARN of an IAM role that AWS Cloudformation assumes when executing a deployment change set.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* metadata:\nMap of metadata to attach to ALL the artifacts that are referenced in the template.\n* notification_arns:\nARNs of SNS topics that AWS Cloudformation associates with the stack.\n* tags:\nList of tags to associate with the stack.\n* capabilities:\nList of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html\n* save_params:\nSave the parameters provided via the command line to the configuration file.", "type": "object", "properties": { "template_file": { @@ -1853,6 +1853,11 @@ "type": "string", "description": "AWS S3 bucket where artifacts referenced in the template are uploaded." }, + "force_upload": { + "title": "force_upload", + "type": "boolean", + "description": "Indicates whether to override existing files in the S3 bucket. Specify this flag to upload artifacts even if they match existing artifacts in the S3 bucket." + }, "s3_prefix": { "title": "s3_prefix", "type": "string", diff --git a/tests/unit/commands/samconfig/test_samconfig.py b/tests/unit/commands/samconfig/test_samconfig.py index 99152dd0928..2ba4323f206 100644 --- a/tests/unit/commands/samconfig/test_samconfig.py +++ b/tests/unit/commands/samconfig/test_samconfig.py @@ -1313,6 +1313,7 @@ def test_sync( "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1", None, "mybucket", + False, "myprefix", "mykms", ["cap1", "cap2"], diff --git a/tests/unit/commands/sync/test_command.py b/tests/unit/commands/sync/test_command.py index ba6147149dc..bd4a6982fdd 100644 --- a/tests/unit/commands/sync/test_command.py +++ b/tests/unit/commands/sync/test_command.py @@ -43,6 +43,7 @@ def setUp(self): self.image_repositories = None self.mode = "mode" self.s3_bucket = "s3-bucket" + self.force_upload = False self.s3_prefix = "s3-prefix" self.kms_key_id = "kms-key-id" self.notification_arns = [] @@ -136,6 +137,7 @@ def test_infra_must_succeed_sync( self.image_repository, self.image_repositories, self.s3_bucket, + self.force_upload, self.s3_prefix, self.kms_key_id, self.capabilities, @@ -190,7 +192,7 @@ def test_infra_must_succeed_sync( region=self.region, profile=self.profile, use_json=False, - force_upload=True, + force_upload=self.force_upload, ) DeployContextMock.assert_called_with( @@ -213,7 +215,7 @@ def test_infra_must_succeed_sync( fail_on_empty_changeset=True, confirm_changeset=False, use_changeset=False, - force_upload=True, + force_upload=self.force_upload, signing_profiles=None, disable_rollback=False, poll_delay=10, @@ -304,6 +306,7 @@ def test_watch_must_succeed_sync( self.image_repository, self.image_repositories, self.s3_bucket, + self.force_upload, self.s3_prefix, self.kms_key_id, self.capabilities, @@ -354,7 +357,7 @@ def test_watch_must_succeed_sync( region=self.region, profile=self.profile, use_json=False, - force_upload=True, + force_upload=self.force_upload, ) DeployContextMock.assert_called_with( @@ -377,7 +380,7 @@ def test_watch_must_succeed_sync( fail_on_empty_changeset=True, confirm_changeset=False, use_changeset=False, - force_upload=True, + force_upload=self.force_upload, signing_profiles=None, disable_rollback=False, poll_delay=0.5, @@ -456,6 +459,7 @@ def test_code_must_succeed_sync( self.image_repository, self.image_repositories, self.s3_bucket, + self.force_upload, self.s3_prefix, self.kms_key_id, self.capabilities, @@ -481,6 +485,77 @@ def test_code_must_succeed_sync( auto_dependency_layer=auto_dependency_layer, ) + @parameterized.expand([(True,), (False,)]) + @patch("samcli.commands.sync.command.click") + @patch("samcli.commands.sync.command.execute_infra_contexts") + @patch("samcli.commands.sync.command.execute_code_sync") + @patch("samcli.commands.build.build_context.BuildContext") + @patch("samcli.commands.package.package_context.PackageContext") + @patch("samcli.commands.deploy.deploy_context.DeployContext") + @patch("samcli.commands.sync.command.manage_stack") + @patch("samcli.commands.sync.command.SyncContext") + @patch("samcli.commands.sync.command.check_enable_dependency_layer") + def test_force_upload_flag_propagates_to_package_and_deploy_contexts( + self, + force_upload, + check_enable_adl_mock, + SyncContextMock, + manage_stack_mock, + DeployContextMock, + PackageContextMock, + BuildContextMock, + execute_code_sync_mock, + execute_infra_mock, + click_mock, + ): + # `force_upload` was previously hardcoded to True in sam sync, which + # bypassed the S3Uploader SHA-based dedup and made every sync re-upload + # all artifacts. The flag must now flow through to both PackageContext + # and DeployContext so users can opt back into the old behavior. + BuildContextMock.return_value.__enter__.return_value = Mock() + PackageContextMock.return_value.__enter__.return_value = Mock() + DeployContextMock.return_value.__enter__.return_value = Mock() + SyncContextMock.return_value.__enter__.return_value = Mock() + check_enable_adl_mock.return_value = False + execute_infra_mock.return_value = InfraSyncResult(True) + + do_cli( + self.template_file, + False, + False, + self.resource_id, + self.resource, + False, + True, + self.stack_name, + self.region, + self.profile, + self.base_dir, + self.parameter_overrides, + self.mode, + self.image_repository, + self.image_repositories, + self.s3_bucket, + force_upload, + self.s3_prefix, + self.kms_key_id, + self.capabilities, + self.role_arn, + self.notification_arns, + self.tags, + self.metadata, + False, + self.container_env_var_file, + self.build_image, + self.config_file, + self.config_env, + build_in_source=False, + watch_exclude={}, + ) + + self.assertEqual(PackageContextMock.call_args.kwargs["force_upload"], force_upload) + self.assertEqual(DeployContextMock.call_args.kwargs["force_upload"], force_upload) + class TestSyncCode(TestCase): def setUp(self) -> None: diff --git a/tests/unit/lib/sync/flows/test_zip_function_sync_flow.py b/tests/unit/lib/sync/flows/test_zip_function_sync_flow.py index 1b79849fc5f..e09d688d923 100644 --- a/tests/unit/lib/sync/flows/test_zip_function_sync_flow.py +++ b/tests/unit/lib/sync/flows/test_zip_function_sync_flow.py @@ -281,6 +281,39 @@ def test_sync_s3(self, session_mock, getsize_mock, uploader_mock, exists_mock, r sync_flow._get_lock_chain.return_value.__exit__.assert_called_once() remove_mock.assert_called_once_with("zip_file") + @parameterized.expand([(True,), (False,)]) + @patch("samcli.lib.sync.flows.function_sync_flow.wait_for_function_update_complete") + @patch("samcli.lib.sync.flows.zip_function_sync_flow.open", mock_open(read_data=b"zip_content"), create=True) + @patch("samcli.lib.sync.flows.zip_function_sync_flow.os.remove") + @patch("samcli.lib.sync.flows.zip_function_sync_flow.os.path.exists") + @patch("samcli.lib.sync.flows.zip_function_sync_flow.S3Uploader") + @patch("samcli.lib.sync.flows.zip_function_sync_flow.os.path.getsize") + @patch("samcli.lib.sync.sync_flow.Session") + def test_s3_uploader_uses_deploy_context_force_upload( + self, force_upload, session_mock, getsize_mock, uploader_mock, exists_mock, remove_mock, wait_mock + ): + # The S3Uploader for oversized zips previously hardcoded + # force_upload=True, which defeated upload_with_dedup's SHA-based skip. + # It must now read the value from the deploy context so the new + # --force-upload sam sync flag actually controls this code path. + getsize_mock.return_value = 51 * 1024 * 1024 + exists_mock.return_value = True + uploader_mock.return_value.upload_with_dedup.return_value = "s3://bucket_name/bucket/key" + sync_flow = self.create_function_sync_flow() + sync_flow._zip_file = "zip_file" + sync_flow._deploy_context.s3_bucket = "bucket_name" + sync_flow._deploy_context.force_upload = force_upload + + sync_flow._get_lock_chain = MagicMock() + sync_flow.has_locks = MagicMock() + sync_flow.get_physical_id = MagicMock() + sync_flow.get_physical_id.return_value = "PhysicalFunction1" + + sync_flow.set_up() + sync_flow.sync() + + self.assertEqual(uploader_mock.call_args.kwargs["force_upload"], force_upload) + @parameterized.expand( [ # publish_to_latest_published, has_capacity_provider_config, expect_api_list