From b6354dfc77cdcdde8e46a4d775c25227d903bb35 Mon Sep 17 00:00:00 2001 From: Stephen Cuppett Date: Fri, 26 Mar 2021 15:38:23 -0400 Subject: [PATCH 1/4] Fix: Interrupted UPDATE/CREATE with DELETE would cause error and orphan state Observed the following sequence: 1) Create long-running stack create 2) During CREATE_IN_PROGRESS, delete k8s Stack 3) Delete timestamps got added, no deleteStack call made to AWS 4) Stack completed 5) Operator stopped following, but left resource as CREATE_IN_PROGRESS The problem was in updates being performed to an out-of-date object handle (in the follower). Now, it fetches a fresh object (cached) for the latest version we have. This will give it more opportunity to work itself out. See also: https://github.com/cuppett/cloudformation-operator/pull/1#discussion_r601849772 --- controllers/stack_follower.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/controllers/stack_follower.go b/controllers/stack_follower.go index a044df2..3209b83 100644 --- a/controllers/stack_follower.go +++ b/controllers/stack_follower.go @@ -32,6 +32,7 @@ import ( cloudformationv1alpha1 "github.com/linki/cloudformation-operator/api/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "reflect" "sigs.k8s.io/controller-runtime/pkg/client" "sync" @@ -69,7 +70,8 @@ func (f *StackFollower) BeingFollowed(stackId string) bool { // Identify if the follower is actively working this one. func (f *StackFollower) startFollowing(stack *cloudformationv1alpha1.Stack) { - f.mapPollingList.Store(stack.Status.StackID, stack) + namespacedName := &types.NamespacedName{Name: stack.Name, Namespace: stack.Namespace} + f.mapPollingList.Store(stack.Status.StackID, namespacedName) f.Log.Info("Now following Stack", "StackID", stack.Status.StackID) } @@ -157,7 +159,21 @@ func (f *StackFollower) UpdateStackStatus(ctx context.Context, instance *cloudfo func (f *StackFollower) processStack(key interface{}, value interface{}) bool { stackId := key.(string) - stack := value.(*cloudformationv1alpha1.Stack) + namespacedName := value.(*types.NamespacedName) + stack := &cloudformationv1alpha1.Stack{} + + // Fetch the Stack instance + err := f.Client.Get(context.TODO(), *namespacedName, stack) + if err != nil { + if errors.IsNotFound(err) { + f.Log.Info("Stack resource not found. Ignoring since object must be deleted") + f.stopFollowing(stackId) + return true + } + // Error reading the object - requeue the request. + f.Log.Error(err, "Failed to get Stack on this pass, requeuing") + return true + } cfs, err := f.CloudFormationHelper.GetStack(context.TODO(), stack) if err != nil { From 51f70574ad83b637f1fc7c20b49d70efab8e6804 Mon Sep 17 00:00:00 2001 From: Yassin Chelabi Date: Sat, 27 Mar 2021 16:58:28 -0400 Subject: [PATCH 2/4] Recreate the CloudFormation during updates when the CF stack was deleted by hand in AWS Signed-off-by: Stephen Cuppett --- controllers/stack_controller.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/controllers/stack_controller.go b/controllers/stack_controller.go index 8ab4dae..2ff0788 100644 --- a/controllers/stack_controller.go +++ b/controllers/stack_controller.go @@ -237,6 +237,9 @@ func (r *StackReconciler) updateStack(loop *StackLoop) error { if strings.Contains(err.Error(), "No updates are to be performed.") { r.Log.WithValues("stack", loop.instance.Name).Info("stack already updated") return nil + } else if strings.Contains(err.Error(), "does not exist") { + r.Log.WithValues("stack", loop.instance.Name).Info("Stack does not exist in AWS. Re-creating it.") + return r.createStack(loop) } return err } From 36ac786e782fabb134a213524813fb268c3b68de Mon Sep 17 00:00:00 2001 From: Stephen Cuppett Date: Fri, 26 Mar 2021 14:56:22 -0400 Subject: [PATCH 3/4] Simplifying the examples to S3 Bucket and CloudFront CDN --- config/samples/cfs-my-bucket-tags.yaml | 17 -- config/samples/cfs-my-bucket-v1.yaml | 15 -- config/samples/cfs-my-bucket-v2.yaml | 15 -- config/samples/cfs-my-bucket-v3.yaml | 26 -- config/samples/cloudfront-cdn.yaml | 231 ++++++++++++++++++ config/samples/kustomization.yaml | 7 +- .../{cfs-my-bucket-v4.yaml => s3-bucket.yaml} | 2 + 7 files changed, 235 insertions(+), 78 deletions(-) delete mode 100644 config/samples/cfs-my-bucket-tags.yaml delete mode 100644 config/samples/cfs-my-bucket-v1.yaml delete mode 100644 config/samples/cfs-my-bucket-v2.yaml delete mode 100644 config/samples/cfs-my-bucket-v3.yaml create mode 100644 config/samples/cloudfront-cdn.yaml rename config/samples/{cfs-my-bucket-v4.yaml => s3-bucket.yaml} (95%) diff --git a/config/samples/cfs-my-bucket-tags.yaml b/config/samples/cfs-my-bucket-tags.yaml deleted file mode 100644 index 74f3e20..0000000 --- a/config/samples/cfs-my-bucket-tags.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: cloudformation.linki.space/v1alpha1 -kind: Stack -metadata: - name: my-bucket -spec: - tags: - foo: dataFromStack - template: | - --- - AWSTemplateFormatVersion: '2010-09-09' - - Resources: - S3Bucket: - Type: AWS::S3::Bucket - Properties: - VersioningConfiguration: - Status: Suspended diff --git a/config/samples/cfs-my-bucket-v1.yaml b/config/samples/cfs-my-bucket-v1.yaml deleted file mode 100644 index 781f9ef..0000000 --- a/config/samples/cfs-my-bucket-v1.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: cloudformation.linki.space/v1alpha1 -kind: Stack -metadata: - name: my-bucket -spec: - template: | - --- - AWSTemplateFormatVersion: '2010-09-09' - - Resources: - S3Bucket: - Type: AWS::S3::Bucket - Properties: - VersioningConfiguration: - Status: Suspended diff --git a/config/samples/cfs-my-bucket-v2.yaml b/config/samples/cfs-my-bucket-v2.yaml deleted file mode 100644 index 8a969b0..0000000 --- a/config/samples/cfs-my-bucket-v2.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: cloudformation.linki.space/v1alpha1 -kind: Stack -metadata: - name: my-bucket -spec: - template: | - --- - AWSTemplateFormatVersion: '2010-09-09' - - Resources: - S3Bucket: - Type: AWS::S3::Bucket - Properties: - VersioningConfiguration: - Status: Enabled diff --git a/config/samples/cfs-my-bucket-v3.yaml b/config/samples/cfs-my-bucket-v3.yaml deleted file mode 100644 index 0a67620..0000000 --- a/config/samples/cfs-my-bucket-v3.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: cloudformation.linki.space/v1alpha1 -kind: Stack -metadata: - name: my-bucket -spec: - parameters: - VersioningConfiguration: Enabled - template: | - --- - AWSTemplateFormatVersion: '2010-09-09' - - Parameters: - VersioningConfiguration: - Type: String - Default: Suspended - AllowedValues: - - Enabled - - Suspended - - Resources: - S3Bucket: - Type: AWS::S3::Bucket - Properties: - VersioningConfiguration: - Status: - Ref: VersioningConfiguration diff --git a/config/samples/cloudfront-cdn.yaml b/config/samples/cloudfront-cdn.yaml new file mode 100644 index 0000000..29fa79b --- /dev/null +++ b/config/samples/cloudfront-cdn.yaml @@ -0,0 +1,231 @@ +apiVersion: cloudformation.linki.space/v1alpha1 +kind: Stack +metadata: + name: my-cdn +spec: + template: | + --- + AWSTemplateFormatVersion: 2010-09-09 + Description: 'Content distribution template for AWS public zones (S3 bucket, CloudFront distribution, etc.)' + Parameters: + ResourceSuffix: + Description: >- + (Optional) A LOWER CASE suffix for any resources created by this region script - this + allows multiple sets of resources to be in one region. + Type: String + MinLength: '0' + MaxLength: '255' + AllowedPattern: '[_a-z0-9-]*' + Default: '' + ConstraintDescription: contain only lower case alphanumeric characters. + DnsZone: + Description: >- + (Optional) Amazon Route53 ZONE Name. This is the zone where a DNS record will be + created for the web app. The name should NOT end with a period. + Type: String + Default: '' + AcmCertificateArn: + Description: >- + (Optional) Amazon Certificate Manager ARN (us-east-1) used by CloudFront to protect this distribution + Type: String + Default: '' + Conditions: + HasResourceSuffix: !Not + - !Equals + - !Ref ResourceSuffix + - '' + HasDnsZone: !Not + - !Equals + - !Ref DnsZone + - '' + HasCert: !Not + - !Equals + - !Ref AcmCertificateArn + - '' + Resources: + ContentBucket: + Type: 'AWS::S3::Bucket' + Properties: + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + BucketName: !Join + - '-' + - - cdn + - !Ref 'AWS::Region' + - !Ref 'AWS::AccountId' + - !If + - HasResourceSuffix + - !Ref ResourceSuffix + - !Ref 'AWS::NoValue' + LifecycleConfiguration: + Rules: + - Status: Enabled + AbortIncompleteMultipartUpload: + DaysAfterInitiation: 7 + CloudFrontOriginAccessIdentity: + Type: AWS::CloudFront::CloudFrontOriginAccessIdentity + Properties: + CloudFrontOriginAccessIdentityConfig: + Comment: Identity for CDN + ContentBucketPolicy: + Type: 'AWS::S3::BucketPolicy' + Properties: + Bucket: !Ref ContentBucket + PolicyDocument: + Version: 2012-10-17 + Id: BucketAccessPolicy + Statement: + - Sid: AllowCloudFrontAccess + Effect: Allow + Principal: + CanonicalUser: !GetAtt + - CloudFrontOriginAccessIdentity + - S3CanonicalUserId + Action: 's3:GetObject' + Resource: + - !Join + - '' + - - 'arn:aws:s3:::' + - !Ref ContentBucket + - /* + ContentCachePolicy: + Type: AWS::CloudFront::CachePolicy + Properties: + CachePolicyConfig: + Comment: Default caching policy + Name: !Join + - '-' + - - 'cdn-master' + - !Ref 'AWS::StackName' + DefaultTTL: 86400 + MinTTL: 300 + MaxTTL: 86400 + ParametersInCacheKeyAndForwardedToOrigin: + EnableAcceptEncodingBrotli: true + EnableAcceptEncodingGzip: true + CookiesConfig: + CookieBehavior: none + HeadersConfig: + HeaderBehavior: none + QueryStringsConfig: + QueryStringBehavior: none + ContentDistribution: + Type: 'AWS::CloudFront::Distribution' + Properties: + DistributionConfig: + Aliases: + - !If + - HasDnsZone + - !Join + - '' + - - cdn + - !If + - HasResourceSuffix + - !Join + - '' + - - '-' + - !Ref ResourceSuffix + - !Ref 'AWS::NoValue' + - . + - !Ref DnsZone + - !Ref 'AWS::NoValue' + Origins: + - DomainName: !GetAtt + - ContentBucket + - DomainName + Id: !Join + - '-' + - - s3 + - !Ref ContentBucket + S3OriginConfig: + OriginAccessIdentity: !Join + - '/' + - - 'origin-access-identity' + - 'cloudfront' + - !Ref CloudFrontOriginAccessIdentity + Enabled: 'true' + Comment: S3 bucket content + DefaultCacheBehavior: + AllowedMethods: + - GET + - HEAD + CachePolicyId: !Ref ContentCachePolicy + Compress: true + TargetOriginId: !Join + - '-' + - - s3 + - !Ref ContentBucket + ViewerProtocolPolicy: redirect-to-https + PriceClass: PriceClass_100 + HttpVersion: http2 + ViewerCertificate: + SslSupportMethod: !If + - HasCert + - sni-only + - !Ref 'AWS::NoValue' + MinimumProtocolVersion: TLSv1.2_2019 + AcmCertificateArn: !If + - HasCert + - AcmCertificateArn + - !Ref 'AWS::NoValue' + CloudFrontDefaultCertificate: !If + - HasCert + - false + - true + CdnRecordset: + Type: 'AWS::Route53::RecordSet' + Condition: HasDnsZone + Properties: + HostedZoneName: !Join + - '' + - - !Ref DnsZone + - . + Name: !Join + - '' + - - cdn + - !If + - HasResourceSuffix + - !Join + - '' + - - '-' + - !Ref ResourceSuffix + - !Ref 'AWS::NoValue' + - . + - !Ref DnsZone + - . + Type: A + AliasTarget: + HostedZoneId: Z2FDTNDATAQYW2 + DNSName: !GetAtt + - ContentDistribution + - DomainName + Outputs: + ContentBucket: + Description: Content S3 bucket name + Value: !Ref ContentBucket + Export: + Name: !Join + - '-' + - - !Ref 'AWS::StackName' + - ContentBucket + CdnUrl: + Description: The base CDN URL where content from the bucket will be accessible + Value: !If + - HasDnsZone + - !Join + - '' + - - 'https://' + - !Ref CdnRecordset + - / + - !Join + - '' + - - 'https://' + - !GetAtt ContentDistribution.DomainName + - / + Export: + Name: !Join + - '-' + - - !Ref 'AWS::StackName' + - CdnUrl diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index f27d0f3..0a64e71 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,8 +1,5 @@ ## Append samples you want in your CSV to this file as resources ## resources: -- cfs-my-bucket-tags.yaml -- cfs-my-bucket-v1.yaml -- cfs-my-bucket-v2.yaml -- cfs-my-bucket-v3.yaml -- cfs-my-bucket-v4.yaml +- s3-bucket.yaml +- cloudfront-cdn.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/cfs-my-bucket-v4.yaml b/config/samples/s3-bucket.yaml similarity index 95% rename from config/samples/cfs-my-bucket-v4.yaml rename to config/samples/s3-bucket.yaml index 4c59ce8..804353b 100644 --- a/config/samples/cfs-my-bucket-v4.yaml +++ b/config/samples/s3-bucket.yaml @@ -3,6 +3,8 @@ kind: Stack metadata: name: my-bucket spec: + tags: + foo: dataFromStack parameters: VersioningConfiguration: Enabled template: | From b02e676c79b3be2e63eb2d09bea9b12cb17e8c92 Mon Sep 17 00:00:00 2001 From: Stephen Cuppett Date: Sun, 2 May 2021 12:02:00 -0400 Subject: [PATCH 4/4] Adding SQS example Signed-off-by: Stephen Cuppett --- config/samples/sqs-queue.yaml | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 config/samples/sqs-queue.yaml diff --git a/config/samples/sqs-queue.yaml b/config/samples/sqs-queue.yaml new file mode 100644 index 0000000..dfdd566 --- /dev/null +++ b/config/samples/sqs-queue.yaml @@ -0,0 +1,41 @@ +apiVersion: cloudformation.linki.space/v1alpha1 +kind: Stack +metadata: + name: my-queue +spec: + template: | + --- + AWSTemplateFormatVersion: "2010-09-09" + Resources: + MySourceQueue: + Type: AWS::SQS::Queue + Properties: + RedrivePolicy: + deadLetterTargetArn: + Fn::GetAtt: + - "MyDeadLetterQueue" + - "Arn" + maxReceiveCount: 5 + MyDeadLetterQueue: + Type: AWS::SQS::Queue + Outputs: + SourceQueueURL: + Description: "URL of source queue" + Value: + Ref: "MySourceQueue" + SourceQueueARN: + Description: "ARN of source queue" + Value: + Fn::GetAtt: + - "MySourceQueue" + - "Arn" + DeadLetterQueueURL: + Description: "URL of dead-letter queue" + Value: + Ref: "MyDeadLetterQueue" + DeadLetterQueueARN: + Description: "ARN of dead-letter queue" + Value: + Fn::GetAtt: + - "MyDeadLetterQueue" + - "Arn"