AWSTemplateFormatVersion: ‘2010-09-09’
Description: “Template to setup the AWS CloudWatch Logs collection using Amazon Kinesis Firehose and send the data to provided Sumo Logic URL.”
Metadata:
AWS::ServerlessRepo::Application:
Author: Sumo Logic
Description: This solution helps to setup the AWS CloudWatch Logs collection using Amazon Kinesis Firehose and send the data to provided Sumo Logic URL.
HomePageUrl:
https://github.com/SumoLogic/sumologic-aws-lambda
Labels:
- lambda
- sumologic
- serverless
- kinesis
- firehose
- cloudwatch
- logs
Name: sumologic-kinesis-cloudwatch-logs
SemanticVersion: 1.0.0
SourceCodeUrl:
https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/kinesis-firehose-cloudwatch-collection/logs
LicenseUrl: ./LICENSE
ReadmeUrl: ./README.md
SpdxLicenseId: Apache-2.0
‘AWS::CloudFormation::Interface’:
ParameterGroups:
- Label:
default: “1. Sumo Logic Kinesis Firehose Logs Configuration”
Parameters:
- Section1aSumoLogicKinesisLogsURL
- Label:
default: “2. Failed Data AWS S3 Bucket Configuration”
Parameters:
- Section2aCreateS3Bucket
- Section2bFailedDataS3Bucket
ParameterLabels:
Section1aSumoLogicKinesisLogsURL:
default: “Sumo Logic AWS Kinesis Firehose for Logs Source URL”
Section2aCreateS3Bucket:
default: “Create AWS S3 Bucket”
Section2bFailedDataS3Bucket:
default: “AWS S3 Bucket Name for Failed Data”
Parameters:
Section1aSumoLogicKinesisLogsURL:
Type: String
Description: “Provide HTTP Source Address from AWS Kinesis Firehose for Logs source created on your Sumo Logic account.”
AllowedPattern: “.+”
ConstraintDescription: “Sumo Logic AWS Kinesis Firehose for Logs Source URL can not be empty.”
Section2aCreateS3Bucket:
Type: String
Default: ‘Yes’
Description: “Yes - Create a new AWS S3 Bucket to store failed data.’.
No - Use an existing AWS S3 Bucket to store failed data.”
AllowedValues:
- ‘Yes’
- ‘No’
Section2bFailedDataS3Bucket:
Type: String
Description: “Provide a unique name of AWS S3 bucket where you would like to store Failed data. In case of existing AWS S3 bucket, provide the bucket from the current AWS Account. For Logs, failed data will be stored in folder prefix as SumoLogic-Kinesis-Failed-Logs.”
AllowedPattern: “[a-z0-9-.]{3,63}$”
ConstraintDescription: “3-63 characters; must contain only lowercase letters, numbers, hyphen or period. For more details -
https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html”
Conditions:
create_bucket: !Equals [ !Ref Section2aCreateS3Bucket, ‘Yes’ ]
Mappings:
ArnValue:
us-east-1:
ArnValue: “aws”
us-east-2:
ArnValue: “aws”
us-west-1:
ArnValue: “aws”
us-west-2:
ArnValue: “aws”
af-south-1:
ArnValue: “aws”
ca-central-1:
ArnValue: “aws”
eu-central-1:
ArnValue: “aws”
eu-west-1:
ArnValue: “aws”
eu-west-2:
ArnValue: “aws”
eu-south-1:
ArnValue: “aws”
eu-west-3:
ArnValue: “aws”
eu-north-1:
ArnValue: “aws”
ap-east-1:
ArnValue: “aws”
ap-northeast-1:
ArnValue: “aws”
ap-northeast-2:
ArnValue: “aws”
ap-northeast-3:
ArnValue: “aws”
ap-southeast-1:
ArnValue: “aws”
ap-southeast-2:
ArnValue: “aws”
ap-south-1:
ArnValue: “aws”
me-south-1:
ArnValue: “aws”
sa-east-1:
ArnValue: “aws”
us-gov-west-1:
ArnValue: “aws-us-gov”
us-gov-east-1:
ArnValue: “aws-us-gov”
cn-north-1:
ArnValue: “aws”
cn-northwest-1:
ArnValue: “aws”
Resources:
# Common Resources including creating bucket, create logs role and attach bucket policy.
FailedDataBucket:
Type: AWS::S3::Bucket
Condition: create_bucket
Metadata:
cfn_nag:
rules_to_suppress:
- id: W51
reason: “Role has been assigned permission to put logs in AWS S3 bucket.”
- id: W35
reason: “Access logging not required for AWS S3 Bucket.”
- id: W41
reason: “Encryption not required for AWS S3 Bucket.”
Properties:
BucketName: !Ref Section2bFailedDataS3Bucket
AccessControl: Private
PublicAccessBlockConfiguration:
BlockPublicAcls: true
BlockPublicPolicy: true
IgnorePublicAcls: true
RestrictPublicBuckets: true
FirehoseLogsRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: ‘2012-10-17’
Statement:
- Sid: ‘FirehoseAssumeRole’
Effect: Allow
Principal:
Service:
firehose.amazonaws.com
Action: sts:AssumeRole
Condition:
StringEquals:
sts:ExternalId: !Ref “AWS::AccountId”
AttachBucketPolicyToFirehoseLogsRole:
Type: AWS::IAM::Policy
Properties:
PolicyName: “S3BucketPermissionPolicy”
PolicyDocument:
Version: ‘2012-10-17’
Statement:
- Effect: Allow
Action:
- s3:AbortMultipartUpload
- s3:GetBucketLocation
- s3:GetObject
- s3:ListBucket
- s3:ListBucketMultipartUploads
- s3:PutObject
Resource:
- !Sub
- “arn:${arn}😒3:::${S3Bucket}/*”
- arn: !FindInMap [ArnValue, !Ref “AWS::Region”, ArnValue]
S3Bucket: !Ref Section2bFailedDataS3Bucket
- !Sub
- “arn:${arn}😒3:::${S3Bucket}”
- arn: !FindInMap [ArnValue, !Ref “AWS::Region”, ArnValue]
S3Bucket: !Ref Section2bFailedDataS3Bucket
Roles:
- Ref: FirehoseLogsRole
# Resources for AWS Kinesis Logs including log group, stream, delivery stream, subscription filter, policy and role.
KinesisLogsLogGroup:
Type: AWS::Logs::LogGroup
Properties:
LogGroupName: !Sub
- “/aws/kinesisfirehose/Kinesis-Logs-${StackID}”
- StackID: !Select
- 0
- !Split
- “-”
- !Select
- 2
- !Split [“/”, !Ref “AWS::StackId”]
RetentionInDays: 7
KinesisLogsLogStream:
Type: AWS::Logs::LogStream
Properties:
LogGroupName: !Ref KinesisLogsLogGroup
LogStreamName: “HttpEndpointDelivery”
KinesisLogsLogStreamS3:
Type: AWS::Logs::LogStream
Properties:
LogGroupName: !Ref KinesisLogsLogGroup
LogStreamName: “S3Delivery”
KinesisLogsRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: ‘2012-10-17’
Statement:
- Sid: ‘LogsAssumeRole’
Effect: Allow
Principal:
Service: !Sub “logs.${AWS::Region}.
amazonaws.com”
Action: sts:AssumeRole
KinesisLogsRolePolicy:
Type: AWS::IAM::Policy
Properties:
PolicyName: “KinesisFirehoseLogsPolicy”
PolicyDocument:
Version: ‘2012-10-17’
Statement:
- Effect: Allow
Action:
- firehose:PutRecord
- firehose:PutRecordBatch
- kinesis:PutRecord
- kinesis:PutRecordBatch
Resource: !Sub
- “arn:${arn}:firehose:${Region}:${AccountId}:*”
- arn: !FindInMap [ArnValue, !Ref “AWS::Region”, ArnValue]
Region: !Ref “AWS::Region”
AccountId: !Ref “AWS::AccountId”
- Effect: Allow
Action:
- iam:PassRole
Resource: !Sub
- “arn:${arn}:iam::${AccountId}:role/${KinesisLogsRole}”
- arn: !FindInMap [ArnValue, !Ref “AWS::Region”, ArnValue]
AccountId: !Ref “AWS::AccountId”
KinesisLogsRole: !Ref KinesisLogsRole
Roles:
- Ref: KinesisLogsRole
KinesisLogsDeliveryStream:
Type: AWS::KinesisFirehose::DeliveryStream
Properties:
DeliveryStreamName: !Sub
- “Kinesis-Logs-${StackID}”
- StackID: !Select
- 0
- !Split
- “-”
- !Select
- 2
- !Split [“/”, !Ref “AWS::StackId”]
DeliveryStreamType: DirectPut
HttpEndpointDestinationConfiguration:
RoleARN: !GetAtt FirehoseLogsRole.Arn
EndpointConfiguration:
Url: !Ref Section1aSumoLogicKinesisLogsURL
Name: !Sub “${AWS::StackName}-sumologic-logs-endpoint”
RequestConfiguration:
ContentEncoding: GZIP
CloudWatchLoggingOptions:
Enabled: true
LogGroupName: !Ref KinesisLogsLogGroup
LogStreamName: !Ref KinesisLogsLogStream
BufferingHints:
IntervalInSeconds: 60
SizeInMBs: 4
RetryOptions:
DurationInSeconds: 60
S3BackupMode: FailedDataOnly
S3Configuration:
BucketARN: !If
- create_bucket
- !GetAtt FailedDataBucket.Arn
- !Sub
- “arn:${arn}😒3:::${S3Bucket}”
- arn: !FindInMap [ArnValue, !Ref “AWS::Region”, ArnValue]
S3Bucket: !Ref Section2bFailedDataS3Bucket
CompressionFormat: UNCOMPRESSED
ErrorOutputPrefix: “SumoLogic-Kinesis-Failed-Logs/”
RoleARN: !GetAtt FirehoseLogsRole.Arn
CloudWatchLoggingOptions:
Enabled: true
LogGroupName: !Ref KinesisLogsLogGroup
LogStreamName: !Ref KinesisLogsLogStreamS3
KinesisLogsFirehoseRolePolicy:
Type: AWS::IAM::Policy
Properties:
PolicyName: “KinesisLogsLogStreamPermissionPolicy”
PolicyDocument:
Version: ‘2012-10-17’
Statement:
- Effect: Allow
Action:
- logs:PutLogEvents
Resource:
- !GetAtt KinesisLogsLogGroup.Arn
- !Sub “${KinesisLogsLogGroup.Arn}:log-stream:*”
- Effect: Allow
Action:
- kinesis:DescribeStream
- kinesis:GetShardIterator
- kinesis:GetRecords
- kinesis:ListShards
Resource:
- !GetAtt KinesisLogsDeliveryStream.Arn
Roles:
- Ref: FirehoseLogsRole
SubscriptionFilter:
Type: AWS::Logs::SubscriptionFilter
Properties:
RoleArn: !GetAtt KinesisLogsRole.Arn
LogGroupName: !Ref KinesisLogsLogGroup
FilterPattern: ‘’
DestinationArn: !GetAtt KinesisLogsDeliveryStream.Arn
Outputs:
FailedDataBucketArn:
Description: “S3 Bucket Arn where failed deliveries will be saved”
Condition: create_bucket
Value: !GetAtt FailedDataBucket.Arn
KinesisLogsDeliveryStreamARN:
Description: “The ARN for your Kinesis Firehose Delivery Stream, use this as the destination when adding CloudWatch Logs subscription filters.”
Value: !GetAtt KinesisLogsDeliveryStream.Arn
KinesisLogsRoleARN:
Description: “The ARN for your CloudWatch Logs role to write to your delivery stream, use this as the role-arn when adding CloudWatch Logs subscription filters.”
Value: !GetAtt KinesisLogsRole.Arn
KinesisLogsLogGroupARN:
Description: “The CloudWatch log group name where kinesis stream logs will be sent.”
Value: !Ref KinesisLogsLogGroup