https://pulumi.com logo
Join Slack
Powered by
Hey there! I have a question, need help. I need to...
# aws
s
Hey there! I have a question, need help. I need to configure Standard logging in Cloudfront to send logs to Firehose datastreem. I found only options to send to s3 with standard logging and to kinesis with Real time logging.
What can be an issue? Curently my config is:
Copy code
package cloudfront

import (
    "ddos-protection-pulumi/config"
    "ddos-protection-pulumi/provider"
    "fmt"
    "<http://github.com/pulumi/pulumi-aws/sdk/v7/go/aws/cloudfront|github.com/pulumi/pulumi-aws/sdk/v7/go/aws/cloudfront>"
    "<http://github.com/pulumi/pulumi-aws/sdk/v7/go/aws/cloudwatch|github.com/pulumi/pulumi-aws/sdk/v7/go/aws/cloudwatch>"
    "<http://github.com/pulumi/pulumi-aws/sdk/v7/go/aws/iam|github.com/pulumi/pulumi-aws/sdk/v7/go/aws/iam>"
    "<http://github.com/pulumi/pulumi-aws/sdk/v7/go/aws/kinesis|github.com/pulumi/pulumi-aws/sdk/v7/go/aws/kinesis>"
    "<http://github.com/pulumi/pulumi/sdk/v3/go/pulumi|github.com/pulumi/pulumi/sdk/v3/go/pulumi>"
    log "<http://github.com/sirupsen/logrus|github.com/sirupsen/logrus>"
)

func CreateFirehoseToVector(ctx *pulumi.Context) (*kinesis.FirehoseDeliveryStream, error) {
    <http://log.Info|log.Info>("Creating Kinesis Data Firehose delivery stream to Vector HTTP endpoint")

    // Reuse existing IAM role
    //firehoseRoleARN := "arn:aws:iam::300926307005:role/firehose_delivery_role"
    firehoseRole, err := CreateMinimalFirehoseRole(ctx)
    if err != nil {
       return nil, err
    }
    // S3 Backup bucket ARN
    backupBucketArn := pulumi.String("arn:aws:s3:::<http://firehose-logs-backup.nanocosmos.de|firehose-logs-backup.nanocosmos.de>")

    // Create Firehose Delivery Stream to HTTP endpoint (Vector)
    firehoseStream, err := kinesis.NewFirehoseDeliveryStream(ctx, "nanoplayer1-firehose", &kinesis.FirehoseDeliveryStreamArgs{
       Name:        pulumi.String("nanoplayer1"),
       Destination: pulumi.String("http_endpoint"),

       // HTTP Endpoint Destination Configuration
       HttpEndpointConfiguration: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationArgs{
          Url:           pulumi.String("<https://vector-aws-kinesis-firehose.nanocosmos.cloud>"),
          Name:          pulumi.String("vector-http-endpoint"),
          RetryDuration: <http://pulumi.Int|pulumi.Int>(300), // Max retry duration (5 min)
          RoleArn:       firehoseRole.Arn,
          S3Configuration: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationS3ConfigurationArgs{
             BucketArn:         backupBucketArn,
             RoleArn:           firehoseRole.Arn,
             Prefix:            pulumi.String("nanoplayer1/"),
             BufferingSize:     <http://pulumi.Int|pulumi.Int>(8),   // In MB
             BufferingInterval: <http://pulumi.Int|pulumi.Int>(300), // In seconds
             CompressionFormat: pulumi.String("GZIP"),
          },
          ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationArgs{
             Enabled: pulumi.Bool(false), // No Lambda processing
          },
          AccessKey: pulumi.String("TempPassword"), // Sensitive — consider using Secrets Manager in production
          // Content encoding
          CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationCloudwatchLoggingOptionsArgs{
             Enabled:       pulumi.Bool(true),
             LogGroupName:  pulumi.String("/aws/firehose/nanoplayer1"),
             LogStreamName: pulumi.String("delivery-stream"),
          },
       },
    }, pulumi.Provider(provider.GetUsEast1Provider(ctx)))

    if err != nil {
       log.Errorf("Failed to create Firehose delivery stream: %v", err)
       return nil, fmt.Errorf("failed to create Firehose stream 'nanoplayer1': %w", err)
    }

    <http://log.Info|log.Info>("Successfully created Firehose delivery stream: nanoplayer1")
    return firehoseStream, nil
}

func SetupLogForwarderToFirehose(ctx *pulumi.Context, cfVars config.CfVars, distr *cloudfront.Distribution, firehose *kinesis.FirehoseDeliveryStream) error {
    log.Infof("Setting up CloudFront V2 log delivery to Firehose: nanoplayer1")

    // Ensure all AWS resources for V2 logging are in us-east-1
    usEast1Provider := provider.GetUsEast1Provider(ctx)

    // Step 1: Create Log Delivery Source (CloudFront distribution)
    logSource, err := cloudwatch.NewLogDeliverySource(ctx, "cf-log-source", &cloudwatch.LogDeliverySourceArgs{
       Region:      pulumi.String("us-east-1"),
       Name:        pulumi.String(cfVars.OriginId + "-log-source"),
       LogType:     pulumi.String("ACCESS_LOGS"),
       ResourceArn: distr.Arn,
    }, pulumi.Provider(usEast1Provider))
    if err != nil {
       log.Errorf("Failed to create log delivery source: %v", err)
       return fmt.Errorf("failed to create log delivery source: %w", err)
    }

    // Step 2: Create Log Delivery Destination (Firehose) with proper role configuration
    logDestination, err := cloudwatch.NewLogDeliveryDestination(ctx, "cf-firehose-destination", &cloudwatch.LogDeliveryDestinationArgs{
       Region:       pulumi.String("us-east-1"),
       Name:         pulumi.String("firehose-nanoplayer1-destination"),
       OutputFormat: pulumi.String("json"), // Structured JSON logs
       DeliveryDestinationConfiguration: &cloudwatch.LogDeliveryDestinationDeliveryDestinationConfigurationArgs{
          DestinationResourceArn: firehose.Arn,
       },
       // Add tags if needed
       Tags: pulumi.StringMap{
          "Purpose": pulumi.String("CloudFront-Logs-to-Vector"),
       },
    }, pulumi.Provider(usEast1Provider))
    if err != nil {
       log.Errorf("Failed to create log delivery destination: %v", err)
       return fmt.Errorf("failed to create log delivery destination: %w", err)
    }

    // Step 3: Enable Log Delivery (connect source → destination) with explicit role
    _, err = cloudwatch.NewLogDelivery(ctx, "cf-to-firehose-delivery", &cloudwatch.LogDeliveryArgs{
       Region:                 pulumi.String("us-east-1"),
       DeliverySourceName:     logSource.Name,
       DeliveryDestinationArn: logDestination.Arn,
       // Add tags if needed
       Tags: pulumi.StringMap{
          "Purpose": pulumi.String("CloudFront-to-Firehose-Delivery"),
       },
    }, pulumi.Provider(usEast1Provider), pulumi.DependsOn([]pulumi.Resource{logSource, logDestination}))
    if err != nil {
       log.Errorf("Failed to enable log delivery: %v", err)
       return fmt.Errorf("failed to enable log delivery to firehose: %w", err)
    }

    log.Infof("Successfully enabled real-time log delivery from CloudFront to Firehose (nanoplayer)")
    return nil
}

func CreateMinimalFirehoseRole(ctx *pulumi.Context) (*iam.Role, error) {
    <http://log.Info|log.Info>("Creating minimal IAM role for Firehose delivery stream")

    // Step 1: Create the IAM role that Firehose can assume
    role, err := iam.NewRole(ctx, "firehose-nanoplayer-role", &iam.RoleArgs{
       AssumeRolePolicy: pulumi.String(`{
          "Version": "2012-10-17",
          "Statement": [
             {
                "Effect": "Allow",
                "Principal": {
                   "Service": "<http://firehose.amazonaws.com|firehose.amazonaws.com>"
                },
                "Action": "sts:AssumeRole"
             }
          ]
       }`),
       Description: pulumi.String("Minimal IAM role for Firehose to deliver CloudFront logs to Vector endpoint"),
    }, pulumi.Provider(provider.GetUsEast1Provider(ctx)))
    if err != nil {
       return nil, fmt.Errorf("failed to create IAM role: %w", err)
    }

    // Step 2: Attach minimal required policies

    // Inline policy: S3 + Firehose + CloudWatch Logs
    policyDoc := pulumi.String(`{
          "Version": "2012-10-17",
          "Statement": [
             {
                "Sid": "S3BackupPermissions",
                "Effect": "Allow",
                "Action": [
                   "s3:AbortMultipartUpload",
                   "s3:GetBucketLocation",
                   "s3:ListBucket",
                   "s3:ListBucketMultipartUploads",
                   "s3:PutObject"
                ],
                "Resource": [
                   "arn:aws:s3:::<http://firehose-logs-backup.nanocosmos.de|firehose-logs-backup.nanocosmos.de>",
                   "arn:aws:s3:::<http://firehose-logs-backup.nanocosmos.de/*|firehose-logs-backup.nanocosmos.de/*>"
                ]
             },
             {
                "Sid": "FirehoseDeliveryToHTTP",
                "Effect": "Allow",
                "Action": [
                   "firehose:PutRecord",
                   "firehose:PutRecordBatch"
                ],
                "Resource": "*"
             },
             {
                "Sid": "CloudWatchLogsDeliverySetup",
                "Effect": "Allow",
                "Action": [
                   "logs:CreateLogDelivery",
                   "logs:UpdateLogDelivery",
                   "logs:DeleteLogDelivery",
                   "logs:GetLogDelivery",
                   "logs:ListLogDeliveries",
                   "logs:PutResourcePolicy"
                ],
                "Resource": "*"
             },
             {
                "Sid": "CloudWatchLogging",
                "Effect": "Allow",
                "Action": [
                   "logs:PutLogEvents"
                ],
                "Resource": "arn:aws:logs:us-east-1:*:*"
             }
          ]
       }`)

    _, err = iam.NewRolePolicy(ctx, "firehose-nanoplayer-policy", &iam.RolePolicyArgs{
       Role:   role.Name,
       Policy: policyDoc,
    }, pulumi.Provider(provider.GetUsEast1Provider(ctx)))
    if err != nil {
       return nil, fmt.Errorf("failed to attach policy to IAM role: %w", err)
    }

    <http://log.Info|log.Info>("Successfully created minimal Firehose IAM role")
    return role, nil
}
But I got an error: 
aws:cloudwatch:LogDelivery (cf-to-firehose-delivery):
error: creating CloudWatch Logs Delivery: operation error CloudWatch Logs: CreateDelivery, https response error StatusCode: 400, RequestID: e656faef-aefe-445f-a36e-7d300d133e0c, AccessDeniedException: Access Denied for this Delivery Destination. Please make sure that you have correct permissions to access the Log Destination Resource.
b
Have you tried messing around with the delivery destination? Someone else mentioned double checking the name conventions and names of everything too. I think that would be a good place to start
Failing that, maybe set up all the infrastructure and comment out the part that is failing, and then log into the account with the credential you are using (or a similarly scoped one) and try to set up the log firehose manually. That will let you know if: 1. There is a legitimate problem with the credentials you are using (insufficient permissions or what have you) 2. If AWS was rejecting the format of your endpoint or one of your variable names or something If you can do that and it is still failing maybe ping the thread again and I can look again
Open in Slack
PreviousNext
Powered by