Mastering Secrets Management at Scale: Vault, AWS Secrets Manager, and Parameter Store

Introduction

Every organization eventually faces the "where do we put secrets?" question. The answer is usually "it depends" - different use cases call for different solutions. I built a unified secrets management architecture that leverages the strengths of each tool:

• HashiCorp Vault: Dynamic secrets, PKI, encryption as a service
• AWS Secrets Manager: Managed rotation, RDS integration, cross-account sharing
• Parameter Store: Configuration values, feature flags, non-sensitive config
• External Secrets Operator: Kubernetes-native secret synchronization

Architecture Overview

When to Use What

HashiCorp Vault Setup

Vault on EKS with Raft Storage

# vault/main.tf

resource "helm_release" "vault" {
  name       = "vault"
  repository = "https://helm.releases.hashicorp.com"
  chart      = "vault"
  namespace  = "vault"
  version    = "0.27.0"

  values = [
    yamlencode({
      global = {
        enabled = true
        tlsDisable = false
      }

      server = {
        image = {
          repository = "hashicorp/vault"
          tag        = "1.15.4"
        }

        resources = {
          requests = {
            memory = "256Mi"
            cpu    = "250m"
          }
          limits = {
            memory = "512Mi"
            cpu    = "500m"
          }
        }

        # High availability with Raft
        ha = {
          enabled  = true
          replicas = 3
          raft = {
            enabled   = true
            setNodeId = true
            config    = <<-EOF
              ui = true
              
              listener "tcp" {
                tls_disable = 0
                address = "[::]:8200"
                cluster_address = "[::]:8201"
                tls_cert_file = "/vault/userconfig/vault-tls/tls.crt"
                tls_key_file = "/vault/userconfig/vault-tls/tls.key"
              }

              storage "raft" {
                path = "/vault/data"
                retry_join {
                  leader_api_addr = "https://vault-0.vault-internal:8200"
                  leader_ca_cert_file = "/vault/userconfig/vault-tls/ca.crt"
                }
                retry_join {
                  leader_api_addr = "https://vault-1.vault-internal:8200"
                  leader_ca_cert_file = "/vault/userconfig/vault-tls/ca.crt"
                }
                retry_join {
                  leader_api_addr = "https://vault-2.vault-internal:8200"
                  leader_ca_cert_file = "/vault/userconfig/vault-tls/ca.crt"
                }
              }

              seal "awskms" {
                region     = "${var.region}"
                kms_key_id = "${var.kms_key_id}"
              }

              service_registration "kubernetes" {}
            EOF
          }
        }

        # Auto-unseal with AWS KMS
        extraEnvironmentVars = {
          VAULT_SEAL_TYPE = "awskms"
          AWS_REGION      = var.region
        }

        # Service account for IRSA
        serviceAccount = {
          create = true
          annotations = {
            "eks.amazonaws.com/role-arn" = var.vault_role_arn
          }
        }

        # Persistent storage
        dataStorage = {
          enabled      = true
          size         = "10Gi"
          storageClass = "gp3"
        }

        # Audit logging
        auditStorage = {
          enabled      = true
          size         = "10Gi"
          storageClass = "gp3"
        }
      }

      injector = {
        enabled = true
        resources = {
          requests = {
            memory = "64Mi"
            cpu    = "50m"
          }
        }
      }
    })
  ]
}

Database Secrets Engine

# vault/database-engine.tf

resource "vault_database_secrets_mount" "postgres" {
  path = "database"

  postgresql {
    name              = "prod-postgres"
    username          = "vault_admin"
    password          = var.postgres_admin_password
    connection_url    = "postgresql://{{username}}:{{password}}@${var.postgres_host}:5432/postgres?sslmode=require"
    verify_connection = true
    allowed_roles     = ["app-readonly", "app-readwrite", "admin"]
  }
}

resource "vault_database_secret_backend_role" "app_readonly" {
  backend = vault_database_secrets_mount.postgres.path
  name    = "app-readonly"
  db_name = "prod-postgres"

  creation_statements = [
    "CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';",
    "GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"{{name}}\";",
    "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO \"{{name}}\";"
  ]

  revocation_statements = [
    "REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM \"{{name}}\";",
    "DROP ROLE IF EXISTS \"{{name}}\";"
  ]

  default_ttl = 3600   # 1 hour
  max_ttl     = 86400  # 24 hours
}

resource "vault_database_secret_backend_role" "app_readwrite" {
  backend = vault_database_secrets_mount.postgres.path
  name    = "app-readwrite"
  db_name = "prod-postgres"

  creation_statements = [
    "CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';",
    "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO \"{{name}}\";",
    "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO \"{{name}}\";"
  ]

  default_ttl = 3600
  max_ttl     = 86400
}

Kubernetes Authentication

# vault/kubernetes-auth.tf

resource "vault_auth_backend" "kubernetes" {
  type = "kubernetes"
  path = "kubernetes"
}

resource "vault_kubernetes_auth_backend_config" "config" {
  backend         = vault_auth_backend.kubernetes.path
  kubernetes_host = var.kubernetes_host
  
  # Use Vault's service account for token review
  disable_local_ca_jwt = true
}

resource "vault_kubernetes_auth_backend_role" "app" {
  backend                          = vault_auth_backend.kubernetes.path
  role_name                        = "app"
  bound_service_account_names      = ["app-sa"]
  bound_service_account_namespaces = ["production", "staging"]
  token_ttl                        = 3600
  token_policies                   = ["app-policy"]
}

resource "vault_policy" "app" {
  name = "app-policy"

  policy = <<-EOF
    # Read database credentials
    path "database/creds/app-readonly" {
      capabilities = ["read"]
    }

    # Read KV secrets
    path "secret/data/app/*" {
      capabilities = ["read", "list"]
    }

    # Use transit encryption
    path "transit/encrypt/app-key" {
      capabilities = ["update"]
    }

    path "transit/decrypt/app-key" {
      capabilities = ["update"]
    }
  EOF
}

AWS Secrets Manager

Secrets with Automatic Rotation

# aws-secrets/rds-secret.tf

resource "aws_secretsmanager_secret" "rds_master" {
  name        = "${var.environment}/rds/master-credentials"
  description = "RDS master credentials for ${var.environment}"
  
  kms_key_id = var.kms_key_arn

  tags = {
    Environment = var.environment
    ManagedBy   = "terraform"
  }
}

resource "aws_secretsmanager_secret_version" "rds_master" {
  secret_id = aws_secretsmanager_secret.rds_master.id
  secret_string = jsonencode({
    username = "admin"
    password = random_password.rds_master.result
    engine   = "postgres"
    host     = var.rds_endpoint
    port     = 5432
    dbname   = var.database_name
  })
}

# Automatic rotation
resource "aws_secretsmanager_secret_rotation" "rds_master" {
  secret_id           = aws_secretsmanager_secret.rds_master.id
  rotation_lambda_arn = aws_lambda_function.rotation.arn

  rotation_rules {
    automatically_after_days = 30
  }
}

# Rotation Lambda
resource "aws_lambda_function" "rotation" {
  function_name = "${var.environment}-rds-secret-rotation"
  role          = aws_iam_role.rotation_lambda.arn
  handler       = "lambda_function.lambda_handler"
  runtime       = "python3.11"
  timeout       = 30

  filename         = data.archive_file.rotation_lambda.output_path
  source_code_hash = data.archive_file.rotation_lambda.output_base64sha256

  vpc_config {
    subnet_ids         = var.private_subnet_ids
    security_group_ids = [aws_security_group.rotation_lambda.id]
  }

  environment {
    variables = {
      SECRETS_MANAGER_ENDPOINT = "https://secretsmanager.${var.region}.amazonaws.com"
    }
  }
}

Cross-Account Secret Sharing

# aws-secrets/cross-account.tf

resource "aws_secretsmanager_secret" "shared_api_key" {
  name       = "shared/api-keys/payment-gateway"
  kms_key_id = aws_kms_key.secrets.arn
}

resource "aws_secretsmanager_secret_policy" "cross_account" {
  secret_arn = aws_secretsmanager_secret.shared_api_key.arn

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Sid    = "AllowCrossAccountAccess"
        Effect = "Allow"
        Principal = {
          AWS = [
            "arn:aws:iam::${var.prod_account_id}:role/AppRole",
            "arn:aws:iam::${var.staging_account_id}:role/AppRole",
          ]
        }
        Action = [
          "secretsmanager:GetSecretValue",
          "secretsmanager:DescribeSecret"
        ]
        Resource = "*"
      }
    ]
  })
}

# KMS key policy for cross-account decryption
resource "aws_kms_key_policy" "secrets" {
  key_id = aws_kms_key.secrets.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Sid    = "AllowRootAccess"
        Effect = "Allow"
        Principal = {
          AWS = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"
        }
        Action   = "kms:*"
        Resource = "*"
      },
      {
        Sid    = "AllowCrossAccountDecrypt"
        Effect = "Allow"
        Principal = {
          AWS = [
            "arn:aws:iam::${var.prod_account_id}:role/AppRole",
            "arn:aws:iam::${var.staging_account_id}:role/AppRole",
          ]
        }
        Action = [
          "kms:Decrypt",
          "kms:DescribeKey"
        ]
        Resource = "*"
      }
    ]
  })
}

External Secrets Operator

Installation and Configuration

# external-secrets/main.tf

resource "helm_release" "external_secrets" {
  name       = "external-secrets"
  repository = "https://charts.external-secrets.io"
  chart      = "external-secrets"
  namespace  = "external-secrets"
  version    = "0.9.11"

  create_namespace = true

  values = [
    yamlencode({
      installCRDs = true
      
      serviceAccount = {
        create = true
        annotations = {
          "eks.amazonaws.com/role-arn" = var.external_secrets_role_arn
        }
      }

      webhook = {
        port = 9443
      }

      certController = {
        requeueInterval = "5m"
      }
    })
  ]
}

Secret Store Configuration

# external-secrets/cluster-secret-store.yaml
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: aws-secrets-manager
spec:
  provider:
    aws:
      service: SecretsManager
      region: us-east-1
      auth:
        jwt:
          serviceAccountRef:
            name: external-secrets
            namespace: external-secrets
---
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: aws-parameter-store
spec:
  provider:
    aws:
      service: ParameterStore
      region: us-east-1
      auth:
        jwt:
          serviceAccountRef:
            name: external-secrets
            namespace: external-secrets
---
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: vault
spec:
  provider:
    vault:
      server: "https://vault.vault.svc:8200"
      path: "secret"
      version: "v2"
      auth:
        kubernetes:
          mountPath: "kubernetes"
          role: "external-secrets"
          serviceAccountRef:
            name: external-secrets
            namespace: external-secrets

External Secret Examples

# external-secrets/database-secret.yaml
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: database-credentials
  namespace: production
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: aws-secrets-manager
    kind: ClusterSecretStore
  target:
    name: database-credentials
    creationPolicy: Owner
    template:
      type: Opaque
      data:
        DB_HOST: "{{ .host }}"
        DB_PORT: "{{ .port }}"
        DB_NAME: "{{ .dbname }}"
        DB_USER: "{{ .username }}"
        DB_PASSWORD: "{{ .password }}"
  data:
    - secretKey: host
      remoteRef:
        key: prod/rds/master-credentials
        property: host
    - secretKey: port
      remoteRef:
        key: prod/rds/master-credentials
        property: port
    - secretKey: dbname
      remoteRef:
        key: prod/rds/master-credentials
        property: dbname
    - secretKey: username
      remoteRef:
        key: prod/rds/master-credentials
        property: username
    - secretKey: password
      remoteRef:
        key: prod/rds/master-credentials
        property: password
---
# Vault dynamic database credentials
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: vault-db-creds
  namespace: production
spec:
  refreshInterval: 30m
  secretStoreRef:
    name: vault
    kind: ClusterSecretStore
  target:
    name: vault-db-creds
    creationPolicy: Owner
  dataFrom:
    - extract:
        key: database/creds/app-readonly

Secret Synchronization Flow

Parameter Store for Configuration

# parameter-store/main.tf

resource "aws_ssm_parameter" "app_config" {
  for_each = var.app_parameters

  name        = "/${var.environment}/app/${each.key}"
  description = each.value.description
  type        = each.value.secure ? "SecureString" : "String"
  value       = each.value.value
  key_id      = each.value.secure ? var.kms_key_id : null

  tags = {
    Environment = var.environment
    Application = "app"
  }
}

# Example parameters
variable "app_parameters" {
  default = {
    "log-level" = {
      value       = "INFO"
      description = "Application log level"
      secure      = false
    }
    "feature-flags/new-checkout" = {
      value       = "true"
      description = "Enable new checkout flow"
      secure      = false
    }
    "api-endpoint" = {
      value       = "https://api.internal.company.com"
      description = "Internal API endpoint"
      secure      = false
    }
    "encryption-key" = {
      value       = "base64-encoded-key"
      description = "Application encryption key"
      secure      = true
    }
  }
}

Parameter Store External Secret

apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: app-config
  namespace: production
spec:
  refreshInterval: 5m
  secretStoreRef:
    name: aws-parameter-store
    kind: ClusterSecretStore
  target:
    name: app-config
    creationPolicy: Owner
  data:
    - secretKey: LOG_LEVEL
      remoteRef:
        key: /prod/app/log-level
    - secretKey: FEATURE_NEW_CHECKOUT
      remoteRef:
        key: /prod/app/feature-flags/new-checkout
    - secretKey: API_ENDPOINT
      remoteRef:
        key: /prod/app/api-endpoint
    - secretKey: ENCRYPTION_KEY
      remoteRef:
        key: /prod/app/encryption-key

Vault Agent Sidecar for EC2

# vault-agent/userdata.sh.tpl

#!/bin/bash
set -e

# Install Vault
curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
apt-get update && apt-get install -y vault

# Configure Vault Agent
cat > /etc/vault-agent.hcl <<EOF
pid_file = "/var/run/vault-agent.pid"

vault {
  address = "${vault_addr}"
}

auto_auth {
  method "aws" {
    mount_path = "auth/aws"
    config = {
      type = "iam"
      role = "${vault_role}"
    }
  }

  sink "file" {
    config = {
      path = "/var/run/vault-token"
      mode = 0640
    }
  }
}

template {
  source      = "/etc/vault-templates/database.tpl"
  destination = "/etc/app/database.env"
  perms       = 0640
  command     = "systemctl reload app"
}

template {
  source      = "/etc/vault-templates/api-keys.tpl"
  destination = "/etc/app/api-keys.env"
  perms       = 0640
  command     = "systemctl reload app"
}
EOF

# Database credentials template
cat > /etc/vault-templates/database.tpl <<'EOF'
{{ with secret "database/creds/app-readonly" }}
DB_USER={{ .Data.username }}
DB_PASSWORD={{ .Data.password }}
{{ end }}
EOF

# Start Vault Agent
systemctl enable vault-agent
systemctl start vault-agent

Best Practices

Troubleshooting

"ExternalSecret not syncing"

# Check ExternalSecret status
kubectl get externalsecret -n production database-credentials -o yaml

# Check ESO logs
kubectl logs -n external-secrets -l app.kubernetes.io/name=external-secrets

# Verify ClusterSecretStore health
kubectl get clustersecretstore aws-secrets-manager -o yaml

"Vault authentication failing"

# Test Kubernetes auth from pod
vault write auth/kubernetes/login role=app jwt=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)

# Check Vault audit logs
vault audit list

"Secret rotation not working"

• Check Lambda execution logs in CloudWatch
• Verify Lambda has network access to RDS
• Check Secrets Manager rotation status in console

Conclusion

A unified secrets management strategy eliminates the "where did we put that secret?" problem. By using:

• Vault for dynamic, short-lived credentials
• Secrets Manager for AWS-integrated rotation
• Parameter Store for configuration
• External Secrets Operator for Kubernetes sync

You get the best of each tool while maintaining a consistent interface for applications. The key is choosing the right tool for each use case rather than forcing everything into one solution.