Implementing Private EKS with Transit Gateway and Hybrid Connectivity

Introduction

Running EKS with public endpoints is a non-starter for regulated industries. When I was tasked with building a Kubernetes platform for financial services, the requirements were clear:

• No public IP addresses - anywhere
• All traffic stays on private networks
• Connectivity to on-premises data centers
• Centralized egress for compliance logging

This post covers how I architected a fully private EKS deployment using Transit Gateway as the backbone for multi-VPC and hybrid connectivity.

Architecture Overview

Network CIDR Design

Transit Gateway Configuration

# transit-gateway/main.tf

resource "aws_ec2_transit_gateway" "main" {
  description                     = "Central Transit Gateway"
  default_route_table_association = "disable"
  default_route_table_propagation = "disable"
  dns_support                     = "enable"
  vpn_ecmp_support               = "enable"

  tags = {
    Name = "central-tgw"
  }
}

# Route Tables for different traffic patterns
resource "aws_ec2_transit_gateway_route_table" "production" {
  transit_gateway_id = aws_ec2_transit_gateway.main.id
  
  tags = {
    Name = "tgw-rt-production"
  }
}

resource "aws_ec2_transit_gateway_route_table" "shared_services" {
  transit_gateway_id = aws_ec2_transit_gateway.main.id
  
  tags = {
    Name = "tgw-rt-shared-services"
  }
}

resource "aws_ec2_transit_gateway_route_table" "egress" {
  transit_gateway_id = aws_ec2_transit_gateway.main.id
  
  tags = {
    Name = "tgw-rt-egress"
  }
}

# VPC Attachments
resource "aws_ec2_transit_gateway_vpc_attachment" "production" {
  subnet_ids         = var.production_tgw_subnet_ids
  transit_gateway_id = aws_ec2_transit_gateway.main.id
  vpc_id             = var.production_vpc_id
  
  dns_support        = "enable"
  
  tags = {
    Name = "tgw-attach-production"
  }
}

resource "aws_ec2_transit_gateway_vpc_attachment" "shared_services" {
  subnet_ids         = var.shared_services_tgw_subnet_ids
  transit_gateway_id = aws_ec2_transit_gateway.main.id
  vpc_id             = var.shared_services_vpc_id
  
  dns_support        = "enable"
  
  tags = {
    Name = "tgw-attach-shared-services"
  }
}

resource "aws_ec2_transit_gateway_vpc_attachment" "egress" {
  subnet_ids         = var.egress_tgw_subnet_ids
  transit_gateway_id = aws_ec2_transit_gateway.main.id
  vpc_id             = var.egress_vpc_id
  
  appliance_mode_support = "enable"  # For firewall inspection
  dns_support            = "enable"
  
  tags = {
    Name = "tgw-attach-egress"
  }
}

# Direct Connect Gateway attachment
resource "aws_dx_gateway_association" "main" {
  dx_gateway_id         = var.dx_gateway_id
  associated_gateway_id = aws_ec2_transit_gateway.main.id
  
  allowed_prefixes = [
    "10.0.0.0/8",
  ]
}

Transit Gateway Route Tables

# Transit Gateway Routes

# Production can reach Shared Services
resource "aws_ec2_transit_gateway_route" "prod_to_shared" {
  destination_cidr_block         = "10.200.0.0/16"
  transit_gateway_attachment_id  = aws_ec2_transit_gateway_vpc_attachment.shared_services.id
  transit_gateway_route_table_id = aws_ec2_transit_gateway_route_table.production.id
}

# Production default route to Egress for internet
resource "aws_ec2_transit_gateway_route" "prod_to_egress" {
  destination_cidr_block         = "0.0.0.0/0"
  transit_gateway_attachment_id  = aws_ec2_transit_gateway_vpc_attachment.egress.id
  transit_gateway_route_table_id = aws_ec2_transit_gateway_route_table.production.id
}

# Production to on-premises via Direct Connect
resource "aws_ec2_transit_gateway_route" "prod_to_onprem" {
  destination_cidr_block         = "172.16.0.0/12"
  transit_gateway_attachment_id  = aws_ec2_transit_gateway_dx_gateway_attachment.main.id
  transit_gateway_route_table_id = aws_ec2_transit_gateway_route_table.production.id
}

# Route table associations
resource "aws_ec2_transit_gateway_route_table_association" "production" {
  transit_gateway_attachment_id  = aws_ec2_transit_gateway_vpc_attachment.production.id
  transit_gateway_route_table_id = aws_ec2_transit_gateway_route_table.production.id
}

Private EKS Cluster

# eks/main.tf

resource "aws_eks_cluster" "private" {
  name     = var.cluster_name
  role_arn = aws_iam_role.cluster.arn
  version  = var.kubernetes_version

  vpc_config {
    subnet_ids              = var.private_subnet_ids
    endpoint_private_access = true
    endpoint_public_access  = false
    security_group_ids      = [aws_security_group.cluster.id]
  }

  # Enable secrets encryption
  encryption_config {
    provider {
      key_arn = var.kms_key_arn
    }
    resources = ["secrets"]
  }

  # Control plane logging
  enabled_cluster_log_types = [
    "api",
    "audit",
    "authenticator",
    "controllerManager",
    "scheduler"
  ]

  depends_on = [
    aws_iam_role_policy_attachment.cluster_policy,
    aws_cloudwatch_log_group.cluster,
  ]
}

# Security group for cluster
resource "aws_security_group" "cluster" {
  name_prefix = "${var.cluster_name}-cluster-"
  vpc_id      = var.vpc_id

  # Allow nodes to communicate with control plane
  ingress {
    from_port       = 443
    to_port         = 443
    protocol        = "tcp"
    security_groups = [aws_security_group.nodes.id]
  }

  # Allow control plane to communicate with nodes
  egress {
    from_port       = 1025
    to_port         = 65535
    protocol        = "tcp"
    security_groups = [aws_security_group.nodes.id]
  }

  tags = {
    Name = "${var.cluster_name}-cluster-sg"
  }
}

VPC Endpoints for Private Access

# vpc-endpoints/main.tf

locals {
  interface_endpoints = [
    "ec2",
    "ecr.api",
    "ecr.dkr",
    "sts",
    "logs",
    "ssm",
    "ssmmessages",
    "ec2messages",
    "secretsmanager",
    "kms",
    "elasticloadbalancing",
    "autoscaling",
  ]
}

# Interface endpoints
resource "aws_vpc_endpoint" "interface" {
  for_each = toset(local.interface_endpoints)

  vpc_id              = var.vpc_id
  service_name        = "com.amazonaws.${var.region}.${each.value}"
  vpc_endpoint_type   = "Interface"
  subnet_ids          = var.endpoint_subnet_ids
  security_group_ids  = [aws_security_group.endpoints.id]
  private_dns_enabled = true

  tags = {
    Name = "vpce-${each.value}"
  }
}

# S3 Gateway endpoint (free, no PrivateLink charges)
resource "aws_vpc_endpoint" "s3" {
  vpc_id            = var.vpc_id
  service_name      = "com.amazonaws.${var.region}.s3"
  vpc_endpoint_type = "Gateway"
  route_table_ids   = var.private_route_table_ids

  tags = {
    Name = "vpce-s3"
  }
}

# Security group for VPC endpoints
resource "aws_security_group" "endpoints" {
  name_prefix = "vpc-endpoints-"
  vpc_id      = var.vpc_id

  ingress {
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr, "10.100.0.0/16"]  # Include EKS VPC
  }

  tags = {
    Name = "vpc-endpoints-sg"
  }
}

Hybrid DNS Resolution

# dns/route53-resolver.tf

# Inbound endpoint - allows on-prem to resolve AWS private zones
resource "aws_route53_resolver_endpoint" "inbound" {
  name      = "inbound-resolver"
  direction = "INBOUND"

  security_group_ids = [aws_security_group.resolver.id]

  dynamic "ip_address" {
    for_each = var.resolver_subnet_ids
    content {
      subnet_id = ip_address.value
    }
  }

  tags = {
    Name = "r53-resolver-inbound"
  }
}

# Outbound endpoint - allows AWS to resolve on-prem domains
resource "aws_route53_resolver_endpoint" "outbound" {
  name      = "outbound-resolver"
  direction = "OUTBOUND"

  security_group_ids = [aws_security_group.resolver.id]

  dynamic "ip_address" {
    for_each = var.resolver_subnet_ids
    content {
      subnet_id = ip_address.value
    }
  }

  tags = {
    Name = "r53-resolver-outbound"
  }
}

# Forward rule for on-prem domain
resource "aws_route53_resolver_rule" "forward_corp" {
  domain_name          = "corp.internal"
  name                 = "forward-to-corp-dns"
  rule_type            = "FORWARD"
  resolver_endpoint_id = aws_route53_resolver_endpoint.outbound.id

  target_ip {
    ip   = "172.16.1.10"
    port = 53
  }

  target_ip {
    ip   = "172.16.1.11"
    port = 53
  }

  tags = {
    Name = "forward-corp-internal"
  }
}

# Associate rule with Production VPC
resource "aws_route53_resolver_rule_association" "production" {
  resolver_rule_id = aws_route53_resolver_rule.forward_corp.id
  vpc_id           = var.production_vpc_id
}

# Share resolver rules via RAM
resource "aws_ram_resource_share" "resolver_rules" {
  name                      = "resolver-rules-share"
  allow_external_principals = false

  tags = {
    Name = "resolver-rules-share"
  }
}

resource "aws_ram_resource_association" "resolver_rule" {
  resource_arn       = aws_route53_resolver_rule.forward_corp.arn
  resource_share_arn = aws_ram_resource_share.resolver_rules.arn
}

Centralized Egress with Inspection

# egress-vpc/main.tf

# NAT Gateway for outbound internet
resource "aws_nat_gateway" "egress" {
  count         = length(var.public_subnet_ids)
  allocation_id = aws_eip.nat[count.index].id
  subnet_id     = var.public_subnet_ids[count.index]

  tags = {
    Name = "nat-egress-${count.index + 1}"
  }
}

# Network Firewall for traffic inspection
resource "aws_networkfirewall_firewall" "main" {
  name                = "egress-firewall"
  firewall_policy_arn = aws_networkfirewall_firewall_policy.main.arn
  vpc_id              = var.vpc_id

  dynamic "subnet_mapping" {
    for_each = var.firewall_subnet_ids
    content {
      subnet_id = subnet_mapping.value
    }
  }

  tags = {
    Name = "egress-network-firewall"
  }
}

# Firewall policy
resource "aws_networkfirewall_firewall_policy" "main" {
  name = "egress-policy"

  firewall_policy {
    stateless_default_actions          = ["aws:forward_to_sfe"]
    stateless_fragment_default_actions = ["aws:forward_to_sfe"]

    stateful_rule_group_reference {
      resource_arn = aws_networkfirewall_rule_group.allow_domains.arn
    }

    stateful_rule_group_reference {
      resource_arn = aws_networkfirewall_rule_group.deny_all.arn
      priority     = 100
    }
  }
}

# Allow specific domains
resource "aws_networkfirewall_rule_group" "allow_domains" {
  capacity = 100
  name     = "allow-domains"
  type     = "STATEFUL"

  rule_group {
    rules_source {
      rules_source_list {
        generated_rules_type = "ALLOWLIST"
        target_types         = ["HTTP_HOST", "TLS_SNI"]
        targets = [
          ".amazonaws.com",
          ".docker.io",
          ".docker.com",
          "ghcr.io",
          ".github.com",
          ".githubusercontent.com",
          "registry.k8s.io",
        ]
      }
    }
  }
}

# Deny everything else
resource "aws_networkfirewall_rule_group" "deny_all" {
  capacity = 10
  name     = "deny-all"
  type     = "STATEFUL"

  rule_group {
    rules_source {
      stateful_rule {
        action = "DROP"
        header {
          destination      = "ANY"
          destination_port = "ANY"
          direction        = "ANY"
          protocol         = "IP"
          source           = "ANY"
          source_port      = "ANY"
        }
        rule_option {
          keyword = "sid:100"
        }
      }
    }
  }
}

Traffic Flow Diagram

EKS Node Security Group

# eks/security-groups.tf

resource "aws_security_group" "nodes" {
  name_prefix = "${var.cluster_name}-nodes-"
  vpc_id      = var.vpc_id

  # Node to node communication
  ingress {
    from_port = 0
    to_port   = 0
    protocol  = "-1"
    self      = true
  }

  # Control plane to nodes (kubelet)
  ingress {
    from_port       = 10250
    to_port         = 10250
    protocol        = "tcp"
    security_groups = [aws_security_group.cluster.id]
  }

  # Control plane to nodes (extensions)
  ingress {
    from_port       = 443
    to_port         = 443
    protocol        = "tcp"
    security_groups = [aws_security_group.cluster.id]
  }

  # CoreDNS
  ingress {
    from_port = 53
    to_port   = 53
    protocol  = "tcp"
    self      = true
  }

  ingress {
    from_port = 53
    to_port   = 53
    protocol  = "udp"
    self      = true
  }

  # Allow traffic from on-premises (for hybrid workloads)
  ingress {
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = ["172.16.0.0/12"]
    description = "On-premises access"
  }

  # Egress to VPC endpoints
  egress {
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = ["10.200.0.0/16"]
    description = "VPC endpoints in shared services"
  }

  # Egress to control plane
  egress {
    from_port       = 443
    to_port         = 443
    protocol        = "tcp"
    security_groups = [aws_security_group.cluster.id]
  }

  # Egress to Transit Gateway (for all other traffic)
  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = {
    Name = "${var.cluster_name}-nodes-sg"
    "kubernetes.io/cluster/${var.cluster_name}" = "owned"
  }
}

Accessing Private Cluster

# access/client-vpn.tf

resource "aws_ec2_client_vpn_endpoint" "main" {
  description            = "EKS cluster access VPN"
  server_certificate_arn = var.vpn_server_cert_arn
  client_cidr_block      = "10.254.0.0/16"
  split_tunnel           = true

  authentication_options {
    type                       = "certificate-authentication"
    root_certificate_chain_arn = var.vpn_client_cert_arn
  }

  connection_log_options {
    enabled               = true
    cloudwatch_log_group  = aws_cloudwatch_log_group.vpn.name
    cloudwatch_log_stream = aws_cloudwatch_log_stream.vpn.name
  }

  dns_servers = [
    "10.200.10.10",  # Route 53 Resolver inbound
    "10.200.11.10",
  ]

  tags = {
    Name = "eks-access-vpn"
  }
}

resource "aws_ec2_client_vpn_network_association" "main" {
  count                  = length(var.vpn_subnet_ids)
  client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.main.id
  subnet_id              = var.vpn_subnet_ids[count.index]
}

# Authorization rule for EKS VPC
resource "aws_ec2_client_vpn_authorization_rule" "eks" {
  client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.main.id
  target_network_cidr    = "10.100.0.0/16"
  authorize_all_groups   = true
}

Best Practices

Troubleshooting

"EKS nodes can't pull images"

# Check VPC endpoint connectivity
aws ec2 describe-vpc-endpoints --filters "Name=service-name,Values=*ecr*"

# Test from node (via SSM)
curl -v https://api.ecr.<region>.amazonaws.com/

# Check security groups allow 443 to endpoint

"Pods can't resolve on-prem DNS"

# Verify resolver rules
aws route53resolver list-resolver-rules

# Check rule associations
aws route53resolver list-resolver-rule-associations

# Test DNS from pod
kubectl run -it --rm debug --image=busybox -- nslookup app.corp.internal

"Traffic not reaching firewall"

• Verify Transit Gateway route tables
• Check appliance mode is enabled on egress attachment
• Verify return routes in egress VPC

Conclusion

Building a fully private EKS cluster requires careful network architecture, but the security benefits are substantial. The combination of:

• Transit Gateway for scalable multi-VPC connectivity
• VPC Endpoints for private AWS service access
• Route 53 Resolver for hybrid DNS
• Network Firewall for egress inspection

Creates an enterprise-grade platform that satisfies even the most stringent compliance requirements. No public IPs, all traffic inspected, and full connectivity to on-premises systems.