EC2 can't get to internet? SG and ACL seem OK

Question

I've applied the terraform code below (variables.tf and main.tf are, of course, separate IRL) to setup an AWS VPC, private and public subnets, routes, NAT gateway, Internet Gateway, Network ACLs, an EC2 instance (in the public subnet) with a Security Group attached.

I can ssh into the EC2 just fine, but apt update won't complete. tcpdump shows SYN packets going out eth0 but nothing coming back. DNS resolution works, and I can ping 10.12.100.1 which is something upstream from eth0, I'm not sure what, but that's about it. The EC2 has a private IP on eth0 which is in the 10.12.100.x CIDR and an IPv6 address which god only knows what IP is on that.

I've spent the better part of a day redoing aws_route and aws_route_table resources trying to figure out if I have something wrong there and comparing my resources to the terraform vpc module code and I just can't figure it out.

It seems like it should be an SG or ACL issue but when I look at those configurations in the AWS console, they are as configured, and allowing all egress traffic.

Anyone know what's missing, or how I can troubleshoot this more efficiently? Thanks

############################################################################# 
#                         variables.tf 
#############################################################################

variable "aws_region" {
  description = "AWS region where resources will be created"
  type        = string
  default     = "us-east-2"
}

variable "project" {
  description = "Project tag value for resources"
  type        = string
  default     = "vpc-test"
}

variable "environment" {
  description = "Environment tag value for resources"
  type        = string
  default     = "test"
}

variable "owner" {
  description = "The owner of the project"
  type        = string
  default     = "xyz"  # initials
}

variable "ssh_pub_key" {
  description = "pubkey of the SSH key pair for EC2 instances"
  type        = string
  default     = "ssh-ed25519 AAAAC...9"
}

variable "ingress_cidr_blocks" {
  description = "CIDR blocks to allow for ingress traffic"
  type        = list(string)
  default     = ["63.9.0.0/16", "64.10.1.1/32"]  
}

variable "vpc_dns_support" {
  description = "Enable DNS support for the VPC"
  type        = bool
  default     = true
}

variable "vpc_dns_hostnames" {
  description = "Enable DNS hostnames for the VPC"
  type        = bool
  default     = true
}

variable "vpc_cidr_block" {
  description = "IPv4 CIDR block for the VPC"
  type        = string
  default     = "10.12.0.0/16"
}

variable "subnet_cidr_public" {
  description = "CIDR block for the public end of the subnet"
  type        = string
  default     = "10.12.100.0/24"
}

variable "subnet_cidr_private" {
  description = "CIDR block for the private end of the subnet"
  type        = string
  default     = "10.12.1.0/24"
}

variable "availability_zone" {
  description = "Availability zone for the subnet"
  type        = string
  default     = "us-east-2a"
}

variable "instance_type" {
  description = "EC2 instance type"
  type        = string
  default     = "t2.micro"  # Free tier - could be a policy in spacelift
}

variable "ami_instance" {
  description  = "AMI Instance identifier"
  type         = string
  default      = "ami-024e6efaf93d85776" # Ubuntu 22.04 Server in us-east-2
}


variable "ec2_webserver_data" {
  description = "User data script to initialize EC2 instances"
  type        = string
  default     = <<-EOF
    #!/bin/bash

    test ! -d /var/www/html && mkdir -p /var/www/html

    # Update package repositories and install tmux and vim
    apt-get update -y
    apt-get install apache2 tmux vim dig -y
    
    echo "<b>Private appserver instance B</b>" > /var/www/html/index.html
    systemctl enable apache2

    # Disable web caching in apache config so subsequent page loads don't conflict 
    cat <<EOT >> /etc/apache2/apache2.conf
    <Directory /var/www/html>
        <IfModule mod_headers.c>
            Header unset ETag
            Header set Cache-Control "max-age=0, no-cache, no-store, must-revalidate"
            Header set Pragma "no-cache"
            Header set Expires "Wed, 11 Jan 1984 05:00:00 GMT"
        </IfModule>
    </Directory>
    EOT               

    systemctl start apache2       
    EOF
}

variable "ec2_std_data" {
  description = "User data script to initialize EC2 instances"
  type        = string
  default     = <<-EOF
    #!/bin/bash

    # Update package repositories and install tmux and vim
    apt-get update -y
    apt-get install apache2 tmux vim dig -y
    EOF
}


############################################################################# 
#                         main.tf 
#############################################################################


# Define the provider (AWS)
provider "aws" {
  region = var.aws_region
}


# the ssh pub key to use --  get from environment variable TV_VAR_ssh_pub_key
resource "aws_key_pair" "my_key_pair" {
  key_name   = "${var.project}-key-pair"  # Replace with the desired key name
  public_key = var.ssh_pub_key
}



# VPC 
resource "aws_vpc" "my_vpc" {
  cidr_block           = var.vpc_cidr_block
  enable_dns_support   = var.vpc_dns_support
  enable_dns_hostnames = var.vpc_dns_hostnames

  tags = {
    Name        = "${var.project}-TEST-VPC"
    Environment = var.environment
    Project     = var.project
    Owner         = var.owner    
  }
}


# private subnet within the VPC
resource "aws_subnet" "private_subnet" {
  vpc_id                  = aws_vpc.my_vpc.id
  cidr_block              = var.subnet_cidr_private
  availability_zone       = var.availability_zone
  map_public_ip_on_launch = false

  tags = {
    Name        = "${var.project}-PRIV-SUB"
    Environment = var.environment
    Project     = var.project
    Owner         = var.owner    
  }
}

# public subnet within VPC
resource "aws_subnet" "public_subnet" {
  vpc_id                  = aws_vpc.my_vpc.id
  cidr_block              = var.subnet_cidr_public
  availability_zone       = var.availability_zone
  map_public_ip_on_launch = true     # For public subnet

  tags = {
    Name        = "${var.project}-PUB-SUB"
    Environment = var.environment
    Project     = var.project
    Owner         = var.owner    
  }
}


resource "aws_network_acl" "priv_sub_acl" {
  vpc_id      = aws_vpc.my_vpc.id
  subnet_ids  = [aws_subnet.private_subnet.id]

  # Define ingress for trusted networks
  ingress {
    rule_no       = 100
    protocol      = "-1"
    action        = "allow"
    cidr_block    = aws_vpc.my_vpc.cidr_block
    from_port     = 0
    to_port       = 0
  }

  # Define ingress (inbound) rules
  egress {
    rule_no       = 100
    protocol      = "-1"
    action        = "allow"
    cidr_block    = "0.0.0.0/0"
    from_port     = 0
    to_port       = 0
  }
  
  # Tags
  tags = {
    Name          = "${var.project}-PRIV-ACL"
    Environment   = var.environment
    Project       = var.project
    Owner         = var.owner
  }
}


resource "aws_network_acl" "pub_sub_acl" {
  vpc_id      = aws_vpc.my_vpc.id
  subnet_ids  = [aws_subnet.public_subnet.id]

  # allow trusted networks ingress
  ingress {
    rule_no       = 100
    protocol      = "-1"
    action        = "allow"
    cidr_block    = var.ingress_cidr_blocks[0]
    from_port     = 0
    to_port       = 0
  }

  # dynamic config for multiple elements of var.ingress_cidr_blocks
  dynamic "ingress" {
    for_each  = range(length(var.ingress_cidr_blocks))
    iterator = i
    content {
      rule_no       = 100 + i.value * 100
      protocol      = "-1"
      action        = "allow"
      cidr_block    = var.ingress_cidr_blocks[i.value]
      from_port     = 0
      to_port       = 0
    }
  }

  # Define egress
  egress {
    rule_no       = 100
    protocol      = "-1"
    action        = "allow"
    cidr_block    = "0.0.0.0/0"
    from_port     = 0
    to_port       = 0
  }

  # Define IPv6 egress rule
  egress {
    rule_no             = 101
    protocol            = "-1"
    action              = "allow"
    ipv6_cidr_block     = "::/0"  # IPv6 CIDR block
    from_port           = 0
    to_port             = 0
  }
  
  # Tags
  tags = {
    Name          = "${var.project}-PUB-ACL"
    Environment   = var.environment
    Project       = var.project
    Owner         = var.owner
  }
}



# Create an Internet Gateway
resource "aws_internet_gateway" "my_igw" {
  vpc_id       = aws_vpc.my_vpc.id

  tags = {
    Name        = "${var.project}-IGW"
    Environment = var.environment
    Project     = var.project
    Owner         = var.owner    
  }
}

# Elastic IP for the NAT Gateway
resource "aws_eip" "nat_eip" {
  domain           = "vpc"

  # Tags
  tags = {
    Name          = "${var.project}-NATG-EIP"
    Environment   = var.environment
    Project       = var.project
    Owner         = var.owner
  }
}

# NAT Gateway in the public subnet for private resources to reach internet resources
resource "aws_nat_gateway" "my_natg" {
  allocation_id     = aws_eip.nat_eip.id
  subnet_id         = aws_subnet.public_subnet.id  #< NATG needs to reside in pub sub

  # Tags
  tags = {
    Name          = "${var.project}-NATG"
    Environment   = var.environment
    Project       = var.project
    Owner         = var.owner
  }  
  
  depends_on = [aws_internet_gateway.my_igw]
}


# route for private subnet - uses NATG
resource "aws_route_table" "private_route_table" {
  vpc_id = aws_vpc.my_vpc.id

  route {
    cidr_block = "0.0.0.0/0"
    nat_gateway_id = aws_nat_gateway.my_natg.id
  }

  tags = {
   Name = "${var.project}-PRIV-RT"
    Environment = var.environment
    Project     = var.project
    Owner       = var.owner
  }
}

# route to internet for vpc resources
resource "aws_route_table" "public_route_table" {
  vpc_id = aws_vpc.my_vpc.id

  route {
      cidr_block = "0.0.0.0/0"
      gateway_id = aws_internet_gateway.my_igw.id
  }

  tags = {
    Name        = "${var.project}-PUB-RT"
    Environment = var.environment
    Project     = var.project
    Owner       = var.owner
  }
}


resource "aws_default_route_table" "default_route_table" {
  default_route_table_id = aws_vpc.my_vpc.default_route_table_id

  tags = {
    Name        = "${var.project}-DEF-RT"
    Environment = var.environment
    Project     = var.project
    Owner       = var.owner
  }
}


# Associate pubsub with pub route
resource "aws_route_table_association" "public_association" {
  subnet_id      = aws_subnet.public_subnet.id
  route_table_id = aws_route_table.public_route_table.id
  
  # no tags
}

# Associate privsub with priv route
resource "aws_route_table_association" "private_association" {
  subnet_id      = aws_subnet.private_subnet.id
  route_table_id = aws_route_table.private_route_table.id

  # no tags
}


# Define a security group to allow incoming SSH traffic (adjust rules as needed)
resource "aws_security_group" "ec2_sg" {
  name                = "${var.project}-SG"
  description         = "Security group allowing SSH traffic from campus only, egress anywhere"
  vpc_id              = aws_vpc.my_vpc.id 

  # Ingress rule for SSH
  ingress {
    from_port         = 22
    to_port           = 22
    protocol          = "tcp"
    cidr_blocks       = var.ingress_cidr_blocks
  }

  # Egress rule for all IPv4 traffic anywhere
  egress {
    from_port         = 0
    to_port           = 0
    protocol          = "-1"
    cidr_blocks       = ["0.0.0.0/0"]
  }

  # Egress rule for all IPv6 traffic anywhere
  egress {
    from_port         = 0
    to_port           = 0
    protocol          = "-1"
    ipv6_cidr_blocks  = ["::/0"]
  }

  # Tags
  tags = {
    Name          = "${var.project}-EC2-SG"
    Environment   = var.environment
    Project       = var.project
    Owner         = var.owner
  }
}


/*

# Create an EC2 instance in privsub
resource "aws_instance" "my_ec2_instance" {
  ami                     = var.ami_instance
  instance_type           = var.instance_type
  key_name                = aws_key_pair.my_key_pair.key_name
  user_data               = var.ec2_std_data
  vpc_security_group_ids  = [aws_security_group.ec2_sg.id]
  subnet_id               = aws_subnet.private_subnet.id

  metadata_options {
    http_tokens          = "required"
  }

  # Tags
  tags = {
    Name          = "${var.project}-priv1-ec2"
    Environment   = var.environment
    Project       = var.project
    Owner         = var.owner

  }
}


# Create a second EC2 instance in privsub
resource "aws_instance" "my_ec2_instance_b" {
  ami                     = var.ami_instance
  instance_type           = var.instance_type
  key_name                = aws_key_pair.my_key_pair.key_name
  user_data               = var.ec2_std_data
  vpc_security_group_ids  = [aws_security_group.ec2_sg.id]
  subnet_id               = aws_subnet.private_subnet.id

  metadata_options {
    http_tokens          = "required"
  }


  # Tags
  tags = {
    Name          = "${var.project}-priv2-ec2"
    Environment   = var.environment
    Project       = var.project
    Owner         = var.owner
  }
}
*/


# Create a second EC2 instance in pubsub
resource "aws_instance" "my_ec2_instance_c" {
  ami                     = var.ami_instance
  instance_type           = var.instance_type
  key_name                = aws_key_pair.my_key_pair.key_name  # Reference the key name from the aws_key_pair 
  user_data               = var.ec2_webserver_data
  vpc_security_group_ids  = [aws_security_group.ec2_sg.id]
  subnet_id               = aws_subnet.public_subnet.id

  metadata_options {
    http_tokens          = "required"
  }

  # Tags
  tags = {
    Name          = "${var.project}-pub1-ec2"
    Environment   = var.environment
    Project       = var.project
    Owner         = var.owner
  }
}

Answer 1

So, for anyone else who stumbles across this, the ACLs don't know that the SYN-ACK packets coming back from a connection initiated from within an EC2 is traffic that is related to that connection (stateless).

This means you can't just apt update && apt dist-upgrade -y without first opening 0/0 on the inbound side of the ACL.