Security Groups AWS with referenced Security Group

Question

I have the following minimal reproducable setup:

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "5.21.0"
    }
  }
}

provider "aws" {
  region = "eu-central-1"
}

data "aws_ami" "amazon_linux" {
  most_recent = true

  owners = ["amazon"]

  filter {
    name   = "image-id"
    values = ["ami-0a23a9827c6dab833"]
  }
}

resource "aws_vpc" "main" {
  cidr_block = "10.0.0.0/16"
}

resource "aws_subnet" "public" {
  cidr_block        = "10.0.1.0/24"
  vpc_id            = aws_vpc.main.id
  availability_zone = "eu-central-1a"
}

resource "aws_internet_gateway" "main" {
  vpc_id = aws_vpc.main.id
}

resource "aws_route_table" "public" {
  vpc_id = aws_vpc.main.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.main.id
  }
}

resource "aws_route_table_association" "public" {
  subnet_id      = aws_subnet.public.id
  route_table_id = aws_route_table.public.id
}

resource "aws_security_group" "inbound" {
  name        = "ingress-ec2"
  description = "For the EC2 that has the inbound traffic"
  vpc_id      = aws_vpc.main.id
}

resource "aws_security_group" "outbound" {
  name        = "egress-ec2"
  description = "For the EC2 that has the outbound traffic"
  vpc_id      = aws_vpc.main.id
}

resource "aws_vpc_security_group_ingress_rule" "main" {
  security_group_id            = aws_security_group.inbound.id
  from_port                    = 5000
  to_port                      = 5000
  ip_protocol                  = "tcp"
  referenced_security_group_id = aws_security_group.outbound.id
}

resource "aws_vpc_security_group_egress_rule" "main" {
  security_group_id            = aws_security_group.outbound.id
  from_port                    = 5000
  to_port                      = 5000
  ip_protocol                  = "tcp"
  referenced_security_group_id = aws_security_group.inbound.id
}

resource "aws_vpc_security_group_ingress_rule" "ssh_inbound" {
  security_group_id = aws_security_group.inbound.id
  from_port         = 22
  to_port           = 22
  ip_protocol       = "tcp"
  cidr_ipv4         = "0.0.0.0/0"
}

resource "aws_vpc_security_group_ingress_rule" "ssh_outbound" {
  security_group_id = aws_security_group.outbound.id
  from_port         = 22
  to_port           = 22
  ip_protocol       = "tcp"
  cidr_ipv4         = "0.0.0.0/0"
}

resource "aws_vpc_security_group_egress_rule" "internet_inbound" {
  security_group_id = aws_security_group.inbound.id
  ip_protocol       = -1
  cidr_ipv4         = "0.0.0.0/0"
}

resource "aws_vpc_security_group_egress_rule" "internet_outbound" {
  security_group_id = aws_security_group.outbound.id
  ip_protocol       = -1
  cidr_ipv4         = "0.0.0.0/0"
}

resource "aws_key_pair" "inbound" {
  key_name   = "ingress-ec2"
  public_key = file(var.public_ssh_key)
}

resource "aws_instance" "inbound" {
  ami                         = data.aws_ami.amazon_linux.id
  instance_type               = "t2.micro"
  key_name                    = aws_key_pair.inbound.id
  subnet_id                   = aws_subnet.public.id
  security_groups             = [aws_security_group.inbound.id]
  associate_public_ip_address = true
}

resource "aws_key_pair" "outbound" {
  key_name   = "egress-ec2"
  public_key = file(var.public_ssh_key)
}

resource "aws_instance" "outbound" {
  ami                         = data.aws_ami.amazon_linux.id
  instance_type               = "t2.micro"
  key_name                    = aws_key_pair.outbound.id
  subnet_id                   = aws_subnet.public.id
  security_groups             = [aws_security_group.outbound.id]
  associate_public_ip_address = true
}

variable "public_ssh_key" {
  type    = string
  default = "~/.ssh/test_key.pub"
}

output "ingress-ec2" {
  value = aws_instance.inbound.public_ip
}

output "egress-ec2" {
  value = aws_instance.outbound.public_ip
}

when start a simple flask application into the ingress EC2 instance I get timeout errors when I try to access it from the egress EC2 instance. The problem lays in the Security Group since I have a similar setup for my RDS instance. Why is the egress EC2 instance blocked by the Security Groups here?

When I change

resource "aws_vpc_security_group_ingress_rule" "main" {
  security_group_id            = aws_security_group.inbound.id
  from_port                    = 5000
  to_port                      = 5000
  ip_protocol                  = "tcp"
  referenced_security_group_id = aws_security_group.outbound.id
}

to

resource "aws_vpc_security_group_ingress_rule" "main" {
  security_group_id            = aws_security_group.inbound.id
  from_port                    = 5000
  to_port                      = 5000
  ip_protocol                  = "tcp"
  cidr_ipv4                    = "0.0.0.0/0"
}

it works, which further supports my assumptions that the traffic in the initial setup get blocked by the security groups.

Answer 1

I don't think there's anything wrong with the Terraform code. Your issue is most likely because your "egress" EC2 instance is trying to access the "ingress" EC2 instance via the "ingress" instance's public IP address. When an EC2 server tries to access another EC2 server via public IP, the network traffic exits the VPC and comes back into the VPC, at which point the reference to the security group the traffic originated from is lost.

In order for security group references inside security group rules to work, all the traffic must be over the EC2 instance private IPs so that the traffic stays inside the VPC and retains the security group reference.