Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: exposed port validation #5303

Merged
merged 14 commits into from
Sep 18, 2023
Merged

refactor: exposed port validation #5303

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
881ebe3
sort errors alphabetically, fix test order
Aug 25, 2023
a2e5242
reset commit on forked mainline
Aug 25, 2023
52d7cd1
Merge branch 'mainline' of /~https://github.com/CaptainCarpensir/copilo…
Aug 25, 2023
13db40f
Merge branch 'mainline' of /~https://github.com/CaptainCarpensir/copilo…
Aug 28, 2023
b356514
Merge branch 'mainline' of /~https://github.com/CaptainCarpensir/copilo…
Sep 14, 2023
71bdf85
move changes to independent branch
Sep 14, 2023
fc6ed62
nit
Sep 14, 2023
311f399
give nlb same fallback plan, add more tests
Sep 14, 2023
573aeb9
address feedback, nlb protocol fallback
Sep 14, 2023
d8e215f
fix error
Sep 14, 2023
d3faec3
revert order of port validation
Sep 14, 2023
66ce171
let main container use non-default protocol
Sep 15, 2023
1e4fd9d
address nit, add same protocol err test
Sep 18, 2023
e0bf921
Merge branch 'mainline' into refactor/exposed-port-validation
mergify[bot] Sep 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions internal/pkg/manifest/errors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,3 +236,14 @@ type errContainersExposingSamePort struct {
func (e *errContainersExposingSamePort) Error() string {
return fmt.Sprintf(`containers %q and %q are exposing the same port %d`, e.firstContainer, e.secondContainer, e.port)
}

type errContainerPortExposedWithMultipleProtocol struct {
container string
port uint16
firstProtocol string
secondProtocol string
}

func (e *errContainerPortExposedWithMultipleProtocol) Error() string {
return fmt.Sprintf(`container %q is exposing the same port %d with protocol %s and %s`, e.container, e.port, e.firstProtocol, e.secondProtocol)
}
146 changes: 97 additions & 49 deletions internal/pkg/manifest/validate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ const (
awsXRAY = "awsxray"
)

const (
defaultProtocol = TCP
)

const (
// Listener rules have a quota of five condition values per rule.
// Please refer to https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-limits.html.
Expand Down Expand Up @@ -2050,29 +2054,44 @@ func validateDepsForEssentialContainers(deps map[string]containerDependency) err
return nil
}

type containerNameAndProtocol struct {
containerName string
containerProtocol string
}

func validateExposedPorts(opts validateExposedPortsOpts) error {
containerNameFor := make(map[uint16]string)
populateMainContainerPort(containerNameFor, opts)
if err := populateSidecarContainerPortsAndValidate(containerNameFor, opts); err != nil {
portExposedTo := make(map[uint16]containerNameAndProtocol)

if err := populateAndValidateSidecarContainerPorts(portExposedTo, opts); err != nil {
return err
}
if err := populateALBPortsAndValidate(containerNameFor, opts); err != nil {
if err := populateAndValidateALBPorts(portExposedTo, opts); err != nil {
return err
}
if err := populateNLBPortsAndValidate(containerNameFor, opts); err != nil {
if err := populateAndValidateNLBPorts(portExposedTo, opts); err != nil {
return err
}
if err := populateAndValidateMainContainerPort(portExposedTo, opts); err != nil {
return err
}
return nil
}

func populateMainContainerPort(containerNameFor map[uint16]string, opts validateExposedPortsOpts) {
func populateAndValidateMainContainerPort(portExposedTo map[uint16]containerNameAndProtocol, opts validateExposedPortsOpts) error {
if opts.mainContainerPort == nil {
return
return nil
}
containerNameFor[aws.Uint16Value(opts.mainContainerPort)] = opts.mainContainerName

targetPort := aws.Uint16Value(opts.mainContainerPort)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just sharing my thought here, this is not a request 👀 I personally would refrain from using the term target in the variable names, because up until this point, target means that a port a) not only is exposed, b) but also functions as a port that receives traffic from an NLB/ALB.

A port could be exposed, but only receive traffic from within the task - for example, a sidecar could communicate with another sidecar through a port. These port satisfy a), but not b). They are generally not considered "a target port", because they are not the target of anything.

http:
  target_container: nginx
sidecars:
  logging:
    port: 81 # Exposed, but not an HTTP target nor an NLB target.
  nginx:
    port: 82

targetProtocol := defaultProtocol
if existingContainerNameAndProtocol, ok := portExposedTo[targetPort]; ok {
targetProtocol = existingContainerNameAndProtocol.containerProtocol
}
Comment on lines +2087 to +2089
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this added so that populateAndValidateMainContainerPort works no matter what order it is called in relative to the other populateAndValidateXPort functions? Based on the order in which these functions are called, it seems like ok would never be true 👀 !

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's for the case where the NLB specifies a protocol that's not TCP to be used for the main container. In this case the NLB specifies that and it gets populated in the map.
An example manifest:

image:
  port: 80
nlb:
  port: 80/udp

Without this statement here the program would error out saying container "mockMainContainer" is exposing the same port 80 with protocol TCP and UDP

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh I just realized you moved the function call from the first to the last! Then I think we are good 👍🏼


return validateAndPopulateExposedPortMapping(portExposedTo, targetPort, targetProtocol, opts.mainContainerName)
}

func populateSidecarContainerPortsAndValidate(containerNameFor map[uint16]string, opts validateExposedPortsOpts) error {
func populateAndValidateSidecarContainerPorts(portExposedTo map[uint16]containerNameAndProtocol, opts validateExposedPortsOpts) error {
for name, sidecar := range opts.sidecarConfig {
if sidecar.Port == nil {
continue
Expand All @@ -2081,100 +2100,129 @@ func populateSidecarContainerPortsAndValidate(containerNameFor map[uint16]string
if err != nil {
return err
}
port, err := strconv.ParseUint(aws.StringValue(sidecarPort), 10, 16)
parsedPort, err := strconv.ParseUint(aws.StringValue(sidecarPort), 10, 16)
if err != nil {
return err
}
if _, ok := containerNameFor[uint16(port)]; ok {
return &errContainersExposingSamePort{
firstContainer: name,
secondContainer: containerNameFor[uint16(port)],
port: uint16(port),
}

if err = validateAndPopulateExposedPortMapping(portExposedTo, uint16(parsedPort), defaultProtocol, name); err != nil {
return err
}
containerNameFor[uint16(port)] = name
}
return nil
}

func populateALBPortsAndValidate(containerNameFor map[uint16]string, opts validateExposedPortsOpts) error {
// This condition takes care of the use case where target_container is set to x container and
// target_port exposing port 80 which is already exposed by container y.That means container x
// is trying to expose the port that is already being exposed by container y, so error out.
func populateAndValidateALBPorts(portExposedTo map[uint16]containerNameAndProtocol, opts validateExposedPortsOpts) error {
if opts.alb == nil || opts.alb.IsEmpty() {
return nil
}

alb := opts.alb
for _, rule := range alb.RoutingRules() {
if rule.TargetPort == nil {
continue
}
if err := validateContainersNotExposingSamePort(containerNameFor, aws.Uint16Value(rule.TargetPort), rule.TargetContainer); err != nil {
return err
targetPort := aws.Uint16Value(rule.TargetPort)

// Prefer `http.target_container`, then existing exposed port mapping, then fallback on name of main container
targetContainer := opts.mainContainerName
if existingContainerNameAndProtocol, ok := portExposedTo[targetPort]; ok {
targetContainer = existingContainerNameAndProtocol.containerName
}
targetContainerName := opts.mainContainerName
if rule.TargetContainer != nil {
targetContainerName = aws.StringValue(rule.TargetContainer)
targetContainer = aws.StringValue(rule.TargetContainer)
}
containerNameFor[aws.Uint16Value(rule.TargetPort)] = targetContainerName
}

return nil
}

func validateContainersNotExposingSamePort(containerNameFor map[uint16]string, targetPort uint16, targetContainer *string) error {
container, exists := containerNameFor[targetPort]
if !exists {
return nil
}
if targetContainer != nil && container != aws.StringValue(targetContainer) {
return &errContainersExposingSamePort{
firstContainer: aws.StringValue(targetContainer),
secondContainer: container,
port: targetPort,
if err := validateAndPopulateExposedPortMapping(portExposedTo, targetPort, defaultProtocol, targetContainer); err != nil {
return err
}
}
return nil
}

func populateNLBPortsAndValidate(containerNameFor map[uint16]string, opts validateExposedPortsOpts) error {
func populateAndValidateNLBPorts(portExposedTo map[uint16]containerNameAndProtocol, opts validateExposedPortsOpts) error {
if opts.nlb == nil || opts.nlb.IsEmpty() {
return nil
}

nlb := opts.nlb
if err := populateAndValidateNLBPorts(nlb.Listener, containerNameFor, opts.mainContainerName); err != nil {
if err := populateAndValidateNLBListenerPorts(nlb.Listener, portExposedTo, opts.mainContainerName); err != nil {
return fmt.Errorf(`validate "nlb": %w`, err)
}

for idx, listener := range nlb.AdditionalListeners {
if err := populateAndValidateNLBPorts(listener, containerNameFor, opts.mainContainerName); err != nil {
if err := populateAndValidateNLBListenerPorts(listener, portExposedTo, opts.mainContainerName); err != nil {
return fmt.Errorf(`validate "nlb.additional_listeners[%d]": %w`, idx, err)
}
}
return nil
}

func populateAndValidateNLBPorts(listener NetworkLoadBalancerListener, containerNameFor map[uint16]string, mainContainerName string) error {
nlbPort, _, err := ParsePortMapping(listener.Port)
func populateAndValidateNLBListenerPorts(listener NetworkLoadBalancerListener, portExposedTo map[uint16]containerNameAndProtocol, mainContainerName string) error {
nlbRecieverPort, nlbProtocol, err := ParsePortMapping(listener.Port)
if err != nil {
return err
}
port, err := strconv.ParseUint(aws.StringValue(nlbPort), 10, 16)
port, err := strconv.ParseUint(aws.StringValue(nlbRecieverPort), 10, 16)
if err != nil {
return err
}

targetPort := uint16(port)
if listener.TargetPort != nil {
targetPort = uint16(aws.IntValue(listener.TargetPort))
}
if err = validateContainersNotExposingSamePort(containerNameFor, uint16(aws.IntValue(listener.TargetPort)), listener.TargetContainer); err != nil {
return err

// Prefer `nlb.port`, then existing exposed port mapping, then fallback on default protocol
targetProtocol := defaultProtocol
if existingContainerNameAndProtocol, ok := portExposedTo[targetPort]; ok {
targetProtocol = existingContainerNameAndProtocol.containerProtocol
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like nlb is a little different than the other. Without protocol, default protocol for nlb should be tcp, instead of any protocol that is already exposed. Consider this manifest:

nlb:
  port: 81

sidecars:
  nginx:
    port: 81/udp

I would expect this manifest to error out, honestly. The nlb.port being 81 means it listens on 81 for TCP port, and route the traffic to port 81 of the nginx container, and expect that target port to be tcp. However, the same port is already exposed as udp. So error.

I think basically, the question is, when we think of the protocol for nlb, do we think of it in a NLB-first way, or a container-first way. My proposal above is NLB-first, the code that I'm commenting on is container-first.

}
if nlbProtocol != nil {
targetProtocol = strings.ToUpper(aws.StringValue(nlbProtocol))
}

// Prefer `nlb.target_container`, then existing exposed port mapping, then fallback on name of main container
targetContainer := mainContainerName
if existingContainerNameAndProtocol, ok := portExposedTo[targetPort]; ok {
targetContainer = existingContainerNameAndProtocol.containerName
}
if listener.TargetContainer != nil {
targetContainer = aws.StringValue(listener.TargetContainer)
}
containerNameFor[targetPort] = targetContainer

return validateAndPopulateExposedPortMapping(portExposedTo, targetPort, targetProtocol, targetContainer)
}

func validateAndPopulateExposedPortMapping(portExposedTo map[uint16]containerNameAndProtocol, targetPort uint16, targetProtocol string, targetContainer string) error {
exposedContainerAndProtocol, alreadyExposed := portExposedTo[targetPort]

// Port is not associated with container and protocol, populate map
if !alreadyExposed {
portExposedTo[targetPort] = containerNameAndProtocol{
containerName: targetContainer,
containerProtocol: targetProtocol,
}
return nil
}

exposedContainer := exposedContainerAndProtocol.containerName
exposedProtocol := exposedContainerAndProtocol.containerProtocol
if exposedContainer != targetContainer {
return &errContainersExposingSamePort{
firstContainer: targetContainer,
secondContainer: exposedContainer,
port: targetPort,
}
}
if exposedProtocol != targetProtocol {
return &errContainerPortExposedWithMultipleProtocol{
container: exposedContainer,
port: targetPort,
firstProtocol: targetProtocol,
secondProtocol: exposedProtocol,
}
}
return nil
}

Expand Down
58 changes: 57 additions & 1 deletion internal/pkg/manifest/validate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3985,7 +3985,20 @@ func TestValidateExposedPorts(t *testing.T) {
},
},
},
wanted: fmt.Errorf(`containers "foo" and "mockMainContainer" are exposing the same port 80`),
wanted: fmt.Errorf(`containers "mockMainContainer" and "foo" are exposing the same port 80`),
},
"should not error out when main container uses non-default protocol": {
in: validateExposedPortsOpts{
mainContainerName: "mockMainContainer",
mainContainerPort: aws.Uint16(80),
nlb: &NetworkLoadBalancerConfiguration{
Listener: NetworkLoadBalancerListener{
Port: aws.String("8080/udp"),
TargetPort: aws.Int(80),
},
},
},
wanted: nil,
},
"should not error out when alb target_port is same as that of sidecar container port but target_container is empty": {
in: validateExposedPortsOpts{
Expand All @@ -4004,6 +4017,42 @@ func TestValidateExposedPorts(t *testing.T) {
},
wanted: nil,
},
"should not error out when nlb target_port is same as that of sidecar container port but target_container is empty": {
in: validateExposedPortsOpts{
mainContainerName: "mockMainContainer",
mainContainerPort: aws.Uint16(8080),
sidecarConfig: map[string]*SidecarConfig{
"foo": {
Port: aws.String("80"),
},
},
nlb: &NetworkLoadBalancerConfiguration{
Listener: NetworkLoadBalancerListener{
Port: aws.String("8080/tcp"),
TargetPort: aws.Int(80),
},
},
},
wanted: nil,
},
"should not error out when nlb target_port is same as that of sidecar container port but target_container and protocol is empty": {
in: validateExposedPortsOpts{
mainContainerName: "mockMainContainer",
mainContainerPort: aws.Uint16(8080),
sidecarConfig: map[string]*SidecarConfig{
"foo": {
Port: aws.String("80"),
},
},
nlb: &NetworkLoadBalancerConfiguration{
Listener: NetworkLoadBalancerListener{
Port: aws.String("8080"),
TargetPort: aws.Int(80),
},
},
},
wanted: nil,
},
"should return an error if alb target_port points to one sidecar container port and target_container points to another sidecar container": {
in: validateExposedPortsOpts{
mainContainerName: "mockMainContainer",
Expand Down Expand Up @@ -4070,6 +4119,13 @@ func TestValidateExposedPorts(t *testing.T) {
TargetContainer: aws.String("foo"),
},
},
nlb: &NetworkLoadBalancerConfiguration{
Listener: NetworkLoadBalancerListener{
Port: aws.String("8080/tcp"),
TargetPort: aws.Int(80),
TargetContainer: aws.String("foo"),
},
},
},
wanted: nil,
},
Expand Down