chore(manifest): add from_tag for workload (#3727)

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the Apache 2.0 License.

Makefile

@@ -201,7 +201,7 @@ gen-mocks: tools
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/template/mocks/mock_template.go -source=./internal/pkg/template/template.go
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/task/mocks/mock_task.go -source=./internal/pkg/task/task.go
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/repository/mocks/mock_repository.go -source=./internal/pkg/repository/repository.go
-	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/logging/mocks/mock_service.go -source=./internal/pkg/logging/service.go
+	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/logging/mocks/mock_workload.go -source=./internal/pkg/logging/workload.go
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/logging/mocks/mock_task.go -source=./internal/pkg/logging/task.go
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/cli/list/mocks/mock_list.go -source=./internal/pkg/cli/list/list.go
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/cli/deploy/mocks/mock_svc.go -source=./internal/pkg/cli/deploy/svc.go
@@ -211,3 +211,4 @@ gen-mocks: tools
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/apprunner/mocks/mock_apprunner.go -source=./internal/pkg/apprunner/apprunner.go
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/ecs/mocks/mock_run_task_request.go -source=./internal/pkg/ecs/run_task_request.go
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/runner/jobrunner/mocks/mock.go -source=./internal/pkg/runner/jobrunner/jobrunner.go
+	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/manifest/mocks/mock.go -source=./internal/pkg/manifest/loader.go

go.mod

@@ -54,6 +54,8 @@ require (
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/tools v0.1.1 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -1533,6 +1533,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/pkg/aws/ec2/ec2.go

@@ -18,8 +18,9 @@ const (
 	internetGatewayIDPrefix  = "igw-"
 	cloudFrontPrefixListName = "com.amazonaws.global.cloudfront.origin-facing"
 
-	// TagFilterName is the filter name format for tag filters
-	TagFilterName = "tag:%s"
+	// FmtTagFilter is the filter name format for tag filters
+	FmtTagFilter = "tag:%s"
+	tagKeyFilter = "tag-key"
 )
 
 var (
@@ -50,6 +51,17 @@ type Filter struct {
 	Values []string
 }
 
+// FilterForTags takes a key and optional values to construct an EC2 filter.
+func FilterForTags(key string, values ...string) Filter {
+	if len(values) == 0 {
+		return Filter{Name: tagKeyFilter, Values: []string{key}}
+	}
+	return Filter{
+		Name:   fmt.Sprintf(FmtTagFilter, key),
+		Values: values,
+	}
+}
+
 // EC2 wraps an AWS EC2 client.
 type EC2 struct {
 	client api
@@ -322,7 +334,6 @@ func (c *EC2) subnets(filters ...Filter) ([]*ec2.Subnet, error) {
 		return nil, fmt.Errorf("describe subnets: %w", err)
 	}
 	subnets = append(subnets, response.Subnets...)
-
 	for response.NextToken != nil {
 		response, err = c.client.DescribeSubnets(&ec2.DescribeSubnetsInput{
 			Filters:   inputFilters,
@@ -333,7 +344,9 @@ func (c *EC2) subnets(filters ...Filter) ([]*ec2.Subnet, error) {
 		}
 		subnets = append(subnets, response.Subnets...)
 	}
-
+	if len(subnets) == 0 {
+		return nil, fmt.Errorf("cannot find any subnets")
+	}
 	return subnets, nil
 }
 

internal/pkg/aws/ec2/ec2_test.go

@@ -11,19 +11,18 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/copilot-cli/internal/pkg/aws/ec2/mocks"
-	"github.com/aws/copilot-cli/internal/pkg/deploy"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 )
 
 var (
 	inAppEnvFilters = []Filter{
 		{
-			Name:   fmt.Sprintf(TagFilterName, deploy.AppTagKey),
+			Name:   fmt.Sprintf(FmtTagFilter, "copilot-application"),
 			Values: []string{"my-app"},
 		},
 		{
-			Name:   fmt.Sprintf(TagFilterName, deploy.EnvTagKey),
+			Name:   fmt.Sprintf(FmtTagFilter, "copilot-environment"),
 			Values: []string{"my-env"},
 		},
 	}
@@ -42,6 +41,33 @@ var (
 	}
 )
 
+func TestEC2_FilterForTags(t *testing.T) {
+	testCases := map[string]struct {
+		inValues []string
+		wanted   Filter
+	}{
+		"with no values": {
+			wanted: Filter{
+				Name:   "tag-key",
+				Values: []string{"mockKey"},
+			},
+		},
+		"with values": {
+			inValues: []string{"foo", "bar"},
+			wanted: Filter{
+				Name:   "tag:mockKey",
+				Values: []string{"foo", "bar"},
+			},
+		},
+	}
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			filter := FilterForTags("mockKey", tc.inValues...)
+			require.Equal(t, tc.wanted, filter)
+		})
+	}
+}
+
 func TestEC2_extractResource(t *testing.T) {
 	testCases := map[string]struct {
 		displayString  string
@@ -802,6 +828,17 @@ func TestEC2_SubnetIDs(t *testing.T) {
 			},
 			wantedError: fmt.Errorf("describe subnets: error describing subnets"),
 		},
+		"cannot get any subnets": {
+			inFilter: inAppEnvFilters,
+			mockEC2Client: func(m *mocks.Mockapi) {
+				m.EXPECT().DescribeSubnets(&ec2.DescribeSubnetsInput{
+					Filters: toEC2Filter(inAppEnvFilters),
+				}).Return(&ec2.DescribeSubnetsOutput{
+					Subnets: []*ec2.Subnet{},
+				}, nil)
+			},
+			wantedError: fmt.Errorf("cannot find any subnets"),
+		},
 		"successfully get subnets": {
 			inFilter: inAppEnvFilters,
 			mockEC2Client: func(m *mocks.Mockapi) {

internal/pkg/cli/job_deploy.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/aws/copilot-cli/internal/pkg/describe"
 
@@ -44,6 +45,7 @@ type deployJobOpts struct {
 	// cached variables
 	targetApp       *config.Application
 	targetEnv       *config.Environment
+	envSess         *session.Session
 	appliedManifest interface{}
 	rootUserARN     string
 }
@@ -149,6 +151,7 @@ func (o *deployJobOpts) Execute() error {
 		interpolator: o.newInterpolator(o.appName, o.envName),
 		ws:           o.ws,
 		unmarshal:    o.unmarshal,
+		sess:         o.envSess,
 	})
 	if err != nil {
 		return err
@@ -220,6 +223,11 @@ func (o *deployJobOpts) configureClients() error {
 	if err != nil {
 		return fmt.Errorf("create default session: %w", err)
 	}
+	envSess, err := o.sessProvider.FromRole(env.ManagerRoleARN, env.Region)
+	if err != nil {
+		return err
+	}
+	o.envSess = envSess
 
 	// client to retrieve caller identity.
 	caller, err := identity.New(defaultSess).Get()

internal/pkg/cli/svc_deploy.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/aws/copilot-cli/internal/pkg/aws/identity"
 	"github.com/aws/copilot-cli/internal/pkg/aws/tags"
@@ -63,6 +64,7 @@ type deploySvcOpts struct {
 	// cached variables
 	targetApp       *config.Application
 	targetEnv       *config.Environment
+	envSess         *session.Session
 	svcType         string
 	appliedManifest interface{}
 	rootUserARN     string
@@ -185,6 +187,7 @@ func (o *deploySvcOpts) Execute() error {
 		interpolator: o.newInterpolator(o.appName, o.envName),
 		ws:           o.ws,
 		unmarshal:    o.unmarshal,
+		sess:         o.envSess,
 	})
 	if err != nil {
 		return err
@@ -327,6 +330,11 @@ func (o *deploySvcOpts) configureClients() error {
 	if err != nil {
 		return fmt.Errorf("create default session: %w", err)
 	}
+	envSess, err := o.sessProvider.FromRole(env.ManagerRoleARN, env.Region)
+	if err != nil {
+		return err
+	}
+	o.envSess = envSess
 
 	// client to retrieve caller identity.
 	caller, err := identity.New(defaultSess).Get()
@@ -353,6 +361,7 @@ type workloadManifestInput struct {
 	envName      string
 	ws           wsWlDirReader
 	interpolator interpolator
+	sess         *session.Session
 	unmarshal    func([]byte) (manifest.WorkloadManifest, error)
 }
 
@@ -371,10 +380,13 @@ func workloadManifest(in *workloadManifestInput) (manifest.WorkloadManifest, err
 	}
 	envMft, err := mft.ApplyEnv(in.envName)
 	if err != nil {
-		return nil, fmt.Errorf("apply environment %s override: %s", in.envName, err)
+		return nil, fmt.Errorf("apply environment %s override: %w", in.envName, err)
 	}
 	if err := envMft.Validate(); err != nil {
-		return nil, fmt.Errorf("validate manifest against environment %s: %s", in.envName, err)
+		return nil, fmt.Errorf("validate manifest against environment %q: %w", in.envName, err)
+	}
+	if err := envMft.Load(in.sess); err != nil {
+		return nil, fmt.Errorf("load dynamic content: %w", err)
 	}
 	return envMft, nil
 }

internal/pkg/cli/svc_deploy_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/copilot-cli/internal/pkg/manifest"
 	"github.com/aws/copilot-cli/internal/pkg/template"
 	"github.com/golang/mock/gomock"
@@ -348,6 +349,14 @@ func (m *mockWorkloadMft) Validate() error {
 	return nil
 }
 
+func (m *mockWorkloadMft) Load(sess *session.Session) error {
+	return nil
+}
+
+func (m *mockWorkloadMft) Manifest() interface{} {
+	return nil
+}
+
 func (m *mockWorkloadMft) RequiredEnvironmentFeatures() []string {
 	return m.mockRequiredEnvironmentFeatures()
 }

internal/pkg/cli/svc_package.go

@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	clideploy "github.com/aws/copilot-cli/internal/pkg/cli/deploy"
 	"github.com/aws/copilot-cli/internal/pkg/deploy"
@@ -78,6 +79,7 @@ type packageSvcOpts struct {
 	// cached variables
 	targetApp       *config.Application
 	targetEnv       *config.Environment
+	envSess         *session.Session
 	appliedManifest manifest.WorkloadManifest
 	rootUserARN     string
 }
@@ -136,10 +138,10 @@ func newWkldTplGenerator(o *packageSvcOpts) (workloadTemplateGenerator, error) {
 		App:             targetApp,
 		Env:             targetEnv,
 		ImageTag:        o.tag,
-		Mft:             o.appliedManifest,
+		Mft:             o.appliedManifest.Manifest(),
 		RawMft:          raw,
 	}
-	switch t := o.appliedManifest.(type) {
+	switch t := o.appliedManifest.Manifest().(type) {
 	case *manifest.LoadBalancedWebService:
 		deployer, err = clideploy.NewLBWSDeployer(&in)
 	case *manifest.BackendService:
@@ -275,6 +277,15 @@ func (o *packageSvcOpts) configureClients() error {
 	if err != nil {
 		return fmt.Errorf("create default session: %w", err)
 	}
+	targetEnv, err := o.getTargetEnv()
+	if err != nil {
+		return err
+	}
+	envSess, err := o.sessProvider.FromRole(targetEnv.ManagerRoleARN, targetEnv.Region)
+	if err != nil {
+		return err
+	}
+	o.envSess = envSess
 	// client to retrieve caller identity.
 	caller, err := identity.New(defaultSess).Get()
 	if err != nil {
@@ -308,6 +319,7 @@ func (o *packageSvcOpts) getSvcTemplates(env *config.Environment) (*wkldCfnTempl
 		interpolator: o.newInterpolator(o.appName, o.envName),
 		ws:           o.ws,
 		unmarshal:    o.unmarshal,
+		sess:         o.envSess,
 	})
 	if err != nil {
 		return nil, err

internal/pkg/deploy/cloudformation/stack/backend_svc_test.go

@@ -542,9 +542,10 @@ func TestBackendService_TemplateAndParamsGeneration(t *testing.T) {
 			paramsBytes, err := ioutil.ReadFile(tc.ParamsPath)
 			require.NoError(t, err)
 
-			mft, err := manifest.UnmarshalWorkload([]byte(manifestBytes))
+			dynamicMft, err := manifest.UnmarshalWorkload([]byte(manifestBytes))
 			require.NoError(t, err)
-			require.NoError(t, mft.Validate())
+			require.NoError(t, dynamicMft.Validate())
+			mft := dynamicMft.Manifest()
 
 			envConfig := &manifest.Environment{
 				Workload: manifest.Workload{

internal/pkg/deploy/cloudformation/stack/lb_grpc_web_service_integration_test.go

@@ -1,5 +1,4 @@
-//go:build localintegration
-// +build localintegration
+//go:build integration || localintegration
 
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
@@ -48,11 +47,11 @@ func TestGrpcLoadBalancedWebService_Template(t *testing.T) {
 		require.NoError(t, err)
 		envMft, err := mft.ApplyEnv(tc.envName)
 		require.NoError(t, err)
-
 		err = envMft.Validate()
 		require.NoError(t, err)
+		content := envMft.Manifest()
 
-		v, ok := envMft.(*manifest.LoadBalancedWebService)
+		v, ok := content.(*manifest.LoadBalancedWebService)
 		require.True(t, ok)
 
 		envConfig := &manifest.Environment{

internal/pkg/deploy/cloudformation/stack/lb_network_web_service_integration_test.go

@@ -67,11 +67,11 @@ func TestNetworkLoadBalancedWebService_Template(t *testing.T) {
 		require.NoError(t, err)
 		envMft, err := mft.ApplyEnv(tc.envName)
 		require.NoError(t, err)
-
 		err = envMft.Validate()
 		require.NoError(t, err)
+		content := envMft.Manifest()
 
-		v, ok := envMft.(*manifest.LoadBalancedWebService)
+		v, ok := content.(*manifest.LoadBalancedWebService)
 		require.True(t, ok)
 
 		svcDiscoveryEndpointName := fmt.Sprintf("%s.%s.local", tc.envName, appName)

internal/pkg/deploy/cloudformation/stack/lb_web_service_integration_test.go

@@ -67,11 +67,11 @@ func TestLoadBalancedWebService_Template(t *testing.T) {
 		require.NoError(t, err)
 		envMft, err := mft.ApplyEnv(tc.envName)
 		require.NoError(t, err)
-
 		err = envMft.Validate()
 		require.NoError(t, err)
+		content := envMft.Manifest()
 
-		v, ok := envMft.(*manifest.LoadBalancedWebService)
+		v, ok := content.(*manifest.LoadBalancedWebService)
 		require.True(t, ok)
 
 		svcDiscoveryEndpointName := fmt.Sprintf("%s.%s.local", tc.envName, appName)