chore(manifest): add DynamicWorkloadManifest

Makefile

@@ -211,3 +211,4 @@ gen-mocks: tools
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/apprunner/mocks/mock_apprunner.go -source=./internal/pkg/apprunner/apprunner.go
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/ecs/mocks/mock_run_task_request.go -source=./internal/pkg/ecs/run_task_request.go
 	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/runner/jobrunner/mocks/mock.go -source=./internal/pkg/runner/jobrunner/jobrunner.go
+	${GOBIN}/mockgen -package=mocks -destination=./internal/pkg/manifest/mocks/mock.go -source=./internal/pkg/manifest/loader.go

internal/pkg/aws/ec2/ec2.go

@@ -18,8 +18,9 @@ const (
 	internetGatewayIDPrefix  = "igw-"
 	cloudFrontPrefixListName = "com.amazonaws.global.cloudfront.origin-facing"
 
-	// TagFilterName is the filter name format for tag filters
-	TagFilterName = "tag:%s"
+	// FmtTagFilter is the filter name format for tag filters
+	FmtTagFilter = "tag:%s"
+	tagKeyFilter = "tag-key"
 )
 
 var (
@@ -50,6 +51,17 @@ type Filter struct {
 	Values []string
 }
 
+// FilterForTags takes a key and optional values to construct an EC2 filter.
+func FilterForTags(key string, values ...string) Filter {
+	if len(values) == 0 {
+		return Filter{Name: tagKeyFilter, Values: []string{key}}
+	}
+	return Filter{
+		Name:   fmt.Sprintf(FmtTagFilter, key),
+		Values: values,
+	}
+}
+
 // EC2 wraps an AWS EC2 client.
 type EC2 struct {
 	client api
@@ -322,7 +334,6 @@ func (c *EC2) subnets(filters ...Filter) ([]*ec2.Subnet, error) {
 		return nil, fmt.Errorf("describe subnets: %w", err)
 	}
 	subnets = append(subnets, response.Subnets...)
-
 	for response.NextToken != nil {
 		response, err = c.client.DescribeSubnets(&ec2.DescribeSubnetsInput{
 			Filters:   inputFilters,
@@ -333,7 +344,9 @@ func (c *EC2) subnets(filters ...Filter) ([]*ec2.Subnet, error) {
 		}
 		subnets = append(subnets, response.Subnets...)
 	}
-
+	if len(subnets) == 0 {
+		return nil, fmt.Errorf("cannot find any subnets")
+	}
 	return subnets, nil
 }
 

internal/pkg/aws/ec2/ec2_test.go

@@ -11,19 +11,18 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/copilot-cli/internal/pkg/aws/ec2/mocks"
-	"github.com/aws/copilot-cli/internal/pkg/deploy"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 )
 
 var (
 	inAppEnvFilters = []Filter{
 		{
-			Name:   fmt.Sprintf(TagFilterName, deploy.AppTagKey),
+			Name:   fmt.Sprintf(FmtTagFilter, "copilot-application"),
 			Values: []string{"my-app"},
 		},
 		{
-			Name:   fmt.Sprintf(TagFilterName, deploy.EnvTagKey),
+			Name:   fmt.Sprintf(FmtTagFilter, "copilot-environment"),
 			Values: []string{"my-env"},
 		},
 	}
@@ -802,6 +801,17 @@ func TestEC2_SubnetIDs(t *testing.T) {
 			},
 			wantedError: fmt.Errorf("describe subnets: error describing subnets"),
 		},
+		"cannot get any subnets": {
+			inFilter: inAppEnvFilters,
+			mockEC2Client: func(m *mocks.Mockapi) {
+				m.EXPECT().DescribeSubnets(&ec2.DescribeSubnetsInput{
+					Filters: toEC2Filter(inAppEnvFilters),
+				}).Return(&ec2.DescribeSubnetsOutput{
+					Subnets: []*ec2.Subnet{},
+				}, nil)
+			},
+			wantedError: fmt.Errorf("cannot find any subnets"),
+		},
 		"successfully get subnets": {
 			inFilter: inAppEnvFilters,
 			mockEC2Client: func(m *mocks.Mockapi) {

internal/pkg/cli/job_deploy.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/aws/copilot-cli/internal/pkg/describe"
 
@@ -44,6 +45,7 @@ type deployJobOpts struct {
 	// cached variables
 	targetApp       *config.Application
 	targetEnv       *config.Environment
+	envSess         *session.Session
 	appliedManifest interface{}
 	rootUserARN     string
 }
@@ -149,6 +151,7 @@ func (o *deployJobOpts) Execute() error {
 		interpolator: o.newInterpolator(o.appName, o.envName),
 		ws:           o.ws,
 		unmarshal:    o.unmarshal,
+		sess:         o.envSess,
 	})
 	if err != nil {
 		return err
@@ -220,6 +223,11 @@ func (o *deployJobOpts) configureClients() error {
 	if err != nil {
 		return fmt.Errorf("create default session: %w", err)
 	}
+	envSess, err := o.sessProvider.FromRole(env.ManagerRoleARN, env.Region)
+	if err != nil {
+		return err
+	}
+	o.envSess = envSess
 
 	// client to retrieve caller identity.
 	caller, err := identity.New(defaultSess).Get()

internal/pkg/cli/svc_deploy.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/aws/copilot-cli/internal/pkg/aws/identity"
 	"github.com/aws/copilot-cli/internal/pkg/aws/tags"
@@ -63,6 +64,7 @@ type deploySvcOpts struct {
 	// cached variables
 	targetApp       *config.Application
 	targetEnv       *config.Environment
+	envSess         *session.Session
 	svcType         string
 	appliedManifest interface{}
 	rootUserARN     string
@@ -185,6 +187,7 @@ func (o *deploySvcOpts) Execute() error {
 		interpolator: o.newInterpolator(o.appName, o.envName),
 		ws:           o.ws,
 		unmarshal:    o.unmarshal,
+		sess:         o.envSess,
 	})
 	if err != nil {
 		return err
@@ -327,6 +330,11 @@ func (o *deploySvcOpts) configureClients() error {
 	if err != nil {
 		return fmt.Errorf("create default session: %w", err)
 	}
+	envSess, err := o.sessProvider.FromRole(env.ManagerRoleARN, env.Region)
+	if err != nil {
+		return err
+	}
+	o.envSess = envSess
 
 	// client to retrieve caller identity.
 	caller, err := identity.New(defaultSess).Get()
@@ -353,6 +361,7 @@ type workloadManifestInput struct {
 	envName      string
 	ws           wsWlDirReader
 	interpolator interpolator
+	sess         *session.Session
 	unmarshal    func([]byte) (manifest.WorkloadManifest, error)
 }
 
@@ -371,10 +380,13 @@ func workloadManifest(in *workloadManifestInput) (manifest.WorkloadManifest, err
 	}
 	envMft, err := mft.ApplyEnv(in.envName)
 	if err != nil {
-		return nil, fmt.Errorf("apply environment %s override: %s", in.envName, err)
+		return nil, fmt.Errorf("apply environment %s override: %w", in.envName, err)
 	}
 	if err := envMft.Validate(); err != nil {
-		return nil, fmt.Errorf("validate manifest against environment %s: %s", in.envName, err)
+		return nil, fmt.Errorf("validate manifest against environment %s: %w", in.envName, err)
+	}
+	if err := envMft.Load(in.sess); err != nil {
+		return nil, fmt.Errorf("load dynamic content: %w", err)
 	}
 	return envMft, nil
 }

internal/pkg/cli/svc_deploy_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/copilot-cli/internal/pkg/manifest"
 	"github.com/aws/copilot-cli/internal/pkg/template"
 	"github.com/golang/mock/gomock"
@@ -348,6 +349,14 @@ func (m *mockWorkloadMft) Validate() error {
 	return nil
 }
 
+func (m *mockWorkloadMft) Load(sess *session.Session) error {
+	return nil
+}
+
+func (m *mockWorkloadMft) Manifest() interface{} {
+	return nil
+}
+
 func (m *mockWorkloadMft) RequiredEnvironmentFeatures() []string {
 	return m.mockRequiredEnvironmentFeatures()
 }

internal/pkg/cli/svc_package.go

@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	clideploy "github.com/aws/copilot-cli/internal/pkg/cli/deploy"
 	"github.com/aws/copilot-cli/internal/pkg/deploy"
@@ -78,6 +79,7 @@ type packageSvcOpts struct {
 	// cached variables
 	targetApp       *config.Application
 	targetEnv       *config.Environment
+	envSess         *session.Session
 	appliedManifest manifest.WorkloadManifest
 	rootUserARN     string
 }
@@ -136,10 +138,10 @@ func newWkldTplGenerator(o *packageSvcOpts) (workloadTemplateGenerator, error) {
 		App:             targetApp,
 		Env:             targetEnv,
 		ImageTag:        o.tag,
-		Mft:             o.appliedManifest,
+		Mft:             o.appliedManifest.Manifest(),
 		RawMft:          raw,
 	}
-	switch t := o.appliedManifest.(type) {
+	switch t := o.appliedManifest.Manifest().(type) {
 	case *manifest.LoadBalancedWebService:
 		deployer, err = clideploy.NewLBWSDeployer(&in)
 	case *manifest.BackendService:
@@ -275,6 +277,15 @@ func (o *packageSvcOpts) configureClients() error {
 	if err != nil {
 		return fmt.Errorf("create default session: %w", err)
 	}
+	targetEnv, err := o.getTargetEnv()
+	if err != nil {
+		return err
+	}
+	envSess, err := o.sessProvider.FromRole(targetEnv.ManagerRoleARN, targetEnv.Region)
+	if err != nil {
+		return err
+	}
+	o.envSess = envSess
 	// client to retrieve caller identity.
 	caller, err := identity.New(defaultSess).Get()
 	if err != nil {
@@ -308,6 +319,7 @@ func (o *packageSvcOpts) getSvcTemplates(env *config.Environment) (*wkldCfnTempl
 		interpolator: o.newInterpolator(o.appName, o.envName),
 		ws:           o.ws,
 		unmarshal:    o.unmarshal,
+		sess:         o.envSess,
 	})
 	if err != nil {
 		return nil, err

internal/pkg/deploy/cloudformation/stack/backend_svc_test.go

@@ -542,9 +542,10 @@ func TestBackendService_TemplateAndParamsGeneration(t *testing.T) {
 			paramsBytes, err := ioutil.ReadFile(tc.ParamsPath)
 			require.NoError(t, err)
 
-			mft, err := manifest.UnmarshalWorkload([]byte(manifestBytes))
+			dynamicMft, err := manifest.UnmarshalWorkload([]byte(manifestBytes))
 			require.NoError(t, err)
-			require.NoError(t, mft.Validate())
+			require.NoError(t, dynamicMft.Validate())
+			mft := dynamicMft.Manifest()
 
 			envConfig := &manifest.Environment{
 				Workload: manifest.Workload{

internal/pkg/deploy/cloudformation/stack/lb_grpc_web_service_integration_test.go

@@ -1,5 +1,4 @@
-//go:build localintegration
-// +build localintegration
+//go:build integration || localintegration
 
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
@@ -48,11 +47,11 @@ func TestGrpcLoadBalancedWebService_Template(t *testing.T) {
 		require.NoError(t, err)
 		envMft, err := mft.ApplyEnv(tc.envName)
 		require.NoError(t, err)
-
 		err = envMft.Validate()
 		require.NoError(t, err)
+		content := envMft.Manifest()
 
-		v, ok := envMft.(*manifest.LoadBalancedWebService)
+		v, ok := content.(*manifest.LoadBalancedWebService)
 		require.True(t, ok)
 
 		envConfig := &manifest.Environment{

internal/pkg/deploy/cloudformation/stack/lb_network_web_service_integration_test.go

@@ -70,11 +70,11 @@ func TestNetworkLoadBalancedWebService_Template(t *testing.T) {
 		require.NoError(t, err)
 		envMft, err := mft.ApplyEnv(tc.envName)
 		require.NoError(t, err)
-
 		err = envMft.Validate()
 		require.NoError(t, err)
+		content := envMft.Manifest()
 
-		v, ok := envMft.(*manifest.LoadBalancedWebService)
+		v, ok := content.(*manifest.LoadBalancedWebService)
 		require.True(t, ok)
 
 		svcDiscoveryEndpointName := fmt.Sprintf("%s.%s.local", tc.envName, appName)

internal/pkg/deploy/cloudformation/stack/lb_web_service_integration_test.go

@@ -70,11 +70,11 @@ func TestLoadBalancedWebService_Template(t *testing.T) {
 		require.NoError(t, err)
 		envMft, err := mft.ApplyEnv(tc.envName)
 		require.NoError(t, err)
-
 		err = envMft.Validate()
 		require.NoError(t, err)
+		content := envMft.Manifest()
 
-		v, ok := envMft.(*manifest.LoadBalancedWebService)
+		v, ok := content.(*manifest.LoadBalancedWebService)
 		require.True(t, ok)
 
 		svcDiscoveryEndpointName := fmt.Sprintf("%s.%s.local", tc.envName, appName)

internal/pkg/deploy/cloudformation/stack/rd_web_svc_integration_test.go

@@ -38,17 +38,16 @@ func TestRDWS_Template(t *testing.T) {
 	// Read manifest.
 	manifestBytes, err := ioutil.ReadFile(filepath.Join("testdata", "workloads", manifestFileName))
 	require.NoError(t, err, "read manifest file")
-
 	mft, err := manifest.UnmarshalWorkload(manifestBytes)
 	require.NoError(t, err, "unmarshal manifest file")
 	for _, tc := range testCases {
 		envMft, err := mft.ApplyEnv(tc.envName)
 		require.NoError(t, err, "apply test env to manifest")
-
 		err = envMft.Validate()
 		require.NoError(t, err)
+		content := envMft.Manifest()
 
-		v, ok := envMft.(*manifest.RequestDrivenWebService)
+		v, ok := content.(*manifest.RequestDrivenWebService)
 		require.True(t, ok)
 
 		// Read wanted stack template.
@@ -70,7 +69,6 @@ func TestRDWS_Template(t *testing.T) {
 		require.NoError(t, err, "create rdws serializer")
 		actualTemplate, err := serializer.Template()
 		require.NoError(t, err, "get cloudformation template for rdws")
-
 		// Compare the two.
 		wanted := make(map[interface{}]interface{})
 		require.NoError(t, yaml.Unmarshal(wantedTemplate, wanted), "unmarshal wanted template to map[interface{}]interface{}")

internal/pkg/deploy/cloudformation/stack/scheduled_job_integration_test.go

@@ -30,17 +30,15 @@ func TestScheduledJob_Template(t *testing.T) {
 	path := filepath.Join("testdata", "workloads", jobManifestPath)
 	manifestBytes, err := ioutil.ReadFile(path)
 	require.NoError(t, err)
-
 	mft, err := manifest.UnmarshalWorkload(manifestBytes)
 	require.NoError(t, err)
-
 	envMft, err := mft.ApplyEnv(envName)
 	require.NoError(t, err)
-
 	err = envMft.Validate()
 	require.NoError(t, err)
+	content := envMft.Manifest()
 
-	v, ok := envMft.(*manifest.ScheduledJob)
+	v, ok := content.(*manifest.ScheduledJob)
 	require.True(t, ok)
 
 	serializer, err := stack.NewScheduledJob(stack.ScheduledJobConfig{