Cloud: Make e2e Tests Less Chatty, and More Stable

go.mod

@@ -43,7 +43,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.0
 	github.com/hashicorp/go-tfe v0.20.1-0.20211110172530-c43c6b574caa
 	github.com/hashicorp/go-uuid v1.0.2
-	github.com/hashicorp/go-version v1.2.1
+	github.com/hashicorp/go-version v1.3.0
 	github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f
 	github.com/hashicorp/hcl/v2 v2.10.1
 	github.com/hashicorp/terraform-config-inspect v0.0.0-20210209133302-4fd17a0faac2

go.sum

@@ -388,8 +388,9 @@ github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw=
+github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=

internal/cloud/e2e/README.md

@@ -0,0 +1,24 @@
+# How to run tests
+
+To run them, use:
+```
+TFE_TOKEN=<token> TFE_HOSTNAME=<hostname> TF_ACC=1 go test  ./internal/cloud/e2e/... -ldflags "-X \"github.com/hashicorp/terraform/version.Prerelease=<PRE-RELEASE>\""
+```
+
+Required flags
+* `TF_ACC=1`. This variable is used as part of terraform for tests that make 
+  external network calls. This is needed to run these tests. Without it, the
+  tests do not run.
+* `TFE_TOKEN=<admin token>` and `TFE_HOSTNAME=<hostname>`. The helpers
+for these tests require admin access to a TFC/TFE instance.
+* `-timeout=30m`. Some of these tests take longer than the default 10m timeout for `go test`.
+
+### Flags
+
+* Use the `-v` flag for normal verbose mode.
+* Use the `-tfoutput` flag to print the terraform output to standard out.
+*  Use `-ldflags` to change the version Prerelease to match a version
+available remotely. Some behaviors rely on the exact local version Terraform
+being available in TFC/TFE, and manipulating the Prerelease during build is
+often the only way to ensure this.
+[(More on `-ldflags`.)](https://www.digitalocean.com/community/tutorials/using-ldflags-to-set-version-information-for-go-applications)

internal/cloud/e2e/apply_auto_approve_test.go

@@ -1,12 +1,8 @@
-//go:build e2e
-// +build e2e
-
 package main
 
 import (
 	"context"
 	"io/ioutil"
-	"log"
 	"os"
 	"testing"
 
@@ -17,7 +13,7 @@ import (
 )
 
 func Test_terraform_apply_autoApprove(t *testing.T) {
-	t.Parallel()
+	skipIfMissingEnvVar(t)
 	skipWithoutRemoteTerraformVersion(t)
 
 	ctx := context.Background()
@@ -183,12 +179,10 @@ func Test_terraform_apply_autoApprove(t *testing.T) {
 			},
 		},
 	}
-	for name, tc := range cases {
-		log.Println("Test: ", name)
-
+	for _, tc := range cases {
 		organization, cleanup := createOrganization(t)
 		defer cleanup()
-		exp, err := expect.NewConsole(expect.WithStdout(os.Stdout), expect.WithDefaultTimeout(expectConsoleTimeout))
+		exp, err := expect.NewConsole(defaultOpts()...)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -201,7 +195,6 @@ func Test_terraform_apply_autoApprove(t *testing.T) {
 		defer os.RemoveAll(tmpDir)
 
 		tf := e2e.NewBinary(terraformBin, tmpDir)
-		tf.AddEnv("TF_LOG=info")
 		tf.AddEnv(cliConfigFileEnv)
 		defer tf.Close()
 

internal/cloud/e2e/backend_apply_before_init_test.go

@@ -1,10 +1,6 @@
-//go:build e2e
-// +build e2e
-
 package main
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -14,6 +10,7 @@ import (
 )
 
 func Test_backend_apply_before_init(t *testing.T) {
+	skipIfMissingEnvVar(t)
 	t.Parallel()
 	skipWithoutRemoteTerraformVersion(t)
 
@@ -51,10 +48,8 @@ func Test_backend_apply_before_init(t *testing.T) {
 							expectedCmdOutput: `Successfully configured the backend "local"!`,
 						},
 						{
-							command:           []string{"apply"},
-							expectedCmdOutput: `Do you want to perform these actions?`,
-							userInput:         []string{"yes"},
-							postInputOutput:   []string{`Apply complete!`},
+							command:         []string{"apply", "-auto-approve"},
+							postInputOutput: []string{`Apply complete!`},
 						},
 					},
 				},
@@ -77,68 +72,70 @@ func Test_backend_apply_before_init(t *testing.T) {
 	}
 
 	for name, tc := range cases {
-		fmt.Println("Test: ", name)
-		organization, cleanup := createOrganization(t)
-		defer cleanup()
-		exp, err := expect.NewConsole(expect.WithStdout(os.Stdout), expect.WithDefaultTimeout(expectConsoleTimeout))
-		if err != nil {
-			t.Fatal(err)
-		}
-		defer exp.Close()
-
-		tmpDir, err := ioutil.TempDir("", "terraform-test")
-		if err != nil {
-			t.Fatal(err)
-		}
-		defer os.RemoveAll(tmpDir)
+		tc := tc
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			organization, cleanup := createOrganization(t)
+			defer cleanup()
+			exp, err := expect.NewConsole(defaultOpts()...)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer exp.Close()
 
-		tf := e2e.NewBinary(terraformBin, tmpDir)
-		tf.AddEnv("TF_LOG=info")
-		tf.AddEnv(cliConfigFileEnv)
-		defer tf.Close()
+			tmpDir, err := ioutil.TempDir("", "terraform-test")
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer os.RemoveAll(tmpDir)
 
-		for _, op := range tc.operations {
-			op.prep(t, organization.Name, tf.WorkDir())
-			for _, tfCmd := range op.commands {
-				cmd := tf.Cmd(tfCmd.command...)
-				cmd.Stdin = exp.Tty()
-				cmd.Stdout = exp.Tty()
-				cmd.Stderr = exp.Tty()
+			tf := e2e.NewBinary(terraformBin, tmpDir)
+			tf.AddEnv(cliConfigFileEnv)
+			defer tf.Close()
 
-				err = cmd.Start()
-				if err != nil {
-					t.Fatal(err)
-				}
+			for _, op := range tc.operations {
+				op.prep(t, organization.Name, tf.WorkDir())
+				for _, tfCmd := range op.commands {
+					cmd := tf.Cmd(tfCmd.command...)
+					cmd.Stdin = exp.Tty()
+					cmd.Stdout = exp.Tty()
+					cmd.Stderr = exp.Tty()
 
-				if tfCmd.expectedCmdOutput != "" {
-					_, err := exp.ExpectString(tfCmd.expectedCmdOutput)
+					err = cmd.Start()
 					if err != nil {
 						t.Fatal(err)
 					}
-				}
 
-				lenInput := len(tfCmd.userInput)
-				lenInputOutput := len(tfCmd.postInputOutput)
-				if lenInput > 0 {
-					for i := 0; i < lenInput; i++ {
-						input := tfCmd.userInput[i]
-						exp.SendLine(input)
-						// use the index to find the corresponding
-						// output that matches the input.
-						if lenInputOutput-1 >= i {
-							output := tfCmd.postInputOutput[i]
-							_, err := exp.ExpectString(output)
-							if err != nil {
-								t.Fatal(err)
+					if tfCmd.expectedCmdOutput != "" {
+						_, err := exp.ExpectString(tfCmd.expectedCmdOutput)
+						if err != nil {
+							t.Fatal(err)
+						}
+					}
+
+					lenInput := len(tfCmd.userInput)
+					lenInputOutput := len(tfCmd.postInputOutput)
+					if lenInput > 0 {
+						for i := 0; i < lenInput; i++ {
+							input := tfCmd.userInput[i]
+							exp.SendLine(input)
+							// use the index to find the corresponding
+							// output that matches the input.
+							if lenInputOutput-1 >= i {
+								output := tfCmd.postInputOutput[i]
+								_, err := exp.ExpectString(output)
+								if err != nil {
+									t.Fatal(err)
+								}
 							}
 						}
 					}
-				}
-				err = cmd.Wait()
-				if err != nil && !tfCmd.expectError {
-					t.Fatal(err)
+					err = cmd.Wait()
+					if err != nil && !tfCmd.expectError {
+						t.Fatal(err)
+					}
 				}
 			}
-		}
+		})
 	}
 }

internal/cloud/e2e/helper_test.go

@@ -1,6 +1,3 @@
-//go:build e2e
-// +build e2e
-
 package main
 
 import (
@@ -10,8 +7,10 @@ import (
 	"testing"
 	"time"
 
+	expect "github.com/Netflix/go-expect"
 	tfe "github.com/hashicorp/go-tfe"
 	"github.com/hashicorp/go-uuid"
+	goversion "github.com/hashicorp/go-version"
 	tfversion "github.com/hashicorp/terraform/version"
 )
 
@@ -22,7 +21,6 @@ const (
 type tfCommand struct {
 	command           []string
 	expectedCmdOutput string
-	expectedErr       string
 	expectError       bool
 	userInput         []string
 	postInputOutput   []string
@@ -38,6 +36,16 @@ type testCases map[string]struct {
 	validations func(t *testing.T, orgName string)
 }
 
+func defaultOpts() []expect.ConsoleOpt {
+	opts := []expect.ConsoleOpt{
+		expect.WithDefaultTimeout(expectConsoleTimeout),
+	}
+	if verboseMode {
+		opts = append(opts, expect.WithStdout(os.Stdout))
+	}
+	return opts
+}
+
 func createOrganization(t *testing.T) (*tfe.Organization, func()) {
 	ctx := context.Background()
 	org, err := tfeClient.Organizations.Create(ctx, tfe.OrganizationCreateOptions{
@@ -93,7 +101,7 @@ func randomString(t *testing.T) string {
 }
 
 func terraformConfigLocalBackend() string {
-	return fmt.Sprintf(`
+	return `
 terraform {
   backend "local" {
   }
@@ -102,7 +110,7 @@ terraform {
 output "val" {
   value = "${terraform.workspace}"
 }
-`)
+`
 }
 
 func terraformConfigRemoteBackendName(org, name string) string {
@@ -193,9 +201,16 @@ func writeMainTF(t *testing.T, block string, dir string) {
 	f.Close()
 }
 
-// Ensure that TFC/E has a particular terraform version.
+// The e2e tests rely on the fact that the terraform version in TFC/E is able to
+// run the `cloud` configuration block, which is available in 1.1 and will
+// continue to be available in later versions. So this function checks that
+// there is a version that is >= 1.1.
 func skipWithoutRemoteTerraformVersion(t *testing.T) {
-	version := tfversion.String()
+	version := tfversion.Version
+	baseVersion, err := goversion.NewVersion(version)
+	if err != nil {
+		t.Fatalf(fmt.Sprintf("Error instantiating go-version for %s", version))
+	}
 	opts := tfe.AdminTerraformVersionsListOptions{
 		ListOptions: tfe.ListOptions{
 			PageNumber: 1,
@@ -213,7 +228,12 @@ findTfVersion:
 			t.Fatalf("Could not retrieve list of terraform versions: %v", err)
 		}
 		for _, item := range tfVersionList.Items {
-			if item.Version == version {
+			availableVersion, err := goversion.NewVersion(item.Version)
+			if err != nil {
+				t.Logf("Error instantiating go-version for %s", item.Version)
+				continue
+			}
+			if availableVersion.Core().GreaterThanOrEqual(baseVersion.Core()) {
 				hasVersion = true
 				break findTfVersion
 			}