Skip to content

Commit

Permalink
e2e/framework: split up fixtures
Browse files Browse the repository at this point in the history
  • Loading branch information
sttts committed Aug 1, 2022
1 parent abe4772 commit 632953e
Show file tree
Hide file tree
Showing 6 changed files with 522 additions and 450 deletions.
52 changes: 52 additions & 0 deletions test/e2e/framework/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,34 @@ limitations under the License.
package framework

import (
"context"
"errors"
"flag"
"fmt"
"path/filepath"
"strings"
"testing"

"github.com/kcp-dev/logicalcluster/v2"
"github.com/stretchr/testify/require"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/klog/v2"

tenancyv1alpha1 "github.com/kcp-dev/kcp/pkg/apis/tenancy/v1alpha1"
kcpclientset "github.com/kcp-dev/kcp/pkg/client/clientset/versioned"
)

func init() {
klog.InitFlags(flag.CommandLine)
if err := flag.Lookup("v").Value.Set("2"); err != nil {
panic(err)
}
}

type testConfig struct {
syncerImage string
kcpTestImage string
Expand Down Expand Up @@ -78,3 +101,32 @@ func registerFlags(c *testConfig) {
flag.StringVar(&c.kcpTestImage, "kcp-test-image", "", "The test image to use with the pcluster. Requires --pcluster-kubeconfig")
flag.BoolVar(&c.useDefaultKCPServer, "use-default-kcp-server", false, "Whether to use server configuration from .kcp/admin.kubeconfig.")
}

// WriteLogicalClusterConfig creates a logical cluster config for the given config and
// cluster name and writes it to the test's artifact path. Useful for configuring the
// workspace plugin with --kubeconfig.
func WriteLogicalClusterConfig(t *testing.T, rawConfig clientcmdapi.Config, contextName string, clusterName logicalcluster.Name) (clientcmd.ClientConfig, string) {
logicalRawConfig := LogicalClusterRawConfig(rawConfig, clusterName, contextName)
artifactDir, err := CreateTempDirForTest(t, "artifacts")
require.NoError(t, err)
pathSafeClusterName := strings.ReplaceAll(clusterName.String(), ":", "_")
kubeconfigPath := filepath.Join(artifactDir, fmt.Sprintf("%s.kubeconfig", pathSafeClusterName))
err = clientcmd.WriteToFile(logicalRawConfig, kubeconfigPath)
require.NoError(t, err)
logicalConfig := clientcmd.NewNonInteractiveClientConfig(logicalRawConfig, logicalRawConfig.CurrentContext, &clientcmd.ConfigOverrides{}, nil)
return logicalConfig, kubeconfigPath
}

// ShardConfig returns a rest config that talk directly to the given shard.
func ShardConfig(t *testing.T, kcpClusterClient kcpclientset.ClusterInterface, shardName string, cfg *rest.Config) *rest.Config {
ctx, cancelFunc := context.WithCancel(context.Background())
t.Cleanup(cancelFunc)

shard, err := kcpClusterClient.Cluster(tenancyv1alpha1.RootCluster).TenancyV1alpha1().ClusterWorkspaceShards().Get(ctx, shardName, metav1.GetOptions{})
require.NoError(t, err)

shardCfg := rest.CopyConfig(cfg)
shardCfg.Host = shard.Spec.BaseURL

return shardCfg
}
133 changes: 132 additions & 1 deletion test/e2e/framework/kcp.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,137 @@ import (
kubefixtures "github.com/kcp-dev/kcp/test/e2e/fixtures/kube"
)

// TestServerArgs returns the set of kcp args used to start a test
// server using the token auth file from the working tree.
func TestServerArgs() []string {
return TestServerArgsWithTokenAuthFile("test/e2e/framework/auth-tokens.csv")
}

// TestServerArgsWithTokenAuthFile returns the set of kcp args used to
// start a test server with the given token auth file.
func TestServerArgsWithTokenAuthFile(tokenAuthFile string) []string {
return []string{
"-v=4",
"--token-auth-file", tokenAuthFile,
}
}

// KcpFixture manages the lifecycle of a set of kcp servers.
//
// Deprecated for use outside this package. Prefer PrivateKcpServer().
type kcpFixture struct {
Servers map[string]RunningServer
}

// PrivateKcpServer returns a new kcp server fixture managing a new
// server process that is not intended to be shared between tests.
func PrivateKcpServer(t *testing.T, args ...string) RunningServer {
serverName := "main"
f := newKcpFixture(t, kcpConfig{
Name: serverName,
Args: args,
})
return f.Servers[serverName]
}

// SharedKcpServer returns a kcp server fixture intended to be shared
// between tests. A persistent server will be configured if
// `--kcp-kubeconfig` or `--use-default-kcp-server` is supplied to the test
// runner. Otherwise a test-managed server will be started. Only tests
// that are known to be hermetic are compatible with shared fixture.
func SharedKcpServer(t *testing.T) RunningServer {
serverName := "shared"
kubeconfig := TestConfig.KCPKubeconfig()
if len(kubeconfig) > 0 {
// Use a persistent server

t.Logf("shared kcp server will target configuration %q", kubeconfig)
server, err := newPersistentKCPServer(serverName, kubeconfig, TestConfig.RootShardKubeconfig())
require.NoError(t, err, "failed to create persistent server fixture")
return server
}

// Use a test-provisioned server
//
// TODO(marun) Enable non-persistent fixture to be shared across
// tests. This will likely require composing tests into a suite that
// initializes the shared fixture before tests that rely on the
// fixture.

tokenAuthFile := WriteTokenAuthFile(t)
f := newKcpFixture(t, kcpConfig{
Name: serverName,
Args: TestServerArgsWithTokenAuthFile(tokenAuthFile),
})
return f.Servers[serverName]
}

// Deprecated for use outside this package. Prefer PrivateKcpServer().
func newKcpFixture(t *testing.T, cfgs ...kcpConfig) *kcpFixture {
f := &kcpFixture{}

artifactDir, dataDir, err := ScratchDirs(t)
require.NoError(t, err, "failed to create scratch dirs: %v", err)

// Initialize servers from the provided configuration
var servers []*kcpServer
f.Servers = map[string]RunningServer{}
for _, cfg := range cfgs {
server, err := newKcpServer(t, cfg, artifactDir, dataDir)
require.NoError(t, err)

servers = append(servers, server)
f.Servers[server.name] = server
}

// Launch kcp servers and ensure they are ready before starting the test
start := time.Now()
t.Log("Starting kcp servers...")
wg := sync.WaitGroup{}
wg.Add(len(servers))
for i, srv := range servers {
var opts []RunOption
if LogToConsoleEnvSet() || cfgs[i].LogToConsole {
opts = append(opts, WithLogStreaming)
}
if InProcessEnvSet() || cfgs[i].RunInProcess {
opts = append(opts, RunInProcess)
}
err := srv.Run(opts...)
require.NoError(t, err)

// Wait for the server to become ready
go func(s *kcpServer, i int) {
defer wg.Done()
err := s.Ready(!cfgs[i].RunInProcess)
require.NoError(t, err, "kcp server %s never became ready: %v", s.name, err)
}(srv, i)
}
wg.Wait()

if t.Failed() {
t.Fatal("Fixture setup failed: one or more servers did not become ready")
}

t.Logf("Started kcp servers after %s", time.Since(start))

return f
}

func InProcessEnvSet() bool {
inProcess, _ := strconv.ParseBool(os.Getenv("INPROCESS"))
return inProcess
}

func LogToConsoleEnvSet() bool {
inProcess, _ := strconv.ParseBool(os.Getenv("LOG_TO_CONSOLE"))
return inProcess
}

func preserveTestResources() bool {
return os.Getenv("PRESERVE") != ""
}

type RunningServer interface {
Name() string
KubeconfigPath() string
Expand Down Expand Up @@ -644,7 +775,7 @@ func NewFakeWorkloadServer(t *testing.T, server RunningServer, org logicalcluste
logicalClusterName := NewWorkspaceFixture(t, server, org)
rawConfig, err := server.RawConfig()
require.NoError(t, err, "failed to read config for server")
logicalConfig, kubeconfigPath := WriteLogicalClusterConfig(t, rawConfig, logicalClusterName, "base")
logicalConfig, kubeconfigPath := WriteLogicalClusterConfig(t, rawConfig, "base", logicalClusterName)
fakeServer := &unmanagedKCPServer{
name: logicalClusterName.String(),
cfg: logicalConfig,
Expand Down
136 changes: 136 additions & 0 deletions test/e2e/framework/kubectl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
/*
Copyright 2022 The KCP Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"

"github.com/stretchr/testify/require"
)

// KcpCliPluginCommand returns the cli args to run the workspace plugin directly or
// via go run (depending on whether NO_GORUN is set).
func KcpCliPluginCommand() []string {
if NoGoRunEnvSet() {
return []string{"kubectl", "kcp"}

} else {
cmdPath := filepath.Join(RepositoryDir(), "cmd", "kubectl-kcp")
return []string{"go", "run", cmdPath}
}
}

// RunKcpCliPlugin runs the kcp workspace plugin with the provided subcommand and
// returns the combined stderr and stdout output.
func RunKcpCliPlugin(t *testing.T, kubeconfigPath string, subcommand []string) []byte {
ctx, cancelFunc := context.WithCancel(context.Background())
t.Cleanup(cancelFunc)

cmdParts := append(KcpCliPluginCommand(), subcommand...)
cmd := exec.CommandContext(ctx, cmdParts[0], cmdParts[1:]...)

cmd.Env = os.Environ()
// TODO(marun) Consider configuring the workspace plugin with args instead of this env
cmd.Env = append(cmd.Env, fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath))

t.Logf("running: KUBECONFIG=%s %s", kubeconfigPath, strings.Join(cmdParts, " "))

var output, _, combined bytes.Buffer
var lock sync.Mutex
cmd.Stdout = split{a: locked{mu: &lock, w: &combined}, b: &output}
cmd.Stderr = locked{mu: &lock, w: &combined}
err := cmd.Run()
if err != nil {
t.Logf("kcp plugin output:\n%s", combined.String())
}
require.NoError(t, err, "error running kcp plugin command")
return output.Bytes()
}

// KubectlApply runs kubectl apply -f with the supplied input piped to stdin and returns
// the combined stderr and stdout output.
func KubectlApply(t *testing.T, kubeconfigPath string, input []byte) []byte {
ctx, cancelFunc := context.WithCancel(context.Background())
t.Cleanup(cancelFunc)

cmdParts := []string{"kubectl", "--kubeconfig", kubeconfigPath, "apply", "-f", "-"}
cmd := exec.CommandContext(ctx, cmdParts[0], cmdParts[1:]...)
stdin, err := cmd.StdinPipe()
require.NoError(t, err)
_, err = stdin.Write(input)
require.NoError(t, err)
// Close to ensure kubectl doesn't keep waiting for input
err = stdin.Close()
require.NoError(t, err)

t.Logf("running: %s", strings.Join(cmdParts, " "))

output, err := cmd.CombinedOutput()
if err != nil {
t.Logf("kubectl apply output:\n%s", output)
}
require.NoError(t, err)

return output
}

// Kubectl runs kubectl with the given arguments and returns the combined stderr and stdout.
func Kubectl(t *testing.T, kubeconfigPath string, args ...string) []byte {
ctx, cancelFunc := context.WithCancel(context.Background())
t.Cleanup(cancelFunc)

cmdParts := append([]string{"kubectl", "--kubeconfig", kubeconfigPath}, args...)
cmd := exec.CommandContext(ctx, cmdParts[0], cmdParts[1:]...)
t.Logf("running: %s", strings.Join(cmdParts, " "))

output, err := cmd.CombinedOutput()
if err != nil {
t.Logf("kubectl output:\n%s", output)
}
require.NoError(t, err)

return output
}

type locked struct {
mu *sync.Mutex
w io.Writer
}

func (w locked) Write(p []byte) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
return w.w.Write(p)
}

type split struct {
a, b io.Writer
}

func (w split) Write(p []byte) (int, error) {
w.a.Write(p) // nolint: errcheck
return w.b.Write(p)
}
Loading

0 comments on commit 632953e

Please sign in to comment.