Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Part 10: scope cross_logical_cluster tests #1667

Merged
merged 2 commits into from
Aug 5, 2022
Merged

Part 10: scope cross_logical_cluster tests #1667

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e20fc91
scope cross_logical_cluster tests
varshaprasad96 Aug 2, 2022
1b33bc9
Part 10: Scope clients in test/e2e
varshaprasad96 Aug 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 22 additions & 23 deletions test/e2e/conformance/cross_logical_cluster_list_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"testing"
"time"

kcpdynamic "github.com/kcp-dev/apimachinery/pkg/dynamic"
"github.com/kcp-dev/logicalcluster/v2"
"github.com/stretchr/testify/require"

Expand All @@ -36,7 +37,6 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"

configcrds "github.com/kcp-dev/kcp/config/crds"
Expand All @@ -61,41 +61,40 @@ func TestCrossLogicalClusterList(t *testing.T) {
cfg := server.BaseConfig(t)
rootShardCfg := server.RootShardSystemMasterBaseConfig(t)

kcpClients, err := kcpclientset.NewClusterForConfig(cfg)
kcpClusterClient, err := kcpclientset.NewForConfig(cfg)
require.NoError(t, err, "failed to construct kcp client for server")

// Note: we put all consumer workspaces onto root shard in order to enforce conflicts.

logicalClusters := []logicalcluster.Name{
framework.NewOrganizationFixture(t, server, framework.WithShardConstraints(tenancyapi.ShardConstraints{Name: "root"})),
framework.NewOrganizationFixture(t, server, framework.WithShardConstraints(tenancyapi.ShardConstraints{Name: "root"})),
}
expectedWorkspaces := sets.NewString()
for i, logicalCluster := range logicalClusters {
for i, lcluster := range logicalClusters {
wsName := fmt.Sprintf("ws-%d", i)

t.Logf("Creating ClusterWorkspace CRs in logical cluster %s", logicalCluster)
kcpClient := kcpClients.Cluster(logicalCluster)
clustername := lcluster
t.Logf("Creating ClusterWorkspace CRs in logical cluster %s", lcluster)
sourceWorkspace := &tenancyapi.ClusterWorkspace{
ObjectMeta: metav1.ObjectMeta{
Name: wsName,
},
}
_, err = kcpClient.TenancyV1alpha1().ClusterWorkspaces().Create(ctx, sourceWorkspace, metav1.CreateOptions{})
_, err = kcpClusterClient.TenancyV1alpha1().ClusterWorkspaces().Create(logicalcluster.WithCluster(ctx, lcluster), sourceWorkspace, metav1.CreateOptions{})
require.NoError(t, err, "error creating source workspace")

expectedWorkspaces.Insert(logicalCluster.Join(wsName).String())

expectedWorkspaces.Insert(lcluster.Join(wsName).String())
server.Artifact(t, func() (runtime.Object, error) {
return kcpClient.TenancyV1alpha1().ClusterWorkspaces().Get(ctx, sourceWorkspace.Name, metav1.GetOptions{})
obj, err := kcpClusterClient.TenancyV1alpha1().ClusterWorkspaces().Get(logicalcluster.WithCluster(ctx, clustername), sourceWorkspace.Name, metav1.GetOptions{})
return obj, err
})
}

t.Logf("Listing ClusterWorkspace CRs across logical clusters with identity")
tenancyExport, err := kcpClients.Cluster(tenancyapi.RootCluster).ApisV1alpha1().APIExports().Get(ctx, "tenancy.kcp.dev", metav1.GetOptions{})
tenancyExport, err := kcpClusterClient.ApisV1alpha1().APIExports().Get(logicalcluster.WithCluster(ctx, tenancyapi.RootCluster), "tenancy.kcp.dev", metav1.GetOptions{})
require.NoError(t, err, "error getting tenancy API export")
require.NotEmptyf(t, tenancyExport.Status.IdentityHash, "tenancy API export has no identity hash")
dynamicClusterClient, err := dynamic.NewClusterForConfig(rootShardCfg)
dynamicClusterClient, err := kcpdynamic.NewClusterDynamicClientForConfig(rootShardCfg)
require.NoError(t, err, "failed to construct kcp client for server")
client := dynamicClusterClient.Cluster(logicalcluster.Wildcard).Resource(tenancyv1alpha1.SchemeGroupVersion.WithResource(fmt.Sprintf("clusterworkspaces:%s", tenancyExport.Status.IdentityHash)))
workspaces, err := client.List(ctx, metav1.ListOptions{})
Expand All @@ -111,13 +110,13 @@ func TestCrossLogicalClusterList(t *testing.T) {
func bootstrapCRD(
t *testing.T,
clusterName logicalcluster.Name,
client apiextensionsv1client.CustomResourceDefinitionInterface,
clusterClient apiextensionsv1client.CustomResourceDefinitionInterface,
crd *apiextensionsv1.CustomResourceDefinition,
) {
ctx, cancelFunc := context.WithTimeout(context.Background(), wait.ForeverTestTimeout)
ctx, cancelFunc := context.WithTimeout(logicalcluster.WithCluster(context.Background(), clusterName), wait.ForeverTestTimeout)
t.Cleanup(cancelFunc)

err := configcrds.CreateSingle(ctx, client, crd)
err := configcrds.CreateSingle(ctx, clusterClient, crd)
require.NoError(t, err, "error bootstrapping CRD %s in cluster %s", crd.Name, clusterName)
}

Expand Down Expand Up @@ -160,10 +159,10 @@ func TestCRDCrossLogicalClusterListPartialObjectMetadata(t *testing.T) {
cfg := server.BaseConfig(t)
rootShardConfig := server.RootShardSystemMasterBaseConfig(t)

crdClusterClient, err := apiextensionsclient.NewClusterForConfig(cfg)
crdClusterClient, err := apiextensionsclient.NewForConfig(cfg)
require.NoError(t, err, "failed to construct apiextensions client for server")

dynamicClusterClient, err := dynamic.NewClusterForConfig(cfg)
dynamicClusterClient, err := kcpdynamic.NewClusterDynamicClientForConfig(cfg)
require.NoError(t, err, "failed to construct dynamic client for server")

kcpClusterClient, err := kcpclientset.NewForConfig(cfg)
Expand All @@ -177,21 +176,21 @@ func TestCRDCrossLogicalClusterListPartialObjectMetadata(t *testing.T) {
sheriffsGVR := schema.GroupVersionResource{Group: sheriffCRD1.Spec.Group, Resource: "sheriffs", Version: "v1"}

t.Logf("Install a normal sheriffs CRD into workspace %q", wsNormalCRD1a)
bootstrapCRD(t, wsNormalCRD1a, crdClusterClient.Cluster(wsNormalCRD1a).ApiextensionsV1().CustomResourceDefinitions(), sheriffCRD1)
bootstrapCRD(t, wsNormalCRD1a, crdClusterClient.ApiextensionsV1().CustomResourceDefinitions(), sheriffCRD1)

t.Logf("Install another normal sheriffs CRD into workspace %q", wsNormalCRD1b)
bootstrapCRD(t, wsNormalCRD1b, crdClusterClient.Cluster(wsNormalCRD1b).ApiextensionsV1().CustomResourceDefinitions(), sheriffCRD1)
bootstrapCRD(t, wsNormalCRD1b, crdClusterClient.ApiextensionsV1().CustomResourceDefinitions(), sheriffCRD1)

t.Logf("Create a root shard client that is able to do wildcard requests")
rootShardDynamicClients, err := dynamic.NewClusterForConfig(rootShardConfig)
rootShardDynamicClients, err := kcpdynamic.NewClusterDynamicClientForConfig(rootShardConfig)
require.NoError(t, err)

t.Logf("Trying to wildcard list without identity. It should fail.")
_, err = rootShardDynamicClients.Cluster(logicalcluster.Wildcard).Resource(sheriffsGVR).List(ctx, metav1.ListOptions{})
require.Error(t, err, "expected wildcard list to fail because CRD have no identity cross-workspace")

t.Logf("Install a different sheriffs CRD into workspace %q", wsNormalCRD2)
bootstrapCRD(t, wsNormalCRD2, crdClusterClient.Cluster(wsNormalCRD2).ApiextensionsV1().CustomResourceDefinitions(), sheriffCRD2)
bootstrapCRD(t, wsNormalCRD2, crdClusterClient.ApiextensionsV1().CustomResourceDefinitions(), sheriffCRD2)

apifixtures.CreateSheriff(ctx, t, dynamicClusterClient, wsNormalCRD1a, group, wsNormalCRD1a.String())
apifixtures.CreateSheriff(ctx, t, dynamicClusterClient, wsNormalCRD1b, group, wsNormalCRD1b.String())
Expand Down Expand Up @@ -275,7 +274,7 @@ func TestBuiltInCrossLogicalClusterListPartialObjectMetadata(t *testing.T) {
cfg := server.BaseConfig(t)
rootShardCfg := server.RootShardSystemMasterBaseConfig(t)

kubeClusterClient, err := kubernetes.NewClusterForConfig(cfg)
kubeClusterClient, err := kubernetes.NewForConfig(cfg)
require.NoError(t, err, "error creating kube cluster client")

for i := 0; i < 3; i++ {
Expand All @@ -289,7 +288,7 @@ func TestBuiltInCrossLogicalClusterListPartialObjectMetadata(t *testing.T) {
}

t.Logf("Creating configmap %s|default/%s", ws, configMapName)
_, err = kubeClusterClient.Cluster(ws).CoreV1().ConfigMaps("default").Create(ctx, configMap, metav1.CreateOptions{})
_, err = kubeClusterClient.CoreV1().ConfigMaps("default").Create(logicalcluster.WithCluster(ctx, ws), configMap, metav1.CreateOptions{})
require.NoError(t, err, "error creating configmap %s", configMapName)
}

Expand Down
32 changes: 16 additions & 16 deletions test/e2e/conformance/webhook_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,18 +88,18 @@ func TestMutatingWebhookInWorkspace(t *testing.T) {
framework.NewWorkspaceFixture(t, server, organization),
}

kubeClusterClient, err := kubernetes.NewClusterForConfig(cfg)
kubeClusterClient, err := kubernetes.NewForConfig(cfg)
require.NoError(t, err, "failed to construct client for server")
cowbyClients, err := client.NewClusterForConfig(cfg)
cowbyClusterClient, err := client.NewForConfig(cfg)
require.NoError(t, err, "failed to construct cowboy client for server")
apiExtensionsClients, err := apiextensionsclient.NewClusterForConfig(cfg)
apiExtensionsClients, err := apiextensionsclient.NewForConfig(cfg)
require.NoError(t, err, "failed to construct apiextensions client for server")

t.Logf("Install the Cowboy resources into logical clusters")
for _, logicalCluster := range logicalClusters {
t.Logf("Bootstrapping ClusterWorkspace CRDs in logical cluster %s", logicalCluster)
crdClient := apiExtensionsClients.Cluster(logicalCluster).ApiextensionsV1().CustomResourceDefinitions()
wildwest.Create(t, crdClient, metav1.GroupResource{Group: "wildwest.dev", Resource: "cowboys"})
crdClient := apiExtensionsClients.ApiextensionsV1().CustomResourceDefinitions()
wildwest.Create(t, logicalCluster, crdClient, metav1.GroupResource{Group: "wildwest.dev", Resource: "cowboys"})
}

t.Logf("Installing webhook into the first workspace")
Expand Down Expand Up @@ -128,7 +128,7 @@ func TestMutatingWebhookInWorkspace(t *testing.T) {
AdmissionReviewVersions: []string{"v1"},
}},
}
_, err = kubeClusterClient.Cluster(logicalClusters[0]).AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, webhook, metav1.CreateOptions{})
_, err = kubeClusterClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(logicalcluster.WithCluster(ctx, logicalClusters[0]), webhook, metav1.CreateOptions{})
require.NoError(t, err, "failed to add validating webhook configurations")

cowboy := v1alpha1.Cowboy{
Expand All @@ -140,7 +140,7 @@ func TestMutatingWebhookInWorkspace(t *testing.T) {

t.Logf("Creating cowboy resource in first logical cluster")
require.Eventually(t, func() bool {
_, err = cowbyClients.Cluster(logicalClusters[0]).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{})
_, err = cowbyClusterClient.WildwestV1alpha1().Cowboys("default").Create(logicalcluster.WithCluster(ctx, logicalClusters[0]), &cowboy, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return false
}
Expand All @@ -151,7 +151,7 @@ func TestMutatingWebhookInWorkspace(t *testing.T) {
// Avoid race condition here by making sure that CRD is served after installing the types into logical clusters
t.Logf("Creating cowboy resource in second logical cluster")
require.Eventually(t, func() bool {
_, err = cowbyClients.Cluster(logicalClusters[1]).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{})
_, err = cowbyClusterClient.WildwestV1alpha1().Cowboys("default").Create(logicalcluster.WithCluster(ctx, logicalClusters[1]), &cowboy, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return false
}
Expand Down Expand Up @@ -207,18 +207,18 @@ func TestValidatingWebhookInWorkspace(t *testing.T) {
framework.NewWorkspaceFixture(t, server, organization),
}

kubeClusterClient, err := kubernetes.NewClusterForConfig(cfg)
kubeClusterClient, err := kubernetes.NewForConfig(cfg)
require.NoError(t, err, "failed to construct client for server")
cowbyClients, err := client.NewClusterForConfig(cfg)
cowbyClusterClient, err := client.NewForConfig(cfg)
require.NoError(t, err, "failed to construct cowboy client for server")
apiExtensionsClients, err := apiextensionsclient.NewClusterForConfig(cfg)
apiExtensionsClients, err := apiextensionsclient.NewForConfig(cfg)
require.NoError(t, err, "failed to construct apiextensions client for server")

t.Logf("Install the Cowboy resources into logical clusters")
for _, logicalCluster := range logicalClusters {
t.Logf("Bootstrapping ClusterWorkspace CRDs in logical cluster %s", logicalCluster)
crdClient := apiExtensionsClients.Cluster(logicalCluster).ApiextensionsV1().CustomResourceDefinitions()
wildwest.Create(t, crdClient, metav1.GroupResource{Group: "wildwest.dev", Resource: "cowboys"})
crdClient := apiExtensionsClients.ApiextensionsV1().CustomResourceDefinitions()
wildwest.Create(t, logicalCluster, crdClient, metav1.GroupResource{Group: "wildwest.dev", Resource: "cowboys"})
}

t.Logf("Installing webhook into the first workspace")
Expand Down Expand Up @@ -247,7 +247,7 @@ func TestValidatingWebhookInWorkspace(t *testing.T) {
AdmissionReviewVersions: []string{"v1"},
}},
}
_, err = kubeClusterClient.Cluster(logicalClusters[0]).AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, webhook, metav1.CreateOptions{})
_, err = kubeClusterClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(logicalcluster.WithCluster(ctx, logicalClusters[0]), webhook, metav1.CreateOptions{})
require.NoError(t, err, "failed to add validating webhook configurations")

cowboy := v1alpha1.Cowboy{
Expand All @@ -259,7 +259,7 @@ func TestValidatingWebhookInWorkspace(t *testing.T) {

t.Logf("Creating cowboy resource in first logical cluster")
require.Eventually(t, func() bool {
_, err = cowbyClients.Cluster(logicalClusters[0]).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{})
_, err = cowbyClusterClient.WildwestV1alpha1().Cowboys("default").Create(logicalcluster.WithCluster(ctx, logicalClusters[0]), &cowboy, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return false
}
Expand All @@ -268,7 +268,7 @@ func TestValidatingWebhookInWorkspace(t *testing.T) {

// Avoid race condition here by making sure that CRD is served after installing the types into logical clusters
t.Logf("Creating cowboy resource in second logical cluster")
_, err = cowbyClients.Cluster(logicalClusters[1]).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{})
_, err = cowbyClusterClient.WildwestV1alpha1().Cowboys("default").Create(logicalcluster.WithCluster(ctx, logicalClusters[1]), &cowboy, metav1.CreateOptions{})
require.NoError(t, err, "failed to create cowboy resource in second logical cluster")
require.Equal(t, 1, testWebhook.Calls(), "expected that the webhook is not called for logical cluster where webhook is not installed")
}
7 changes: 4 additions & 3 deletions test/e2e/fixtures/wildwest/bootstrap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"embed"
"testing"

"github.com/kcp-dev/logicalcluster/v2"
"github.com/stretchr/testify/require"

apiextensionsv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
Expand All @@ -33,10 +34,10 @@ import (
//go:embed *.yaml
var rawCustomResourceDefinitions embed.FS

func Create(t *testing.T, client apiextensionsv1client.CustomResourceDefinitionInterface, grs ...metav1.GroupResource) {
ctx, cancelFunc := context.WithTimeout(context.Background(), wait.ForeverTestTimeout)
func Create(t *testing.T, clustername logicalcluster.Name, client apiextensionsv1client.CustomResourceDefinitionInterface, grs ...metav1.GroupResource) {
ctx, cancelFunc := context.WithTimeout(logicalcluster.WithCluster(context.Background(), clustername), wait.ForeverTestTimeout)
t.Cleanup(cancelFunc)

err := configcrds.CreateFromFS(ctx, client, rawCustomResourceDefinitions, grs...)
err := configcrds.CreateFromFS(logicalcluster.WithCluster(ctx, clustername), client, rawCustomResourceDefinitions, grs...)
require.NoError(t, err)
}
2 changes: 1 addition & 1 deletion test/e2e/reconciler/cluster/controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ func TestClusterController(t *testing.T) {
sinkCrdClient, err := apiextensionsclientset.NewForConfig(config)
require.NoError(t, err)
t.Log("Installing test CRDs into sink cluster...")
fixturewildwest.Create(t, sinkCrdClient.ApiextensionsV1().CustomResourceDefinitions(), metav1.GroupResource{Group: wildwest.GroupName, Resource: "cowboys"})
fixturewildwest.Create(t, logicalcluster.Name{}, sinkCrdClient.ApiextensionsV1().CustomResourceDefinitions(), metav1.GroupResource{Group: wildwest.GroupName, Resource: "cowboys"})
})).Start(t)

sinkWildwestClient, err := wildwestclientset.NewForConfig(syncerFixture.DownstreamConfig)
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/virtual/syncer/virtualworkspace_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -614,7 +614,7 @@ func TestSyncerVirtualWorkspace(t *testing.T) {
sourceCrdClient, err := apiextensionsclientset.NewClusterForConfig(server.BaseConfig(t))
require.NoError(t, err)

fixturewildwest.Create(t, sourceCrdClient.Cluster(unrelatedWorkspace).ApiextensionsV1().CustomResourceDefinitions(), metav1.GroupResource{Group: wildwest.GroupName, Resource: "cowboys"})
fixturewildwest.Create(t, logicalcluster.Name{}, sourceCrdClient.Cluster(unrelatedWorkspace).ApiextensionsV1().CustomResourceDefinitions(), metav1.GroupResource{Group: wildwest.GroupName, Resource: "cowboys"})

t.Log("Waiting for cowboys crd to be imported and available in the kubelike workspace...")
require.Eventually(t, func() bool {
Expand Down Expand Up @@ -650,7 +650,7 @@ func TestSyncerVirtualWorkspace(t *testing.T) {
sinkCrdClient, err := apiextensionsclientset.NewForConfig(config)
require.NoError(t, err)
t.Log("Installing test CRDs into sink cluster...")
fixturewildwest.Create(t, sinkCrdClient.ApiextensionsV1().CustomResourceDefinitions(), metav1.GroupResource{Group: wildwest.GroupName, Resource: "cowboys"})
fixturewildwest.Create(t, logicalcluster.Name{}, sinkCrdClient.ApiextensionsV1().CustomResourceDefinitions(), metav1.GroupResource{Group: wildwest.GroupName, Resource: "cowboys"})
}),
).Start(t)

Expand Down
2 changes: 1 addition & 1 deletion test/e2e/watchcache/watchcache_enabled_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func TestWatchCacheEnabledForCRD(t *testing.T) {
crdClient := apiExtensionsClients.Cluster(cluster).ApiextensionsV1().CustomResourceDefinitions()

t.Log("Creating wildwest.dev.cowboys CR")
wildwest.Create(t, crdClient, cowBoysGR)
wildwest.Create(t, logicalcluster.Name{}, crdClient, cowBoysGR)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cluster, not empty.

Copy link
Contributor Author

@varshaprasad96 varshaprasad96 Aug 5, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is temporary. I'll create a follow up PR to modify clients in watchcache/ and that will make sure to pass the right cluster name while creating the object. Doing it here, will transitively involve changes in other files, so splitting it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ack

wildwestClusterClient, err := wildwestclientset.NewClusterForConfig(rootShardConfig)
require.NoError(t, err)
_, err = wildwestClusterClient.Cluster(cluster).WildwestV1alpha1().Cowboys("default").Create(ctx, &wildwestv1alpha1.Cowboy{
Expand Down