From 022a97d412ecd4e43155b517377be86d7e9db437 Mon Sep 17 00:00:00 2001 From: Heiko Kast <63250259+H777K@users.noreply.github.com> Date: Wed, 5 Feb 2025 12:30:37 +0100 Subject: [PATCH] feat: allow override of volume and volumemount (#83) This PR ensures, that the default volume and volumemount wont be created, if they are defined in podtemplate to e.g. use an PVC instead of an emptydir --- internal/provider/provider.go | 11 +- internal/provider/provider_test.go | 434 +++++++++++++++++++++++++++++ internal/spec/spec.go | 45 ++- 3 files changed, 483 insertions(+), 7 deletions(-) diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 8671eb3..48941ce 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -22,6 +22,10 @@ import ( "github.com/mercedes-benz/garm-provider-k8s/pkg/diff" ) +const ( + runnerContainerName = "runner" +) + type Provider struct { ControllerID string ClientSet kubernetes.Interface @@ -54,7 +58,7 @@ func (p Provider) CreateInstance(_ context.Context, bootstrapParams params.Boots RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{ { - Name: "runner", + Name: runnerContainerName, Image: bootstrapParams.Image, Resources: resourceRequirements, Env: envs, @@ -74,6 +78,11 @@ func (p Provider) CreateInstance(_ context.Context, bootstrapParams params.Boots return params.ProviderInstance{}, err } + err = spec.CreateRunnerVolumeMount(pod, runnerContainerName) + if err != nil { + return params.ProviderInstance{}, err + } + mergedPod, err := mergePodSpecs(pod, config.Config.PodTemplate) if err != nil { return params.ProviderInstance{}, err diff --git a/internal/provider/provider_test.go b/internal/provider/provider_test.go index 41a224c..bc82108 100644 --- a/internal/provider/provider_test.go +++ b/internal/provider/provider_test.go @@ -1172,6 +1172,436 @@ func TestCreateInstance(t *testing.T) { runtimeObjects: []runtime.Object{}, err: nil, }, + { + name: "Overwrite default runner emptyDir volume with a PVC", + config: &config.ProviderConfig{ + KubeConfigPath: "", + RunnerNamespace: "runner", + PodTemplate: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "runner", + VolumeSource: corev1.VolumeSource{ + Ephemeral: &corev1.EphemeralVolumeSource{ + VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + StorageClassName: toPointer("cinder"), + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "runner", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }, + }, + }, + }, + }, + bootstrapParams: params.BootstrapInstance{ + Name: instanceName, + PoolID: poolID, + Flavor: "small", + RepoURL: "/~https://github.com/testorg", + InstanceToken: "test-token", + MetadataURL: "https://metadata.test", + CallbackURL: "https://callback.test/status", + Image: "localhost:5000/runner:ubuntu-22.04", + OSType: "linux", + OSArch: "arm64", + Labels: []string{"road-runner", "linux", "arm64", "kubernetes"}, + JitConfigEnabled: true, + }, + expectedProviderInstance: params.ProviderInstance{ + ProviderID: providerID, + Name: instanceName, + OSType: "linux", + OSName: "", + OSVersion: "", + OSArch: "arm64", + Status: "running", + }, + expectedPodInstance: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerID, + Namespace: "runner", + Labels: map[string]string{ + spec.GarmInstanceNameLabel: instanceName, + spec.GarmFlavorLabel: "small", + spec.GarmOSArchLabel: "arm64", + spec.GarmOSTypeLabel: "linux", + spec.GarmPoolIDLabel: "ddce45e7-1bbb-4ecd-92cb-c733372b5cde", + spec.GarmControllerIDLabel: controllerID, + spec.GarmRunnerGroupLabel: "", + }, + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "runner", + VolumeSource: corev1.VolumeSource{ + Ephemeral: &corev1.EphemeralVolumeSource{ + VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + StorageClassName: toPointer("cinder"), + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "runner", + Image: "localhost:5000/runner:ubuntu-22.04", + Env: []corev1.EnvVar{ + { + Name: "RUNNER_ORG", + Value: "testorg", + }, + { + Name: "RUNNER_REPO", + Value: "", + }, + { + Name: "RUNNER_ENTERPRISE", + Value: "", + }, + { + Name: "RUNNER_GROUP", + Value: "", + }, + { + Name: "RUNNER_NAME", + Value: instanceName, + }, + { + Name: "RUNNER_LABELS", + Value: "road-runner,linux,arm64,kubernetes", + }, + { + Name: "RUNNER_NO_DEFAULT_LABELS", + Value: "true", + }, + { + Name: "DISABLE_RUNNER_UPDATE", + Value: "true", + }, + { + Name: "RUNNER_WORKDIR", + Value: "/runner/_work/", + }, + { + Name: "GITHUB_URL", + Value: "https://github.com", + }, + { + Name: "RUNNER_EPHEMERAL", + Value: "true", + }, + { + Name: "RUNNER_TOKEN", + Value: "dummy", + }, + { + Name: "METADATA_URL", + Value: "https://metadata.test", + }, + { + Name: "BEARER_TOKEN", + Value: "test-token", + }, + { + Name: "CALLBACK_URL", + Value: "https://callback.test/status", + }, + { + Name: "JIT_CONFIG_ENABLED", + Value: "true", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "runner", + ReadOnly: false, + MountPath: "/runner", + }, + }, + ImagePullPolicy: "Always", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }, + }, + }, + }, + runtimeObjects: []runtime.Object{}, + err: nil, + }, + { + name: "Use new runner volume and corresponding volumemount with custom name", + config: &config.ProviderConfig{ + KubeConfigPath: "", + RunnerNamespace: "runner", + PodTemplate: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "runner-vol", + VolumeSource: corev1.VolumeSource{ + Ephemeral: &corev1.EphemeralVolumeSource{ + VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + StorageClassName: toPointer("cinder"), + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "runner", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "runner-vol", + ReadOnly: false, + MountPath: "/runner", + }, + }, + }, + }, + }, + }, + }, + bootstrapParams: params.BootstrapInstance{ + Name: instanceName, + PoolID: poolID, + Flavor: "small", + RepoURL: "/~https://github.com/testorg", + InstanceToken: "test-token", + MetadataURL: "https://metadata.test", + CallbackURL: "https://callback.test/status", + Image: "localhost:5000/runner:ubuntu-22.04", + OSType: "linux", + OSArch: "arm64", + Labels: []string{"road-runner", "linux", "arm64", "kubernetes"}, + JitConfigEnabled: true, + }, + expectedProviderInstance: params.ProviderInstance{ + ProviderID: providerID, + Name: instanceName, + OSType: "linux", + OSName: "", + OSVersion: "", + OSArch: "arm64", + Status: "running", + }, + expectedPodInstance: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerID, + Namespace: "runner", + Labels: map[string]string{ + spec.GarmInstanceNameLabel: instanceName, + spec.GarmFlavorLabel: "small", + spec.GarmOSArchLabel: "arm64", + spec.GarmOSTypeLabel: "linux", + spec.GarmPoolIDLabel: "ddce45e7-1bbb-4ecd-92cb-c733372b5cde", + spec.GarmControllerIDLabel: controllerID, + spec.GarmRunnerGroupLabel: "", + }, + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "runner-vol", + VolumeSource: corev1.VolumeSource{ + Ephemeral: &corev1.EphemeralVolumeSource{ + VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + StorageClassName: toPointer("cinder"), + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }, + { + Name: "runner", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: "", + SizeLimit: nil, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "runner", + Image: "localhost:5000/runner:ubuntu-22.04", + Env: []corev1.EnvVar{ + { + Name: "RUNNER_ORG", + Value: "testorg", + }, + { + Name: "RUNNER_REPO", + Value: "", + }, + { + Name: "RUNNER_ENTERPRISE", + Value: "", + }, + { + Name: "RUNNER_GROUP", + Value: "", + }, + { + Name: "RUNNER_NAME", + Value: instanceName, + }, + { + Name: "RUNNER_LABELS", + Value: "road-runner,linux,arm64,kubernetes", + }, + { + Name: "RUNNER_NO_DEFAULT_LABELS", + Value: "true", + }, + { + Name: "DISABLE_RUNNER_UPDATE", + Value: "true", + }, + { + Name: "RUNNER_WORKDIR", + Value: "/runner/_work/", + }, + { + Name: "GITHUB_URL", + Value: "https://github.com", + }, + { + Name: "RUNNER_EPHEMERAL", + Value: "true", + }, + { + Name: "RUNNER_TOKEN", + Value: "dummy", + }, + { + Name: "METADATA_URL", + Value: "https://metadata.test", + }, + { + Name: "BEARER_TOKEN", + Value: "test-token", + }, + { + Name: "CALLBACK_URL", + Value: "https://callback.test/status", + }, + { + Name: "JIT_CONFIG_ENABLED", + Value: "true", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "runner-vol", + ReadOnly: false, + MountPath: "/runner", + }, + }, + ImagePullPolicy: "Always", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }, + }, + }, + }, + runtimeObjects: []runtime.Object{}, + err: nil, + }, } for _, tc := range testCases { @@ -1827,3 +2257,7 @@ func TestRemoveAllInstances(t *testing.T) { }) } } + +func toPointer(str string) *string { + return &str +} diff --git a/internal/spec/spec.go b/internal/spec/spec.go index 5b84c6c..9e40893 100644 --- a/internal/spec/spec.go +++ b/internal/spec/spec.go @@ -5,6 +5,7 @@ package spec import ( "fmt" "net/url" + "path/filepath" "strings" "unicode" @@ -15,6 +16,12 @@ import ( "github.com/mercedes-benz/garm-provider-k8s/pkg/config" ) +var ( + runnerVolumeName = "runner" + runnerVolumeMountPath = "/runner" + runnerVolumeEmptyDir = &corev1.EmptyDirVolumeSource{} +) + const ( GarmInstanceNameLabel = "garm/instance-name" GarmControllerIDLabel = "garm/controllerID" @@ -181,15 +188,16 @@ func ToValidLabel(input string) string { } func CreateRunnerVolume(pod *corev1.Pod) error { - runnerVolumeName := "runner" - runnerVolumeMountPath := "/runner" - runnerVolumeEmptyDir := &corev1.EmptyDirVolumeSource{} - if len(pod.Spec.Containers) < 1 { return fmt.Errorf("pod %s has no runner container spec", pod.Name) } - runnerContainer := &pod.Spec.Containers[0] + // Skip volume creation if a volume with the default name already exists in podTemplate + for _, vol := range config.Config.PodTemplate.Spec.Volumes { + if vol.Name == runnerVolumeName { + return nil + } + } volume := corev1.Volume{ Name: runnerVolumeName, @@ -197,14 +205,39 @@ func CreateRunnerVolume(pod *corev1.Pod) error { EmptyDir: runnerVolumeEmptyDir, }, } + pod.Spec.Volumes = append(pod.Spec.Volumes, volume) + return nil +} + +func CreateRunnerVolumeMount(pod *corev1.Pod, runnerContainerName string) error { + if len(pod.Spec.Containers) < 1 { + return fmt.Errorf("pod %s has no runner container spec", pod.Name) + } + + // Skip volumemount creation if a volumemount with the same path already exists + // in podTemplate for the default container + for _, container := range config.Config.PodTemplate.Spec.Containers { + if container.Name == runnerContainerName { + for _, volMounts := range container.VolumeMounts { + // Volumemount paths e.g. /runner and /runner/ are threated equal + // The last one in the pod spec will take precedence, which can lead to unexpected behavior + if volMounts.MountPath == filepath.Clean(runnerVolumeMountPath) { + return nil + } + } + } + } + + runnerContainer := &pod.Spec.Containers[0] + volumeMount := corev1.VolumeMount{ Name: runnerVolumeName, MountPath: runnerVolumeMountPath, } - runnerContainer.VolumeMounts = append(runnerContainer.VolumeMounts, volumeMount) + return nil }