Skip to content

Commit

Permalink
Merge pull request #79983 from pohly/persistent-and-ephemeral-csi-vol…
Browse files Browse the repository at this point in the history 
…umes

persistent and ephemeral csi volumes
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Jul 25, 2019
2 parents f30af9d + 4bc5d06 commit a375050
Show file tree
Hide file tree
Showing 13 changed files with 560 additions and 176 deletions.
19 changes: 12 additions & 7 deletions pkg/volume/csi/csi_mounter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"os"
"path"
"path/filepath"
"strconv"

"k8s.io/klog"

Expand All @@ -44,14 +45,14 @@ var (
driverName,
nodeName,
attachmentID,
driverMode string
csiVolumeMode string
}{
"specVolID",
"volumeHandle",
"driverName",
"nodeName",
"attachmentID",
"driverMode",
"csiVolumeMode",
}
)

Expand All @@ -60,7 +61,7 @@ type csiMountMgr struct {
k8s kubernetes.Interface
plugin *csiPlugin
driverName csiDriverName
driverMode driverMode
csiVolumeMode csiVolumeMode
volumeID string
specVolumeID string
readOnly bool
Expand Down Expand Up @@ -146,8 +147,8 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error
if !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) {
return fmt.Errorf("CSIInlineVolume feature required")
}
if c.driverMode != ephemeralDriverMode {
return fmt.Errorf("unexpected driver mode: %s", c.driverMode)
if c.csiVolumeMode != ephemeralVolumeMode {
return fmt.Errorf("unexpected volume mode: %s", c.csiVolumeMode)
}
if volSrc.FSType != nil {
fsType = *volSrc.FSType
Expand All @@ -161,8 +162,8 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error
secretRef = &api.SecretReference{Name: secretName, Namespace: ns}
}
case pvSrc != nil:
if c.driverMode != persistentDriverMode {
return fmt.Errorf("unexpected driver mode: %s", c.driverMode)
if c.csiVolumeMode != persistentVolumeMode {
return fmt.Errorf("unexpected driver mode: %s", c.csiVolumeMode)
}

fsType = pvSrc.FSType
Expand Down Expand Up @@ -324,6 +325,10 @@ func (c *csiMountMgr) podAttributes() (map[string]string, error) {
"csi.storage.k8s.io/pod.uid": string(c.pod.UID),
"csi.storage.k8s.io/serviceAccount.name": c.pod.Spec.ServiceAccountName,
}
if utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) {
attrs["csi.storage.k8s.io/ephemeral"] = strconv.FormatBool(c.csiVolumeMode == ephemeralVolumeMode)
}

klog.V(4).Infof(log("CSIDriver %q requires pod information", c.driverName))
return attrs, nil
}
Expand Down
43 changes: 27 additions & 16 deletions pkg/volume/csi/csi_mounter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) {
driver string
volumeContext map[string]string
expectedVolumeContext map[string]string
csiInlineVolume bool
}{
{
name: "no pod info",
Expand Down Expand Up @@ -136,13 +137,23 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) {
volumeContext: map[string]string{"foo": "bar"},
expectedVolumeContext: map[string]string{"foo": "bar", "csi.storage.k8s.io/pod.uid": "test-pod", "csi.storage.k8s.io/serviceAccount.name": "test-service-account", "csi.storage.k8s.io/pod.name": "test-pod", "csi.storage.k8s.io/pod.namespace": "test-ns"},
},
{
name: "CSIInlineVolume pod info",
driver: "info",
volumeContext: nil,
expectedVolumeContext: map[string]string{"csi.storage.k8s.io/pod.uid": "test-pod", "csi.storage.k8s.io/serviceAccount.name": "test-service-account", "csi.storage.k8s.io/pod.name": "test-pod", "csi.storage.k8s.io/pod.namespace": "test-ns", "csi.storage.k8s.io/ephemeral": "false"},
csiInlineVolume: true,
},
}

noPodMountInfo := false
currentPodInfoMount := true
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
klog.Infof("Starting test %s", test.name)
if test.csiInlineVolume {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)()
}
fakeClient := fakeclient.NewSimpleClientset(
getTestCSIDriver("no-info", &noPodMountInfo, nil),
getTestCSIDriver("info", &currentPodInfoMount, nil),
Expand Down Expand Up @@ -267,7 +278,7 @@ func TestMounterSetUpSimple(t *testing.T) {
testCases := []struct {
name string
podUID types.UID
mode driverMode
mode csiVolumeMode
fsType string
options []string
spec func(string, []string) *volume.Spec
Expand All @@ -276,7 +287,7 @@ func TestMounterSetUpSimple(t *testing.T) {
{
name: "setup with vol source",
podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
mode: ephemeralDriverMode,
mode: ephemeralVolumeMode,
fsType: "ext4",
shouldFail: true,
spec: func(fsType string, options []string) *volume.Spec {
Expand All @@ -288,7 +299,7 @@ func TestMounterSetUpSimple(t *testing.T) {
{
name: "setup with persistent source",
podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
mode: persistentDriverMode,
mode: persistentVolumeMode,
fsType: "zfs",
spec: func(fsType string, options []string) *volume.Spec {
pvSrc := makeTestPV("pv1", 20, testDriver, "vol1")
Expand All @@ -300,7 +311,7 @@ func TestMounterSetUpSimple(t *testing.T) {
{
name: "setup with persistent source without unspecified fstype and options",
podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
mode: persistentDriverMode,
mode: persistentVolumeMode,
spec: func(fsType string, options []string) *volume.Spec {
return volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol2"), false)
},
Expand Down Expand Up @@ -334,8 +345,8 @@ func TestMounterSetUpSimple(t *testing.T) {
csiMounter := mounter.(*csiMountMgr)
csiMounter.csiClient = setupClient(t, true)

if csiMounter.driverMode != persistentDriverMode {
t.Fatal("unexpected driver mode: ", csiMounter.driverMode)
if csiMounter.csiVolumeMode != persistentVolumeMode {
t.Fatal("unexpected volume mode: ", csiMounter.csiVolumeMode)
}

attachID := getAttachmentName(csiMounter.volumeID, string(csiMounter.driverName), string(plug.host.GetNodeName()))
Expand Down Expand Up @@ -393,7 +404,7 @@ func TestMounterSetUpWithInline(t *testing.T) {
testCases := []struct {
name string
podUID types.UID
mode driverMode
mode csiVolumeMode
fsType string
options []string
spec func(string, []string) *volume.Spec
Expand All @@ -402,7 +413,7 @@ func TestMounterSetUpWithInline(t *testing.T) {
{
name: "setup with vol source",
podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
mode: ephemeralDriverMode,
mode: ephemeralVolumeMode,
fsType: "ext4",
spec: func(fsType string, options []string) *volume.Spec {
volSrc := makeTestVol("pv1", testDriver)
Expand All @@ -413,7 +424,7 @@ func TestMounterSetUpWithInline(t *testing.T) {
{
name: "setup with persistent source",
podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
mode: persistentDriverMode,
mode: persistentVolumeMode,
fsType: "zfs",
spec: func(fsType string, options []string) *volume.Spec {
pvSrc := makeTestPV("pv1", 20, testDriver, "vol1")
Expand All @@ -425,7 +436,7 @@ func TestMounterSetUpWithInline(t *testing.T) {
{
name: "setup with persistent source without unspecified fstype and options",
podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
mode: persistentDriverMode,
mode: persistentVolumeMode,
spec: func(fsType string, options []string) *volume.Spec {
return volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol2"), false)
},
Expand Down Expand Up @@ -459,15 +470,15 @@ func TestMounterSetUpWithInline(t *testing.T) {
csiMounter := mounter.(*csiMountMgr)
csiMounter.csiClient = setupClient(t, true)

if csiMounter.driverMode != tc.mode {
t.Fatal("unexpected driver mode: ", csiMounter.driverMode)
if csiMounter.csiVolumeMode != tc.mode {
t.Fatal("unexpected volume mode: ", csiMounter.csiVolumeMode)
}

if csiMounter.driverMode == ephemeralDriverMode && csiMounter.volumeID != makeVolumeHandle(string(tc.podUID), csiMounter.specVolumeID) {
if csiMounter.csiVolumeMode == ephemeralVolumeMode && csiMounter.volumeID != makeVolumeHandle(string(tc.podUID), csiMounter.specVolumeID) {
t.Fatal("unexpected generated volumeHandle:", csiMounter.volumeID)
}

if csiMounter.driverMode == persistentDriverMode {
if csiMounter.csiVolumeMode == persistentVolumeMode {
attachID := getAttachmentName(csiMounter.volumeID, string(csiMounter.driverName), string(plug.host.GetNodeName()))
attachment := makeTestAttachment(attachID, "test-node", csiMounter.spec.Name())
_, err = csiMounter.k8s.StorageV1().VolumeAttachments().Create(attachment)
Expand All @@ -492,10 +503,10 @@ func TestMounterSetUpWithInline(t *testing.T) {
}

// validate stagingTargetPath
if tc.mode == ephemeralDriverMode && vol.DeviceMountPath != "" {
if tc.mode == ephemeralVolumeMode && vol.DeviceMountPath != "" {
t.Errorf("unexpected devicePathTarget sent to driver: %s", vol.DeviceMountPath)
}
if tc.mode == persistentDriverMode {
if tc.mode == persistentVolumeMode {
devicePath, err := makeDeviceMountPath(plug, csiMounter.spec)
if err != nil {
t.Fatal(err)
Expand Down
96 changes: 56 additions & 40 deletions pkg/volume/csi/csi_plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ type driverMode string

const persistentDriverMode driverMode = "persistent"
const ephemeralDriverMode driverMode = "ephemeral"
const combinedDriverMode driverMode = "persistent+ephemeral"

type csiVolumeMode string

const persistentVolumeMode csiVolumeMode = "persistent"
const ephemeralVolumeMode csiVolumeMode = "ephemeral"

// ProbeVolumePlugins returns implemented plugins
func ProbeVolumePlugins() []volume.VolumePlugin {
Expand Down Expand Up @@ -381,11 +387,16 @@ func (p *csiPlugin) NewMounter(
return nil, fmt.Errorf("volume source not found in volume.Spec")
}

driverMode, err := p.getDriverMode(spec)
csiVolumeMode, err := p.getCSIVolumeMode(spec)
if err != nil {
return nil, err
}

// TODO(pohly): check CSIDriver.Spec.Mode to ensure that the CSI driver
// supports the current csiVolumeMode.
// In alpha it is assumed that drivers are used correctly without
// the additional sanity check.

k8s := p.host.GetKubeClient()
if k8s == nil {
klog.Error(log("failed to get a kubernetes client"))
Expand All @@ -399,17 +410,17 @@ func (p *csiPlugin) NewMounter(
}

mounter := &csiMountMgr{
plugin: p,
k8s: k8s,
spec: spec,
pod: pod,
podUID: pod.UID,
driverName: csiDriverName(driverName),
driverMode: driverMode,
volumeID: volumeHandle,
specVolumeID: spec.Name(),
readOnly: readOnly,
kubeVolHost: kvh,
plugin: p,
k8s: k8s,
spec: spec,
pod: pod,
podUID: pod.UID,
driverName: csiDriverName(driverName),
csiVolumeMode: csiVolumeMode,
volumeID: volumeHandle,
specVolumeID: spec.Name(),
readOnly: readOnly,
kubeVolHost: kvh,
}
mounter.csiClientGetter.driverName = csiDriverName(driverName)

Expand All @@ -428,11 +439,11 @@ func (p *csiPlugin) NewMounter(
// persist volume info data for teardown
node := string(p.host.GetNodeName())
volData := map[string]string{
volDataKey.specVolID: spec.Name(),
volDataKey.volHandle: volumeHandle,
volDataKey.driverName: driverName,
volDataKey.nodeName: node,
volDataKey.driverMode: string(driverMode),
volDataKey.specVolID: spec.Name(),
volDataKey.volHandle: volumeHandle,
volDataKey.driverName: driverName,
volDataKey.nodeName: node,
volDataKey.csiVolumeMode: string(csiVolumeMode),
}

attachID := getAttachmentName(volumeHandle, driverName, node)
Expand Down Expand Up @@ -496,16 +507,13 @@ func (p *csiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.S
var spec *volume.Spec
inlineEnabled := utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume)

// If inlineEnabled is true and mode is ephemeralDriverMode,
// If inlineEnabled is true and mode is ephemeralVolumeMode,
// use constructVolSourceSpec to construct volume source spec.
// If inlineEnabled is false or mode is persistentDriverMode,
// If inlineEnabled is false or mode is persistentVolumeMode,
// use constructPVSourceSpec to construct volume construct pv source spec.
if inlineEnabled {
if driverMode(volData[volDataKey.driverMode]) == ephemeralDriverMode {
spec = p.constructVolSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName])
return spec, nil

}
if inlineEnabled && csiVolumeMode(volData[volDataKey.csiVolumeMode]) == ephemeralVolumeMode {
spec = p.constructVolSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName])
return spec, nil
}
spec = p.constructPVSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName], volData[volDataKey.volHandle])

Expand Down Expand Up @@ -576,14 +584,17 @@ func (p *csiPlugin) NewDetacher() (volume.Detacher, error) {
}

func (p *csiPlugin) CanAttach(spec *volume.Spec) (bool, error) {
driverMode, err := p.getDriverMode(spec)
if err != nil {
return false, err
}
inlineEnabled := utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume)
if inlineEnabled {
csiVolumeMode, err := p.getCSIVolumeMode(spec)
if err != nil {
return false, err
}

if driverMode == ephemeralDriverMode {
klog.V(5).Info(log("plugin.CanAttach = false, ephemeral mode detected for spec %v", spec.Name()))
return false, nil
if csiVolumeMode == ephemeralVolumeMode {
klog.V(5).Info(log("plugin.CanAttach = false, ephemeral mode detected for spec %v", spec.Name()))
return false, nil
}
}

pvSrc, err := getCSISourceFromSpec(spec)
Expand All @@ -603,16 +614,23 @@ func (p *csiPlugin) CanAttach(spec *volume.Spec) (bool, error) {

// CanDeviceMount returns true if the spec supports device mount
func (p *csiPlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
driverMode, err := p.getDriverMode(spec)
inlineEnabled := utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume)
if !inlineEnabled {
// No need to check anything, we assume it is a persistent volume.
return true, nil
}

csiVolumeMode, err := p.getCSIVolumeMode(spec)
if err != nil {
return false, err
}

if driverMode == ephemeralDriverMode {
if csiVolumeMode == ephemeralVolumeMode {
klog.V(5).Info(log("plugin.CanDeviceMount skipped ephemeral mode detected for spec %v", spec.Name()))
return false, nil
}

// Persistent volumes support device mount.
return true, nil
}

Expand Down Expand Up @@ -783,13 +801,11 @@ func (p *csiPlugin) skipAttach(driver string) (bool, error) {
return false, nil
}

// getDriverMode returns the driver mode for the specified spec: {persistent|ephemeral}.
// getCSIVolumeMode returns the mode for the specified spec: {persistent|ephemeral}.
// 1) If mode cannot be determined, it will default to "persistent".
// 2) If Mode cannot be resolved to either {persistent | ephemeral}, an error is returned
// See /~https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md
func (p *csiPlugin) getDriverMode(spec *volume.Spec) (driverMode, error) {
// TODO (vladimirvivien) ultimately, mode will be retrieved from CSIDriver.Spec.Mode.
// However, in alpha version, mode is determined by the volume source:
func (p *csiPlugin) getCSIVolumeMode(spec *volume.Spec) (csiVolumeMode, error) {
// 1) if volume.Spec.Volume.CSI != nil -> mode is ephemeral
// 2) if volume.Spec.PersistentVolume.Spec.CSI != nil -> persistent
volSrc, _, err := getSourceFromSpec(spec)
Expand All @@ -798,9 +814,9 @@ func (p *csiPlugin) getDriverMode(spec *volume.Spec) (driverMode, error) {
}

if volSrc != nil && utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) {
return ephemeralDriverMode, nil
return ephemeralVolumeMode, nil
}
return persistentDriverMode, nil
return persistentVolumeMode, nil
}

func (p *csiPlugin) getPublishContext(client clientset.Interface, handle, driver, nodeName string) (map[string]string, error) {
Expand Down
Loading

0 comments on commit a375050

Please sign in to comment.