Merge pull request kubernetes#36398 from saad-ali/automated-cherry-pi…

…ck-of-#33976-kubernetes#35560-upstream-release-1.4

Automatic merge from submit-queue

Automated cherry pick of kubernetes#33976 kubernetes#35560 upstream release 1.4

Cherry pick PRs kubernetes#33976 and kubernetes#35560 on behalf of @jingxu97 to 1.4 branch to fix `Pod disks...` flakes.

test/e2e/framework/framework.go

@@ -570,6 +570,16 @@ func (f *Framework) ReadFileViaContainer(podName, containerName string, path str
 	return string(stdout), err
 }
 
+func (f *Framework) CheckFileSizeViaContainer(podName, containerName, path string) (string, error) {
+	By("checking a file size in the container")
+
+	stdout, stderr, err := kubectlExecWithRetry(f.Namespace.Name, podName, containerName, "--", "ls", "-l", path)
+	if err != nil {
+		Logf("error running kubectl exec to read file: %v\nstdout=%v\nstderr=%v)", err, string(stdout), string(stderr))
+	}
+	return string(stdout), err
+}
+
 // CreateServiceForSimpleAppWithPods is a convenience wrapper to create a service and its matching pods all at once.
 func (f *Framework) CreateServiceForSimpleAppWithPods(contPort int, svcPort int, appName string, podSpec func(n api.Node) api.PodSpec, count int, block bool) (error, *api.Service) {
 	var err error = nil

test/e2e/pd.go

@@ -48,6 +48,7 @@ const (
 	nodeStatusPollTime  = 1 * time.Second
 	gcePDRetryTimeout   = 5 * time.Minute
 	gcePDRetryPollTime  = 5 * time.Second
+	maxReadRetry        = 3
 )
 
 var _ = framework.KubeDescribe("Pod Disks", func() {
@@ -449,13 +450,28 @@ func deletePDWithRetry(diskName string) {
 
 func verifyPDContentsViaContainer(f *framework.Framework, podName, containerName string, fileAndContentToVerify map[string]string) {
 	for filePath, expectedContents := range fileAndContentToVerify {
-		v, err := f.ReadFileViaContainer(podName, containerName, filePath)
-		if err != nil {
-			framework.Logf("Error reading file: %v", err)
+		var value string
+		// Add a retry to avoid temporal failure in reading the content
+		for i := 0; i < maxReadRetry; i++ {
+			v, err := f.ReadFileViaContainer(podName, containerName, filePath)
+			value = v
+			if err != nil {
+				framework.Logf("Error reading file: %v", err)
+			}
+			framework.ExpectNoError(err)
+			framework.Logf("Read file %q with content: %v (iteration %d)", filePath, v, i)
+			if strings.TrimSpace(v) != strings.TrimSpace(expectedContents) {
+				framework.Logf("Warning: read content <%q> does not match execpted content <%q>.", v, expectedContents)
+				size, err := f.CheckFileSizeViaContainer(podName, containerName, filePath)
+				if err != nil {
+					framework.Logf("Error checking file size: %v", err)
+				}
+				framework.Logf("Check file %q size: %q", filePath, size)
+			} else {
+				break
+			}
 		}
-		framework.ExpectNoError(err)
-		framework.Logf("Read file %q with content: %v", filePath, v)
-		Expect(strings.TrimSpace(v)).To(Equal(strings.TrimSpace(expectedContents)))
+		Expect(strings.TrimSpace(value)).To(Equal(strings.TrimSpace(expectedContents)))
 	}
 }