test: Catch vfkit process exiting unexpectedly

This commit an a go routine which will `Wait()` for the vfkit process.
When it exits, it will push the corresponding error (or nil) to a
channel.
The code waiting for the VM to startup can then check for errors in
their select{} retry loop, and fail acccordingly.

This will make debugging test failures easier, as currently if the vfkit
process dies early, the test appears to be taking a very long time
without any indication as to what's going on.

Signed-off-by: Christophe Fergeau <cfergeau@redhat.com>

test/vm_helpers.go

@@ -17,14 +17,16 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-func retryIPFromMAC(macAddress string) (string, error) {
+func retryIPFromMAC(errCh chan error, macAddress string) (string, error) {
 	var (
 		err error
 		ip  string
 	)
 
 	for {
 		select {
+		case err := <-errCh:
+			return "", err
 		case <-time.After(1 * time.Second):
 			ip, err = vfkithelpers.GetIPAddressByMACAddress(macAddress)
 			if err == nil {
@@ -37,13 +39,15 @@ func retryIPFromMAC(macAddress string) (string, error) {
 	}
 }
 
-func retrySSHDial(scheme string, address string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
+func retrySSHDial(errCh chan error, scheme string, address string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
 	var (
 		sshClient *ssh.Client
 		err       error
 	)
 	for {
 		select {
+		case err := <-errCh:
+			return nil, err
 		case <-time.After(1 * time.Second):
 			log.Debugf("trying ssh dial")
 			sshClient, err = ssh.Dial(scheme, address, sshConfig)
@@ -60,6 +64,7 @@ func retrySSHDial(scheme string, address string, sshConfig *ssh.ClientConfig) (*
 
 type vfkitRunner struct {
 	*exec.Cmd
+	errCh              chan error
 	gracefullyShutdown bool
 }
 
@@ -75,15 +80,25 @@ func startVfkit(t *testing.T, vm *config.VirtualMachine) *vfkitRunner {
 
 	err = vfkitCmd.Start()
 	require.NoError(t, err)
+	errCh := make(chan error)
+	go func() {
+		err := vfkitCmd.Wait()
+		if err != nil {
+			log.Infof("vfkitCmd.Wait() returned %v", err)
+		}
+		errCh <- err
+		close(errCh)
+	}()
 
 	return &vfkitRunner{
 		vfkitCmd,
+		errCh,
 		false,
 	}
 }
 
 func (cmd *vfkitRunner) Wait(t *testing.T) {
-	err := cmd.Cmd.Wait()
+	err := <-cmd.errCh
 	require.NoError(t, err)
 	cmd.gracefullyShutdown = true
 }
@@ -196,12 +211,12 @@ func (vm *testVM) WaitForSSH(t *testing.T) {
 	)
 	switch vm.sshNetwork {
 	case "tcp":
-		ip, err := retryIPFromMAC(vm.macAddress)
+		ip, err := retryIPFromMAC(vm.vfkitCmd.errCh, vm.macAddress)
 		require.NoError(t, err)
-		sshClient, err = retrySSHDial("tcp", net.JoinHostPort(ip, strconv.Itoa(vm.port)), vm.provider.SSHConfig())
+		sshClient, err = retrySSHDial(vm.vfkitCmd.errCh, "tcp", net.JoinHostPort(ip, strconv.Itoa(vm.port)), vm.provider.SSHConfig())
 		require.NoError(t, err)
 	case "vsock":
-		sshClient, err = retrySSHDial("unix", vm.vsockPath, vm.provider.SSHConfig())
+		sshClient, err = retrySSHDial(vm.vfkitCmd.errCh, "unix", vm.vsockPath, vm.provider.SSHConfig())
 		require.NoError(t, err)
 	default:
 		t.FailNow()

test/vm_test.go

@@ -16,6 +16,29 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestFailedVfkitStart(t *testing.T) {
+	puipuiProvider := NewPuipuiProvider()
+	log.Info("fetching os image")
+	err := puipuiProvider.Fetch(t.TempDir())
+	require.NoError(t, err)
+
+	vm := NewTestVM(t, puipuiProvider)
+	defer vm.Close(t)
+	require.NotNil(t, vm)
+
+	vm.AddSSH(t, "vsock")
+
+	dev, err := config.NVMExpressControllerNew("/a/b")
+	require.NoError(t, err)
+	vm.AddDevice(t, dev)
+
+	vm.Start(t)
+
+	log.Infof("waiting for SSH")
+	_, err = retrySSHDial(vm.vfkitCmd.errCh, "unix", vm.vsockPath, vm.provider.SSHConfig())
+	require.Error(t, err)
+}
+
 func testSSHAccess(t *testing.T, vm *testVM, network string) {
 	log.Infof("testing SSH access over %s", network)
 	vm.AddSSH(t, network)