From 17b02611e438e0e5e43c1127337d26f0fd4a7bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 11 Aug 2024 20:11:50 +0200 Subject: [PATCH] Add new macOS vfkit driver, like hyperkit and qemu It uses the new Virtualization.framework from macOS 11, instead of the older Hypervisor.framework (hvf) in QEMU. --- pkg/drivers/vfkit/iso.go | 65 +++ pkg/drivers/vfkit/iso_test.go | 85 ++++ pkg/drivers/vfkit/iso_test.iso | Bin 0 -> 362496 bytes pkg/drivers/vfkit/vfkit.go | 508 ++++++++++++++++++++++ pkg/minikube/driver/driver.go | 2 + pkg/minikube/driver/driver_darwin.go | 2 + pkg/minikube/registry/drvs/init.go | 1 + pkg/minikube/registry/drvs/vfkit/doc.go | 17 + pkg/minikube/registry/drvs/vfkit/vfkit.go | 92 ++++ 9 files changed, 772 insertions(+) create mode 100644 pkg/drivers/vfkit/iso.go create mode 100644 pkg/drivers/vfkit/iso_test.go create mode 100644 pkg/drivers/vfkit/iso_test.iso create mode 100644 pkg/drivers/vfkit/vfkit.go create mode 100644 pkg/minikube/registry/drvs/vfkit/doc.go create mode 100644 pkg/minikube/registry/drvs/vfkit/vfkit.go diff --git a/pkg/drivers/vfkit/iso.go b/pkg/drivers/vfkit/iso.go new file mode 100644 index 000000000000..a92af67f884c --- /dev/null +++ b/pkg/drivers/vfkit/iso.go @@ -0,0 +1,65 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vfkit + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/hooklift/iso9660" +) + +// ExtractFile extracts a file from an ISO +func ExtractFile(isoPath, srcPath, destPath string) error { + iso, err := os.Open(isoPath) + if err != nil { + return err + } + defer iso.Close() + + r, err := iso9660.NewReader(iso) + if err != nil { + return err + } + + f, err := findFile(r, srcPath) + if err != nil { + return err + } + + dst, err := os.Create(destPath) + if err != nil { + return err + } + defer dst.Close() + + _, err = io.Copy(dst, f.Sys().(io.Reader)) + return err +} + +func findFile(r *iso9660.Reader, path string) (os.FileInfo, error) { + // Look through the ISO for a file with a matching path. + for f, err := r.Next(); err != io.EOF; f, err = r.Next() { + // For some reason file paths in the ISO sometimes contain a '.' character at the end, so strip that off. + if strings.TrimSuffix(f.Name(), ".") == path { + return f, nil + } + } + return nil, fmt.Errorf("unable to find file %s", path) +} diff --git a/pkg/drivers/vfkit/iso_test.go b/pkg/drivers/vfkit/iso_test.go new file mode 100644 index 000000000000..cc6884c22c29 --- /dev/null +++ b/pkg/drivers/vfkit/iso_test.go @@ -0,0 +1,85 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vfkit + +import ( + "testing" +) + +func TestExtractFile(t *testing.T) { + testDir := t.TempDir() + + tests := []struct { + name string + isoPath string + srcPath string + destPath string + expectedError bool + }{ + { + name: "all is right", + isoPath: "iso_test.iso", + srcPath: "/test1.txt", + destPath: testDir + "/test1.txt", + expectedError: false, + }, + { + name: "isoPath is error", + isoPath: "tests.iso", + srcPath: "/test1.txt", + destPath: testDir + "/test1.txt", + expectedError: true, + }, + { + name: "srcPath is empty", + isoPath: "iso_tests.iso", + srcPath: "", + destPath: testDir + "/test1.txt", + expectedError: true, + }, + { + name: "srcPath is error", + isoPath: "iso_tests.iso", + srcPath: "/t1.txt", + destPath: testDir + "/test1.txt", + expectedError: true, + }, + { + name: "destPath is empty", + isoPath: "iso_test.iso", + srcPath: "/test1.txt", + destPath: "", + expectedError: true, + }, + { + name: "find files in a folder", + isoPath: "./iso_test.iso", + srcPath: "/test2/test2.txt", + destPath: testDir + "/test2.txt", + expectedError: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ExtractFile(tt.isoPath, tt.srcPath, tt.destPath) + if (nil != err) != tt.expectedError { + t.Errorf("expectedError = %v, get = %v", tt.expectedError, err) + return + } + }) + } +} diff --git a/pkg/drivers/vfkit/iso_test.iso b/pkg/drivers/vfkit/iso_test.iso new file mode 100644 index 0000000000000000000000000000000000000000..dbed69dd413fd8b02f0f019e33ac700e6f75d699 GIT binary patch literal 362496 zcmeI(+fEZf7y#f|4Wu5pQp4fGOt>(bVA-|=(sot@pze!J2-Y%)CV4T@MbVt02p-#$E;#e?Cf zn4Qha;yCuFMX#JpEwAHZ*)|h*GjlCA`A1G zo3s8e0`-5IErCZM@arP}KZ2>40s#U92oNAZfB*pk1PBlyFeicRe2u+YO%F@w>+o49 zi&@#8)2}%#0RjXF5FkK+009C72oNCf5CZAVzghyQs-@Ig3z_%-1fdWhK!5-N0t5&U zAV7csfkhIyJIC&p5pS=u@UpG2UX!dZ*Y$GB(|frbt8Xb!??qZtXf?~z@>A>GRkq*X zbNZGnKh%b{1PBlyKwypnn|H+H?Cvc5D*kWOGu*fq|6j+d_bUFk>-ay%OL1xf1PBly zK;WJN-@ktQ-g 0 { + log.Info("Creating extra disk images...") + for i := 0; i < d.ExtraDisks; i++ { + path := pkgdrivers.ExtraDiskPath(d.BaseDriver, i) + if err := pkgdrivers.CreateRawDisk(path, d.DiskSize); err != nil { + return err + } + } + } + + log.Info("Starting vfkit VM...") + return d.Start() +} + +func (d *Driver) Start() error { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + + var startCmd []string + + startCmd = append(startCmd, + "--memory", fmt.Sprintf("%d", d.Memory), + "--cpus", fmt.Sprintf("%d", d.CPU), + "--restful-uri", fmt.Sprintf("unix://%s", d.sockfilePath())) + var isoPath = filepath.Join(machineDir, isoFilename) + startCmd = append(startCmd, + "--device", fmt.Sprintf("virtio-blk,path=%s", isoPath)) + + var mac = d.MACAddress + startCmd = append(startCmd, + "--device", fmt.Sprintf("virtio-net,nat,mac=%s", mac)) + + startCmd = append(startCmd, + "--device", "virtio-rng") + + startCmd = append(startCmd, + "--kernel", d.ResolveStorePath("bzimage")) + startCmd = append(startCmd, + "--kernel-cmdline", d.Cmdline) + startCmd = append(startCmd, + "--initrd", d.ResolveStorePath("initrd")) + + for i := 0; i < d.ExtraDisks; i++ { + startCmd = append(startCmd, + "--device", fmt.Sprintf("virtio-blk,path=%s", pkgdrivers.ExtraDiskPath(d.BaseDriver, i))) + } + + startCmd = append(startCmd, + "--device", fmt.Sprintf("virtio-blk,path=%s", d.diskPath())) + + log.Debugf("executing: vfkit %s", strings.Join(startCmd, " ")) + os.Remove(d.sockfilePath()) + cmd := exec.Command("vfkit", startCmd...) + if err := cmd.Start(); err != nil { + return err + } + pid := cmd.Process.Pid + if err := os.WriteFile(d.pidfilePath(), []byte(fmt.Sprintf("%v", pid)), 0600); err != nil { + return err + } + + var err error + getIP := func() error { + mac := pkgdrivers.TrimMacAddress(d.MACAddress) + d.IPAddress, err = pkgdrivers.GetIPAddressByMACAddress(mac) + if err != nil { + return errors.Wrap(err, "failed to get IP address") + } + d.SSHPort = 22 + return nil + } + // Implement a retry loop because IP address isn't added to dhcp leases file immediately + for i := 0; i < 60; i++ { + log.Debugf("Attempt %d", i) + err = getIP() + if err == nil { + break + } + time.Sleep(2 * time.Second) + } + + if err != nil { + return fmt.Errorf("IP address never found in dhcp leases file %v", err) + } + if err == nil { + log.Debugf("IP: %s", d.IPAddress) + } + + log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress) + + return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second) +} + +func (d *Driver) Stop() error { + if err := d.SetVFKitState("HardStop"); err != nil { + return err + } + return nil +} + +func (d *Driver) Remove() error { + s, err := d.GetState() + if err != nil { + return errors.Wrap(err, "get state") + } + if s == state.Running { + if err := d.Kill(); err != nil { + return errors.Wrap(err, "kill") + } + } + if s != state.Stopped { + if err := d.SetVFKitState("Stop"); err != nil { + return errors.Wrap(err, "quit") + } + } + return nil +} + +func (d *Driver) Restart() error { + s, err := d.GetState() + if err != nil { + return err + } + + if s == state.Running { + if err := d.Stop(); err != nil { + return err + } + } + return d.Start() +} + +func (d *Driver) extractKernel(isoPath string) error { + for _, f := range []struct { + pathInIso string + destPath string + }{ + {"/boot/bzimage", "bzimage"}, + {"/boot/initrd", "initrd"}, + } { + fullDestPath := d.ResolveStorePath(f.destPath) + if err := ExtractFile(isoPath, f.pathInIso, fullDestPath); err != nil { + return err + } + } + return nil +} + +func (d *Driver) Kill() error { + if err := d.SetVFKitState("HardStop"); err != nil { + return err + } + return nil +} + +func (d *Driver) StartDocker() error { + return fmt.Errorf("hosts without a driver cannot start docker") +} + +func (d *Driver) StopDocker() error { + return fmt.Errorf("hosts without a driver cannot stop docker") +} + +func (d *Driver) GetDockerConfigDir() string { + return "" +} + +func (d *Driver) Upgrade() error { + return fmt.Errorf("hosts without a driver cannot be upgraded") +} + +func (d *Driver) sshKeyPath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "id_rsa") +} + +func (d *Driver) publicSSHKeyPath() string { + return d.sshKeyPath() + ".pub" +} + +func (d *Driver) diskPath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "disk.img") +} + +func (d *Driver) monitorPath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "monitor") +} + +func (d *Driver) pidfilePath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, pidFileName) +} + +func (d *Driver) sockfilePath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, sockFilename) +} + +// Make a boot2docker VM disk image. +func (d *Driver) generateDiskImage(size int) error { + log.Debugf("Creating %d MB hard disk image...", size) + + magicString := "boot2docker, please format-me" + + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + // magicString first so the automount script knows to format the disk + file := &tar.Header{Name: magicString, Size: int64(len(magicString))} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write([]byte(magicString)); err != nil { + return err + } + // .ssh/key.pub => authorized_keys + file = &tar.Header{Name: ".ssh", Typeflag: tar.TypeDir, Mode: 0700} + if err := tw.WriteHeader(file); err != nil { + return err + } + pubKey, err := os.ReadFile(d.publicSSHKeyPath()) + if err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write(pubKey); err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys2", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write(pubKey); err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + rawFile := d.diskPath() + if err := os.WriteFile(rawFile, buf.Bytes(), 0644); err != nil { + return nil + } + if err := os.Truncate(rawFile, int64(size)*int64(1024*1024)); err != nil { + return nil + } + log.Debugf("DONE writing to %s and %s", rawFile, d.diskPath()) + return nil +} + +func httpUnixClient(path string) http.Client { + return http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", path) + }, + }, + } +} + +type VMState struct { + State string `json:"state"` +} + +func (d *Driver) GetVFKitState() (string, error) { + httpc := httpUnixClient(d.sockfilePath()) + var vmstate VMState + response, err := httpc.Get("http://_/vm/state") + if err != nil { + return "", err + } + defer response.Body.Close() + err = json.NewDecoder(response.Body).Decode(&vmstate) + if err != nil { + return "", err + } + log.Debugf("get state: %+v", vmstate) + return vmstate.State, nil +} + +func (d *Driver) SetVFKitState(state string) error { + httpc := httpUnixClient(d.sockfilePath()) + var vmstate VMState + vmstate.State = state + data, err := json.Marshal(&vmstate) + if err != nil { + return err + } + _, err = httpc.Post("http://_/vm/state", "application/json", bytes.NewReader(data)) + if err != nil { + return err + } + log.Debugf("set state: %+v", vmstate) + return nil +} + +func WaitForTCPWithDelay(addr string, duration time.Duration) error { + for { + conn, err := net.Dial("tcp", addr) + if err != nil { + continue + } + defer conn.Close() + if _, err := conn.Read(make([]byte, 1)); err != nil && err != io.EOF { + time.Sleep(duration) + continue + } + break + } + return nil +} diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index a24c72aeed02..7f721e5c955f 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -61,6 +61,8 @@ const ( HyperV = "hyperv" // Parallels driver Parallels = "parallels" + // VFKit driver + VFKit = "vfkit" // AliasKVM is driver name alias for kvm2 AliasKVM = "kvm" diff --git a/pkg/minikube/driver/driver_darwin.go b/pkg/minikube/driver/driver_darwin.go index cc2270797039..6bf5f20d7c5b 100644 --- a/pkg/minikube/driver/driver_darwin.go +++ b/pkg/minikube/driver/driver_darwin.go @@ -28,6 +28,7 @@ var supportedDrivers = func() []string { // on darwin/arm64 only docker and ssh are supported yet return []string{ QEMU2, + VFKit, Parallels, Docker, Podman, @@ -51,6 +52,7 @@ var supportedDrivers = func() []string { HyperKit, VMware, QEMU2, + VFKit, Docker, Podman, SSH, diff --git a/pkg/minikube/registry/drvs/init.go b/pkg/minikube/registry/drvs/init.go index 63a7f291214f..3f19b06e27a5 100644 --- a/pkg/minikube/registry/drvs/init.go +++ b/pkg/minikube/registry/drvs/init.go @@ -19,6 +19,7 @@ package drvs import ( // Register all of the drvs we know of _ "k8s.io/minikube/pkg/minikube/registry/drvs/docker" + _ "k8s.io/minikube/pkg/minikube/registry/drvs/vfkit" _ "k8s.io/minikube/pkg/minikube/registry/drvs/hyperkit" _ "k8s.io/minikube/pkg/minikube/registry/drvs/hyperv" _ "k8s.io/minikube/pkg/minikube/registry/drvs/kvm2" diff --git a/pkg/minikube/registry/drvs/vfkit/doc.go b/pkg/minikube/registry/drvs/vfkit/doc.go new file mode 100644 index 000000000000..26f1ffd65af5 --- /dev/null +++ b/pkg/minikube/registry/drvs/vfkit/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vfkit diff --git a/pkg/minikube/registry/drvs/vfkit/vfkit.go b/pkg/minikube/registry/drvs/vfkit/vfkit.go new file mode 100644 index 000000000000..1f2bb8c094dc --- /dev/null +++ b/pkg/minikube/registry/drvs/vfkit/vfkit.go @@ -0,0 +1,92 @@ +//go:build darwin + +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vfkit + +import ( + "crypto/rand" + "fmt" + "os/exec" + + "github.com/docker/machine/libmachine/drivers" + + "k8s.io/minikube/pkg/drivers/vfkit" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/registry" +) + +const ( + docURL = "https://minikube.sigs.k8s.io/docs/reference/drivers/vfkit/" +) + +func init() { + if err := registry.Register(registry.DriverDef{ + Name: driver.VFKit, + Init: func() drivers.Driver { return vfkit.NewDriver("", "") }, + Config: configure, + Status: status, + Default: true, + Priority: registry.Experimental, + }); err != nil { + panic(fmt.Sprintf("register failed: %v", err)) + } +} + +func configure(cfg config.ClusterConfig, n config.Node) (interface{}, error) { + mac, err := generateMACAddress() + if err != nil { + return nil, fmt.Errorf("generating MAC address: %v", err) + } + + return &vfkit.Driver{ + BaseDriver: &drivers.BaseDriver{ + MachineName: config.MachineName(cfg, n), + StorePath: localpath.MiniPath(), + SSHUser: "docker", + }, + Boot2DockerURL: download.LocalISOResource(cfg.MinikubeISO), + DiskSize: cfg.DiskSize, + Memory: cfg.Memory, + CPU: cfg.CPUs, + MACAddress: mac, + Cmdline: "", + ExtraDisks: cfg.ExtraDisks, + }, nil +} + +func status() registry.State { + _, err := exec.LookPath("vfkit") + if err != nil { + return registry.State{Error: err, Fix: "Run 'brew tap cfergeau/crc && brew install vfkit'", Doc: docURL} + } + return registry.State{Installed: true, Healthy: true, Running: true} +} + +func generateMACAddress() (string, error) { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + return "", err + } + // Set local bit, ensure unicast address + buf[0] = (buf[0] | 2) & 0xfe + mac := fmt.Sprintf("%02x:%02x:%02x:%02x:%02x:%02x", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]) + return mac, nil +}