migrate the internal kube-apiserver to kwok

README.md

@@ -23,6 +23,8 @@ That's why we are developing a simulator for kube-scheduler
 ```shell
 git clone git@github.com:kubernetes-sigs/kube-scheduler-simulator.git
 cd kube-scheduler-simulator
+# switch to the version you want to use.
+git switch simulator/v0.1.1
 # pull images from the registry and run up all components
 make docker_up 
 # All things up! 
@@ -41,12 +43,13 @@ We have several components:
 
 ### Simulator
 
-Simulator is kube-apiserver with some controllers + debuggable scheduler + the HTTP server which mainly for the web UI.
+Simulator is composed of debuggable scheduler + the HTTP server which mainly for the web UI.
 
 There are several ways to integrate your scheduler into the simulator.
 See [integrate-your-scheduler.md](simulator/docs/integrate-your-scheduler.md).
 
-You can create any resources by any ways (kubectl, k8s client library, or web UI described next).
+Simulator runs with a fake cluster powered by [KWOK](/~https://github.com/kubernetes-sigs/kwok)
+You can create any resources in KWOK cluster via any clients (e.g. kubectl, k8s client library, or web UI described next).
 And when you create Pods, 
 Pods will be scheduled by the [debuggable scheduler](./simulator/docs/debuggable-scheduler.md),
 and they'll get the annotations that explain how each Pod was evaluated by each scheduler plugin.

docker-compose-local.yml

@@ -5,13 +5,10 @@ services:
     container_name: simulator-server
     environment:
       - PORT=1212
-      - KUBE_SCHEDULER_SIMULATOR_ETCD_URL=http://simulator-etcd:2379
-      - CORS_ALLOWED_ORIGIN_LIST=http://${SIMULATOR_EXTERNAL_IP:-localhost}:3000
-      - KUBE_API_HOST=0.0.0.0
-      - KUBE_API_PORT=3131
+      - KUBE_SCHEDULER_SIMULATOR_ETCD_URL=http://simulator-cluster:2379
+      - KUBE_APISERVER_URL=http://simulator-cluster:3131
     ports:
       - "1212:1212"
-      - "3131:3131"
     restart: always
     tty: true
     networks:
@@ -27,17 +24,19 @@ services:
     ports:
     - "3000:3000"
     tty: true
-  simulator-etcd:
-    image: quay.io/coreos/etcd:v3.4.26
-    container_name: simulator-etcd
+  simulator-cluster:
+    image: registry.k8s.io/kwok/cluster:v0.5.0-k8s.v1.29.0
+    container_name: simulator-cluster
     restart: always
+    ports:
+      - "3131:3131"
     volumes:
       - simulator-etcd-data:/var/lib/etcd
-    command: etcd --advertise-client-urls http://simulator-etcd:2379 --data-dir /var/lib/etcd --listen-client-urls http://0.0.0.0:2379 --initial-cluster-state new --initial-cluster-token tkn
+      - ./kwok.yaml:/root/.kwok/kwok.yaml
     networks:
       - simulator-internal-network
-volumes:
-  simulator-etcd-data:
 networks:
   simulator-internal-network:
     driver: bridge
+volumes:
+  simulator-etcd-data:

docker-compose.yml

@@ -5,13 +5,10 @@ services:
     container_name: simulator-server
     environment:
       - PORT=1212
-      - KUBE_SCHEDULER_SIMULATOR_ETCD_URL=http://simulator-etcd:2379
-      - CORS_ALLOWED_ORIGIN_LIST=http://${SIMULATOR_EXTERNAL_IP:-localhost}:3000
-      - KUBE_API_HOST=0.0.0.0
-      - KUBE_API_PORT=3131
+      - KUBE_SCHEDULER_SIMULATOR_ETCD_URL=http://simulator-cluster:2379
+      - KUBE_APISERVER_URL=http://simulator-cluster:3131
     ports:
       - "1212:1212"
-      - "3131:3131"
     restart: always
     tty: true
     networks:
@@ -27,17 +24,19 @@ services:
     ports:
     - "3000:3000"
     tty: true
-  simulator-etcd:
-    image: quay.io/coreos/etcd:v3.4.26
-    container_name: simulator-etcd
+  simulator-cluster:
+    image: registry.k8s.io/kwok/cluster:v0.5.0-k8s.v1.29.0
+    container_name: simulator-cluster
     restart: always
+    ports:
+      - "3131:3131"
     volumes:
       - simulator-etcd-data:/var/lib/etcd
-    command: etcd --advertise-client-urls http://simulator-etcd:2379 --data-dir /var/lib/etcd --listen-client-urls http://0.0.0.0:2379 --initial-cluster-state new --initial-cluster-token tkn
+      - ./kwok.yaml:/root/.kwok/kwok.yaml
     networks:
       - simulator-internal-network
-volumes:
-  simulator-etcd-data:
 networks:
   simulator-internal-network:
     driver: bridge
+volumes:
+  simulator-etcd-data:

kwok.yaml

@@ -0,0 +1,12 @@
+kind: KwokctlConfiguration
+apiVersion: config.kwok.x-k8s.io/v1alpha1
+options:
+  kubeApiserverPort: 3131
+  etcdPort: 2379
+  etcdPrefix: /kube-scheduler-simulator
+  disableKubeScheduler: true
+componentsPatches:
+- name: kube-apiserver
+  extraArgs:
+  - key: cors-allowed-origins
+    value: ^*$

simulator/config.yaml

@@ -24,13 +24,9 @@ corsAllowedOriginList:
 # cluster for importing resources to scheduler simulator.
 kubeConfig: ""
 
-# This is the host of kube-apiserver which the simulator
-# starts internally. Its default value is 127.0.0.1.
-kubeApiHost: "127.0.0.1"
-
-# This is the port of kube-apiserver. Its default value is 3131.
-kubeApiPort: 3131
-
+# This is the url of kube-apiserver.
+# This variable is used to connect to the user cluster's kube-apiserver.
+kubeApiServerUrl: ""
 
 # The path to a KubeSchedulerConfiguration file.
 # If passed, the simulator will start the scheduler
@@ -45,4 +41,4 @@ externalImportEnabled: false
 
 # This variable indicates whether an external scheduler
 # is used.
-externalSchedulerEnabled: false
+externalSchedulerEnabled: false

simulator/config/config.go

@@ -68,7 +68,10 @@ func NewConfig() (*Config, error) {
 		return nil, xerrors.Errorf("get frontend URL: %w", err)
 	}
 
-	apiurl := getKubeAPIServerURL()
+	apiurl, err := getKubeAPIServerURL()
+	if err != nil {
+		return nil, xerrors.Errorf("get kube API server URL: %w", err)
+	}
 
 	externalimportenabled := getExternalImportEnabled()
 	externalKubeClientCfg := &rest.Config{}
@@ -136,23 +139,15 @@ func getPort() (int, error) {
 }
 
 // getKubeAPIServerURL gets KubeAPIServerURL from environment variable first, if empty from the config file.
-func getKubeAPIServerURL() string {
-	p := os.Getenv("KUBE_API_PORT")
-	if p == "" {
-		p = strconv.Itoa(configYaml.KubeAPIPort)
-		if p == "" {
-			p = "3131"
-		}
-	}
-
-	h := os.Getenv("KUBE_API_HOST")
-	if h == "" {
-		h = configYaml.KubeAPIHost
-		if h == "" {
-			h = "127.0.0.1"
+func getKubeAPIServerURL() (string, error) {
+	url := os.Getenv("KUBE_APISERVER_URL")
+	if url == "" {
+		url = configYaml.KubeAPIServerURL
+		if url == "" {
+			return "", xerrors.Errorf("get KUBE_APISERVER_URL from config: %w", ErrEmptyConfig)
 		}
 	}
-	return h + ":" + p
+	return url, nil
 }
 
 // getExternalSchedulerEnabled gets ExternalSchedulerEnabled from environment variable first,

simulator/config/v1alpha1/types.go

@@ -27,8 +27,7 @@ type SimulatorConfiguration struct {
 	// server is started.
 	Port int `json:"port,omitempty"`
 
-	// This is the URL for etcd. The simulator runs kube-apiserver
-	// internally, and the kube-apiserver uses this etcd.
+	// This is the URL for etcd.
 	EtcdURL string `json:"etcdURL,omitempty"`
 
 	// This URL represents the URL once web UI is started.
@@ -41,6 +40,9 @@ type SimulatorConfiguration struct {
 	// cluster for importing resources to scheduler simulator.
 	KubeConfig string `json:"kubeConfig,omitempty"`
 
+	// This is the URL for kube-apiserver.
+	KubeAPIServerURL string `json:"kubeApiServerUrl,omitempty"`
+
 	// This is the host of kube-apiserver which the simulator
 	// starts internally. Its default value is 127.0.0.1.
 	KubeAPIHost string `json:"kubeApiHost,omitempty"`