Skip to content

Commit

Permalink
Merge branch 'main' into dag
Browse files Browse the repository at this point in the history
  • Loading branch information
@6543
6543 committed Dec 22, 2023
2 parents bf1ecce + f72ca04 commit a0fe800
Show file tree
Hide file tree
Showing 14 changed files with 219 additions and 154 deletions.
2 changes: 1 addition & 1 deletion docs/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"dependencies": {
"@docusaurus/core": "^3.0.0",
"@docusaurus/preset-classic": "^3.0.0",
"@easyops-cn/docusaurus-search-local": "^0.38.0",
"@easyops-cn/docusaurus-search-local": "^0.40.0",
"@mdx-js/react": "^3.0.0",
"@svgr/webpack": "^8.1.0",
"clsx": "^2.0.0",
Expand Down
8 changes: 4 additions & 4 deletions docs/pnpm-lock.yaml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion pipeline/backend/docker/convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,12 @@ func toHostConfig(step *types.Step) *container.HostConfig {
if len(step.DNSSearch) != 0 {
config.DNSSearch = step.DNSSearch
}
extraHosts := []string{}
for _, hostAlias := range step.ExtraHosts {
extraHosts = append(extraHosts, hostAlias.Name+":"+hostAlias.IP)
}
if len(step.ExtraHosts) != 0 {
config.ExtraHosts = step.ExtraHosts
config.ExtraHosts = extraHosts
}
if len(step.Devices) != 0 {
config.Devices = toDev(step.Devices)
Expand Down
10 changes: 4 additions & 6 deletions pipeline/backend/kubernetes/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"io"
"os"
"runtime"
"strings"
"time"

"github.com/rs/zerolog/log"
Expand Down Expand Up @@ -166,21 +165,20 @@ func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID s
}
}

extraHosts := []string{}

extraHosts := []types.HostAlias{}
for _, stage := range conf.Stages {
if stage.Alias == "services" {
for _, step := range stage.Steps {
svc, err := startService(ctx, e, step)
if err != nil {
return err
}
extraHosts = append(extraHosts, step.Networks[0].Aliases[0]+":"+svc.Spec.ClusterIP)
hostAlias := types.HostAlias{Name: step.Networks[0].Aliases[0], IP: svc.Spec.ClusterIP}
extraHosts = append(extraHosts, hostAlias)
}
}
}

log.Trace().Msgf("Adding extra hosts: %s", strings.Join(extraHosts, ", "))
log.Trace().Msgf("Adding extra hosts: %v", extraHosts)
for _, stage := range conf.Stages {
for _, step := range stage.Steps {
step.ExtraHosts = extraHosts
Expand Down
23 changes: 11 additions & 12 deletions pipeline/backend/kubernetes/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,16 @@ const (

func mkPod(namespace, name, image, workDir, goos, serviceAccountName string,
pool, privileged bool,
commands, vols, extraHosts []string,
commands, vols []string,
labels, annotations, env, nodeSelector map[string]string,
tolerations []types.Toleration, resources types.Resources,
extraHosts []types.HostAlias, tolerations []types.Toleration, resources types.Resources,
securityContext *types.SecurityContext, securityContextConfig SecurityContextConfig,
) (*v1.Pod, error) {
var err error

meta := podMeta(name, namespace, labels, annotations)

spec, err := podSpec(serviceAccountName, vols, extraHosts, env,
nodeSelector, tolerations, securityContext, securityContextConfig)
spec, err := podSpec(serviceAccountName, vols, env, nodeSelector, extraHosts, tolerations, securityContext, securityContextConfig)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -86,7 +85,8 @@ func podMeta(name, namespace string, labels, annotations map[string]string) meta
return meta
}

func podSpec(serviceAccountName string, vols, extraHosts []string, env, backendNodeSelector map[string]string, backendTolerations []types.Toleration,
func podSpec(serviceAccountName string, vols []string, env, backendNodeSelector map[string]string,
extraHosts []types.HostAlias, backendTolerations []types.Toleration,
securityContext *types.SecurityContext, securityContextConfig SecurityContextConfig,
) (v1.PodSpec, error) {
var err error
Expand Down Expand Up @@ -195,7 +195,7 @@ func volumeMount(name, path string) v1.VolumeMount {
}

// Here is the service IPs (placed in /etc/hosts in the Pod)
func hostAliases(extraHosts []string) []v1.HostAlias {
func hostAliases(extraHosts []types.HostAlias) []v1.HostAlias {
hostAliases := []v1.HostAlias{}
for _, extraHost := range extraHosts {
hostAlias := hostAlias(extraHost)
Expand All @@ -204,11 +204,10 @@ func hostAliases(extraHosts []string) []v1.HostAlias {
return hostAliases
}

func hostAlias(extraHost string) v1.HostAlias {
host := strings.Split(extraHost, ":")
func hostAlias(extraHost types.HostAlias) v1.HostAlias {
return v1.HostAlias{
IP: host[1],
Hostnames: []string{host[0]},
IP: extraHost.IP,
Hostnames: []string{extraHost.Name},
}
}

Expand Down Expand Up @@ -358,9 +357,9 @@ func startPod(ctx context.Context, engine *kube, step *types.Step) (*v1.Pod, err

pod, err := mkPod(engine.config.Namespace, podName, step.Image, step.WorkingDir, engine.goos, step.BackendOptions.Kubernetes.ServiceAccountName,
step.Pull, step.Privileged,
step.Commands, step.Volumes, step.ExtraHosts,
step.Commands, step.Volumes,
engine.config.PodLabels, engine.config.PodAnnotations, step.Environment, step.BackendOptions.Kubernetes.NodeSelector,
step.BackendOptions.Kubernetes.Tolerations, step.BackendOptions.Kubernetes.Resources, step.BackendOptions.Kubernetes.SecurityContext, engine.config.SecurityContext)
step.ExtraHosts, step.BackendOptions.Kubernetes.Tolerations, step.BackendOptions.Kubernetes.Resources, step.BackendOptions.Kubernetes.SecurityContext, engine.config.SecurityContext)
if err != nil {
return nil, err
}
Expand Down
18 changes: 14 additions & 4 deletions pipeline/backend/kubernetes/pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,9 @@ func TestTinyPod(t *testing.T) {

pod, err := mkPod("woodpecker", "wp-01he8bebctabr3kgk0qj36d2me-0", "gradle:8.4.0-jdk21", "/woodpecker/src", "linux/amd64", "",
false, false,
[]string{"gradle build"}, []string{"workspace:/woodpecker/src"}, nil,
[]string{"gradle build"}, []string{"workspace:/woodpecker/src"},
nil, nil, map[string]string{"CI": "woodpecker"}, nil,
nil,
nil, nil,
types.Resources{Requests: nil, Limits: nil}, nil, SecurityContextConfig{},
)
assert.NoError(t, err)
Expand Down Expand Up @@ -228,17 +228,27 @@ func TestFullPod(t *testing.T) {
"hostnames": [
"cloudflare"
]
},
{
"ip": "2606:4700:4700::64",
"hostnames": [
"cf.v6"
]
}
]
},
"status": {}
}`

hostAliases := []types.HostAlias{
{Name: "cloudflare", IP: "1.1.1.1"},
{Name: "cf.v6", IP: "2606:4700:4700::64"},
}
pod, err := mkPod("woodpecker", "wp-01he8bebctabr3kgk0qj36d2me-0", "meltwater/drone-cache", "/woodpecker/src", "linux/amd64", "wp-svc-acc",
true, true,
[]string{"go get", "go test"}, []string{"woodpecker-cache:/woodpecker/src/cache"}, []string{"cloudflare:1.1.1.1"},
[]string{"go get", "go test"}, []string{"woodpecker-cache:/woodpecker/src/cache"},
map[string]string{"app": "test"}, map[string]string{"apparmor.security": "runtime/default"}, map[string]string{"CGO": "0"}, map[string]string{"storage": "ssd"},
[]types.Toleration{{Key: "net-port", Value: "100Mbit", Effect: types.TaintEffectNoSchedule}},
hostAliases, []types.Toleration{{Key: "net-port", Value: "100Mbit", Effect: types.TaintEffectNoSchedule}},
types.Resources{Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"}, Limits: map[string]string{"memory": "256Mi", "cpu": "2"}},
&types.SecurityContext{Privileged: newBool(true), RunAsNonRoot: newBool(true), RunAsUser: newInt64(101), RunAsGroup: newInt64(101), FSGroup: newInt64(101)},
SecurityContextConfig{RunAsNonRoot: false},
Expand Down
5 changes: 5 additions & 0 deletions pipeline/backend/types/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,8 @@ package types
type Network struct {
Name string `json:"name,omitempty"`
}

type HostAlias struct {
Name string `json:"name,omitempty"`
IP string `json:"ip,omitempty"`
}
2 changes: 1 addition & 1 deletion pipeline/backend/types/step.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ type Step struct {
Environment map[string]string `json:"environment,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Commands []string `json:"commands,omitempty"`
ExtraHosts []string `json:"extra_hosts,omitempty"`
ExtraHosts []HostAlias `json:"extra_hosts,omitempty"`
Volumes []string `json:"volumes,omitempty"`
Tmpfs []string `json:"tmpfs,omitempty"`
Devices []string `json:"devices,omitempty"`
Expand Down
148 changes: 78 additions & 70 deletions pipeline/frontend/yaml/compiler/compiler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,15 @@ func TestCompilerCompile(t *testing.T) {
Name: "test_clone",
Alias: "clone",
Steps: []*backend_types.Step{{
Name: "test_clone",
Alias: "clone",
Type: backend_types.StepTypeClone,
Image: constant.DefaultCloneImage,
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"clone"}}},
Name: "test_clone",
Alias: "clone",
Type: backend_types.StepTypeClone,
Image: constant.DefaultCloneImage,
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"clone"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}

Expand Down Expand Up @@ -129,14 +130,15 @@ func TestCompilerCompile(t *testing.T) {
Name: "test_stage_0",
Alias: "dummy",
Steps: []*backend_types.Step{{
Name: "test_step_0",
Alias: "dummy",
Type: backend_types.StepTypePlugin,
Image: "dummy_img",
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"dummy"}}},
Name: "test_step_0",
Alias: "dummy",
Type: backend_types.StepTypePlugin,
Image: "dummy_img",
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"dummy"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}},
},
Expand Down Expand Up @@ -165,39 +167,42 @@ func TestCompilerCompile(t *testing.T) {
Name: "test_stage_0",
Alias: "echo env",
Steps: []*backend_types.Step{{
Name: "test_step_0",
Alias: "echo env",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
Name: "test_step_0",
Alias: "echo env",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}, {
Name: "test_stage_1",
Alias: "parallel echo 1",
Steps: []*backend_types.Step{{
Name: "test_step_1",
Alias: "parallel echo 1",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 1"}}},
Name: "test_step_1",
Alias: "parallel echo 1",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 1"}}},
ExtraHosts: []backend_types.HostAlias{},
}, {
Name: "test_step_2",
Alias: "parallel echo 2",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 2"}}},
Name: "test_step_2",
Alias: "parallel echo 2",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 2"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}},
},
Expand Down Expand Up @@ -225,39 +230,42 @@ func TestCompilerCompile(t *testing.T) {
Name: "stage_0",
Alias: "stage_0",
Steps: []*backend_types.Step{{
Name: "test_step_0",
Alias: "echo env",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
Name: "test_step_0",
Alias: "echo env",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
ExtraHosts: []backend_types.HostAlias{},
}, {
Name: "test_step_2",
Alias: "echo 2",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 2"}}},
Name: "test_step_2",
Alias: "echo 2",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 2"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}, {
Name: "stage_1",
Alias: "stage_1",
Steps: []*backend_types.Step{{
Name: "test_step_1",
Alias: "echo 1",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 1"}}},
Name: "test_step_1",
Alias: "echo 1",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 1"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}},
},
Expand Down
Loading

0 comments on commit a0fe800

Please sign in to comment.