From 9028a812363c46806382a3a16d79b5515c39d004 Mon Sep 17 00:00:00 2001 From: Gaius Date: Thu, 26 Sep 2024 19:01:14 +0800 Subject: [PATCH] feat: support preheat with self-signed certs Signed-off-by: Gaius --- deploy/docker-compose/docker-compose.yaml | 48 +-- deploy/docker-compose/run.sh | 21 +- .../template/client.template.yaml | 164 ++++++++ .../template/dfget.template.yaml | 331 ---------------- .../template/manager.template.yaml | 95 +++-- .../template/scheduler.template.yaml | 86 ++-- .../template/seed-client.template.yaml | 174 +++++++++ .../template/seed-peer.template.yaml | 366 ------------------ hack/docker-build.sh | 2 +- internal/job/types.go | 2 + manager/config/config.go | 13 +- manager/job/job.go | 4 +- manager/job/preheat.go | 23 +- scheduler/job/job.go | 2 + 14 files changed, 529 insertions(+), 802 deletions(-) create mode 100644 deploy/docker-compose/template/client.template.yaml delete mode 100644 deploy/docker-compose/template/dfget.template.yaml create mode 100644 deploy/docker-compose/template/seed-client.template.yaml delete mode 100644 deploy/docker-compose/template/seed-peer.template.yaml diff --git a/deploy/docker-compose/docker-compose.yaml b/deploy/docker-compose/docker-compose.yaml index abeab22c9d5..f743eb19f64 100644 --- a/deploy/docker-compose/docker-compose.yaml +++ b/deploy/docker-compose/docker-compose.yaml @@ -22,7 +22,7 @@ services: - MARIADB_DATABASE=manager - MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=yes healthcheck: - test: ["CMD-SHELL", "mysqladmin status"] + test: ["CMD-SHELL", "mysqladmin ping -h localhost"] interval: 1s timeout: 2s retries: 30 @@ -31,13 +31,13 @@ services: manager: image: dragonflyoss/manager:latest - container_name: dragonfly-manager + container_name: manager depends_on: - redis - mysql restart: always volumes: - - ./log/manager:/var/log/dragonfly/manager + - ./log/manager:/var/log/dragonfly - ./config/manager.yaml:/etc/dragonfly/manager.yaml:ro healthcheck: test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=:65003 || exit 1"] @@ -48,32 +48,32 @@ services: - 65003:65003 - 8080:8080 - dfdaemon: - image: dragonflyoss/dfdaemon:latest + client: + image: dragonflyoss/client:latest depends_on: - manager - scheduler - - seed-peer - container_name: dragonfly-seed-peer + - seed-client + container_name: client restart: always healthcheck: - test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=:65000 || exit 1"] + test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=unix:///var/run/dragonfly/dfdaemon.sock || exit 1"] interval: 1s timeout: 2s retries: 30 volumes: - - ./log/peer:/var/log/dragonfly/daemon - - ./config/dfget.yaml:/etc/dragonfly/dfget.yaml:ro + - ./log/client:/var/log/dragonfly + - ./config/client.yaml:/etc/dragonfly/dfdaemon.yaml:ro ports: - - 65000:65000 - - 65001:65001 - - 65002:65002 + - 4000:4000 + - 4001:4001 + - 4002:4002 scheduler: image: dragonflyoss/scheduler:latest depends_on: - manager - container_name: dragonfly-scheduler + container_name: scheduler restart: always healthcheck: test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=:8002 || exit 1"] @@ -81,27 +81,27 @@ services: timeout: 2s retries: 30 volumes: - - ./log/scheduler:/var/log/dragonfly/scheduler + - ./log/scheduler:/var/log/dragonfly - ./config/scheduler.yaml:/etc/dragonfly/scheduler.yaml:ro ports: - 8002:8002 - seed-peer: - image: dragonflyoss/dfdaemon:latest + seed-client: + image: dragonflyoss/client:latest depends_on: - manager - scheduler - container_name: dragonfly-dfdaemon + container_name: seed-client restart: always healthcheck: - test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=:65006 || exit 1"] + test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=unix:///var/run/dragonfly/dfdaemon.sock || exit 1"] interval: 1s timeout: 2s retries: 30 volumes: - - ./log/seed-peer:/var/log/dragonfly/daemon - - ./config/seed-peer.yaml:/etc/dragonfly/dfget.yaml:ro + - ./log/seed-client:/var/log/dragonfly + - ./config/seed-client.yaml:/etc/dragonfly/dfdaemon.yaml:ro ports: - - 65006:65006 - - 65007:65007 - - 65008:65008 + - 4010:4010 + - 4011:4011 + - 4012:4012 diff --git a/deploy/docker-compose/run.sh b/deploy/docker-compose/run.sh index 4106b63de25..d2b1b386366 100755 --- a/deploy/docker-compose/run.sh +++ b/deploy/docker-compose/run.sh @@ -4,6 +4,7 @@ set -e REPO=${REPO:-dragonflyoss} TAG=${TAG:-latest} +CLIENT_TAG=${CLIENT_TAG:-latest} DIR=$(cd "$(dirname "$0")" && pwd) cd $DIR @@ -13,8 +14,8 @@ prepare(){ ip=${IP:-$(hostname -i)} - sed "s,__IP__,$ip," template/dfget.template.yaml > config/dfget.yaml - sed "s,__IP__,$ip," template/seed-peer.template.yaml > config/seed-peer.yaml + sed "s,__IP__,$ip," template/client.template.yaml > config/client.yaml + sed "s,__IP__,$ip," template/seed-client.template.yaml > config/seed-client.yaml sed "s,__IP__,$ip," template/scheduler.template.yaml > config/scheduler.yaml sed "s,__IP__,$ip," template/manager.template.yaml > config/manager.yaml } @@ -25,7 +26,7 @@ delete_container(){ echo try to clean old containers ${RUNTIME} rm -f dragonfly-redis dragonfly-mysql dragonfly-manager dragonfly-scheduler \ - dragonfly-dfdaemon dragonfly-seed-peer + dragonfly-client dragonfly-seed-client } run_container(){ @@ -34,7 +35,7 @@ run_container(){ echo try to clean old containers ${RUNTIME} rm -f dragonfly-redis dragonfly-mysql dragonfly-manager dragonfly-scheduler \ - dragonfly-dfdaemon dragonfly-seed-peer + dragonfly-client dragonfly-seed-client printf "create dragonfly-redis " ${RUNTIME} run -d --name dragonfly-redis --restart=always -p 6379:6379 \ @@ -55,11 +56,11 @@ run_container(){ -v ${DIR}/config/manager.yaml:/etc/dragonfly/manager.yaml \ ${REPO}/manager:${TAG} - printf "create dragonfly-seed-peer " - ${RUNTIME} run -d --name dragonfly-seed-peer --restart=always --net=host \ + printf "create dragonfly-seed-client " + ${RUNTIME} run -d --name dragonfly-seed-client --restart=always --net=host \ -v /tmp/log/dragonfly:/var/log/dragonfly \ -v ${DIR}/config/seed-peer.yaml:/etc/dragonfly/dfget.yaml \ - ${REPO}/dfdaemon:${TAG} + ${REPO}/client:${CLIENT_TAG} printf "create dragonfly-scheduler " ${RUNTIME} run -d --name dragonfly-scheduler --restart=always --net=host \ @@ -67,11 +68,11 @@ run_container(){ -v ${DIR}/config/scheduler.yaml:/etc/dragonfly/scheduler.yaml \ ${REPO}/scheduler:${TAG} - printf "create dragonfly-dfdaemon " - ${RUNTIME} run -d --name dragonfly-dfdaemon --restart=always --net=host \ + printf "create dragonfly-client " + ${RUNTIME} run -d --name dragonfly-client --restart=always --net=host \ -v /tmp/log/dragonfly:/var/log/dragonfly \ -v ${DIR}/config/dfget.yaml:/etc/dragonfly/dfget.yaml \ - ${REPO}/dfdaemon:${TAG} + ${REPO}/client:${CLIENT_TAG} } prepare diff --git a/deploy/docker-compose/template/client.template.yaml b/deploy/docker-compose/template/client.template.yaml new file mode 100644 index 00000000000..9a4908fb59b --- /dev/null +++ b/deploy/docker-compose/template/client.template.yaml @@ -0,0 +1,164 @@ +# verbose prints log to stdout. +verbose: true + +log: + # Specify the logging level [trace, debug, info, warn, error] + level: info + +# host is the host configuration for dfdaemon. +host: + ## idc is the idc of the host. + idc: '' + ## location is the location of the host. + location: '' + ## hostname is the hostname of the host. + # hostname: "" + ## ip is the advertise ip of the host. + ip: __IP__ + +server: + # pluginDir is the directory to store plugins. + pluginDir: /var/lib/dragonfly/plugins/dfdaemon/ + # cacheDir is the directory to store cache files. + cacheDir: /var/cache/dragonfly/dfdaemon/ + +download: + server: + # socketPath is the unix socket path for dfdaemon GRPC service. + socketPath: /var/run/dragonfly/dfdaemon.sock + # rateLimit is the default rate limit of the download speed in KiB/MiB/GiB per second, default is 10GiB/s. + rateLimit: 10GiB + # pieceTimeout is the timeout for downloading a piece from source. + pieceTimeout: 30s + # concurrentPieceCount is the number of concurrent pieces to download. + concurrentPieceCount: 10 + +upload: + server: + # port is the port to the grpc server. + port: 4000 + ## ip is the listen ip of the grpc server. + # ip: "" + # disableShared indicates whether disable to share data for other peers. + disableShared: false + # rateLimit is the default rate limit of the upload speed in KiB/MiB/GiB per second, default is 10GiB/s. + rateLimit: 10GiB + +manager: + # addrs is manager addresses. + addrs: + - http://__IP__:65003 + +scheduler: + # announceInterval is the interval to announce peer to the scheduler. + # Announcer will provide the scheduler with peer information for scheduling, + # peer information includes cpu, memory, etc. + announceInterval: 10s + # scheduleTimeout is the timeout for scheduling. If the scheduling timesout, dfdaemon will back-to-source + # download if enableBackToSource is true, otherwise dfdaemon will return download failed. + scheduleTimeout: 30s + # maxScheduleCount is the max count of schedule. + maxScheduleCount: 5 + # enableBackToSource indicates whether enable back-to-source download, when the scheduling failed. + enableBackToSource: true + +dynconfig: + # refreshInterval is the interval to refresh dynamic configuration from manager. + refreshInterval: 1m + +storage: + # dir is the directory to store task's metadata and content. + dir: /var/lib/dragonfly/ + # keep indicates whether keep the task's metadata and content when the dfdaemon restarts. + keep: true + # writeBufferSize is the buffer size for writing piece to disk, default is 128KB. + writeBufferSize: 131072 + # readBufferSize is the buffer size for reading piece from disk, default is 128KB. + readBufferSize: 131072 + +gc: + # interval is the interval to do gc. + interval: 900s + policy: + # taskTTL is the ttl of the task. + taskTTL: 21600s + # distHighThresholdPercent is the high threshold percent of the disk usage. + # If the disk usage is greater than the threshold, dfdaemon will do gc. + distHighThresholdPercent: 80 + # distLowThresholdPercent is the low threshold percent of the disk usage. + # If the disk usage is less than the threshold, dfdaemon will stop gc. + distLowThresholdPercent: 60 + +proxy: + server: + # port is the port to the proxy server. + port: 4001 + ## ip is the listen ip of the proxy server. + # ip: "" + ## caCert is the root CA cert path with PEM format for the proxy server to generate the server cert. + ## If ca_cert is empty, proxy will generate a smaple CA cert by rcgen::generate_simple_self_signed. + ## When client requests via the proxy, the client should not verify the server cert and set + ## insecure to true. If ca_cert is not empty, proxy will sign the server cert with the CA cert. If openssl is installed, + ## you can use openssl to generate the root CA cert and make the system trust the root CA cert. + ## Then set the ca_cert and ca_key to the root CA cert and key path. Dfdaemon generates the server cert + ## and key, and signs the server cert with the root CA cert. When client requests via the proxy, + ## the proxy can intercept the request by the server cert. + # caCert: "" + ## caKey is the root CA key path with PEM format for the proxy server to generate the server cert. + ## If ca_key is empty, proxy will generate a smaple CA key by rcgen::generate_simple_self_signed. + ## When client requests via the proxy, the client should not verify the server cert and set + ## insecure to true. If ca_key is not empty, proxy will sign the server cert with the CA cert. If openssl is installed, + ## you can use openssl to generate the root CA cert and make the system trust the root CA cert. + ## Then set the ca_cert and ca_key to the root CA cert and key path. Dfdaemon generates the server cert + ## and key, and signs the server cert with the root CA cert. When client requests via the proxy, + ## the proxy can intercept the request by the server cert. + # caKey: "" + # rules is the list of rules for the proxy server. + # regex is the regex of the request url. + # useTLS indicates whether use tls for the proxy backend. + # redirect is the redirect url. + # filteredQueryParams is the filtered query params to generate the task id. + # When filter is ["Signature", "Expires", "ns"], for example: + # http://example.com/xyz?Expires=e1&Signature=s1&ns=docker.io and http://example.com/xyz?Expires=e2&Signature=s2&ns=docker.io + # will generate the same task id. + # Default value includes the filtered query params of s3, gcs, oss, obs, cos. + # `X-Dragonfly-Use-P2P` header can instead of the regular expression of the rule. If the value is "true", + # the request will use P2P technology to distribute the content. If the value is "false", + # but url matches the regular expression in rules. The request will also use P2P technology to distribute the content. + rules: + - regex: 'blobs/sha256.*' + # useTLS: false + # redirect: "" + # filteredQueryParams: [] + registryMirror: + # addr is the default address of the registry mirror. Proxy will start a registry mirror service for the + # client to pull the image. The client can use the default address of the registry mirror in + # configuration to pull the image. The `X-Dragonfly-Registry` header can instead of the default address + # of registry mirror. + addr: https://index.docker.io + ## certs is the client certs path with PEM format for the registry. + ## If registry use self-signed cert, the client should set the + ## cert for the registry mirror. + # certs: "" + # disableBackToSource indicates whether disable to download back-to-source when download failed. + disableBackToSource: false + # prefetch pre-downloads full of the task when download with range request. + prefetch: false + # readBufferSize is the buffer size for reading piece from disk, default is 32KB. + readBufferSize: 32768 + +security: + # enable indicates whether enable security. + enable: false + +metrics: + server: + # port is the port to the metrics server. + port: 4002 + ## ip is the listen ip of the metrics server. + # ip: "" + +## tracing is the tracing configuration for dfdaemon. +# tracing: +## addr is the address to report tracing log. +# addr: "" diff --git a/deploy/docker-compose/template/dfget.template.yaml b/deploy/docker-compose/template/dfget.template.yaml deleted file mode 100644 index 445d42afeb6..00000000000 --- a/deploy/docker-compose/template/dfget.template.yaml +++ /dev/null @@ -1,331 +0,0 @@ -# daemon alive time, when sets 0s, daemon will not auto exit -# it is useful for longtime running -aliveTime: 0s - -# daemon gc task running interval -gcInterval: 1m0s - -# WorkHome is working directory. -# In linux, default value is /usr/local/dragonfly. -# In macos(just for testing), default value is /Users/$USER/.dragonfly. -workHome: '' - -# logDir is the log directory. -# In linux, default value is /var/log/dragonfly. -# In macos(just for testing), default value is /Users/$USER/.dragonfly/logs. -logDir: '' - -# cacheDir is dynconfig cache directory. -# In linux, default value is /var/cache/dragonfly. -# In macos(just for testing), default value is /Users/$USER/.dragonfly/cache. -cacheDir: '' - -# pluginDir is the plugin directory. -# In linux, default value is /usr/local/dragonfly/plugins. -# In macos(just for testing), default value is /Users/$USER/.dragonfly/plugins. -pluginDir: '' - -# dataDir is the download data directory. -# In linux, default value is /var/lib/dragonfly. -# In macos(just for testing), default value is /Users/$USER/.dragonfly/data. -dataDir: '' - -# when daemon exit, keep peer task data or not -# it is usefully when upgrade daemon service, all local cache will be saved -# default is false -keepStorage: true - -# console shows log on console -console: false - -# whether to enable debug level logger and enable pprof -verbose: true - -# listen port for pprof, only valid when the verbose option is true -# default is -1. If it is 0, pprof will use a random port. -pprof-port: -1 - -# jaeger endpoint url, like: http://jaeger.dragonfly.svc:14268/api/traces -jaeger: "" - -# all addresses of all schedulers -# the schedulers of all daemons should be same in one region or zone. -# daemon will send tasks to a fixed scheduler by hashing the task url and meta data -# caution: only tcp is supported -scheduler: - manager: - # get scheduler list dynamically from manager - enable: false - # schedule timeout - scheduleTimeout: 30s - # when true, only scheduler says back source, daemon can back source - disableAutoBackSource: false - # below example is a stand address - netAddrs: - - type: tcp - addr: __IP__:8002 - -# current host info used for scheduler -host: - # access ip for other peers - # when local ip is different with access ip, advertiseIP should be set - advertiseIP: __IP__ - # geographical location, separated by "|" characters - location: "" - # idc deployed by daemon - idc: "" - # daemon hostname - # hostname: "" - -# download service option -download: - concurrent: - # thresholdSize indicates the threshold to download pieces concurrently. - thresholdSize: 10M - # thresholdSpeed indicates the threshold download speed to download pieces concurrently. - thresholdSpeed: 2M - # goroutineCount indicates the concurrent goroutine count for every task. - goroutineCount: 4 - # initBackoff second for every piece failed, default: 0.5. - initBackoff: 0.5 - # maxBackoff second for every piece failed, default: 3. - maxBackoff: 3 - # maxAttempts for every piece failed,default: 3. - maxAttempts: 3 - # calculate digest when transfer files, set false to save memory - calculateDigest: true - # total download limit per second - totalRateLimit: 1024Mi - # per peer task download limit per second - perPeerRateLimit: 512Mi - # traffic shaper type - trafficShaperType: sampling - # download piece timeout - pieceDownloadTimeout: 30s - # When request data with range header, prefetch data not in range. - prefetch: false - # golang transport option - transportOption: - # dial timeout - dialTimeout: 2s - # keep alive - keepAlive: 30s - # same with http.Transport.MaxIdleConns - maxIdleConns: 100 - # same with http.Transport.IdleConnTimeout - idleConnTimeout: 90s - # same with http.Transport.ResponseHeaderTimeout - responseHeaderTimeout: 2s - # same with http.Transport.TLSHandshakeTimeout - tlsHandshakeTimeout: 1s - # same with http.Transport.ExpectContinueTimeout - expectContinueTimeout: 2s - # download grpc option - downloadGRPC: - # Security option. - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: true - tlsConfig: null - # Download service listen address - # current, only support unix domain socket. - unixListen: - # In linux, default value is /var/run/dfdaemon.sock. - # In macos(just for testing), default value is /tmp/dfdaemon.sock. - socket: '' - # peer grpc option - # peer grpc service send pieces info to other peers - peerGRPC: - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: true - tcpListen: - # # Listen address. - # listen: 0.0.0.0 - # Listen port, daemon will try to listen, - # when this port is not available, daemon will try next port. - port: 65000 - # If want to limit upper port, please use blow format. -# port: -# start: 65000 -# end: 65009 - -# upload service option -upload: - # Upload limit per second. - rateLimit: 1024Mi - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: false - tcpListen: - # # Listen address. - # listen: 0.0.0.0 - # Listen port, daemon will try to listen, - # when this port is not available, daemon will try next port. - port: 65002 - # If want to limit upper port, please use blow format. -# port: -# start: 65020 -# end: 65029 -# -# Object storage service. -objectStorage: - # Enable object storage service. - enable: false - # Filter is used to generate a unique Task ID by - # filtering unnecessary query params in the URL, - # it is separated by & character. - # When filter: "Expires&Signature&ns", for example: - # http://localhost/xyz?Expires=111&Signature=222&ns=docker.io and http://localhost/xyz?Expires=333&Signature=999&ns=docker.io - # is same task. - filter: 'Expires&Signature&ns' - # maxReplicas is the maximum number of replicas of an object cache in seed peers. - maxReplicas: 3 - # Object storage service security option. - security: - insecure: true - tlsVerify: true - tcpListen: - # # Listen address. - # listen: 0.0.0.0 - # Listen port. - port: 65004 - -# peer task storage option -storage: - # task data expire time - # when there is no access to a task data, this task will be gc. - taskExpireTime: 6h - # storage strategy when process task data - # io.d7y.storage.v2.simple : download file to data directory first, then copy to output path, this is default action - # the download file in date directory will be the peer data for uploading to other peers - # io.d7y.storage.v2.advance: download file directly to output path with postfix, hard link to final output, - # avoid copy to output path, fast than simple strategy, but: - # the output file with postfix will be the peer data for uploading to other peers - # when user delete or change this file, this peer data will be corrupted - # default is io.d7y.storage.v2.simple - strategy: io.d7y.storage.v2.simple - # disk quota gc threshold, when the quota of all tasks exceeds the gc threshold, the oldest tasks will be reclaimed. - diskGCThreshold: 50Gi - # disk used percent gc threshold, when the disk used percent exceeds, the oldest tasks will be reclaimed. - # eg, diskGCThresholdPercent=80, when the disk usage is above 80%, start to gc the oldest tasks - diskGCThresholdPercent: 80 - # set to ture for reusing underlying storage for same task id - multiplex: true - -# Health service option. -health: - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: false - tcpListen: - # # Listen address. - # listen: 0.0.0.0 - # Listen port, daemon will try to listen, - # when this port is not available, daemon will try next port. - port: 40901 - # If want to limit upper port, please use blow format. -# port: -# start: 40901 -# end: 40901 - -# proxy service detail option -proxy: - # filter for hash url - # when defaultFilter: "Expires&Signature&ns", for example: - # http://localhost/xyz?Expires=111&Signature=222&ns=docker.io and http://localhost/xyz?Expires=333&Signature=999&ns=docker.io - # is same task - defaultFilter: "Expires&Signature&ns" - # Tag the task. - # when the value of the default tag is different, - # the same download url can be divided into different tasks according to the tag, - # it is also possible to override the default tag by adding - # the X-Dragonfly-Tag header through the proxy. - defaultTag: '' - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: false - tcpListen: - # namespace stands the linux net namespace, like /proc/1/ns/net. - # It's useful for running daemon in pod with ip allocated and listening the special port in host net namespace. - # Linux only. - namespace: '' - # # Listen address. - # listen: 0.0.0.0 - # Listen port, daemon will try to listen, - # when this port is not available, daemon will try next port. - port: 65001 - # If want to limit upper port, please use blow format. - # port: - # start: 65020 - # end: 65029 - registryMirror: - # when enable, using header "X-Dragonfly-Registry" for remote instead of url - dynamic: true - # url for the registry mirror - url: https://index.docker.io - # whether to ignore https certificate errors - insecure: true - # optional certificates if the remote server uses self-signed certificates - certs: [] - # whether to request the remote registry directly - direct: false - # whether to use proxies to decide if dragonfly should be used - useProxies: false - - proxies: - # Proxy all http image layer download requests with dfget. - - regx: blobs/sha256.* - # Proxy all http image layer download requests with dfget. - - regx: file-server.* - # Change http requests to some-registry to https and proxy them with dfget. - - regx: some-registry/ - useHTTPS: true - # Proxy requests directly, without dfget. - - regx: no-proxy-reg - direct: true - # Proxy requests with redirect. - - regx: some-registry - redirect: another-registry - # The same with url rewrite like apache ProxyPass directive. - - regx: ^http://some-registry/(.*) - redirect: http://another-registry/$1 - - hijackHTTPS: - # key pair used to hijack https requests - cert: "" - key: "" - hosts: - - regx: mirror.aliyuncs.com:443 # regexp to match request hosts - # whether to ignore https certificate errors - insecure: true - # optional certificates if the host uses self-signed certificates - certs: [] - # max tasks to download same time, 0 is no limit - maxConcurrency: 0 - whiteList: - # the host of the whitelist - - host: "" - # match whitelist hosts - regx: ".*" - # port that need to be added to the whitelist - ports: - -network: - # Enable ipv6. - enableIPv6: false diff --git a/deploy/docker-compose/template/manager.template.yaml b/deploy/docker-compose/template/manager.template.yaml index 4a193e7e9ed..fd0fbf6a7d9 100644 --- a/deploy/docker-compose/template/manager.template.yaml +++ b/deploy/docker-compose/template/manager.template.yaml @@ -1,8 +1,9 @@ -# current server info used for server +# Current server info used for server. server: # GRPC server configure. grpc: - # # Advertise ip. + # # Access ip for other services, + # # when local ip is different with access ip, advertiseIP should be set. advertiseIP: __IP__ # # Listen ip. # listenIP: 0.0.0.0 @@ -15,6 +16,11 @@ server: rest: # REST server address addr: :8080 + # tls: + # # Certificate file path. + # cert: /etc/ssl/certs/server.crt + # # Key file path. + # key: /etc/ssl/private/server.pem # WorkHome is working directory. # In linux, default value is /usr/local/dragonfly. # In macos(just for testing), default value is /Users/$USER/.dragonfly. @@ -32,14 +38,16 @@ server: # In macos(just for testing), default value is /Users/$USER/.dragonfly/plugins. pluginDir: '' +# Auth configuration. auth: + # JWT configuration used for sigining. jwt: # Realm name to display to the user, default value is Dragonfly. - realm: "Dragonfly" + realm: 'Dragonfly' # Key is secret key used for signing, default value is # encoded base64 of dragonfly. # Please change the key in production. - key: "ZHJhZ29uZmx5Cg==" + key: 'ZHJhZ29uZmx5Cg==' # Timeout is duration that a jwt token is valid, # default duration is two days. timeout: 48h @@ -49,8 +57,6 @@ auth: # Database info used for server. database: - # Database type, supported types include mysql, mariadb and postgres. - type: mysql # Mysql configure. mysql: user: dragonfly @@ -67,20 +73,24 @@ database: # key: /etc/ssl/private/key.pem # # CA file path. # ca: /etc/ssl/certs/ca.pem - # # Whether a client verifies the server's certificate chain and host name. + # # Whether a client verifies the server's certificate chain and hostname. # insecureSkipVerify: true # Redis configure. redis: # Redis addresses. addrs: - "__IP__:6379" + # Redis sentinel master name. + masterName: '' + # Redis username. + username: '' # Redis password. password: dragonfly - # Redis DB name. + # Redis DB. db: 0 - # Redis brokerDB name. + # Redis broker DB. brokerDB: 1 - # Redis backendDB name. + # Redis backend DB. backendDB: 2 # Manager server cache. @@ -88,38 +98,56 @@ cache: # Redis cache configure. redis: # Cache ttl configure. - ttl: 30s + ttl: 5m # Local cache configure. local: # LFU cache size. - size: 10000 + size: 200000 # Cache ttl configure. - ttl: 10s + ttl: 3m -# Object storage service. -objectStorage: - # Enable object storage. - enable: false - # Object storage name of type, it can be s3 or oss. - name: s3 - # Storage region. - region: '' - # Datacenter endpoint. - endpoint: '' - # Access key id. - accessKey: '' - # Access key secret. - secretKey: '' +# Job configuration. +job: + # rateLimit configuration. + rateLimit: + # fillInterval is the interval for refilling the bucket. + fillInterval: 1m + # capacity is the maximum number of requests that can be consumed in a single fillInterval. + capacity: 5 + # quantum is the number of tokens taken from the bucket for each request. + quantum: 5 + # gc configuration. + gc: + # Interval is the interval for garbage collection. + interval: 24h + # TTL is the time to live for the job. + ttl: 24h + # Sync peers configuration. + syncPeers: + # Interval is the interval for syncing all peers information from the scheduler and + # display peers information in the manager console. + interval: 24h + # Timeout is the timeout for syncing peers information from the single scheduler. + timeout: 10m + # Preheat configuration. + preheat: + # registryTimeout is the timeout for requesting registry to get token and manifest. + registryTimeout: 1m + tls: + insecureSkipVerify: false + # # caCert is the CA certificate for preheat tls handshake, it can be path or PEM format string. + # caCert: '' # Prometheus metrics. metrics: # Manager enable metrics service. - enable: false + enable: true # Metrics service address. addr: ':8000' # Enable peer gauge metrics. enablePeerGauge: true +# Security configuration. security: # autoIssueCert indicates to issue client certificates for all grpc call. # If AutoIssueCert is false, any other option in Security will be ignored. @@ -147,19 +175,20 @@ security: # validityPeriod is the validity period of certificate. validityPeriod: 87600h +# Network configuration. network: # Enable ipv6. enableIPv6: false -# console shows log on console +# Console shows log on console. console: false -# whether to enable debug level logger and enable pprof +# Whether to enable debug level logger and enable pprof. verbose: true -# listen port for pprof, only valid when the verbose option is true +# Listen port for pprof, only valid when the verbose option is true # default is -1. If it is 0, pprof will use a random port. pprof-port: -1 -# jaeger endpoint url, like: http://jaeger.dragonfly.svc:14268/api/traces -jaeger: "" +# Jaeger endpoint url, like: http://jaeger.dragonfly.svc:14268/api/traces. +jaeger: '' diff --git a/deploy/docker-compose/template/scheduler.template.yaml b/deploy/docker-compose/template/scheduler.template.yaml index c7e5d33c692..e4c0f2b9b64 100644 --- a/deploy/docker-compose/template/scheduler.template.yaml +++ b/deploy/docker-compose/template/scheduler.template.yaml @@ -1,7 +1,11 @@ -# server scheduler instance configuration +# Server scheduler instance configuration. server: - # # Advertise ip. + # # Access ip for other services, + # # when local ip is different with access ip, advertiseIP should be set. advertiseIP: __IP__ + # # Access port for other services, + # # when local ip is different with access port, advertisePort should be set. + # advertisePort: 8002 # # Listen ip. # listenIP: 0.0.0.0 # Port is the ip and port scheduler server listens on. @@ -29,7 +33,7 @@ server: # In macos(just for testing), default value is /Users/$USER/.dragonfly/data. dataDir: '' -# scheduler policy configuration +# Scheduler policy configuration. scheduler: # Algorithm configuration to use different scheduling algorithms, # default configuration supports "default" and "ml" @@ -39,15 +43,14 @@ scheduler: # and the compiled `d7y-scheduler-plugin-evaluator.so` file is added to # the dragonfly working directory plugins. algorithm: default - # backSourceCount is the number of backsource clients - # when the seed peer is unavailable. - backSourceCount: 3 - # Retry scheduling back-to-source limit times. - retryBackSourceLimit: 5 + # backToSourceCount is single task allows the peer to back-to-source count. + backToSourceCount: 200 + # retryBackToSourceLimit reaches the limit, then the peer back-to-source. + retryBackToSourceLimit: 3 # Retry scheduling limit times. - retryLimit: 10 + retryLimit: 5 # Retry scheduling interval. - retryInterval: 50ms + retryInterval: 400ms # GC metadata configuration. gc: # pieceDownloadTimeout is the timeout of downloading piece. @@ -73,14 +76,34 @@ database: # Redis addresses. addrs: - "__IP__:6379" + # Redis sentinel master name. + masterName: '' # Redis username. username: '' # Redis password. password: dragonfly - # Redis brokerDB name. + # Redis broker DB. brokerDB: 1 - # Redis backendDB name. + # Redis backend DB. backendDB: 2 + # Network topology DB. + networkTopologyDB: 3 + +# Resource configuration. +resource: + # Task configuration. + task: + # downloadTiny is the configuration of downloading tiny task by scheduler. + downloadTiny: + # scheme is download tiny task scheme. + scheme: http + # Timeout is http request timeout. + timeout: 1m + # tls is download tiny task TLS configuration. + tls: + # insecureSkipVerify controls whether a client verifies the + # server's certificate chain and hostname. + insecureSkipVerify: true # Dynamic data configuration. dynConfig: @@ -99,7 +122,7 @@ manager: # addr is manager access address. addr: "__IP__:65003" # schedulerClusterID cluster id to which scheduler instance belongs. - schedulerClusterID: "1" + schedulerClusterID: 1 # keepAlive keep alive configuration. keepAlive: # KeepAlive interval. @@ -118,11 +141,23 @@ job: # Scheduler enable job service. enable: true # Number of workers in global queue. - globalWorkerNum: 1 + globalWorkerNum: 500 # Number of workers in scheduler queue. - schedulerWorkerNum: 1 + schedulerWorkerNum: 500 # Number of workers in local queue. - localWorkerNum: 5 + localWorkerNum: 1000 + +# Network topology to collect configuration. +networkTopology: + # enable network topology service, including probe, network topology collection. + enable: true + # collectInterval is the interval of collecting network topology. + collectInterval: 2h + probe: + # queueLength is the length of probe queue. + queueLength: 5 + # count is the number of probing hosts. + count: 10 # Store task download information. storage: @@ -137,7 +172,7 @@ storage: # Enable prometheus metrics. metrics: # Scheduler enable metrics service. - enable: false + enable: true # Metrics service address. addr: ':8000' # Enable host metrics. @@ -160,6 +195,13 @@ security: # The second step is to set tlsPolicy to prefer, and then completely upgrade the dragonfly services. tlsPolicy: 'prefer' certSpec: + # dnsNames is a list of dns names be set on the certificate. + dnsNames: + - 'dragonfly-scheduler' + - 'dragonfly-scheduler.dragonfly-system.svc' + - 'dragonfly-scheduler.dragonfly-system.svc.cluster.local' + # ipAddresses is a list of ip addresses be set on the certificate. + ipAddresses: # validityPeriod is the validity period of certificate. validityPeriod: 4320h @@ -167,15 +209,15 @@ network: # Enable ipv6. enableIPv6: false -# console shows log on console +# Console shows log on console. console: false -# whether to enable debug level logger and enable pprof +# Whether to enable debug level logger and enable pprof. verbose: true -# listen port for pprof, only valid when the verbose option is true +# Listen port for pprof, only valid when the verbose option is true # default is -1. If it is 0, pprof will use a random port. pprof-port: -1 -# jaeger endpoint url, like: http://jaeger.dragonfly.svc:14268/api/traces -jaeger: "" +# Jaeger endpoint url, like: http://jaeger.dragonfly.svc:14268/api/traces. +jaeger: '' diff --git a/deploy/docker-compose/template/seed-client.template.yaml b/deploy/docker-compose/template/seed-client.template.yaml new file mode 100644 index 00000000000..46d30165dda --- /dev/null +++ b/deploy/docker-compose/template/seed-client.template.yaml @@ -0,0 +1,174 @@ +# verbose prints log to stdout. +verbose: true + +log: + # Specify the logging level [trace, debug, info, warn, error] + level: info + +# host is the host configuration for dfdaemon. +host: + ## idc is the idc of the host. + idc: '' + ## location is the location of the host. + location: '' + ## hostname is the hostname of the host. + # hostname: "" + ## ip is the advertise ip of the host. + ip: __IP__ + +server: + # pluginDir is the directory to store plugins. + pluginDir: /var/lib/dragonfly/plugins/dfdaemon/ + # cacheDir is the directory to store cache files. + cacheDir: /var/cache/dragonfly/dfdaemon/ + +download: + server: + # socketPath is the unix socket path for dfdaemon GRPC service. + socketPath: /var/run/dragonfly/dfdaemon.sock + # rateLimit is the default rate limit of the download speed in KiB/MiB/GiB per second, default is 10GiB/s. + rateLimit: 10GiB + # pieceTimeout is the timeout for downloading a piece from source. + pieceTimeout: 30s + # concurrentPieceCount is the number of concurrent pieces to download. + concurrentPieceCount: 10 + +upload: + server: + # port is the port to the grpc server. + port: 4010 + ## ip is the listen ip of the grpc server. + # ip: "" + # disableShared indicates whether disable to share data for other peers. + disableShared: false + # rateLimit is the default rate limit of the upload speed in KiB/MiB/GiB per second, default is 10GiB/s. + rateLimit: 10GiB + +manager: + # addrs is manager addresses. + addrs: + - http://__IP__:65003 + +scheduler: + # announceInterval is the interval to announce peer to the scheduler. + # Announcer will provide the scheduler with peer information for scheduling, + # peer information includes cpu, memory, etc. + announceInterval: 10s + # scheduleTimeout is the timeout for scheduling. If the scheduling timesout, dfdaemon will back-to-source + # download if enableBackToSource is true, otherwise dfdaemon will return download failed. + scheduleTimeout: 30s + # maxScheduleCount is the max count of schedule. + maxScheduleCount: 5 + # enableBackToSource indicates whether enable back-to-source download, when the scheduling failed. + enableBackToSource: true + +seedPeer: + # enable indicates whether enable seed peer. + enable: true + # type is the type of seed peer. + type: super + # clusterID is the cluster id of the seed peer cluster. + clusterID: 1 + # keepaliveInterval is the interval to keep alive with manager. + keepaliveInterval: 15s + +dynconfig: + # refreshInterval is the interval to refresh dynamic configuration from manager. + refreshInterval: 1m + +storage: + # dir is the directory to store task's metadata and content. + dir: /var/lib/dragonfly/ + # keep indicates whether keep the task's metadata and content when the dfdaemon restarts. + keep: true + # writeBufferSize is the buffer size for writing piece to disk, default is 128KB. + writeBufferSize: 131072 + # readBufferSize is the buffer size for reading piece from disk, default is 128KB. + readBufferSize: 131072 + +gc: + # interval is the interval to do gc. + interval: 900s + policy: + # taskTTL is the ttl of the task. + taskTTL: 21600s + # distHighThresholdPercent is the high threshold percent of the disk usage. + # If the disk usage is greater than the threshold, dfdaemon will do gc. + distHighThresholdPercent: 80 + # distLowThresholdPercent is the low threshold percent of the disk usage. + # If the disk usage is less than the threshold, dfdaemon will stop gc. + distLowThresholdPercent: 60 + +proxy: + server: + # port is the port to the proxy server. + port: 4011 + ## ip is the listen ip of the proxy server. + # ip: "" + ## caCert is the root CA cert path with PEM format for the proxy server to generate the server cert. + ## If ca_cert is empty, proxy will generate a smaple CA cert by rcgen::generate_simple_self_signed. + ## When client requests via the proxy, the client should not verify the server cert and set + ## insecure to true. If ca_cert is not empty, proxy will sign the server cert with the CA cert. If openssl is installed, + ## you can use openssl to generate the root CA cert and make the system trust the root CA cert. + ## Then set the ca_cert and ca_key to the root CA cert and key path. Dfdaemon generates the server cert + ## and key, and signs the server cert with the root CA cert. When client requests via the proxy, + ## the proxy can intercept the request by the server cert. + # caCert: "" + ## caKey is the root CA key path with PEM format for the proxy server to generate the server cert. + ## If ca_key is empty, proxy will generate a smaple CA key by rcgen::generate_simple_self_signed. + ## When client requests via the proxy, the client should not verify the server cert and set + ## insecure to true. If ca_key is not empty, proxy will sign the server cert with the CA cert. If openssl is installed, + ## you can use openssl to generate the root CA cert and make the system trust the root CA cert. + ## Then set the ca_cert and ca_key to the root CA cert and key path. Dfdaemon generates the server cert + ## and key, and signs the server cert with the root CA cert. When client requests via the proxy, + ## the proxy can intercept the request by the server cert. + # caKey: "" + # rules is the list of rules for the proxy server. + # regex is the regex of the request url. + # useTLS indicates whether use tls for the proxy backend. + # redirect is the redirect url. + # filteredQueryParams is the filtered query params to generate the task id. + # When filter is ["Signature", "Expires", "ns"], for example: + # http://example.com/xyz?Expires=e1&Signature=s1&ns=docker.io and http://example.com/xyz?Expires=e2&Signature=s2&ns=docker.io + # will generate the same task id. + # Default value includes the filtered query params of s3, gcs, oss, obs, cos. + # `X-Dragonfly-Use-P2P` header can instead of the regular expression of the rule. If the value is "true", + # the request will use P2P technology to distribute the content. If the value is "false", + # but url matches the regular expression in rules. The request will also use P2P technology to distribute the content. + rules: + - regex: 'blobs/sha256.*' + # useTLS: false + # redirect: "" + # filteredQueryParams: [] + registryMirror: + # addr is the default address of the registry mirror. Proxy will start a registry mirror service for the + # client to pull the image. The client can use the default address of the registry mirror in + # configuration to pull the image. The `X-Dragonfly-Registry` header can instead of the default address + # of registry mirror. + addr: https://index.docker.io + ## certs is the client certs path with PEM format for the registry. + ## If registry use self-signed cert, the client should set the + ## cert for the registry mirror. + # certs: "" + # disableBackToSource indicates whether disable to download back-to-source when download failed. + disableBackToSource: false + # prefetch pre-downloads full of the task when download with range request. + prefetch: false + # readBufferSize is the buffer size for reading piece from disk, default is 32KB. + readBufferSize: 32768 + +security: + # enable indicates whether enable security. + enable: false + +metrics: + server: + # port is the port to the metrics server. + port: 4012 + ## ip is the listen ip of the metrics server. + # ip: "" + +## tracing is the tracing configuration for dfdaemon. +# tracing: +## addr is the address to report tracing log. +# addr: "" diff --git a/deploy/docker-compose/template/seed-peer.template.yaml b/deploy/docker-compose/template/seed-peer.template.yaml deleted file mode 100644 index 181e61d2da3..00000000000 --- a/deploy/docker-compose/template/seed-peer.template.yaml +++ /dev/null @@ -1,366 +0,0 @@ -# daemon alive time, when sets 0s, daemon will not auto exit -# it is useful for longtime running -aliveTime: 0s - -# daemon gc task running interval -gcInterval: 1m0s - -# WorkHome is working directory. -# In linux, default value is /usr/local/dragonfly. -# In macos(just for testing), default value is /Users/$USER/.dragonfly. -workHome: '' - -# logDir is the log directory. -# In linux, default value is /var/log/dragonfly. -# In macos(just for testing), default value is /Users/$USER/.dragonfly/logs. -logDir: '' - -# cacheDir is dynconfig cache directory. -# In linux, default value is /var/cache/dragonfly. -# In macos(just for testing), default value is /Users/$USER/.dragonfly/cache. -cacheDir: '' - -# pluginDir is the plugin directory. -# In linux, default value is /usr/local/dragonfly/plugins. -# In macos(just for testing), default value is /Users/$USER/.dragonfly/plugins. -pluginDir: '' - -# dataDir is the download data directory. -# In linux, default value is /var/lib/dragonfly. -# In macos(just for testing), default value is /Users/$USER/.dragonfly/data. -dataDir: '' - -# when daemon exit, keep peer task data or not -# it is usefully when upgrade daemon service, all local cache will be saved -# default is false -keepStorage: true - -# console shows log on console -console: false - -# whether to enable debug level logger and enable pprof -verbose: true - -# listen port for pprof, only valid when the verbose option is true -# default is -1. If it is 0, pprof will use a random port. -pprof-port: -1 - -# jaeger endpoint url, like: http://jaeger.dragonfly.svc:14268/api/traces -jaeger: "" - -# all addresses of all schedulers -# the schedulers of all daemons should be same in one region or zone. -# daemon will send tasks to a fixed scheduler by hashing the task url and meta data -# caution: only tcp is supported -scheduler: - manager: - # get scheduler list dynamically from manager - enable: true - # manager service addresses - netAddrs: - - type: tcp - addr: __IP__:65003 - # scheduler list refresh interval - refreshInterval: 10s - seedPeer: - # Dfdaemon enabled seed peer mode. - enable: true - # Seed peer type includes super, strong and weak. - type: super - # Seed peer cluster id. - clusterID: 1 - keepAlive: - # Keep alive internal. - internal: 5s - # schedule timeout - scheduleTimeout: 30s - # when true, only scheduler says back source, daemon can back source - disableAutoBackSource: false - -# Current host info used for scheduler. -host: - # # Access ip for other peers, - # # when local ip is different with access ip, advertiseIP should be set. - advertiseIP: __IP__ - # Geographical location, separated by "|" characters. - location: '' - # IDC deployed by daemon. - idc: '' - # Daemon hostname. - # hostname: "" - -# Download service option. -download: - # Calculate digest when transfer files, set false to save memory. - calculateDigest: true - # Total download limit per second. - totalRateLimit: 2048Mi - # Per peer task download limit per second. - perPeerRateLimit: 1024Mi - # Download piece timeout. - pieceDownloadTimeout: 30s - # When request data with range header, prefetch data not in range. - prefetch: false - # Golang transport option. - transportOption: - # Ddial timeout. - dialTimeout: 2s - # Keep alive. - keepAlive: 30s - # Same with http.Transport.MaxIdleConns. - maxIdleConns: 100 - # Same with http.Transport.IdleConnTimeout. - idleConnTimeout: 90s - # Same with http.Transport.ResponseHeaderTimeout. - responseHeaderTimeout: 2s - # Same with http.Transport.TLSHandshakeTimeout. - tlsHandshakeTimeout: 1s - # Same with http.Transport.ExpectContinueTimeout. - expectContinueTimeout: 2s - # Concurrent option for back source, default: empty - # if you want to enable concurrent option, thresholdSize and goroutineCount is enough, keep other options empty is okay. - concurrent: - # thresholdSize indicates the threshold to download pieces concurrently. - thresholdSize: 10M - # thresholdSpeed indicates the threshold download speed to download pieces concurrently. - thresholdSpeed: 2M - # goroutineCount indicates the concurrent goroutine count for every task. - goroutineCount: 4 - # initBackoff second for every piece failed, default: 0.5. - initBackoff: 0.5 - # maxBackoff second for every piece failed, default: 3. - maxBackoff: 3 - # maxAttempts for every piece failed,default: 3. - maxAttempts: 3 - # Download grpc option. - downloadGRPC: - # Security option. - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: true - tlsConfig: null - # Download service listen address - # current, only support unix domain socket. - unixListen: - # In linux, default value is /var/run/dfdaemon.sock. - # In macos(just for testing), default value is /tmp/dfdaemon.sock. - socket: '' - # Peer grpc option. - # Peer grpc service send pieces info to other peers. - peerGRPC: - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: true - tcpListen: - # # Listen address. - # listen: 0.0.0.0 - # Listen port, daemon will try to listen, - # when this port is not available, daemon will try next port. - port: 65006 - # If want to limit upper port, please use blow format. -# port: -# start: 65000 -# end: 65009 - -# Upload service option. -upload: - # Upload limit per second. - rateLimit: 2048Mi - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: false - tcpListen: - # # Listen address. - # listen: 0.0.0.0 - # Listen port, daemon will try to listen, - # when this port is not available, daemon will try next port. - port: 65008 - # If want to limit upper port, please use blow format. -# port: -# start: 65020 -# end: 65029 - -# Object storage service. -objectStorage: - # Enable object storage service. - enable: false - # Filter is used to generate a unique Task ID by - # filtering unnecessary query params in the URL, - # it is separated by & character. - # When filter: "Expires&Signature&ns", for example: - # http://localhost/xyz?Expires=111&Signature=222&ns=docker.io and http://localhost/xyz?Expires=333&Signature=999&ns=docker.io - # is same task. - filter: 'Expires&Signature&ns' - # maxReplicas is the maximum number of replicas of an object cache in seed peers. - maxReplicas: 3 - # Object storage service security option. - security: - insecure: true - tlsVerify: true - tcpListen: - # # Listen address. - # listen: 0.0.0.0 - # Listen port. - port: 65004 - -# Peer task storage option. -storage: - # Task data expire time, - # when there is no access to a task data, this task will be gc. - taskExpireTime: 6h - # Storage strategy when process task data. - # io.d7y.storage.v2.simple : download file to data directory first, then copy to output path, this is default action - # the download file in date directory will be the peer data for uploading to other peers. - # io.d7y.storage.v2.advance: download file directly to output path with postfix, hard link to final output, - # avoid copy to output path, fast than simple strategy, but: - # the output file with postfix will be the peer data for uploading to other peers - # when user delete or change this file, this peer data will be corrupted. - # default is io.d7y.storage.v2.simple. - strategy: io.d7y.storage.v2.simple - # Disk quota gc threshold, when the quota of all tasks exceeds the gc threshold, the oldest tasks will be reclaimed. - diskGCThreshold: 50Gi - # Disk used percent gc threshold, when the disk used percent exceeds, the oldest tasks will be reclaimed. - # eg, diskGCThresholdPercent=80, when the disk usage is above 80%, start to gc the oldest tasks. - diskGCThresholdPercent: 80 - # Set to ture for reusing underlying storage for same task id. - multiplex: true - -# Health service option. -health: - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: false - tcpListen: - # # Listen address. - # listen: 0.0.0.0 - # Listen port, daemon will try to listen, - # when this port is not available, daemon will try next port. - port: 40902 - # If want to limit upper port, please use blow format. -# port: -# start: 40901 -# end: 40901 - -# Proxy service detail option. -proxy: - # Filter for hash url. - # when defaultFilter: "Expires&Signature&ns", for example: - # http://localhost/xyz?Expires=111&Signature=222&ns=docker.io and http://localhost/xyz?Expires=333&Signature=999&ns=docker.io - # is same task, it is also possible to override the default filter by adding - # the X-Dragonfly-Filter header through the proxy. - defaultFilter: 'Expires&Signature&ns' - # Tag the task. - # when the value of the default tag is different, - # the same download url can be divided into different tasks according to the tag, - # it is also possible to override the default tag by adding - # the X-Dragonfly-Tag header through the proxy. - defaultTag: '' - security: - insecure: true - cacert: '' - cert: '' - key: '' - tlsVerify: false - tcpListen: - # namespace stands the linux net namespace, like /proc/1/ns/net. - # It's useful for running daemon in pod with ip allocated and listening the special port in host net namespace. - # Linux only. - namespace: '' - # # Listen address. - # listen: 0.0.0.0 - # Listen port, daemon will try to listen, - # when this port is not available, daemon will try next port. - port: 65007 - # If want to limit upper port, please use blow format. - # port: - # start: 65020 - # end: 65029 - registryMirror: - # When enable, using header "X-Dragonfly-Registry" for remote instead of url. - dynamic: true - # URL for the registry mirror. - url: https://index.docker.io - # Whether to ignore https certificate errors. - insecure: true - # Optional certificates if the remote server uses self-signed certificates. - certs: [] - # Whether to request the remote registry directly. - direct: false - # Whether to use proxies to decide if dragonfly should be used. - useProxies: false - - proxies: - # Proxy all http image layer download requests with dfget. - - regx: blobs/sha256.* - # Proxy all http image layer download requests with dfget. - - regx: file-server.* - # Change http requests to some-registry to https and proxy them with dfget. - - regx: some-registry/ - useHTTPS: true - # Proxy requests directly, without dfget. - - regx: no-proxy-reg - direct: true - # Proxy requests with redirect. - - regx: some-registry - redirect: another-registry - # The same with url rewrite like apache ProxyPass directive. - - regx: ^http://some-registry/(.*) - redirect: http://another-registry/$1 - - hijackHTTPS: - # key pair used to hijack https requests - cert: "" - key: "" - hosts: - - regx: mirror.aliyuncs.com:443 # regexp to match request hosts - # whether to ignore https certificate errors - insecure: true - # optional certificates if the host uses self-signed certificates - certs: [] - # max tasks to download same time, 0 is no limit - maxConcurrency: 0 - whiteList: - # the host of the whitelist - - host: "" - # match whitelist hosts - regx: ".*" - # port that need to be added to the whitelist - ports: - -security: - # autoIssueCert indicates to issue client certificates for all grpc call. - # If AutoIssueCert is false, any other option in Security will be ignored. - autoIssueCert: false - # caCert is the root CA certificate for all grpc tls handshake, it can be path or PEM format string. - caCert: '' - # tlsVerify indicates to verify certificates. - tlsVerify: false - # tlsPolicy controls the grpc shandshake behaviors: - # force: both ClientHandshake and ServerHandshake are only support tls - # prefer: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support tls - # default: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support insecure (non-tls) - # Notice: If the drgaonfly service has been deployed, a two-step upgrade is required. - # The first step is to set tlsPolicy to default, and then upgrade the dragonfly services. - # The second step is to set tlsPolicy to prefer, and then completely upgrade the dragonfly services. - tlsPolicy: 'prefer' - certSpec: - # validityPeriod is the validity period of certificate. - validityPeriod: 4320h -# Prometheus metrics address. -# metrics: ':8000' - -network: - # Enable ipv6. - enableIPv6: false diff --git a/hack/docker-build.sh b/hack/docker-build.sh index bd6f413c962..2bf9b7d1ba0 100755 --- a/hack/docker-build.sh +++ b/hack/docker-build.sh @@ -10,7 +10,7 @@ cd "${curDir}/../" || return D7Y_VERSION=${D7Y_VERSION:-"latest"} D7Y_REGISTRY=${D7Y_REGISTRY:-dragonflyoss} IMAGES_DIR="build/images" -BASE_IMAGE=${BASE_IMAGE:-alpine:3.17} +BASE_IMAGE=${BASE_IMAGE:-reg.docker.alibaba-inc.com/dragonflyoss/alpine:3.17} CGO_ENABLED=${CGO_ENABLED:-0} GOPROXY=${GOPROXY:-`go env GOPROXY`} diff --git a/internal/job/types.go b/internal/job/types.go index 4f4f431470f..a0382a964ad 100644 --- a/internal/job/types.go +++ b/internal/job/types.go @@ -31,6 +31,8 @@ type PreheatRequest struct { Priority int32 `json:"priority" validate:"omitempty"` Scope string `json:"scope" validate:"omitempty"` ConcurrentCount int64 `json:"concurrent_count" validate:"omitempty"` + CertificateChain [][]byte `json:"certificate_chain" validate:"omitempty"` + InsecureSkipVerify bool `json:"insecure_skip_verify" validate:"omitempty"` Timeout time.Duration `json:"timeout" validate:"omitempty"` } diff --git a/manager/config/config.go b/manager/config/config.go index c0f131bfb3b..c2e43169b15 100644 --- a/manager/config/config.go +++ b/manager/config/config.go @@ -324,7 +324,7 @@ type PreheatConfig struct { RegistryTimeout time.Duration `yaml:"registryTimeout" mapstructure:"registryTimeout"` // TLS client configuration. - TLS *PreheatTLSClientConfig `yaml:"tls" mapstructure:"tls"` + TLS PreheatTLSClientConfig `yaml:"tls" mapstructure:"tls"` } // RateLimitConfig is the configuration for rate limit. @@ -349,6 +349,10 @@ type SyncPeersConfig struct { } type PreheatTLSClientConfig struct { + // InsecureSkipVerify controls whether a client verifies the + // server's certificate chain and host name. + InsecureSkipVerify bool `yaml:"insecureSkipVerify" mapstructure:"insecureSkipVerify"` + // CACert is the CA certificate for preheat tls handshake, it can be path or PEM format string. CACert types.PEMContent `yaml:"caCert" mapstructure:"caCert"` } @@ -484,6 +488,7 @@ func New() *Config { }, Preheat: PreheatConfig{ RegistryTimeout: DefaultJobPreheatRegistryTimeout, + TLS: PreheatTLSClientConfig{}, }, SyncPeers: SyncPeersConfig{ Interval: DefaultJobSyncPeersInterval, @@ -671,12 +676,6 @@ func (cfg *Config) Validate() error { return errors.New("gc requires parameter ttl") } - if cfg.Job.Preheat.TLS != nil { - if cfg.Job.Preheat.TLS.CACert == "" { - return errors.New("preheat requires parameter caCert") - } - } - if cfg.Job.Preheat.RegistryTimeout == 0 { return errors.New("preheat requires parameter registryTimeout") } diff --git a/manager/job/job.go b/manager/job/job.go index 409e56a265a..f652dd5e741 100644 --- a/manager/job/job.go +++ b/manager/job/job.go @@ -61,14 +61,14 @@ func New(cfg *config.Config, gdb *gorm.DB) (*Job, error) { } var certPool *x509.CertPool - if cfg.Job.Preheat.TLS != nil { + if len(cfg.Job.Preheat.TLS.CACert) != 0 { certPool = x509.NewCertPool() if !certPool.AppendCertsFromPEM([]byte(cfg.Job.Preheat.TLS.CACert)) { return nil, errors.New("invalid CA Cert") } } - preheat, err := newPreheat(j, cfg.Job.Preheat.RegistryTimeout, certPool) + preheat, err := newPreheat(j, cfg.Job.Preheat.RegistryTimeout, certPool, cfg.Job.Preheat.TLS.InsecureSkipVerify) if err != nil { return nil, err } diff --git a/manager/job/preheat.go b/manager/job/preheat.go index 179c40f0139..98f715615f4 100644 --- a/manager/job/preheat.go +++ b/manager/job/preheat.go @@ -76,14 +76,21 @@ type Preheat interface { // preheat is an implementation of Preheat. type preheat struct { - job *internaljob.Job - registryTimeout time.Duration - rootCAs *x509.CertPool + job *internaljob.Job + registryTimeout time.Duration + rootCAs *x509.CertPool + certificateChain [][]byte + insecureSkipVerify bool } // newPreheat creates a new Preheat. -func newPreheat(job *internaljob.Job, registryTimeout time.Duration, rootCAs *x509.CertPool) (Preheat, error) { - return &preheat{job, registryTimeout, rootCAs}, nil +func newPreheat(job *internaljob.Job, registryTimeout time.Duration, rootCAs *x509.CertPool, insecureSkipVerify bool) (Preheat, error) { + var certificateChain [][]byte + if rootCAs != nil { + certificateChain = rootCAs.Subjects() + } + + return &preheat{job, registryTimeout, rootCAs, certificateChain, insecureSkipVerify}, nil } // CreatePreheat creates a preheat job. @@ -112,6 +119,8 @@ func (p *preheat) CreatePreheat(ctx context.Context, schedulers []models.Schedul Headers: json.Headers, Scope: json.Scope, ConcurrentCount: json.ConcurrentCount, + CertificateChain: p.certificateChain, + InsecureSkipVerify: p.insecureSkipVerify, Timeout: json.Timeout, }, } @@ -187,7 +196,7 @@ func (p *preheat) getImageLayers(ctx context.Context, args types.PreheatArgs) ([ Timeout: p.registryTimeout, Transport: &http.Transport{ DialContext: nethttp.NewSafeDialer().DialContext, - TLSClientConfig: &tls.Config{RootCAs: p.rootCAs}, + TLSClientConfig: &tls.Config{RootCAs: p.rootCAs, InsecureSkipVerify: p.insecureSkipVerify}, }, }), withBasicAuth(args.Username, args.Password), @@ -328,6 +337,8 @@ func (p *preheat) parseLayers(manifests []distribution.Manifest, args types.Preh Headers: nethttp.HeaderToMap(header), Scope: args.Scope, ConcurrentCount: args.ConcurrentCount, + CertificateChain: p.certificateChain, + InsecureSkipVerify: p.insecureSkipVerify, Timeout: args.Timeout, } diff --git a/scheduler/job/job.go b/scheduler/job/job.go index 8a9d71a7d66..cc11e2e0069 100644 --- a/scheduler/job/job.go +++ b/scheduler/job/job.go @@ -287,6 +287,7 @@ func (j *job) preheatAllPeers(ctx context.Context, taskID string, req *internalj FilteredQueryParams: strings.Split(req.FilteredQueryParams, idgen.FilteredQueryParamsSeparator), RequestHeader: req.Headers, Timeout: durationpb.New(req.Timeout), + CertificateChain: req.CertificateChain, }}) if err != nil { log.Errorf("preheat failed: %s", err.Error()) @@ -430,6 +431,7 @@ func (j *job) preheatV2(ctx context.Context, taskID string, req *internaljob.Pre Priority: commonv2.Priority(req.Priority), FilteredQueryParams: filteredQueryParams, RequestHeader: req.Headers, + CertificateChain: req.CertificateChain, }}) if err != nil { log.Errorf("preheat failed: %s", err.Error())