feat: support set max threads (#3537)

Signed-off-by: Jim Ma <majinjing3@gmail.com>

client/config/peerhost.go

@@ -51,6 +51,11 @@ type DaemonOption struct {
 	AliveTime  util.Duration `mapstructure:"aliveTime" yaml:"aliveTime"`
 	GCInterval util.Duration `mapstructure:"gcInterval" yaml:"gcInterval"`
 	Metrics    string        `mapstructure:"metrics" yaml:"metrics"`
+	// MaxThreads indicates to invoke debug.SetMaxThreads to set max threads that golang runtime can create,
+	// it's useful for large-scale downloading resource with none native golang source code, like call cgo to invoke c++
+	// one cgo call will create a standalone os tread, the default 10000 limit is to small.
+	// Normally, we can also set seed concurrent to limit for large-scale downloading at same time.
+	MaxThreads int `mapstructure:"maxThreads" yaml:"maxThreads"`
 
 	WorkHome      string `mapstructure:"workHome" yaml:"workHome"`
 	WorkHomeMode  uint32 `mapstructure:"workHomeMode" yaml:"workHomeMode"`

client/daemon/daemon.go

@@ -28,6 +28,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -119,6 +120,11 @@ func New(opt *config.DaemonOption, d dfpath.Dfpath) (Daemon, error) {
 	// update plugin directory
 	source.UpdatePluginDir(d.PluginDir())
 
+	if opt.MaxThreads > 10000 {
+		debug.SetMaxThreads(opt.MaxThreads)
+		logger.Infof("update runtime max threads: %d", opt.MaxThreads)
+	}
+
 	// FIXME the viper casts all case sensitive keys into lower case, but the resource clients option is map[string]interface{}, it should not be casted.
 	// issue: /~https://github.com/spf13/viper/issues/1014
 	tmpOpt := config.NewDaemonConfig()