Skip to content

Commit

Permalink
Fixed API performance issue with too big values for hash generation
Browse files Browse the repository at this point in the history
  • Loading branch information
sparshev committed Sep 3, 2024
1 parent 48bf6f2 commit efb2f68
Show file tree
Hide file tree
Showing 13 changed files with 524 additions and 62 deletions.
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,25 @@ check that the server (or client) is the one is approved in the cluster.
Maybe in the future Fish will allow to manage the cluster CA and issue certificate for a new node,
but for now just check openssl and /~https://github.com/jcmoraisjr/simple-ca for reference.

#### Performance

It really depends on how you want to run the Fish node, in general there are 2 cases:

1. **To serve local resources of the machine**: so you run it on the performant node and don't want
to consume too much of it's precious resources or interfere somehow: then you can use -cpu and -mem
params to limit the node in CPU and RAM utilization. Of course that will impact the API processing
performance, but probably you don't need much since you running a cluster and can distribute the
load across the nodes. You can expect that with 2 vCPU and 512MB of ram it could process ~16 API
requests per second.
2. **To serve remote/cloud resources**: It's worth to set the target on RAM by -mem option, but not
much to CPU. The RAM limit will help you to not get into OOM - just leave ~2GB of RAM for GC and
you will get the maximum performance. With 16 vCPU Fish can serve ~50 API requests per second.

Most of the time during API request processing is wasted on user password validation, so if you
need to squeeze more rps from Fish node you can lower the Argon2id parameters in crypt.go, but with
that you need to make sure you understand the consequences:
https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-argon2-04#section-4

### To run as a cluster

**TODO [#30](/~https://github.com/adobe/aquarium-fish/issues/30):** This functionality is in active
Expand Down
47 changes: 39 additions & 8 deletions cmd/fish/fish.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ import (
"context"
"os"
"path/filepath"
"runtime"
"runtime/debug"
"strconv"
"time"

"github.com/glebarez/sqlite"
Expand All @@ -31,6 +34,7 @@ import (
"github.com/adobe/aquarium-fish/lib/openapi"
"github.com/adobe/aquarium-fish/lib/proxy_socks"
"github.com/adobe/aquarium-fish/lib/proxy_ssh"
"github.com/adobe/aquarium-fish/lib/util"
)

func main() {
Expand All @@ -43,25 +47,28 @@ func main() {
var cluster_join *[]string
var cfg_path string
var dir string
var cpu_limit string
var mem_limit string
var log_verbosity string
var log_timestamp bool

cmd := &cobra.Command{
Use: "aquarium-fish",
Short: "Aquarium fish",
Long: `Part of the Aquarium suite - a distributed resources manager`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
err := log.SetVerbosity(log_verbosity)
if err != nil {
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
if err = log.SetVerbosity(log_verbosity); err != nil {
return err
}
log.UseTimestamp = log_timestamp

return log.InitLoggers()
},
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, args []string) (err error) {
log.Info("Fish init...")

cfg := &fish.Config{}
if err := cfg.ReadConfigFile(cfg_path); err != nil {
if err = cfg.ReadConfigFile(cfg_path); err != nil {
return log.Error("Fish: Unable to apply config file:", cfg_path, err)
}
if api_address != "" {
Expand All @@ -82,9 +89,31 @@ func main() {
if dir != "" {
cfg.Directory = dir
}
if cpu_limit != "" {
val, err := strconv.ParseUint(cpu_limit, 10, 16)
if err != nil {
return log.Errorf("Fish: Unable to parse cpu limit value: %v", err)
}
cfg.CpuLimit = uint16(val)
}
if mem_limit != "" {
if cfg.MemLimit, err = util.NewHumanSize(mem_limit); err != nil {
return log.Errorf("Fish: Unable to parse mem limit value: %v", err)
}
}

// Set Fish Node resources limits
if cfg.CpuLimit > 0 {
log.Info("Fish CPU limited:", cfg.CpuLimit)
runtime.GOMAXPROCS(int(cfg.CpuLimit))
}
if cfg.MemLimit > 0 {
log.Info("Fish MEM limited:", cfg.MemLimit.String())
debug.SetMemoryLimit(int64(cfg.MemLimit.Bytes()))
}

dir := filepath.Join(cfg.Directory, cfg.NodeAddress)
if err := os.MkdirAll(dir, 0o750); err != nil {
if err = os.MkdirAll(dir, 0o750); err != nil {
return log.Errorf("Fish: Can't create working directory %s: %v", dir, err)
}

Expand All @@ -101,7 +130,7 @@ func main() {
if !filepath.IsAbs(cert_path) {
cert_path = filepath.Join(cfg.Directory, cert_path)
}
if err := crypt.InitTlsPairCa([]string{cfg.NodeName, cfg.NodeAddress}, ca_path, key_path, cert_path); err != nil {
if err = crypt.InitTlsPairCa([]string{cfg.NodeName, cfg.NodeAddress}, ca_path, key_path, cert_path); err != nil {
return err
}

Expand Down Expand Up @@ -187,7 +216,9 @@ func main() {
cluster_join = flags.StringSliceP("join", "j", nil, "addresses of existing cluster nodes to join, comma separated")
flags.StringVarP(&cfg_path, "cfg", "c", "", "yaml configuration file")
flags.StringVarP(&dir, "dir", "D", "", "database and other fish files directory")
flags.StringVarP(&log_verbosity, "verbosity", "v", "info", "log level (debug, info, warn, error")
flags.StringVar(&cpu_limit, "cpu", "", "max amount of threads fish node will be able to utilize, default - no limit")
flags.StringVar(&mem_limit, "mem", "", "target memory utilization for fish node to run GC more aggressively when too close")
flags.StringVarP(&log_verbosity, "verbosity", "v", "info", "log level (debug, info, warn, error)")
flags.BoolVar(&log_timestamp, "timestamp", true, "prepend timestamps for each log line")
flags.Lookup("timestamp").NoOptDefVal = "false"

Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ require (
github.com/spf13/cobra v1.7.0
github.com/steinfletcher/apitest v1.5.15
github.com/ulikunitz/xz v0.5.11
golang.org/x/crypto v0.17.0
golang.org/x/net v0.19.0
golang.org/x/crypto v0.26.0
golang.org/x/net v0.21.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/gorm v1.24.6
)
Expand Down Expand Up @@ -67,8 +67,8 @@ require (
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/sys v0.23.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/time v0.5.0 // indirect
gopkg.in/fsnotify.v1 v1.4.7 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
Expand Down
20 changes: 10 additions & 10 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -149,23 +149,23 @@ github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQ
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
60 changes: 41 additions & 19 deletions lib/crypt/crypt.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,20 @@ import (
)

const (
Algo_Argon2 = "Argon2"
Argon2_Memory = 524288
Argon2_Operations = 4
Argon2_Time = 1
Argon2_Threads = 1
Argon2_SaltBytes = 8
Argon2_StrBytes = 128
Argon2_Algo = "Argon2id"
// Default tuned to process at least 20 API requests/sec on 2CPU
Argon2_Memory = 64 * 1024 // 64MB
Argon2_Iterations = 1
Argon2_Threads = 8 // Optimal to quickly execute one request, with not much overhead
Argon2_SaltLen = 8
Argon2_HashLen = 32

// <= v0.7.4 hash params for backward-compatibility
// could easily choke the API system and cause OOMs so not recommended to use them
v074_Argon2_Algo = "Argon2"
v074_Argon2_Memory = 524288
v074_Argon2_Iterations = 1
v074_Argon2_Threads = 1

RandStringCharsetB58 = "abcdefghijkmnopqrstuvwxyz" +
"ABCDEFGHJKLMNPQRSTUVWXYZ123456789" // Base58
Expand All @@ -38,10 +45,18 @@ const (

type Hash struct {
Algo string
Prop properties `gorm:"embedded;embeddedPrefix:prop_"`
Salt []byte
Hash []byte
}

// Properties of Argon2id algo
type properties struct {
Memory uint32
Iterations uint32
Threads uint8
}

// Create random bytes of specified size
func RandBytes(size int) (data []byte) {
data = make([]byte, size)
Expand Down Expand Up @@ -70,27 +85,34 @@ func RandStringCharset(size int, charset string) string {
return string(data)
}

// Generate a salted hash for the input string
func Generate(password string, salt []byte) (hash Hash) {
hash.Algo = Algo_Argon2

// Check salt and if not provided - use generator
// Generate a salted hash for the input string with default parameters
func NewHash(input string, salt []byte) (h Hash) {
h.Algo = Argon2_Algo
if salt != nil {
hash.Salt = salt
h.Salt = salt
} else {
hash.Salt = RandBytes(Argon2_SaltBytes)
h.Salt = RandBytes(Argon2_SaltLen)
}
h.Prop.Iterations = Argon2_Iterations
h.Prop.Memory = Argon2_Memory
h.Prop.Threads = Argon2_Threads

// Create hash data
hash.Hash = argon2.IDKey([]byte(password), hash.Salt,
Argon2_Time, Argon2_Memory, Argon2_Threads, Argon2_StrBytes)
h.Hash = argon2.IDKey([]byte(input), h.Salt, h.Prop.Iterations, h.Prop.Memory, h.Prop.Threads, Argon2_HashLen)

return
}

// Compare string to generated hash
func (hash *Hash) IsEqual(password string) bool {
return bytes.Compare(hash.Hash, Generate(password, hash.Salt).Hash) == 0
// Check the input equal to the current hashed one
func (h *Hash) IsEqual(input string) bool {
if h.Algo == v074_Argon2_Algo {
// Legacy low-performant parameters, not defined in hash
h.Prop.Iterations = v074_Argon2_Iterations
h.Prop.Memory = v074_Argon2_Memory
h.Prop.Threads = v074_Argon2_Threads
}

return bytes.Compare(h.Hash, argon2.IDKey([]byte(input), h.Salt, h.Prop.Iterations, h.Prop.Memory, h.Prop.Threads, uint32(len(h.Hash)))) == 0
}

func (hash *Hash) IsEmpty() bool {
Expand Down
67 changes: 67 additions & 0 deletions lib/crypt/crypt_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/**
* Copyright 2024 Adobe. All rights reserved.
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. You may obtain a copy
* of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
* OF ANY KIND, either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/

package crypt

import (
"fmt"
"testing"
)

// To prevent compiler optimization
var result1 Hash
var result2 bool

// Tests user password hash function performance
func Benchmark_hash_new(b *testing.B) {
// To prevent compiler optimization
var r Hash

b.ResetTimer()
for n := 0; n < b.N; n++ {
r = NewHash(RandString(32), nil)
}
b.StopTimer()

result1 = r
}

// IsEqual is not that different from generating the new hash, but worth to
// check because it's used more often during application execution life
func Benchmark_hash_isequal(b *testing.B) {
h := NewHash(RandString(32), nil)

// To prevent compiler optimization
var r bool

b.ResetTimer()
for n := 0; n < b.N; n++ {
r = h.IsEqual(RandString(32))
}
b.StopTimer()

result2 = r
}

// Make sure the hash generation algo will be the same across the Fish versions to be
// certain the update will not cause issues with users passwords validation compatibility
func Test_ensure_hash_same(t *testing.T) {
input := "abcdefghijklmnopqrstuvwxyz012345"
salt := []byte("abcdefgh")
result := "f3ca14a142596fe4b1c5441cc962cf88b8f7c59243abfbca89f34e24aa8c9e25"

h := NewHash(input, salt)

if fmt.Sprintf("%x", h.Hash) != result {
t.Fatalf("The change of hashing algo props caused incompatibility in hash value")
}
}
12 changes: 7 additions & 5 deletions lib/fish/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@ import (
type Config struct {
Directory string `json:"directory"` // Where to store database and other useful data (if relative - to CWD)

APIAddress string `json:"api_address"` // Where to serve Web UI, API & Meta API
ProxySocksAddress string `json:"proxy_socks_address"` // Where to serve SOCKS5 proxy for the allocated resources
ProxySshAddress string `json:"proxy_ssh_address"` // Where to serve SSH proxy for the allocated resources
NodeAddress string `json:"node_address"` // What is the external address of the node
ClusterJoin []string `json:"cluster_join"` // The node addresses to join the cluster
APIAddress string `json:"api_address"` // Where to serve Web UI, API & Meta API
ProxySocksAddress string `json:"proxy_socks_address"` // Where to serve SOCKS5 proxy for the allocated resources
ProxySshAddress string `json:"proxy_ssh_address"` // Where to serve SSH proxy for the allocated resources
NodeAddress string `json:"node_address"` // What is the external address of the node
CpuLimit uint16 `json:"cpu_limit"` // How many CPU threads Node allowed to use (serve API, ...)
MemLimit util.HumanSize `json:"mem_limit"` // What's the target memory utilization by the Node (GC target where it becomes more aggressive)
ClusterJoin []string `json:"cluster_join"` // The node addresses to join the cluster

TLSKey string `json:"tls_key"` // TLS PEM private key (if relative - to directory)
TLSCrt string `json:"tls_crt"` // TLS PEM public certificate (if relative - to directory)
Expand Down
9 changes: 8 additions & 1 deletion lib/fish/user.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ func (f *Fish) UserAuth(name string, password string) *types.User {
return nil
}

if user.Hash.Algo != crypt.Argon2_Algo {
log.Warnf("Please regenerate password for user %q to improve the API performance", name)
}

if !user.Hash.IsEqual(password) {
log.Warn("Fish: Incorrect user password:", name)
return nil
Expand All @@ -78,7 +82,10 @@ func (f *Fish) UserNew(name string, password string) (string, *types.User, error
password = crypt.RandString(64)
}

user := &types.User{Name: name, Hash: crypt.Generate(password, nil)}
user := &types.User{
Name: name,
Hash: crypt.NewHash(password, nil),
}

if err := f.UserCreate(user); err != nil {
return "", nil, log.Error("Fish: Unable to create new user:", name, err)
Expand Down
Loading

0 comments on commit efb2f68

Please sign in to comment.