Skip to content

Commit

Permalink
chore(file domain): add network.DownloadFile and use go-getter to dow…
Browse files Browse the repository at this point in the history
…nload files (#760)

* chore(file domain): use go-getter to download files
* validate assessment results in file download test
  • Loading branch information
mildwonkey authored Nov 8, 2024
1 parent fae5e4c commit 9654993
Show file tree
Hide file tree
Showing 11 changed files with 261 additions and 103 deletions.
9 changes: 9 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/defenseunicorns/pkg/kubernetes v0.3.0
github.com/evertras/bubble-table v0.17.1
github.com/google/go-cmp v0.6.0
github.com/hashicorp/go-getter/v2 v2.2.3
github.com/hashicorp/go-version v1.7.0
github.com/kyverno/kyverno-json v0.0.3
github.com/mattn/go-runewidth v0.0.16
Expand Down Expand Up @@ -56,6 +57,7 @@ require (
github.com/aymanbagabas/go-udiff v0.2.0 // indirect
github.com/basgys/goxml2json v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/bufbuild/protocompile v0.6.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
Expand Down Expand Up @@ -93,6 +95,10 @@ require (
github.com/gorilla/mux v1.8.1 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-safetemp v1.0.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/hcl/v2 v2.17.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
Expand All @@ -110,6 +116,8 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/buildkit v0.16.0 // indirect
Expand Down Expand Up @@ -150,6 +158,7 @@ require (
github.com/tchap/go-patricia/v2 v2.3.1 // indirect
github.com/tmccombs/hcl2json v0.3.1 // indirect
github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vladimirvivien/gexe v0.3.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
Expand Down
18 changes: 18 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUK
github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y=
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
Expand Down Expand Up @@ -236,6 +238,17 @@ github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJr
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-getter/v2 v2.2.3 h1:6CVzhT0KJQHqd9b0pK3xSP0CM/Cv+bVhk+jcaRJ2pGk=
github.com/hashicorp/go-getter/v2 v2.2.3/go.mod h1:hp5Yy0GMQvwWVUmwLs3ygivz1JSLI323hdIE9J9m7TY=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
Expand Down Expand Up @@ -307,7 +320,10 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
Expand Down Expand Up @@ -461,6 +477,8 @@ github.com/tmccombs/hcl2json v0.3.1/go.mod h1:ljY0/prd2IFUF3cagQjV3cpPEEQKzqyGqn
github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 h1:7I5c2Ig/5FgqkYOh/N87NzoyI9U15qUPXhDD8uCupv8=
github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/vektah/gqlparser v1.2.0/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74=
github.com/vladimirvivien/gexe v0.3.0 h1:4xwiOwGrDob5OMR6E92B9olDXYDglXdHhzR1ggYtWJM=
github.com/vladimirvivien/gexe v0.3.0/go.mod h1:fp7cy60ON1xjhtEI/+bfSEIXX35qgmI+iRYlGOqbBFM=
Expand Down
45 changes: 45 additions & 0 deletions src/pkg/common/network/download.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package network

import (
"context"

"github.com/hashicorp/go-getter/v2"
)

// DownloadFile downloads a file to the requested dst (full file path) from src.
// wd is used when resolving relative paths and should be set to the common root
// directory of relative paths (defaults to the directory the input file is in).
// Go-getter supports the following protocols, though not all are relevant in
// file mode (and Lula can extend the Getter interface if we need to add to this
// list):
//
// - Local files
// - Git
// - Mercurial
// - HTTP
// - Amazon S3
// - Google GCP
// - SMB
// - Source: https://pkg.go.dev/github.com/hashicorp/go-getter/v2#section-readme
//
// DownloadFile returns the actual download path and error. The error returned
// may be a go-multierror.
func DownloadFile(ctx context.Context, dst, src, wd string) (string, error) {
client := getter.Client{
// following symlinks is a security concern
DisableSymlinks: true,
}

rsp, err := client.Get(ctx, &getter.Request{
Src: src,
Dst: dst,
Pwd: wd,
Copy: true, // if we are getting a local file, copy it instead of making a symlink
GetMode: getter.ModeFile,
})
if rsp == nil {
return "", err
}

return rsp.Dst, err
}
68 changes: 38 additions & 30 deletions src/pkg/domains/files/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ import (
"os"
"path/filepath"

"github.com/open-policy-agent/conftest/parser"

"github.com/defenseunicorns/lula/src/pkg/common/network"
"github.com/defenseunicorns/lula/src/types"
"github.com/open-policy-agent/conftest/parser"
)

type Domain struct {
Expand All @@ -24,30 +25,33 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
var errs error
tmpDRs := make(map[string]interface{})
if workDir, ok = ctx.Value(types.LulaValidationWorkDir).(string); !ok {
// if unset, assume lula is working in the same directory the inputFile is in
// if unset, assume lula is already working in the same directory the inputFile is in
workDir = "."
}

// see TODO below: maybe this is a REAL directory?
dst, err := os.MkdirTemp("", "lula-files")
dst, err := os.MkdirTemp("", "lula-files-")
if err != nil {
// allow returning on error here?
return nil, err
}

// TODO? this might be a nice configurable option (for debugging) - store
// the files into a local .lula directory that doesn't necessarily get
// removed.
// (potential) TODO: this could be a nice configurable option (for
// debugging) - store the files into a local .lula directory that doesn't
// get removed.
defer os.RemoveAll(dst)

// make a map of rel filepaths to the user-supplied name, so we can re-key the DomainResources later on.
// filenames stores a map of relative filenames to the user-supplied Name,
// so we can re-key the DomainResources later on.
filenames := make(map[string]string, 0)

// unstructuredFiles is used to store a list of files that Lula needs to parse.
// unstructuredFiles is used to store a list of files that Lula needs to
// parse as strings.
unstructuredFiles := make([]FileInfo, 0)
// filesWithParsers stores files with user-specified parsers to pass to
// conftest.
filesWithParsers := make(map[string][]FileInfo, 0)

// Copy files to a temporary location
// Copy files to a temporary location. In this loop we only grab files that
// don't have configured parsers.
for _, fi := range d.Spec.Filepaths {
if fi.Parser != "" {
if fi.Parser == "string" {
Expand All @@ -59,18 +63,17 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
}
}

file := filepath.Join(workDir, fi.Path)
relname, err := copyFile(dst, file)
realdst := filepath.Join(dst, filepath.Base(fi.Path))
filename, err := copyFile(ctx, realdst, fi.Path, workDir)
if err != nil {
// Assign empty data value for reporting purposes
tmpDRs[fi.Name] = map[string]interface{}{}
filenames[file] = fi.Name
errs = errors.Join(errs, fmt.Errorf("error writing local files: %w", err))
continue
}

// and save this info for later
filenames[relname] = fi.Name
filenames[filename] = fi.Name
}

// get a list of all the files we just downloaded in the temporary directory
Expand Down Expand Up @@ -106,7 +109,6 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
}

// Now for the custom parsing: user-specified parsers and string files.

for parserName, filesByParser := range filesWithParsers {
// make a sub directory by parser name
parserDir, err := os.MkdirTemp(dst, parserName)
Expand All @@ -115,8 +117,8 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
}

for _, fi := range filesByParser {
file := filepath.Join(workDir, fi.Path)
relname, err := copyFile(parserDir, file)
dst := filepath.Join(parserDir, filepath.Base(fi.Path))
relname, err := copyFile(ctx, dst, fi.Path, workDir)
if err != nil {
drs[fi.Name] = map[string]interface{}{}
errs = errors.Join(errs, fmt.Errorf("error writing local files: %w", err))
Expand Down Expand Up @@ -150,15 +152,24 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)

// add the string form of the unstructured files
for _, f := range unstructuredFiles {
// we don't need to copy these files, we'll just slurp the contents into
// a string and append that as one big DomainResource
path := filepath.Clean(filepath.Join(workDir, f.Path))
b, err := os.ReadFile(path)
stringdir, err := os.MkdirTemp(dst, "string")
if err != nil {
errs = errors.Join(errs, fmt.Errorf("error reading source files: %w", err))
drs[f.Name] = ""
continue
}

dst := filepath.Clean(filepath.Join(stringdir, f.Path))
_, err = copyFile(ctx, dst, f.Path, workDir)
if err != nil {
return nil, fmt.Errorf("error writing local files: %w", err)
}

b, err := os.ReadFile(dst)
if err != nil {
return nil, fmt.Errorf("error reading local file: %w", err)
}

drs[f.Name] = string(b)
}

Expand All @@ -179,17 +190,14 @@ func CreateDomain(spec *Spec) (types.Domain, error) {
return Domain{spec}, nil
}

// copyFile is a helper function that copies a file from source to dst, and returns the relative file path between the two.
func copyFile(dst string, src string) (string, error) {
bytes, err := network.Fetch(src)
// copyFile is a helper function that copies a file from source to dst, and
// returns the base filename.
func copyFile(ctx context.Context, dst, src, wd string) (string, error) {
realdst, err := network.DownloadFile(ctx, dst, src, wd)
if err != nil {
return "", fmt.Errorf("error getting source files: %w", err)
}

// We'll use the filename when writing the file so it's easier to reference later
relname := filepath.Base(src)

return relname, os.WriteFile(filepath.Join(dst, relname), bytes, 0600) // G306
return filepath.Base(realdst), nil
}

func listFiles(dir string) ([]string, error) {
Expand Down
12 changes: 5 additions & 7 deletions src/test/e2e/dev_get_resources_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,16 @@ import (
"testing"
"time"

"github.com/defenseunicorns/lula/src/cmd/dev"
"github.com/defenseunicorns/lula/src/pkg/common"
"github.com/defenseunicorns/lula/src/pkg/message"
"github.com/defenseunicorns/lula/src/test/util"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/e2e-framework/klient/wait"
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
"sigs.k8s.io/e2e-framework/pkg/envconf"
"sigs.k8s.io/e2e-framework/pkg/features"

"github.com/defenseunicorns/lula/src/cmd/dev"
"github.com/defenseunicorns/lula/src/pkg/common"
"github.com/defenseunicorns/lula/src/pkg/message"
"github.com/defenseunicorns/lula/src/test/util"
)

func TestGetResources(t *testing.T) {
Expand Down Expand Up @@ -82,9 +83,6 @@ func TestGetResources(t *testing.T) {
if len(collection["empty"].([]map[string]interface{})) != 0 {
t.Fatalf("expected 0 length items in the empty payload - got %v\n", len(collection["empty"].([]map[string]interface{})))
}

message.Info("Successfully validated dev get-resources command")

return ctx
}).
Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
Expand Down
1 change: 0 additions & 1 deletion src/test/e2e/dev_lint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,5 +56,4 @@ func TestLintCommand(t *testing.T) {
}
})
}

}
15 changes: 5 additions & 10 deletions src/test/e2e/dev_validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,16 @@ import (
"testing"
"time"

"github.com/defenseunicorns/lula/src/cmd/dev"
"github.com/defenseunicorns/lula/src/pkg/common"
"github.com/defenseunicorns/lula/src/pkg/message"
"github.com/defenseunicorns/lula/src/test/util"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/e2e-framework/klient/wait"
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
"sigs.k8s.io/e2e-framework/pkg/envconf"
"sigs.k8s.io/e2e-framework/pkg/features"

"github.com/defenseunicorns/lula/src/cmd/dev"
"github.com/defenseunicorns/lula/src/pkg/common"
"github.com/defenseunicorns/lula/src/pkg/message"
"github.com/defenseunicorns/lula/src/test/util"
)

func TestDevValidation(t *testing.T) {
Expand Down Expand Up @@ -63,8 +64,6 @@ func TestDevValidation(t *testing.T) {
t.Errorf("Validation failed")
}

message.Infof("Successfully validated dev validate command with OPA")

return ctx
}).
Assess("Validate DevValidate kyverno", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
Expand All @@ -91,8 +90,6 @@ func TestDevValidation(t *testing.T) {
t.Errorf("Validation failed")
}

message.Infof("Successfully validated dev validate command with kyverno")

return ctx
}).
Assess("Validate with resources", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
Expand Down Expand Up @@ -126,8 +123,6 @@ func TestDevValidation(t *testing.T) {
t.Errorf("expected result to be %t got %t", expectedResult, result)
}

message.Infof("Successfully validated dev validate command with resources")

return ctx
}).
Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
Expand Down
Loading

0 comments on commit 9654993

Please sign in to comment.