From d5b8903397a7acedd052f70d3f5cc2ea0d7a0381 Mon Sep 17 00:00:00 2001 From: Erik Kristensen Date: Tue, 30 Jan 2024 19:02:31 -0700 Subject: [PATCH] feat: split scanner to own package from nuke (#24) * feat: split scanner to own package from nuke * chore: fix golangci-lint errors * fix: rename scanner package to scan, adding more tests * chore: spelling fix * test: disable race detection until registry is not global --- .github/workflows/tests.yml | 2 +- pkg/nuke/nuke.go | 14 ++- pkg/nuke/nuke_filter_test.go | 41 +++--- pkg/nuke/nuke_run_test.go | 19 +-- pkg/nuke/nuke_test.go | 33 ++--- pkg/nuke/testsuite_test.go | 137 ++++++++++++++++++++ pkg/queue/queue.go | 10 +- pkg/{nuke => scan}/scan.go | 46 ++++--- pkg/{nuke => scan}/scan_test.go | 215 ++++++++------------------------ pkg/scan/testsuite_test.go | 159 +++++++++++++++++++++++ 10 files changed, 438 insertions(+), 238 deletions(-) create mode 100644 pkg/nuke/testsuite_test.go rename pkg/{nuke => scan}/scan.go (74%) rename pkg/{nuke => scan}/scan_test.go (59%) create mode 100644 pkg/scan/testsuite_test.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d370811..5bdbe00 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,7 +22,7 @@ jobs: go mod download - name: run go tests run: | - go test -timeout 60s -race -coverprofile=coverage.txt -covermode=atomic ./... + go test -timeout 60s -coverprofile=coverage.txt -covermode=atomic ./... - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 env: diff --git a/pkg/nuke/nuke.go b/pkg/nuke/nuke.go index c098724..8ab2705 100644 --- a/pkg/nuke/nuke.go +++ b/pkg/nuke/nuke.go @@ -13,10 +13,12 @@ import ( "github.com/sirupsen/logrus" liberrors "github.com/ekristen/libnuke/pkg/errors" + libsettings "github.com/ekristen/libnuke/pkg/settings" + "github.com/ekristen/libnuke/pkg/filter" "github.com/ekristen/libnuke/pkg/queue" "github.com/ekristen/libnuke/pkg/resource" - libsettings "github.com/ekristen/libnuke/pkg/settings" + "github.com/ekristen/libnuke/pkg/scan" "github.com/ekristen/libnuke/pkg/types" "github.com/ekristen/libnuke/pkg/utils" ) @@ -69,7 +71,7 @@ type Nuke struct { ValidateHandlers []func() error ResourceTypes map[resource.Scope]types.Collection - Scanners map[resource.Scope][]*Scanner + Scanners map[resource.Scope][]*scan.Scanner Queue queue.Queue // Queue is the queue of resources that will be processed scannerHashes []string // scannerHashes is used to track if a scanner has already been registered @@ -139,12 +141,12 @@ func (n *Nuke) RegisterResourceTypes(scope resource.Scope, resourceTypes ...stri // RegisterScanner is used to register a scanner against a scope. A scope is a string that is used to group resource // types together. A scanner is what is responsible for actually querying the API for resources and adding them to // the queue for processing. -func (n *Nuke) RegisterScanner(scope resource.Scope, scanner *Scanner) error { +func (n *Nuke) RegisterScanner(scope resource.Scope, instance *scan.Scanner) error { if n.Scanners == nil { - n.Scanners = make(map[resource.Scope][]*Scanner) + n.Scanners = make(map[resource.Scope][]*scan.Scanner) } - hashString := fmt.Sprintf("%s-%s", scope, scanner.Owner) + hashString := fmt.Sprintf("%s-%s", scope, instance.Owner) n.log.Debugf("hash: %s", hashString) if slices.Contains(n.scannerHashes, hashString) { return fmt.Errorf("scanner is already registered, you cannot register it twice") @@ -155,7 +157,7 @@ func (n *Nuke) RegisterScanner(scope resource.Scope, scanner *Scanner) error { } n.scannerHashes = append(n.scannerHashes, hashString) - n.Scanners[scope] = append(n.Scanners[scope], scanner) + n.Scanners[scope] = append(n.Scanners[scope], instance) return nil } diff --git a/pkg/nuke/nuke_filter_test.go b/pkg/nuke/nuke_filter_test.go index b980974..df9d611 100644 --- a/pkg/nuke/nuke_filter_test.go +++ b/pkg/nuke/nuke_filter_test.go @@ -12,12 +12,13 @@ import ( "github.com/ekristen/libnuke/pkg/filter" "github.com/ekristen/libnuke/pkg/queue" "github.com/ekristen/libnuke/pkg/resource" + "github.com/ekristen/libnuke/pkg/scan" "github.com/ekristen/libnuke/pkg/types" ) func Test_NukeFiltersBad(t *testing.T) { filters := filter.Filters{ - testResourceType: []filter.Filter{ + TestResourceType: []filter.Filter{ { Type: filter.Exact, }, @@ -35,10 +36,10 @@ func Test_NukeFiltersBad(t *testing.T) { func Test_NukeFiltersMatch(t *testing.T) { resource.ClearRegistry() - resource.Register(testResourceRegistration2) + resource.Register(TestResourceRegistration2) filters := filter.Filters{ - testResourceType2: []filter.Filter{ + TestResourceType2: []filter.Filter{ { Type: filter.Exact, Property: "test", @@ -55,9 +56,9 @@ func Test_NukeFiltersMatch(t *testing.T) { SessionOne: "testing", SecondResource: true, } - scanner := NewScanner("Owner", []string{testResourceType2}, opts) + newScanner := scan.NewScanner("Owner", []string{TestResourceType2}, opts) - sErr := n.RegisterScanner(testScope, scanner) + sErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, sErr) err := n.Scan(context.TODO()) @@ -68,10 +69,10 @@ func Test_NukeFiltersMatch(t *testing.T) { func Test_NukeFiltersMatchInverted(t *testing.T) { resource.ClearRegistry() - resource.Register(testResourceRegistration2) + resource.Register(TestResourceRegistration2) filters := filter.Filters{ - testResourceType2: []filter.Filter{ + TestResourceType2: []filter.Filter{ { Type: filter.Exact, Property: "test", @@ -89,9 +90,9 @@ func Test_NukeFiltersMatchInverted(t *testing.T) { SessionOne: "testing", SecondResource: true, } - scanner := NewScanner("Owner", []string{testResourceType2}, opts) + newScanner := scan.NewScanner("Owner", []string{TestResourceType2}, opts) - sErr := n.RegisterScanner(testScope, scanner) + sErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, sErr) err := n.Scan(context.TODO()) @@ -102,10 +103,10 @@ func Test_NukeFiltersMatchInverted(t *testing.T) { func Test_Nuke_Filters_NoMatch(t *testing.T) { resource.ClearRegistry() - resource.Register(testResourceRegistration2) + resource.Register(TestResourceRegistration2) filters := filter.Filters{ - testResourceType: []filter.Filter{ + TestResourceType: []filter.Filter{ { Type: filter.Exact, Property: "test", @@ -122,9 +123,9 @@ func Test_Nuke_Filters_NoMatch(t *testing.T) { SessionOne: "testing", SecondResource: true, } - scanner := NewScanner("Owner", []string{testResourceType2}, opts) + newScanner := scan.NewScanner("Owner", []string{TestResourceType2}, opts) - sErr := n.RegisterScanner(testScope, scanner) + sErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, sErr) err := n.Scan(context.TODO()) @@ -135,14 +136,14 @@ func Test_Nuke_Filters_NoMatch(t *testing.T) { func Test_Nuke_Filters_ErrorCustomProps(t *testing.T) { resource.ClearRegistry() - resource.Register(testResourceRegistration) + resource.Register(TestResourceRegistration) filters := filter.Filters{ - testResourceType: []filter.Filter{ + TestResourceType: []filter.Filter{ { Type: filter.Exact, Property: "Name", - Value: testResourceType, + Value: TestResourceType, }, }, } @@ -154,9 +155,9 @@ func Test_Nuke_Filters_ErrorCustomProps(t *testing.T) { opts := TestOpts{ SessionOne: "testing", } - scanner := NewScanner("Owner", []string{testResourceType}, opts) + newScanner := scan.NewScanner("Owner", []string{TestResourceType}, opts) - sErr := n.RegisterScanner(testScope, scanner) + sErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, sErr) err := n.Scan(context.TODO()) @@ -183,7 +184,7 @@ func (r *TestResourceFilter) Remove(_ context.Context) error { func Test_Nuke_Filters_Extra(t *testing.T) { filters := filter.Filters{ - testResourceType2: []filter.Filter{ + TestResourceType2: []filter.Filter{ { Type: filter.Glob, Property: "tag:aws:cloudformation:stack-name", @@ -198,7 +199,7 @@ func Test_Nuke_Filters_Extra(t *testing.T) { i := &queue.Item{ Resource: &TestResourceFilter{}, - Type: testResourceType2, + Type: TestResourceType2, } err := n.Filter(i) diff --git a/pkg/nuke/nuke_run_test.go b/pkg/nuke/nuke_run_test.go index 008ec4b..ac1edb9 100644 --- a/pkg/nuke/nuke_run_test.go +++ b/pkg/nuke/nuke_run_test.go @@ -11,6 +11,7 @@ import ( "github.com/ekristen/libnuke/pkg/queue" "github.com/ekristen/libnuke/pkg/resource" + "github.com/ekristen/libnuke/pkg/scan" ) type TestResourceSuccess struct { @@ -62,7 +63,7 @@ func Test_Nuke_Run_Simple(t *testing.T) { Lister: &TestResourceSuccessLister{}, }) - scannerErr := n.RegisterScanner(testScope, NewScanner("Owner", []string{"TestResourceSuccess"}, nil)) + scannerErr := n.RegisterScanner(testScope, scan.NewScanner("Owner", []string{"TestResourceSuccess"}, nil)) assert.NoError(t, scannerErr) runErr := n.Run(context.TODO()) @@ -108,7 +109,7 @@ func Test_NukeRunSimpleWithSecondPromptError(t *testing.T) { Lister: &TestResourceSuccessLister{}, }) - scannerErr := n.RegisterScanner(testScope, NewScanner("Owner", []string{"TestResourceSuccess"}, nil)) + scannerErr := n.RegisterScanner(testScope, scan.NewScanner("Owner", []string{"TestResourceSuccess"}, nil)) assert.NoError(t, scannerErr) runErr := n.Run(context.TODO()) @@ -122,7 +123,7 @@ func Test_Nuke_Run_SimpleWithNoDryRun(t *testing.T) { n.SetLogger(logrus.WithField("test", true)) n.SetRunSleep(time.Millisecond * 5) - scannerErr := n.RegisterScanner(testScope, NewScanner("Owner", []string{"TestResource4"}, nil)) + scannerErr := n.RegisterScanner(testScope, scan.NewScanner("Owner", []string{"TestResource4"}, nil)) assert.NoError(t, scannerErr) runErr := n.Run(context.TODO()) @@ -149,8 +150,8 @@ func Test_Nuke_Run_Failure(t *testing.T) { Lister: &TestResourceFailureLister{}, }) - scanner := NewScanner("Owner", []string{"TestResourceSuccess", "TestResourceFailure"}, nil) - scannerErr := n.RegisterScanner(testScope, scanner) + newScanner := scan.NewScanner("Owner", []string{"TestResourceSuccess", "TestResourceFailure"}, nil) + scannerErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, scannerErr) runErr := n.Run(context.TODO()) @@ -179,8 +180,8 @@ func Test_NukeRunWithMaxWaitRetries(t *testing.T) { Lister: &TestResourceWaitLister{}, }) - scanner := NewScanner("Owner", []string{"TestResourceSuccess"}, nil) - scannerErr := n.RegisterScanner(testScope, scanner) + newScanner := scan.NewScanner("Owner", []string{"TestResourceSuccess"}, nil) + scannerErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, scannerErr) runErr := n.Run(context.TODO()) @@ -233,8 +234,8 @@ func TestNuke_RunWithWaitOnDependencies(t *testing.T) { }, }) - scanner := NewScanner("Owner", []string{"TestResourceAlpha", "TestResourceBeta"}, nil) - scannerErr := n.RegisterScanner(testScope, scanner) + newScanner := scan.NewScanner("Owner", []string{"TestResourceAlpha", "TestResourceBeta"}, nil) + scannerErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, scannerErr) runErr := n.Run(context.TODO()) diff --git a/pkg/nuke/nuke_test.go b/pkg/nuke/nuke_test.go index 24c7c4a..0a91e02 100644 --- a/pkg/nuke/nuke_test.go +++ b/pkg/nuke/nuke_test.go @@ -13,8 +13,10 @@ import ( "github.com/stretchr/testify/assert" liberrors "github.com/ekristen/libnuke/pkg/errors" + "github.com/ekristen/libnuke/pkg/queue" "github.com/ekristen/libnuke/pkg/resource" + "github.com/ekristen/libnuke/pkg/scan" "github.com/ekristen/libnuke/pkg/settings" ) @@ -153,7 +155,7 @@ func Test_Nuke_Scanners(t *testing.T) { name: "test", } - s := NewScanner("test", []string{"TestResource"}, opts) + s := scan.NewScanner("test", []string{"TestResource"}, opts) err := n.RegisterScanner(testScope, s) assert.NoError(t, err) @@ -172,7 +174,7 @@ func Test_Nuke_Scanners_Duplicate(t *testing.T) { name: "test", } - s := NewScanner("test", []string{"TestResource"}, opts) + s := scan.NewScanner("test", []string{"TestResource"}, opts) err := n.RegisterScanner(testScope, s) assert.NoError(t, err) @@ -198,10 +200,10 @@ func TestNuke_RegisterMultipleScanners(t *testing.T) { return o } - s := NewScanner("test", []string{"TestResource"}, opts) + s := scan.NewScanner("test", []string{"TestResource"}, opts) assert.NoError(t, s.RegisterMutateOptsFunc(mutateOpts)) - s2 := NewScanner("test2", []string{"TestResource"}, opts) + s2 := scan.NewScanner("test2", []string{"TestResource"}, opts) assert.NoError(t, s2.RegisterMutateOptsFunc(mutateOpts)) assert.NoError(t, n.RegisterScanner(testScope, s)) @@ -227,9 +229,9 @@ func Test_Nuke_RegisterPrompt(t *testing.T) { func Test_Nuke_Scan(t *testing.T) { resource.ClearRegistry() - resource.Register(testResourceRegistration) + resource.Register(TestResourceRegistration) resource.Register(&resource.Registration{ - Name: testResourceType2, + Name: TestResourceType2, Scope: "account", Lister: TestResourceLister{ Filtered: true, @@ -243,9 +245,9 @@ func Test_Nuke_Scan(t *testing.T) { opts := TestOpts{ SessionOne: "testing", } - scanner := NewScanner("Owner", []string{testResourceType, testResourceType2}, opts) + newScanner := scan.NewScanner("Owner", []string{TestResourceType, TestResourceType2}, opts) - sErr := n.RegisterScanner(testScope, scanner) + sErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, sErr) err := n.Scan(context.TODO()) @@ -303,7 +305,7 @@ func Test_Nuke_HandleRemoveError(t *testing.T) { func Test_Nuke_Run(t *testing.T) { resource.ClearRegistry() - resource.Register(testResourceRegistration) + resource.Register(TestResourceRegistration) p := &Parameters{ Force: true, @@ -319,9 +321,9 @@ func Test_Nuke_Run(t *testing.T) { opts := TestOpts{ SessionOne: "testing", } - scanner := NewScanner("Owner", []string{testResourceType}, opts) + newScanner := scan.NewScanner("Owner", []string{TestResourceType}, opts) - sErr := n.RegisterScanner(testScope, scanner) + sErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, sErr) err := n.Run(context.TODO()) @@ -331,7 +333,7 @@ func Test_Nuke_Run(t *testing.T) { func Test_Nuke_Run_Error(t *testing.T) { resource.ClearRegistry() resource.Register(&resource.Registration{ - Name: testResourceType2, + Name: TestResourceType2, Scope: "account", Lister: TestResourceLister{ RemoveError: true, @@ -351,9 +353,9 @@ func Test_Nuke_Run_Error(t *testing.T) { opts := TestOpts{ SessionOne: "testing", } - scanner := NewScanner("Owner", []string{testResourceType2}, opts) + newScanner := scan.NewScanner("Owner", []string{TestResourceType2}, opts) - sErr := n.RegisterScanner(testScope, scanner) + sErr := n.RegisterScanner(testScope, newScanner) assert.NoError(t, sErr) err := n.Run(context.TODO()) @@ -432,11 +434,10 @@ func Test_Nuke_Run_ItemStateHold(t *testing.T) { Lister: &TestResource4Lister{}, }) - scannerErr := n.RegisterScanner(testScope, NewScanner("Owner", []string{"TestResource4"}, nil)) + scannerErr := n.RegisterScanner(testScope, scan.NewScanner("Owner", []string{"TestResource4"}, nil)) assert.NoError(t, scannerErr) runErr := n.Run(context.TODO()) assert.NoError(t, runErr) - assert.Equal(t, 5, n.Queue.Count(queue.ItemStateFinished)) } diff --git a/pkg/nuke/testsuite_test.go b/pkg/nuke/testsuite_test.go new file mode 100644 index 0000000..f4c5ab0 --- /dev/null +++ b/pkg/nuke/testsuite_test.go @@ -0,0 +1,137 @@ +package nuke + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/ekristen/libnuke/pkg/errors" + "github.com/ekristen/libnuke/pkg/resource" + "github.com/ekristen/libnuke/pkg/settings" + "github.com/ekristen/libnuke/pkg/types" +) + +var ( + TestResourceType = "testResourceType" + TestResourceRegistration = &resource.Registration{ + Name: TestResourceType, + Scope: "account", + Lister: &TestResourceLister{}, + } + + TestResourceType2 = "testResourceType2" + TestResourceRegistration2 = &resource.Registration{ + Name: TestResourceType2, + Scope: "account", + Lister: &TestResourceLister{}, + DependsOn: []string{ + TestResourceType, + }, + } +) + +type TestOpts struct { + Test *testing.T + SessionOne string + SessionTwo string + ThrowError bool + ThrowSkipError bool + ThrowEndpointError bool + Panic bool + SecondResource bool +} + +type TestResourceLister struct { + Filtered bool + RemoveError bool +} + +func (l TestResourceLister) List(_ context.Context, o interface{}) ([]resource.Resource, error) { + opts := o.(TestOpts) + + if opts.ThrowError { + return nil, assert.AnError + } + + if opts.ThrowSkipError { + return nil, errors.ErrSkipRequest("skip request error for testing") + } + + if opts.ThrowEndpointError { + return nil, errors.ErrUnknownEndpoint("unknown endpoint error for testing") + } + + if opts.Panic { + panic(fmt.Errorf("panic error for testing")) + } + + if opts.SecondResource { + return []resource.Resource{ + &TestResource2{ + Filtered: l.Filtered, + RemoveError: l.RemoveError, + }, + }, nil + } + + return []resource.Resource{ + &TestResource{ + Filtered: l.Filtered, + RemoveError: l.RemoveError, + }, + }, nil +} + +// -------------------------------------------------------------------------- + +type TestResource struct { + Filtered bool + RemoveError bool +} + +func (r *TestResource) Filter() error { + if r.Filtered { + return fmt.Errorf("cannot remove default") + } + + return nil +} + +func (r *TestResource) Remove(_ context.Context) error { + if r.RemoveError { + return fmt.Errorf("remove error") + } + return nil +} + +func (r *TestResource) Settings(setting *settings.Setting) { + +} + +type TestResource2 struct { + Filtered bool + RemoveError bool +} + +func (r *TestResource2) Filter() error { + if r.Filtered { + return fmt.Errorf("cannot remove default") + } + + return nil +} + +func (r *TestResource2) Remove(_ context.Context) error { + if r.RemoveError { + return fmt.Errorf("remove error") + } + return nil +} + +func (r *TestResource2) Properties() types.Properties { + props := types.NewProperties() + props.Set("test", "testing") + return props +} diff --git a/pkg/queue/queue.go b/pkg/queue/queue.go index 732af68..1ad537e 100644 --- a/pkg/queue/queue.go +++ b/pkg/queue/queue.go @@ -25,9 +25,9 @@ func (q Queue) Total() int { // Count returns the total number of items in a specific ItemState from the Queue func (q Queue) Count(states ...ItemState) int { count := 0 - for _, item := range q.Items { + for _, i := range q.Items { for _, state := range states { - if item.GetState() == state { + if i.GetState() == state { count++ break } @@ -39,10 +39,10 @@ func (q Queue) Count(states ...ItemState) int { // CountByType returns the total number of items that match a ResourceType and specific ItemState from the Queue func (q Queue) CountByType(resourceType string, states ...ItemState) int { count := 0 - for _, item := range q.Items { - if item.Type == resourceType { + for _, i := range q.Items { + if i.Type == resourceType { for _, state := range states { - if item.GetState() == state { + if i.GetState() == state { count++ break } diff --git a/pkg/nuke/scan.go b/pkg/scan/scan.go similarity index 74% rename from pkg/nuke/scan.go rename to pkg/scan/scan.go index 5b82666..cd67ac4 100644 --- a/pkg/nuke/scan.go +++ b/pkg/scan/scan.go @@ -1,32 +1,35 @@ -package nuke +package scan import ( "context" "errors" "fmt" + "runtime/debug" "github.com/sirupsen/logrus" "golang.org/x/sync/semaphore" - sdkerrors "github.com/ekristen/libnuke/pkg/errors" + liberrors "github.com/ekristen/libnuke/pkg/errors" + "github.com/ekristen/libnuke/pkg/queue" "github.com/ekristen/libnuke/pkg/resource" "github.com/ekristen/libnuke/pkg/utils" ) -// ScannerParallelQueries is the number of parallel queries to run at any given time for a scanner. -const ScannerParallelQueries = 16 +// DefaultParallelQueries is the number of parallel queries to run at any given time for a scanner. +const DefaultParallelQueries = 16 // Scanner is collection of resource types that will be scanned for existing resources and added to the // item queue for processing. These items will be filtered and then processed. type Scanner struct { - Items chan *queue.Item `hash:"ignore"` - semaphore *semaphore.Weighted `hash:"ignore"` - ResourceTypes []string - Options interface{} - Owner string - mutateOptsFunc MutateOptsFunc `hash:"ignore"` + Items chan *queue.Item `hash:"ignore"` + semaphore *semaphore.Weighted `hash:"ignore"` + ResourceTypes []string + Options interface{} + Owner string + mutateOptsFunc MutateOptsFunc `hash:"ignore"` + parallelQueries int64 } // MutateOptsFunc is a function that can mutate the Options for a given resource type. This is useful for when you @@ -37,11 +40,12 @@ type MutateOptsFunc func(opts interface{}, resourceType string) interface{} // NewScanner creates a new scanner for the given resource types. func NewScanner(owner string, resourceTypes []string, opts interface{}) *Scanner { return &Scanner{ - Items: make(chan *queue.Item, 10000), - semaphore: semaphore.NewWeighted(ScannerParallelQueries), - ResourceTypes: resourceTypes, - Options: opts, - Owner: owner, + Items: make(chan *queue.Item, 10000), + semaphore: semaphore.NewWeighted(DefaultParallelQueries), + ResourceTypes: resourceTypes, + Options: opts, + Owner: owner, + parallelQueries: DefaultParallelQueries, } } @@ -60,6 +64,12 @@ func (s *Scanner) RegisterMutateOptsFunc(morph MutateOptsFunc) error { return nil } +// SetParallelQueries changes the number of parallel queries to run at any given time from the default for the scanner. +func (s *Scanner) SetParallelQueries(parallelQueries int64) { + s.parallelQueries = parallelQueries + s.semaphore = semaphore.NewWeighted(s.parallelQueries) +} + // Run starts the scanner and runs the lister for each resource type. func (s *Scanner) Run(ctx context.Context) error { for _, resourceType := range s.ResourceTypes { @@ -76,7 +86,7 @@ func (s *Scanner) Run(ctx context.Context) error { } // Wait for all routines to finish. - if err := s.semaphore.Acquire(ctx, ScannerParallelQueries); err != nil { + if err := s.semaphore.Acquire(ctx, s.parallelQueries); err != nil { return err } @@ -104,14 +114,14 @@ func (s *Scanner) list(ctx context.Context, owner, resourceType string, opts int rs, err := lister.List(ctx, opts) if err != nil { - var errSkipRequest sdkerrors.ErrSkipRequest + var errSkipRequest liberrors.ErrSkipRequest ok := errors.As(err, &errSkipRequest) if ok { logrus.Debugf("skipping request: %v", err) return } - var errUnknownEndpoint sdkerrors.ErrUnknownEndpoint + var errUnknownEndpoint liberrors.ErrUnknownEndpoint ok = errors.As(err, &errUnknownEndpoint) if ok { logrus.Debugf("skipping request: %v", err) diff --git a/pkg/nuke/scan_test.go b/pkg/scan/scan_test.go similarity index 59% rename from pkg/nuke/scan_test.go rename to pkg/scan/scan_test.go index 8d28e80..901d81f 100644 --- a/pkg/nuke/scan_test.go +++ b/pkg/scan/scan_test.go @@ -1,168 +1,18 @@ -package nuke +package scan import ( "context" - "flag" - "fmt" - "io" "strings" + "sync" "testing" + "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "github.com/ekristen/libnuke/pkg/errors" "github.com/ekristen/libnuke/pkg/resource" - "github.com/ekristen/libnuke/pkg/settings" - "github.com/ekristen/libnuke/pkg/types" ) -func init() { - if flag.Lookup("test.v") != nil { - logrus.SetOutput(io.Discard) - } - logrus.SetLevel(logrus.TraceLevel) - logrus.SetReportCaller(true) -} - -var ( - testResourceType = "testResourceType" - testResourceRegistration = &resource.Registration{ - Name: testResourceType, - Scope: "account", - Lister: &TestResourceLister{}, - } - - testResourceType2 = "testResourceType2" - testResourceRegistration2 = &resource.Registration{ - Name: testResourceType2, - Scope: "account", - Lister: &TestResourceLister{}, - DependsOn: []string{ - testResourceType, - }, - } -) - -type TestResource struct { - Filtered bool - RemoveError bool -} - -func (r *TestResource) Filter() error { - if r.Filtered { - return fmt.Errorf("cannot remove default") - } - - return nil -} - -func (r *TestResource) Remove(_ context.Context) error { - if r.RemoveError { - return fmt.Errorf("remove error") - } - return nil -} - -func (r *TestResource) Settings(setting *settings.Setting) { - -} - -type TestResource2 struct { - Filtered bool - RemoveError bool -} - -func (r *TestResource2) Filter() error { - if r.Filtered { - return fmt.Errorf("cannot remove default") - } - - return nil -} - -func (r *TestResource2) Remove(_ context.Context) error { - if r.RemoveError { - return fmt.Errorf("remove error") - } - return nil -} - -func (r *TestResource2) Properties() types.Properties { - props := types.NewProperties() - props.Set("test", "testing") - return props -} - -type TestResourceLister struct { - Filtered bool - RemoveError bool -} - -func (l TestResourceLister) List(_ context.Context, o interface{}) ([]resource.Resource, error) { - opts := o.(TestOpts) - - if opts.ThrowError { - return nil, assert.AnError - } - - if opts.ThrowSkipError { - return nil, errors.ErrSkipRequest("skip request error for testing") - } - - if opts.ThrowEndpointError { - return nil, errors.ErrUnknownEndpoint("unknown endpoint error for testing") - } - - if opts.Panic { - panic(fmt.Errorf("panic error for testing")) - } - - if opts.SecondResource { - return []resource.Resource{ - &TestResource2{ - Filtered: l.Filtered, - RemoveError: l.RemoveError, - }, - }, nil - } - - return []resource.Resource{ - &TestResource{ - Filtered: l.Filtered, - RemoveError: l.RemoveError, - }, - }, nil -} - -type TestOpts struct { - Test *testing.T - SessionOne string - SessionTwo string - ThrowError bool - ThrowSkipError bool - ThrowEndpointError bool - Panic bool - SecondResource bool -} - -type TestGlobalHook struct { - t *testing.T - tf func(t *testing.T, e *logrus.Entry) -} - -func (h *TestGlobalHook) Levels() []logrus.Level { - return logrus.AllLevels -} - -func (h *TestGlobalHook) Fire(e *logrus.Entry) error { - if h.tf != nil { - h.tf(h.t, e) - } - - return nil -} - func Test_NewScannerWithMorphOpts(t *testing.T) { resource.ClearRegistry() resource.Register(testResourceRegistration) @@ -181,6 +31,8 @@ func Test_NewScannerWithMorphOpts(t *testing.T) { mutateErr := scanner.RegisterMutateOptsFunc(morphOpts) assert.NoError(t, mutateErr) + scanner.SetParallelQueries(8) + err := scanner.Run(context.TODO()) assert.NoError(t, err) @@ -308,10 +160,44 @@ func Test_NewScannerWithResourceListerErrorUnknownEndpoint(t *testing.T) { assert.Len(t, scanner.Items, 0) } -/* -TODO: fix - when run as a whole, this panics but doesn't get caught properly instead the test suite panics and exits +func TestRunSemaphoreFirstAcquireError(t *testing.T) { + // Create a new scanner + scanner := NewScanner("owner", []string{testResourceType}, nil) + scanner.SetParallelQueries(0) + + // Create a context that will be canceled immediately + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + defer cancel() + + // Run the scanner + err := scanner.Run(ctx) + assert.Error(t, err) +} + +func TestRunSemaphoreSecondAcquireError(t *testing.T) { + resource.ClearRegistry() + resource.Register(testResourceRegistration) + // Create a new scanner + scanner := NewScanner("owner", []string{testResourceType}, TestOpts{ + Sleep: 45 * time.Second, + }) + + // Create a context that will be canceled immediately + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + + // Run the scanner + err := scanner.Run(ctx) + assert.Error(t, err) +} func Test_NewScannerWithResourceListerPanic(t *testing.T) { + var wg sync.WaitGroup + + wg.Add(2) + + panicCaught := false + resource.ClearRegistry() logrus.AddHook(&TestGlobalHook{ t: t, @@ -319,14 +205,16 @@ func Test_NewScannerWithResourceListerPanic(t *testing.T) { if strings.HasSuffix(e.Caller.File, "pkg/resource/registry.go") { assert.Equal(t, logrus.TraceLevel, e.Level) assert.Equal(t, "registered resource lister", e.Message) + wg.Done() return } - if strings.HasSuffix(e.Caller.File, "pkg/nuke/scan.go") { - assert.Contains(t, e.Message, "Listing testResourceType failed:\n assert.AnError general error for testing") - assert.Contains(t, e.Message, "goroutine") - assert.Contains(t, e.Message, "runtime/debug.Stack()") + if strings.HasSuffix(e.Caller.File, "pkg/scan/scan.go") && e.Caller.Line == 106 { + assert.Contains(t, e.Message, "Listing testResourceType failed") + assert.Contains(t, e.Message, "panic error for testing") logrus.StandardLogger().ReplaceHooks(make(logrus.LevelHooks)) + panicCaught = true + wg.Done() } }, }) @@ -338,9 +226,10 @@ func Test_NewScannerWithResourceListerPanic(t *testing.T) { Panic: true, } - scanner := NewScanner("Owner", []string{testResourceType}, opts, nil) - scanner.Run() + scanner := NewScanner("Owner", []string{testResourceType}, opts) + _ = scanner.Run(context.TODO()) + + wg.Wait() - assert.Len(t, scanner.Items, 0) + assert.True(t, panicCaught) } -*/ diff --git a/pkg/scan/testsuite_test.go b/pkg/scan/testsuite_test.go new file mode 100644 index 0000000..d1a4460 --- /dev/null +++ b/pkg/scan/testsuite_test.go @@ -0,0 +1,159 @@ +package scan + +import ( + "context" + "flag" + "fmt" + "io" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + + "github.com/ekristen/libnuke/pkg/errors" + "github.com/ekristen/libnuke/pkg/resource" + "github.com/ekristen/libnuke/pkg/settings" + "github.com/ekristen/libnuke/pkg/types" +) + +func init() { + if flag.Lookup("test.v") != nil { + logrus.SetOutput(io.Discard) + } + logrus.SetLevel(logrus.TraceLevel) + logrus.SetReportCaller(true) +} + +var ( + testResourceType = "testResourceType" + testResourceRegistration = &resource.Registration{ + Name: testResourceType, + Scope: "account", + Lister: &TestResourceLister{}, + } +) + +type TestResource struct { + Filtered bool + RemoveError bool +} + +func (r *TestResource) Filter() error { + if r.Filtered { + return fmt.Errorf("cannot remove default") + } + + return nil +} + +func (r *TestResource) Remove(_ context.Context) error { + if r.RemoveError { + return fmt.Errorf("remove error") + } + return nil +} + +func (r *TestResource) Settings(setting *settings.Setting) { + +} + +type TestResource2 struct { + Filtered bool + RemoveError bool +} + +func (r *TestResource2) Filter() error { + if r.Filtered { + return fmt.Errorf("cannot remove default") + } + + return nil +} + +func (r *TestResource2) Remove(_ context.Context) error { + if r.RemoveError { + return fmt.Errorf("remove error") + } + return nil +} + +func (r *TestResource2) Properties() types.Properties { + props := types.NewProperties() + props.Set("test", "testing") + return props +} + +type TestResourceLister struct { + Filtered bool + RemoveError bool +} + +func (l TestResourceLister) List(_ context.Context, o interface{}) ([]resource.Resource, error) { + opts := o.(TestOpts) + + if opts.ThrowError { + return nil, assert.AnError + } + + if opts.ThrowSkipError { + return nil, errors.ErrSkipRequest("skip request error for testing") + } + + if opts.ThrowEndpointError { + return nil, errors.ErrUnknownEndpoint("unknown endpoint error for testing") + } + + if opts.Panic { + panic(fmt.Errorf("panic error for testing")) + } + + if opts.Sleep > 0 { + time.Sleep(opts.Sleep) + } + + if opts.SecondResource { + return []resource.Resource{ + &TestResource2{ + Filtered: l.Filtered, + RemoveError: l.RemoveError, + }, + }, nil + } + + return []resource.Resource{ + &TestResource{ + Filtered: l.Filtered, + RemoveError: l.RemoveError, + }, + }, nil +} + +type TestOpts struct { + Test *testing.T + SessionOne string + SessionTwo string + ThrowError bool + ThrowSkipError bool + ThrowEndpointError bool + Panic bool + SecondResource bool + Sleep time.Duration +} + +type TestGlobalHook struct { + t *testing.T + tf func(t *testing.T, e *logrus.Entry) +} + +func (h *TestGlobalHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +func (h *TestGlobalHook) Fire(e *logrus.Entry) error { + if h.tf != nil { + h.tf(h.t, e) + } + + return nil +}