From cd1e30ca0dc1bd6383fed0786c0e7e9481b4dfa3 Mon Sep 17 00:00:00 2001 From: Yusuf Kanchwala Date: Tue, 11 Aug 2020 20:17:43 +0530 Subject: [PATCH 1/4] refactoring policy package --- pkg/cli/run.go | 7 +++-- pkg/policy/interface.go | 8 +++-- pkg/policy/opa/engine.go | 35 +++++++++++++++++---- pkg/results/store.go | 7 +++++ pkg/runtime/executor.go | 67 ++++++++++++++++++---------------------- 5 files changed, 77 insertions(+), 47 deletions(-) diff --git a/pkg/cli/run.go b/pkg/cli/run.go index d766e1b71..280460d59 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -17,7 +17,10 @@ package cli import ( + "os" + "github.com/accurics/terrascan/pkg/runtime" + "github.com/accurics/terrascan/pkg/utils" ) // Run executes terrascan in CLI mode @@ -31,9 +34,9 @@ func Run(iacType, iacVersion, cloudType, iacFilePath, iacDirPath, configFile, po } // executor output - _, err = executor.Execute() + violations, err := executor.Execute() if err != nil { return } - // utils.PrintJSON(violations, os.Stdout) + utils.PrintJSON(violations, os.Stdout) } diff --git a/pkg/policy/interface.go b/pkg/policy/interface.go index 3aa3a548b..c686cd58a 100644 --- a/pkg/policy/interface.go +++ b/pkg/policy/interface.go @@ -16,6 +16,10 @@ package policy +import ( + "github.com/accurics/terrascan/pkg/results" +) + // Manager Policy Manager interface type Manager interface { Import() error @@ -25,9 +29,9 @@ type Manager interface { // Engine Policy Engine interface type Engine interface { - Initialize(policyPath string) error + Init(string) error Configure() error - Evaluate(inputData *interface{}) error + Evaluate(*interface{}) ([]*results.Violation, error) GetResults() error Release() error } diff --git a/pkg/policy/opa/engine.go b/pkg/policy/opa/engine.go index 4e7c756fb..5e0b86095 100644 --- a/pkg/policy/opa/engine.go +++ b/pkg/policy/opa/engine.go @@ -36,6 +36,26 @@ import ( "go.uber.org/zap" ) +var ( + errInitFailed = fmt.Errorf("failed to initialize OPA policy engine") +) + +// NewEngine returns a new OPA policy engine +func NewEngine(policyPath string) (*Engine, error) { + + // opa engine struct + engine := &Engine{} + + // initialize the engine + if err := engine.Init(policyPath); err != nil { + zap.S().Error("failed to initialize OPA policy engine") + return engine, errInitFailed + } + + // successful + return engine, nil +} + // LoadRegoMetadata Loads rego metadata from a given file func (e *Engine) LoadRegoMetadata(metaFilename string) (*RegoMetadata, error) { // Load metadata file if it exists @@ -202,9 +222,9 @@ func (e *Engine) CompileRegoFiles() error { return nil } -// Initialize Initializes the Opa engine +// Init initializes the Opa engine // Handles loading all rules, filtering, compiling, and preparing for evaluation -func (e *Engine) Initialize(policyPath string) error { +func (e *Engine) Init(policyPath string) error { e.Context = context.Background() if err := e.LoadRegoFiles(policyPath); err != nil { @@ -218,6 +238,9 @@ func (e *Engine) Initialize(policyPath string) error { return err } + // initialize ViolationStore + e.ViolationStore = results.NewViolationStore() + return nil } @@ -237,7 +260,7 @@ func (e *Engine) Release() error { } // Evaluate Executes compiled OPA queries against the input JSON data -func (e *Engine) Evaluate(inputData *interface{}) error { +func (e *Engine) Evaluate(inputData *interface{}) ([]*results.Violation, error) { sortedKeys := make([]string, len(e.RegoDataMap)) x := 0 @@ -262,8 +285,8 @@ func (e *Engine) Evaluate(inputData *interface{}) error { // @TODO: Take line number + file info and add to violation regoData := e.RegoDataMap[k] // @TODO: Remove this print, should be done by whomever consumes the results below - fmt.Printf("[%s] [%s] [%s] %s: %s\n", regoData.Metadata.Severity, regoData.Metadata.RuleReferenceID, - regoData.Metadata.Category, regoData.Metadata.RuleName, regoData.Metadata.Description) + // fmt.Printf("[%s] [%s] [%s] %s: %s\n", regoData.Metadata.Severity, regoData.Metadata.RuleReferenceID, + // regoData.Metadata.Category, regoData.Metadata.RuleName, regoData.Metadata.Description) violation := results.Violation{ Name: regoData.Metadata.RuleName, Description: regoData.Metadata.Description, @@ -281,5 +304,5 @@ func (e *Engine) Evaluate(inputData *interface{}) error { } } - return nil + return e.ViolationStore.GetResults(), nil } diff --git a/pkg/results/store.go b/pkg/results/store.go index 799224e85..fec48c4e6 100644 --- a/pkg/results/store.go +++ b/pkg/results/store.go @@ -16,6 +16,13 @@ package results +// NewViolationStore returns a new violation store +func NewViolationStore() *ViolationStore { + return &ViolationStore{ + violations: []*Violation{}, + } +} + // AddResult Adds individual violations into the violation store func (s *ViolationStore) AddResult(violation *Violation) { s.violations = append(s.violations, violation) diff --git a/pkg/runtime/executor.go b/pkg/runtime/executor.go index 1ab8857d7..c1301f9e4 100644 --- a/pkg/runtime/executor.go +++ b/pkg/runtime/executor.go @@ -27,15 +27,16 @@ import ( // Executor object type Executor struct { - filePath string - dirPath string - policyPath string - cloudType string - iacType string - iacVersion string - configFile string - iacProvider iacProvider.IacProvider - notifiers []notifications.Notifier + filePath string + dirPath string + policyPath string + cloudType string + iacType string + iacVersion string + configFile string + iacProvider iacProvider.IacProvider + policyEngine policy.Engine + notifiers []notifications.Notifier } // NewExecutor creates a runtime object @@ -50,7 +51,7 @@ func NewExecutor(iacType, iacVersion, cloudType, filePath, dirPath, configFile, configFile: configFile, } - // initialized executor + // initialize executor if err = e.Init(); err != nil { return e, err } @@ -81,49 +82,41 @@ func (e *Executor) Init() error { return err } + // create a new policy engine based on IaC type + e.policyEngine, err = opa.NewEngine(e.policyPath) + if err != nil { + zap.S().Errorf("failed to create policy engine. error: '%s'", err) + return err + } + zap.S().Debug("initialized executor") return nil } // Execute validates the inputs, processes the IaC, creates json output -func (e *Executor) Execute() (normalized interface{}, err error) { +func (e *Executor) Execute() (results interface{}, err error) { - // create normalized output from Iac + // create results output from Iac if e.dirPath != "" { - normalized, err = e.iacProvider.LoadIacDir(e.dirPath) + results, err = e.iacProvider.LoadIacDir(e.dirPath) } else { - normalized, err = e.iacProvider.LoadIacFile(e.filePath) + results, err = e.iacProvider.LoadIacFile(e.filePath) } if err != nil { - return normalized, err + return results, err } - // create a new policy engine based on IaC type - var engine policy.Engine - - if e.iacType == "terraform" { - engine = &opa.Engine{} - } - - if err = engine.Initialize(e.policyPath); err != nil { - return normalized, err - } - - if err = engine.Evaluate(&normalized); err != nil { - return normalized, err + // evaluate policies + results, err = e.policyEngine.Evaluate(&results) + if err != nil { + return results, err } - // var reporter publish.Reporter = console.Reporter - /// if err = reporter.ImportData() - // if err = reporter.Publish() { - // - // } - // send notifications, if configured - if err = e.SendNotifications(normalized); err != nil { - return normalized, err + if err = e.SendNotifications(results); err != nil { + return results, err } // successful - return normalized, nil + return results, nil } From c9011358d9c6f6ccc8f7a9d1b1bbf54878f7ac30 Mon Sep 17 00:00:00 2001 From: Yusuf Kanchwala Date: Wed, 12 Aug 2020 01:32:00 +0530 Subject: [PATCH 2/4] add support for writer --- pkg/cli/run.go | 4 ++-- pkg/results/types.go | 16 ++++++++-------- pkg/writer/json.go | 38 ++++++++++++++++++++++++++++++++++++++ pkg/writer/register.go | 30 ++++++++++++++++++++++++++++++ pkg/writer/writer.go | 40 ++++++++++++++++++++++++++++++++++++++++ pkg/writer/yaml.go | 39 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 157 insertions(+), 10 deletions(-) create mode 100644 pkg/writer/json.go create mode 100644 pkg/writer/register.go create mode 100644 pkg/writer/writer.go create mode 100644 pkg/writer/yaml.go diff --git a/pkg/cli/run.go b/pkg/cli/run.go index 280460d59..f24ae3fb6 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -20,7 +20,7 @@ import ( "os" "github.com/accurics/terrascan/pkg/runtime" - "github.com/accurics/terrascan/pkg/utils" + "github.com/accurics/terrascan/pkg/writer" ) // Run executes terrascan in CLI mode @@ -38,5 +38,5 @@ func Run(iacType, iacVersion, cloudType, iacFilePath, iacDirPath, configFile, po if err != nil { return } - utils.PrintJSON(violations, os.Stdout) + writer.Write("xml", violations, os.Stdout) } diff --git a/pkg/results/types.go b/pkg/results/types.go index 0dd6377f1..113f8260f 100644 --- a/pkg/results/types.go +++ b/pkg/results/types.go @@ -18,14 +18,14 @@ package results // Violation Contains data for each violation type Violation struct { - Name string - Description string - RuleID string - Category string - RuleData interface{} - InputFile string - InputData interface{} - LineNumber int + Name string `json:"name" yaml:"name" xml:"name,attr"` + Description string `json:"description" yaml:"description" xml:"description, attr"` + RuleID string `json:"rule" yaml:"rule" xml:"rule,attr"` + Category string `json:"category" yaml:"category" xml:"category,attr"` + RuleData interface{} `json:"-" yaml:"-" xml:"-"` + InputFile string `json:"-", yaml:"-", xml:"-"` + InputData interface{} `json:"input_data" yaml:"input_data" xml:"input_data,attr"` + LineNumber int `json:"line" yaml:"line" xml:"line,attr"` } // ViolationStore Storage area for violation data diff --git a/pkg/writer/json.go b/pkg/writer/json.go new file mode 100644 index 000000000..6fbae649f --- /dev/null +++ b/pkg/writer/json.go @@ -0,0 +1,38 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package writer + +import ( + "encoding/json" + "io" +) + +const ( + jsonFormat supportedFormat = "json" +) + +func init() { + RegisterWriter(jsonFormat, JSONWriter) +} + +// JSONWriter prints data in JSON format +func JSONWriter(data interface{}, writer io.Writer) error { + j, _ := json.MarshalIndent(data, "", " ") + writer.Write(j) + writer.Write([]byte{'\n'}) + return nil +} diff --git a/pkg/writer/register.go b/pkg/writer/register.go new file mode 100644 index 000000000..3d7925de1 --- /dev/null +++ b/pkg/writer/register.go @@ -0,0 +1,30 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package writer + +import "io" + +// supportedFormat data type for supported formats +type supportedFormat string + +// writerMap stores mapping of supported writer formats with respective functions +var writerMap = make(map[supportedFormat](func(interface{}, io.Writer) error)) + +// RegisterWriter registers a writer for terrascan +func RegisterWriter(format supportedFormat, writerFunc func(interface{}, io.Writer) error) { + writerMap[format] = writerFunc +} diff --git a/pkg/writer/writer.go b/pkg/writer/writer.go new file mode 100644 index 000000000..97bc799f8 --- /dev/null +++ b/pkg/writer/writer.go @@ -0,0 +1,40 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package writer + +import ( + "fmt" + "io" + + "go.uber.org/zap" +) + +var ( + errNotSupported = fmt.Errorf("output format not supported") +) + +// Write method writes in the given format using the respective writer func +func Write(format supportedFormat, data interface{}, writer io.Writer) error { + + writerFunc, present := writerMap[format] + if !present { + zap.S().Error("output format '%s' not supported", format) + return errNotSupported + } + + return writerFunc(data, writer) +} diff --git a/pkg/writer/yaml.go b/pkg/writer/yaml.go new file mode 100644 index 000000000..2d7f8d7ca --- /dev/null +++ b/pkg/writer/yaml.go @@ -0,0 +1,39 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package writer + +import ( + "io" + + "gopkg.in/yaml.v2" +) + +const ( + yamlFormat supportedFormat = "yaml" +) + +func init() { + RegisterWriter(yamlFormat, YAMLWriter) +} + +// YAMLWriter prints data in YAML format +func YAMLWriter(data interface{}, writer io.Writer) error { + j, _ := yaml.Marshal(data) + writer.Write(j) + writer.Write([]byte{'\n'}) + return nil +} From 483920a15199194df181f6c4196226fca71879a1 Mon Sep 17 00:00:00 2001 From: Yusuf Kanchwala Date: Wed, 12 Aug 2020 10:38:36 +0530 Subject: [PATCH 3/4] changing input/ouput type from interface{} to data specific types --- go.mod | 1 + pkg/cli/run.go | 2 +- pkg/policy/interface.go | 3 ++- pkg/policy/opa/engine.go | 4 ++-- pkg/runtime/executor.go | 11 +++++++---- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 459e68777..7cd2b8da7 100644 --- a/go.mod +++ b/go.mod @@ -17,5 +17,6 @@ require ( golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect golang.org/x/tools v0.0.0-20200809012840-6f4f008689da // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gopkg.in/yaml.v2 v2.3.0 honnef.co/go/tools v0.0.1-2020.1.5 // indirect ) diff --git a/pkg/cli/run.go b/pkg/cli/run.go index f24ae3fb6..963a7c22e 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -38,5 +38,5 @@ func Run(iacType, iacVersion, cloudType, iacFilePath, iacDirPath, configFile, po if err != nil { return } - writer.Write("xml", violations, os.Stdout) + writer.Write("yaml", violations, os.Stdout) } diff --git a/pkg/policy/interface.go b/pkg/policy/interface.go index c686cd58a..62765b331 100644 --- a/pkg/policy/interface.go +++ b/pkg/policy/interface.go @@ -17,6 +17,7 @@ package policy import ( + "github.com/accurics/terrascan/pkg/iac-providers/output" "github.com/accurics/terrascan/pkg/results" ) @@ -31,7 +32,7 @@ type Manager interface { type Engine interface { Init(string) error Configure() error - Evaluate(*interface{}) ([]*results.Violation, error) + Evaluate(output.AllResourceConfigs) ([]*results.Violation, error) GetResults() error Release() error } diff --git a/pkg/policy/opa/engine.go b/pkg/policy/opa/engine.go index 5e0b86095..e3ecc271f 100644 --- a/pkg/policy/opa/engine.go +++ b/pkg/policy/opa/engine.go @@ -28,8 +28,8 @@ import ( "sort" "text/template" + "github.com/accurics/terrascan/pkg/iac-providers/output" "github.com/accurics/terrascan/pkg/results" - "github.com/accurics/terrascan/pkg/utils" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/rego" @@ -260,7 +260,7 @@ func (e *Engine) Release() error { } // Evaluate Executes compiled OPA queries against the input JSON data -func (e *Engine) Evaluate(inputData *interface{}) ([]*results.Violation, error) { +func (e *Engine) Evaluate(inputData output.AllResourceConfigs) ([]*results.Violation, error) { sortedKeys := make([]string, len(e.RegoDataMap)) x := 0 diff --git a/pkg/runtime/executor.go b/pkg/runtime/executor.go index c1301f9e4..983deae3d 100644 --- a/pkg/runtime/executor.go +++ b/pkg/runtime/executor.go @@ -20,9 +20,11 @@ import ( "go.uber.org/zap" iacProvider "github.com/accurics/terrascan/pkg/iac-providers" + "github.com/accurics/terrascan/pkg/iac-providers/output" "github.com/accurics/terrascan/pkg/notifications" "github.com/accurics/terrascan/pkg/policy" opa "github.com/accurics/terrascan/pkg/policy/opa" + "github.com/accurics/terrascan/pkg/results" ) // Executor object @@ -94,20 +96,21 @@ func (e *Executor) Init() error { } // Execute validates the inputs, processes the IaC, creates json output -func (e *Executor) Execute() (results interface{}, err error) { +func (e *Executor) Execute() (results []*results.Violation, err error) { // create results output from Iac + var normalized output.AllResourceConfigs if e.dirPath != "" { - results, err = e.iacProvider.LoadIacDir(e.dirPath) + normalized, err = e.iacProvider.LoadIacDir(e.dirPath) } else { - results, err = e.iacProvider.LoadIacFile(e.filePath) + normalized, err = e.iacProvider.LoadIacFile(e.filePath) } if err != nil { return results, err } // evaluate policies - results, err = e.policyEngine.Evaluate(&results) + results, err = e.policyEngine.Evaluate(normalized) if err != nil { return results, err } From 0c3d58f2d7988266bf41cfb1f3066ccee7f442d8 Mon Sep 17 00:00:00 2001 From: Yusuf Kanchwala Date: Wed, 12 Aug 2020 12:03:20 +0530 Subject: [PATCH 4/4] add unit tests for FindAllDirectories func --- pkg/utils/path_test.go | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/pkg/utils/path_test.go b/pkg/utils/path_test.go index a2efc55b8..5b47d4076 100644 --- a/pkg/utils/path_test.go +++ b/pkg/utils/path_test.go @@ -17,7 +17,9 @@ package utils import ( + "fmt" "os" + "reflect" "testing" ) @@ -67,3 +69,44 @@ func TestGetAbsPath(t *testing.T) { }) } } + +func TestFindAllDirectories(t *testing.T) { + + table := []struct { + name string + basePath string + want []string + wantErr error + }{ + { + name: "happy path", + basePath: "./testdata", + want: []string{"./testdata", "testdata/emptydir", "testdata/testdir1", "testdata/testdir2"}, + wantErr: nil, + }, + { + name: "empty dir", + basePath: "./testdata/emptydir", + want: []string{"./testdata/emptydir"}, + wantErr: nil, + }, + { + name: "invalid dir", + basePath: "./testdata/nothere", + want: []string{}, + wantErr: fmt.Errorf("lstat ./testdata/nothere: no such file or directory"), + }, + } + + for _, tt := range table { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := FindAllDirectories(tt.basePath) + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("gotErr: '%+v', wantErr: '%+v'", gotErr, tt.wantErr) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("got: '%v', want: '%v'", got, tt.want) + } + }) + } +}