Merge pull request #268 from accurics/terrascan-v1.0-policy-support-r…

…efactor

refactoring policy package

go.mod

@@ -17,5 +17,6 @@ require (
 	golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect
 	golang.org/x/tools v0.0.0-20200809012840-6f4f008689da // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 	honnef.co/go/tools v0.0.1-2020.1.5 // indirect
 )

pkg/cli/run.go

@@ -17,7 +17,10 @@
 package cli
 
 import (
+	"os"
+
 	"github.com/accurics/terrascan/pkg/runtime"
+	"github.com/accurics/terrascan/pkg/writer"
 )
 
 // Run executes terrascan in CLI mode
@@ -31,9 +34,9 @@ func Run(iacType, iacVersion, cloudType, iacFilePath, iacDirPath, configFile, po
 	}
 
 	// executor output
-	_, err = executor.Execute()
+	violations, err := executor.Execute()
 	if err != nil {
 		return
 	}
-	//	utils.PrintJSON(violations, os.Stdout)
+	writer.Write("yaml", violations, os.Stdout)
 }

pkg/policy/interface.go

@@ -16,6 +16,11 @@
 
 package policy
 
+import (
+	"github.com/accurics/terrascan/pkg/iac-providers/output"
+	"github.com/accurics/terrascan/pkg/results"
+)
+
 // Manager Policy Manager interface
 type Manager interface {
 	Import() error
@@ -25,9 +30,9 @@ type Manager interface {
 
 // Engine Policy Engine interface
 type Engine interface {
-	Initialize(policyPath string) error
+	Init(string) error
 	Configure() error
-	Evaluate(inputData *interface{}) error
+	Evaluate(output.AllResourceConfigs) ([]*results.Violation, error)
 	GetResults() error
 	Release() error
 }

pkg/policy/opa/engine.go

@@ -28,14 +28,34 @@ import (
 	"sort"
 	"text/template"
 
+	"github.com/accurics/terrascan/pkg/iac-providers/output"
 	"github.com/accurics/terrascan/pkg/results"
-
 	"github.com/accurics/terrascan/pkg/utils"
 	"github.com/open-policy-agent/opa/ast"
 	"github.com/open-policy-agent/opa/rego"
 	"go.uber.org/zap"
 )
 
+var (
+	errInitFailed = fmt.Errorf("failed to initialize OPA policy engine")
+)
+
+// NewEngine returns a new OPA policy engine
+func NewEngine(policyPath string) (*Engine, error) {
+
+	// opa engine struct
+	engine := &Engine{}
+
+	// initialize the engine
+	if err := engine.Init(policyPath); err != nil {
+		zap.S().Error("failed to initialize OPA policy engine")
+		return engine, errInitFailed
+	}
+
+	// successful
+	return engine, nil
+}
+
 // LoadRegoMetadata Loads rego metadata from a given file
 func (e *Engine) LoadRegoMetadata(metaFilename string) (*RegoMetadata, error) {
 	// Load metadata file if it exists
@@ -202,9 +222,9 @@ func (e *Engine) CompileRegoFiles() error {
 	return nil
 }
 
-// Initialize Initializes the Opa engine
+// Init initializes the Opa engine
 // Handles loading all rules, filtering, compiling, and preparing for evaluation
-func (e *Engine) Initialize(policyPath string) error {
+func (e *Engine) Init(policyPath string) error {
 	e.Context = context.Background()
 
 	if err := e.LoadRegoFiles(policyPath); err != nil {
@@ -218,6 +238,9 @@ func (e *Engine) Initialize(policyPath string) error {
 		return err
 	}
 
+	// initialize ViolationStore
+	e.ViolationStore = results.NewViolationStore()
+
 	return nil
 }
 
@@ -237,7 +260,7 @@ func (e *Engine) Release() error {
 }
 
 // Evaluate Executes compiled OPA queries against the input JSON data
-func (e *Engine) Evaluate(inputData *interface{}) error {
+func (e *Engine) Evaluate(inputData output.AllResourceConfigs) ([]*results.Violation, error) {
 
 	sortedKeys := make([]string, len(e.RegoDataMap))
 	x := 0
@@ -262,8 +285,8 @@ func (e *Engine) Evaluate(inputData *interface{}) error {
 				// @TODO: Take line number + file info and add to violation
 				regoData := e.RegoDataMap[k]
 				// @TODO: Remove this print, should be done by whomever consumes the results below
-				fmt.Printf("[%s] [%s] [%s] %s: %s\n", regoData.Metadata.Severity, regoData.Metadata.RuleReferenceID,
-					regoData.Metadata.Category, regoData.Metadata.RuleName, regoData.Metadata.Description)
+				// fmt.Printf("[%s] [%s] [%s] %s: %s\n", regoData.Metadata.Severity, regoData.Metadata.RuleReferenceID,
+				//	regoData.Metadata.Category, regoData.Metadata.RuleName, regoData.Metadata.Description)
 				violation := results.Violation{
 					Name:        regoData.Metadata.RuleName,
 					Description: regoData.Metadata.Description,
@@ -281,5 +304,5 @@ func (e *Engine) Evaluate(inputData *interface{}) error {
 		}
 	}
 
-	return nil
+	return e.ViolationStore.GetResults(), nil
 }

pkg/results/store.go

@@ -16,6 +16,13 @@
 
 package results
 
+// NewViolationStore returns a new violation store
+func NewViolationStore() *ViolationStore {
+	return &ViolationStore{
+		violations: []*Violation{},
+	}
+}
+
 // AddResult Adds individual violations into the violation store
 func (s *ViolationStore) AddResult(violation *Violation) {
 	s.violations = append(s.violations, violation)

pkg/results/types.go

@@ -18,14 +18,14 @@ package results
 
 // Violation Contains data for each violation
 type Violation struct {
-	Name        string
-	Description string
-	RuleID      string
-	Category    string
-	RuleData    interface{}
-	InputFile   string
-	InputData   interface{}
-	LineNumber  int
+	Name        string      `json:"name" yaml:"name" xml:"name,attr"`
+	Description string      `json:"description" yaml:"description" xml:"description, attr"`
+	RuleID      string      `json:"rule" yaml:"rule" xml:"rule,attr"`
+	Category    string      `json:"category" yaml:"category" xml:"category,attr"`
+	RuleData    interface{} `json:"-" yaml:"-" xml:"-"`
+	InputFile   string      `json:"-", yaml:"-", xml:"-"`
+	InputData   interface{} `json:"input_data" yaml:"input_data" xml:"input_data,attr"`
+	LineNumber  int         `json:"line" yaml:"line" xml:"line,attr"`
 }
 
 // ViolationStore Storage area for violation data

pkg/runtime/executor.go

@@ -20,22 +20,25 @@ import (
 	"go.uber.org/zap"
 
 	iacProvider "github.com/accurics/terrascan/pkg/iac-providers"
+	"github.com/accurics/terrascan/pkg/iac-providers/output"
 	"github.com/accurics/terrascan/pkg/notifications"
 	"github.com/accurics/terrascan/pkg/policy"
 	opa "github.com/accurics/terrascan/pkg/policy/opa"
+	"github.com/accurics/terrascan/pkg/results"
 )
 
 // Executor object
 type Executor struct {
-	filePath    string
-	dirPath     string
-	policyPath  string
-	cloudType   string
-	iacType     string
-	iacVersion  string
-	configFile  string
-	iacProvider iacProvider.IacProvider
-	notifiers   []notifications.Notifier
+	filePath     string
+	dirPath      string
+	policyPath   string
+	cloudType    string
+	iacType      string
+	iacVersion   string
+	configFile   string
+	iacProvider  iacProvider.IacProvider
+	policyEngine policy.Engine
+	notifiers    []notifications.Notifier
 }
 
 // NewExecutor creates a runtime object
@@ -50,7 +53,7 @@ func NewExecutor(iacType, iacVersion, cloudType, filePath, dirPath, configFile,
 		configFile: configFile,
 	}
 
-	// initialized executor
+	// initialize executor
 	if err = e.Init(); err != nil {
 		return e, err
 	}
@@ -81,49 +84,42 @@ func (e *Executor) Init() error {
 		return err
 	}
 
+	// create a new policy engine based on IaC type
+	e.policyEngine, err = opa.NewEngine(e.policyPath)
+	if err != nil {
+		zap.S().Errorf("failed to create policy engine. error: '%s'", err)
+		return err
+	}
+
 	zap.S().Debug("initialized executor")
 	return nil
 }
 
 // Execute validates the inputs, processes the IaC, creates json output
-func (e *Executor) Execute() (normalized interface{}, err error) {
+func (e *Executor) Execute() (results []*results.Violation, err error) {
 
-	// create normalized output from Iac
+	// create results output from Iac
+	var normalized output.AllResourceConfigs
 	if e.dirPath != "" {
 		normalized, err = e.iacProvider.LoadIacDir(e.dirPath)
 	} else {
 		normalized, err = e.iacProvider.LoadIacFile(e.filePath)
 	}
 	if err != nil {
-		return normalized, err
+		return results, err
 	}
 
-	// create a new policy engine based on IaC type
-	var engine policy.Engine
-
-	if e.iacType == "terraform" {
-		engine = &opa.Engine{}
-	}
-
-	if err = engine.Initialize(e.policyPath); err != nil {
-		return normalized, err
-	}
-
-	if err = engine.Evaluate(&normalized); err != nil {
-		return normalized, err
+	// evaluate policies
+	results, err = e.policyEngine.Evaluate(normalized)
+	if err != nil {
+		return results, err
 	}
 
-	//	var reporter publish.Reporter = console.Reporter
-	///	if err = reporter.ImportData()
-	//	if err = reporter.Publish() {
-	//
-	//    }
-
 	// send notifications, if configured
-	if err = e.SendNotifications(normalized); err != nil {
-		return normalized, err
+	if err = e.SendNotifications(results); err != nil {
+		return results, err
 	}
 
 	// successful
-	return normalized, nil
+	return results, nil
 }

pkg/utils/path_test.go

@@ -17,7 +17,9 @@
 package utils
 
 import (
+	"fmt"
 	"os"
+	"reflect"
 	"testing"
 )
 
@@ -67,3 +69,44 @@ func TestGetAbsPath(t *testing.T) {
 		})
 	}
 }
+
+func TestFindAllDirectories(t *testing.T) {
+
+	table := []struct {
+		name     string
+		basePath string
+		want     []string
+		wantErr  error
+	}{
+		{
+			name:     "happy path",
+			basePath: "./testdata",
+			want:     []string{"./testdata", "testdata/emptydir", "testdata/testdir1", "testdata/testdir2"},
+			wantErr:  nil,
+		},
+		{
+			name:     "empty dir",
+			basePath: "./testdata/emptydir",
+			want:     []string{"./testdata/emptydir"},
+			wantErr:  nil,
+		},
+		{
+			name:     "invalid dir",
+			basePath: "./testdata/nothere",
+			want:     []string{},
+			wantErr:  fmt.Errorf("lstat ./testdata/nothere: no such file or directory"),
+		},
+	}
+
+	for _, tt := range table {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotErr := FindAllDirectories(tt.basePath)
+			if !reflect.DeepEqual(gotErr, tt.wantErr) {
+				t.Errorf("gotErr: '%+v', wantErr: '%+v'", gotErr, tt.wantErr)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("got: '%v', want: '%v'", got, tt.want)
+			}
+		})
+	}
+}

pkg/writer/json.go

@@ -0,0 +1,38 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package writer
+
+import (
+	"encoding/json"
+	"io"
+)
+
+const (
+	jsonFormat supportedFormat = "json"
+)
+
+func init() {
+	RegisterWriter(jsonFormat, JSONWriter)
+}
+
+// JSONWriter prints data in JSON format
+func JSONWriter(data interface{}, writer io.Writer) error {
+	j, _ := json.MarshalIndent(data, "", "  ")
+	writer.Write(j)
+	writer.Write([]byte{'\n'})
+	return nil
+}