Skip to content

Commit

Permalink
add support for downloading remote modules in terraform v12 Iac scanning
Browse files Browse the repository at this point in the history
  • Loading branch information
Yusuf Kanchwala committed Oct 2, 2020
1 parent c4457da commit 5fd0685
Show file tree
Hide file tree
Showing 3 changed files with 154 additions and 24 deletions.
48 changes: 33 additions & 15 deletions pkg/downloader/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,19 +70,8 @@ var (
ErrEmptyURLTypeDest = fmt.Errorf("empty remote url or type or desitnation dir path")
)

// Download retrieves the remote repository referenced in the given remoteURL
// into the destination path and then returns the full path to any subdir
// indicated in the URL
func (d Downloader) Download(remoteURL, destPath string) (string, error) {

zap.S().Debugf("download with remote url: %q, destination dir: %q",
remoteURL, destPath)

// validations: remote url or destination dir path cannot be empty
if remoteURL == "" || destPath == "" {
zap.S().Error(ErrEmptyURLDest)
return "", ErrEmptyURLDest
}
// GetURLWithType returns the download URL with it's respective type prefix
func GetURLWithType(remoteURL, destPath string) (string, string, error) {

// get subDir, if present
repoURL, subDir := SplitAddrSubdir(remoteURL)
Expand All @@ -92,7 +81,7 @@ func (d Downloader) Download(remoteURL, destPath string) (string, error) {
URLWithType, err := getter.Detect(repoURL, destPath, goGetterDetectors)
if err != nil {
zap.S().Errorf("failed to detect url with type for %q. error: '%v'", remoteURL, err)
return "", err
return "", "", err
}
zap.S().Debugf("remote URL: %q; url with type: %q", remoteURL, URLWithType)

Expand All @@ -106,13 +95,37 @@ func (d Downloader) Download(remoteURL, destPath string) (string, error) {
zap.S().Debugf("detector rewrote %q to %q", repoURL, URLWithType)
}

// successful
return URLWithType, subDir, nil
}

// Download retrieves the remote repository referenced in the given remoteURL
// into the destination path and then returns the full path to any subdir
// indicated in the URL
func (d Downloader) Download(remoteURL, destPath string) (string, error) {

zap.S().Debugf("download with remote url: %q, destination dir: %q",
remoteURL, destPath)

// validations: remote url or destination dir path cannot be empty
if remoteURL == "" || destPath == "" {
zap.S().Error(ErrEmptyURLDest)
return "", ErrEmptyURLDest
}

// get repository url, subdir from given remote url
URLWithType, subDir, err := GetURLWithType(remoteURL, destPath)
if err != nil {
return "", err
}

// downloading from remote addr
client := getter.Client{
Src: URLWithType,
Dst: destPath,
Pwd: destPath,
Mode: getter.ClientModeDir,
Detectors: goGetterNoDetectors, // we already did detection above
Detectors: goGetterNoDetectors,
Decompressors: goGetterDecompressors,
Getters: goGetterGetters,
}
Expand Down Expand Up @@ -170,6 +183,11 @@ func (d Downloader) DownloadWithType(remoteType, remoteURL, destPath string) (st
return d.Download(URLWithType, destPath)
}

// SubDirGlob returns the actual subdir with globbing processed
func SubDirGlob(destPath, subDir string) (string, error) {
return getter.SubdirGlob(destPath, subDir)
}

// SplitAddrSubdir splits the given address (which is assumed to be a
// registry address or go-getter-style address) into a package portion
// and a sub-directory portion.
Expand Down
34 changes: 25 additions & 9 deletions pkg/iac-providers/terraform/v12/load-dir.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@ package tfv12

import (
"fmt"
"os"
"path/filepath"
"strings"

"github.com/accurics/terrascan/pkg/iac-providers/output"
"github.com/accurics/terrascan/pkg/utils"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl/v2"
hclConfigs "github.com/hashicorp/terraform/configs"
Expand Down Expand Up @@ -56,6 +58,9 @@ func (*TfV12) LoadIacDir(absRootDir string) (allResourcesConfig output.AllResour
return allResourcesConfig, errLoadConfigDir
}

// create InstalledCache to track already downloaded remote modules
var c InstalledCache = make(map[string]string)

// using the BuildConfig and ModuleWalkerFunc to traverse through all
// descendant modules from the root module and create a unified
// configuration of type *configs.Config
Expand All @@ -64,15 +69,26 @@ func (*TfV12) LoadIacDir(absRootDir string) (allResourcesConfig output.AllResour
unified, diags := hclConfigs.BuildConfig(rootMod, hclConfigs.ModuleWalkerFunc(
func(req *hclConfigs.ModuleRequest) (*hclConfigs.Module, *version.Version, hcl.Diagnostics) {

// Note: currently only local paths are supported for Module Sources

// determine the absolute path from root module to the sub module
// using *configs.ModuleRequest.Path field

pathArr := strings.Split(req.Path.String(), ".")
pathArr = pathArr[:len(pathArr)-1]

pathToModule := filepath.Join(absRootDir, filepath.Join(pathArr...), req.SourceAddr)
// figure out path sub module directory, if it's remote then download it locally
var pathToModule string
if isLocalSourceAddr(req.SourceAddr) {
// determine the absolute path from root module to the sub module
// using *configs.ModuleRequest.Path field
pathArr := strings.Split(req.Path.String(), ".")
pathArr = pathArr[:len(pathArr)-1]
pathToModule = filepath.Join(absRootDir, filepath.Join(pathArr...), req.SourceAddr)
zap.S().Debugf("processing local module %q", req.SourceAddr)
} else {
// temp dir to download the remote repo
tempDir := filepath.Join(os.TempDir(), utils.GenRandomString(6))
defer os.RemoveAll(tempDir)

// Download remote module
pathToModule, err = c.DownloadModule(req.SourceAddr, tempDir)
if err != nil {
zap.S().Errorf("failed to download remote module %q. error: '%v'", req.SourceAddr, err)
}
}

// load sub module directory
subMod, diags := parser.LoadConfigDir(pathToModule)
Expand Down
96 changes: 96 additions & 0 deletions pkg/iac-providers/terraform/v12/module-download.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/*
Copyright (C) 2020 Accurics, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tfv12

import (
"path/filepath"
"strings"

"github.com/accurics/terrascan/pkg/downloader"
"go.uber.org/zap"
)

var localSourcePrefixes = []string{
"./",
"../",
".\\",
"..\\",
}

func isLocalSourceAddr(addr string) bool {
for _, prefix := range localSourcePrefixes {
if strings.HasPrefix(addr, prefix) {
return true
}
}
return false
}

// InstalledCache remembers the final resolved addresses of all the sources
// already downloaded.
//
// The keys in InstalledCache are resolved and trimmed source addresses
// (with a scheme always present, and without any "subdir" component),
// and the values are the paths where each source was previously installed.
type InstalledCache map[string]string

// DownloadModule retrieves the package referenced in the given address
// into the installation path and then returns the full path to any subdir
// indicated in the address.
func (g InstalledCache) DownloadModule(addr, destPath string) (string, error) {

// get url with type
URLWithType, subDir, err := downloader.GetURLWithType(addr, destPath)
if err != nil {
return "", err
}

// check if the module has already been downloaded
if prevDir, exists := g[URLWithType]; exists {
zap.S().Debugf("module %q already installed at %q", URLWithType, prevDir)
destPath = prevDir
} else {
d := downloader.NewDownloader()
destPath, err := d.Download(URLWithType, destPath)
if err != nil {
zap.S().Debugf("failed to download remote module. error: '%v'", err)
return "", err
}
// Remember where we installed this so we might reuse this directory
// on subsequent calls to avoid re-downloading.
g[URLWithType] = destPath
}

// Our subDir string can contain wildcards until this point, so that
// e.g. a subDir of * can expand to one top-level directory in a .tar.gz
// archive. Now that we've expanded the archive successfully we must
// resolve that into a concrete path.
var finalDir string
if subDir != "" {
finalDir, err = downloader.SubDirGlob(destPath, subDir)
if err != nil {
return "", err
}
zap.S().Debugf("expanded %q to %q", subDir, finalDir)
} else {
finalDir = destPath
}

// If we got this far then we have apparently succeeded in downloading
// the requested object!
return filepath.Clean(finalDir), nil
}

0 comments on commit 5fd0685

Please sign in to comment.