feat(api domain): add support for Post (#801)

* feat(api domain): add support for Post

* don't forget about executable

* add test for executable flag

* reference executable in spec

* query params are for everyone

* add response and raw keys to the api domain response

* adding new response key to documentation

* finish docs

docs/reference/domains/api-domain.md

@@ -1,6 +1,6 @@
 # API Domain
 
-The API Domain allows for collection of data (via HTTP Get Requests) generically from API endpoints. 
+The API Domain allows for collection of data (via HTTP Get or Post Requests) generically from API endpoints.
 
 ## Specification
 The API domain Specification accepts a list of `Requests` and an `Options` block. `Options` can be configured at the top-level and will apply to all requests except those which have embedded `Options`. `Request`-level options will *override* top-level `Options`.
@@ -10,32 +10,39 @@ The API domain Specification accepts a list of `Requests` and an `Options` block
 domain: 
   type: api
   api-spec:
-    # Options specified at this level will apply to all requests except those with an embedded options block.
+    # options (optional): Options specified at this level will apply to all requests except those with an embedded options block.
     options:
-      # Timeout configures the request timeout. The default timeout is 30 seconds (30s). The timeout string is a number followed by a unit suffix (ms, s, m, h, d), such as 30s or 1m.
+      # timeout (optional): configures the request timeout. The default timeout is 30 seconds (30s). The timeout string is a number followed by a unit suffix (ms, s, m, h, d), such as 30s or 1m.
       timeout: 30s
-      # Proxy specifies a proxy server for all requests.
+      # proxy (optional): Specifies a proxy server for all requests.
       proxy: "https://my.proxy"
-      # Headers is a map of key value pairs to send with all requests.
+      # headers (optional):  a map of key value pairs to send with all requests.
       headers: 
         key: "value"
         my-customer-header: "my-custom-value"
     # Requests is a list of URLs to query. The request name is the map key used when referencing the resources returned by the API.
     requests:
-      # A descriptive name for the request.
+      # name (required): A descriptive name for the request.
       - name: "healthcheck" 
-      # The URL of the request. The API domain supports any rfc3986-formatted URI. Lula also supports URL parameters as a separate argument. 
+        # url (required): The URL for the request. The API domain supports any rfc3986-formatted URI. Lula also supports URL parameters as a separate argument.
         url: "https://example.com/health/ready"
-        # Parameters to append to the URL. Lula also supports full URIs in the URL.
+        # method (optional): The HTTP Method to use for the API call. "get" and "post" are supported. Default is "get".
+        method: "get"
+        # parameters (optional): parameters to append to the URL. Lula also supports full URIs in the URL.
         parameters: 
           key: "value"
-        # Request-level options have the same specification as the api-spec-level options. These options apply only to this request.
+        # Body (optional): a json-compatible string to pass into the request as the request body.
+        body: |
+stringjsondata
+        # executable (optional): Lula will request user verification before performing API actions if *any* API request is flagged "executable".
+        executable: true
+        # options (optional): Request-level options have the same specification as the api-spec-level options at the top. These options apply only to this request.
         options:
-          # Configure the request timeout. The default timeout is 30 seconds (30s). The timeout string is a number followed by a unit suffix (ms, s, m, h, d), such as 30s or 1m.
+          # timeout (optional): configures the request timeout. The default timeout is 30 seconds (30s). The timeout string is a number followed by a unit suffix (ms, s, m, h, d), such as 30s or 1m.
           timeout: 30s
-          # Proxy specifies a proxy server for this request.
+          # proxy (optional): Specifies a proxy server for all requests.
           proxy: "https://my.proxy"
-          # Headers is a map of key value pairs to send with this request.
+          # headers (optional):  a map of key value pairs to send with all requests.
           headers: 
             key: "value"
             my-customer-header: "my-custom-value"
@@ -45,13 +52,30 @@ domain:
 
 ## API Domain Resources
 
-The API response body is serialized into a json object with the Request's Name as the top-level key. The API status code is included in the output domain resources.
+The API response body is serialized into a json object with the Request's `Name` as the top-level key. The API status code is included in the output domain resources under `status`. `raw` contains the entire API repsonse in an unmarshalled (`json.RawMessage`) format.
 
 Example output:
 
 ```json
 "healthcheck": {
   "status": 200,
-  "healthy": "ok"
+  "response": {
+    "healthy": true,
+  },
+  "raw": {"healthy": true}
 }
+```
+
+The following example validation verifies that the request named "healthcheck" returns `"healthy": true` 
+
+```
+provider: 
+  type: opa
+  opa-spec:
+    rego: |
+      package validate
+
+      validate {
+        input.healthcheck.response.healthy == true
+      }
 ```

src/pkg/common/schemas/validation.json

@@ -374,6 +374,21 @@
                                 "type": "object",
                                 "additionalProperties": { "type": "string"}
                             },
+                            "body": {
+                                "type": "string"
+                            },
+                            "method": {
+                                "type": "string",
+                                "enum": [
+                                    "post", "POST", "Post",
+                                    "get", "GET", "Get"
+                                ],
+                                "default": "get"
+                            },
+                            "executable": {
+                                "type": "boolean",
+                                "description": "indicates if the request is executable"
+                            },
                             "options": {
                                 "$ref": "#/definitions/api-options"
                             }

src/pkg/domains/api/api.go

@@ -1,13 +1,23 @@
 package api
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
 
 	"github.com/defenseunicorns/lula/src/types"
 )
 
+type APIResponse struct {
+	Status   int
+	Raw      json.RawMessage
+	Response any
+}
+
 func (a ApiDomain) makeRequests(ctx context.Context) (types.DomainResources, error) {
 	select {
 	case <-ctx.Done():
@@ -28,28 +38,39 @@ func (a ApiDomain) makeRequests(ctx context.Context) (types.DomainResources, err
 		}
 
 		// configure the default HTTP client using any top-level Options. Individual
-		// requests with overrides will get bespoke clients.
+		// requests with overrides (in request.Options.Headers) will get bespoke clients.
 		defaultClient := clientFromOpts(defaultOpts)
 		var errs error
 		for _, request := range a.Spec.Requests {
-			var responseType map[string]interface{}
-			var err error
-			var status int
+			var r io.Reader
+			if request.Body != "" {
+				r = bytes.NewBufferString(request.Body)
+			}
+
+			var headers map[string]string
+			var client http.Client
+
 			if request.Options == nil {
-				responseType, status, err = doHTTPReq(ctx, defaultClient, *request.reqURL, defaultOpts.Headers, request.reqParameters, responseType)
+				headers = defaultOpts.Headers
+				client = defaultClient
 			} else {
-				client := clientFromOpts(request.Options)
-				responseType, status, err = doHTTPReq(ctx, client, *request.reqURL, request.Options.Headers, request.reqParameters, responseType)
+				headers = request.Options.Headers
+				client = clientFromOpts(request.Options)
 			}
+
+			response, err := doHTTPReq(ctx, client, request.Method, *request.reqURL, r, headers, request.reqParameters)
 			if err != nil {
 				errs = errors.Join(errs, err)
 			}
-			// Check if the response object is empty and manually add a DR with the status response if so. This is more likely to happen in tests than reality.
-			if responseType != nil {
-				responseType["status"] = status
-				collection[request.Name] = responseType
+			if response != nil {
+				collection[request.Name] = types.DomainResources{
+					"status":   response.Status,
+					"raw":      response.Raw,
+					"response": response.Response,
+				}
 			} else {
-				collection[request.Name] = types.DomainResources{"status": status}
+				// If the entire response is empty, return a validly empty resource
+				collection[request.Name] = types.DomainResources{"status": 0}
 			}
 		}
 		return collection, errs

src/pkg/domains/api/http_request.go

@@ -8,46 +8,52 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+
+	"github.com/defenseunicorns/lula/src/pkg/message"
 )
 
-func doHTTPReq[T any](ctx context.Context, client http.Client, url url.URL, headers map[string]string, queryParameters url.Values, respTy T) (T, int, error) {
-	// append any query parameters.
+func doHTTPReq(ctx context.Context, client http.Client, method string, url url.URL, body io.Reader, headers map[string]string, queryParameters url.Values) (*APIResponse, error) {
+	// append any query parameters
 	q := url.Query()
-
 	for k, v := range queryParameters {
-		// using Add instead of set incase the input URL already had a query encoded
+		// using Add instead of set in case the input URL already had a query encoded
 		q.Add(k, strings.Join(v, ","))
 	}
 	// set the query to the encoded parameters
 	url.RawQuery = q.Encode()
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), body)
 	if err != nil {
-		return respTy, 0, err
+		return nil, err
 	}
 	// add each header to the request
 	for k, v := range headers {
 		req.Header.Set(k, v)
 	}
 
+	// log the request
+	message.Debug("%q %s", method, req.URL.Redacted())
+
 	// do the thing
 	res, err := client.Do(req)
 	if err != nil {
-		return respTy, 0, err
+		return nil, err
 	}
 	if res == nil {
-		return respTy, 0, fmt.Errorf("error: calling %s returned empty response", url.Redacted())
+		return nil, fmt.Errorf("error: calling %s returned empty response", url.Redacted())
 	}
 	defer res.Body.Close()
 
 	responseData, err := io.ReadAll(res.Body)
 	if err != nil {
-		return respTy, 0, err
+		return nil, err
 	}
 
-	var responseObject T
-	err = json.Unmarshal(responseData, &responseObject)
-	return responseObject, res.StatusCode, err
+	var respObj APIResponse
+	respObj.Raw = responseData
+	respObj.Status = res.StatusCode
+	err = json.Unmarshal(responseData, &respObj.Response)
+	return &respObj, err
 }
 
 func clientFromOpts(opts *ApiOpts) http.Client {

src/pkg/domains/api/spec.go

@@ -4,11 +4,17 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"strings"
 	"time"
 )
 
 var defaultTimeout = 30 * time.Second
 
+const (
+	HTTPMethodGet  string = "GET"
+	HTTPMethodPost string = "POST"
+)
+
 // validateAndMutateSpec validates the spec values and applies any defaults or
 // other mutations or normalizations necessary. The original values are not modified.
 // validateAndMutateSpec will validate the entire object and may return multiple
@@ -55,6 +61,21 @@ func validateAndMutateSpec(spec *ApiSpec) (errs error) {
 				errs = errors.Join(errs, err)
 			}
 		}
+
+		switch m := spec.Requests[i].Method; strings.ToLower(m) {
+		case "post":
+			spec.Requests[i].Method = HTTPMethodPost
+		case "get", "":
+			fallthrough
+		default:
+			spec.Requests[i].Method = HTTPMethodGet
+		}
+
+		if !spec.executable { // we only need to set this once
+			if spec.Requests[i].Executable {
+				spec.executable = true
+			}
+		}
 	}
 
 	return errs
@@ -80,8 +101,7 @@ func validateAndMutateOptions(opts *ApiOpts) (errs error) {
 	if opts.Proxy != "" {
 		proxyURL, err := url.Parse(opts.Proxy)
 		if err != nil {
-			// not logging the input URL in case it has embedded credentials
-			errs = errors.Join(errs, errors.New("invalid proxy string"))
+			errs = errors.Join(errs, fmt.Errorf("invalid proxy string %s", proxyURL.Redacted()))
 		}
 		opts.proxyURL = proxyURL
 	}

src/pkg/domains/api/spec_test.go

@@ -108,7 +108,7 @@ func TestValidateAndMutateSpec(t *testing.T) {
 			},
 			1,
 		},
-		"success": {
+		"success (get with params)": {
 			&ApiSpec{
 				Requests: []Request{
 					{
@@ -141,6 +141,43 @@ func TestValidateAndMutateSpec(t *testing.T) {
 							},
 							timeout: &defaultTimeout,
 						},
+						Method: "GET",
+					},
+				},
+				Options: &ApiOpts{timeout: &defaultTimeout},
+			},
+			0,
+		},
+		"success (post with body)": {
+			&ApiSpec{
+				Requests: []Request{
+					{
+						Name: "healthcheck",
+						URL:  "http://example.com/health",
+						Body: `{"some":"thing"}`,
+						Options: &ApiOpts{
+							Headers: map[string]string{
+								"cache-control": "no-hit",
+							},
+						},
+						Method: "POST",
+					},
+				},
+			},
+			&ApiSpec{
+				Requests: []Request{
+					{
+						Name:   "healthcheck",
+						URL:    "http://example.com/health",
+						Body:   `{"some":"thing"}`,
+						reqURL: healthcheckUrl,
+						Options: &ApiOpts{
+							Headers: map[string]string{
+								"cache-control": "no-hit",
+							},
+							timeout: &defaultTimeout,
+						},
+						Method: "POST",
 					},
 				},
 				Options: &ApiOpts{timeout: &defaultTimeout},

src/pkg/domains/api/types.go

@@ -30,9 +30,9 @@ func (a ApiDomain) GetResources(ctx context.Context) (types.DomainResources, err
 	return a.makeRequests(ctx)
 }
 
+// IsExecutable returns true if any of the requests are marked executable
 func (a ApiDomain) IsExecutable() bool {
-	// Domain is not currently executable
-	return false
+	return a.Spec.executable
 }
 
 // ApiSpec contains a list of API requests
@@ -41,13 +41,20 @@ type ApiSpec struct {
 	// Opts will be applied to all requests, except those which have their own
 	// specified ApiOpts
 	Options *ApiOpts `mapstructure:"options" json:"options,omitempty" yaml:"options,omitempty"`
+
+	// internally-managed fields executable will be set to true during spec
+	// validation if *any* of the requests are flagged executable
+	executable bool
 }
 
 // Request is a single API request
 type Request struct {
-	Name   string            `json:"name" yaml:"name"`
-	URL    string            `json:"url" yaml:"url"`
-	Params map[string]string `json:"parameters,omitempty" yaml:"parameters,omitempty"`
+	Name       string            `json:"name" yaml:"name"`
+	URL        string            `json:"url" yaml:"url"`
+	Params     map[string]string `json:"parameters,omitempty" yaml:"parameters,omitempty"`
+	Method     string            `json:"method,omitempty" yaml:"method,omitempty"`
+	Body       string            `json:"body,omitempty" yaml:"body,omitempty"`
+	Executable bool              `json:"executable,omitempty" yaml:"executable,omitempty"`
 	// ApiOpts specific to this request. If ApiOpts is present, values in the
 	// ApiSpec-level Options are ignored for this request.
 	Options *ApiOpts `json:"options,omitempty" yaml:"options,omitempty"`