Merge pull request #513 from pzaino/develop

Some fixes from user's feedback

doc/docker_build.md

@@ -2,19 +2,33 @@
 
 The manual build of the CROWler can be a complex activity, as it requires the
 installation of several dependencies and the configuration of the environment.
-To simplify this process, the `docker-build` script was created. This script
-is responsible for building the CROWler docker images and running them in
-separate containers.
+To simplify this process, the `docker-build.sh` script was created. This script
+is responsible for patching dependencies, building the CROWler docker images,
+and running them in separate containers.
 
 docker-build takes care of building the CROWler docker images using the
-config.yml file and determine the platform (x86_64 or ARM64) you want to use
+config.yaml file and determine the platform (x86_64 or ARM64) you want to use
 and configuring the build appropriately.
 
 ## Before you start
 
 - **Step A** Make sure you have `git`, `make`, `docker` and `docker-compose`
   are installed on your machine, these are ALL required to build the images.
 
+  As for which releases of the above:
+  - Docker >= 26.0
+  - Docker-compose >= 2.27.0
+  - Git >= 2.20.0
+  - GNU Make >= 4.2.1
+
+  As for the minimal requirements for the machine you are building the CROWler:
+  - 4GB of RAM
+  - 2 CPU cores
+  - 20GB of disk space
+
+  **Note**: The above are the minimal requirements, you can run the CROWler on
+  a machine with less resources, but you might experience performance issues.
+
 - **Step B** There are a bunch of ENV variables you can set to
   customize the CROWler deployment. These ENV vars allow you to set up your
   username and password for the database, the database name, the port the API
@@ -85,6 +99,8 @@ Once you've completed the steps above, follow the procedure below:
   and engines. You can simply generate a single engine and VDI and then scale
   up using Kubernetes.
 
+  For more options for this script please check the source code of the script.
+
 - **Step 5**: Generate your **config.yaml** file based on your desired
   configuration. You can simply start from renaming config.default to
   `config.yaml` and then modify it according to your needs. If you are building
@@ -102,13 +118,22 @@ Once you've completed the steps above, follow the procedure below:
 - **Step 6**: To build the CROWler docker images, run the following command:
 
   ```bash
-  ./docker-build build
+  ./docker-build.sh build
   ```
 
-  Please note that the `docker-build` script accepts the docker-compose
+  **Please note (1):** that the `docker-build` script accepts the docker-compose
   arguments, so it's a special wrapper that will ensure that the images are
   configured and generated correctly and then call docker-compose.
 
+  **Please note (2):** On some Linux Distros (or if you do not have your user
+  in the docker group), you may need to run the above command with `sudo`.
+
+  **Please note (3):** If you are using BuildKit and it has a build container
+  running using the 'docker-container' or 'cloud' engines to perform docker
+  container builds, you may need to add the argument `--load` to the command
+  above to load the images into the docker daemon. (thanks @mort666 for the
+  tip and testing the build on that environment).
+
   `docker-build` will build the CROWler docker images and tag them with the
    `crowler` prefix.
 
@@ -127,7 +152,7 @@ procedure (it's the same), and replace Step 6 with executing the following
 command instead of the previous syntax for `docker-build`:
 
 ```bash
-./docker-build up --build
+./docker-build.sh up --build
 ```
 
 This command will build the CROWler docker images and then run them all into 3
@@ -140,7 +165,7 @@ To build the CROWler docker images and run them in detached mode, follow the
 the same, and instead of Step 6, run the following command:
 
 ```bash
-./docker-build up --build -d
+./docker-build.sh up --build -d
 ```
 
 This command will build the CROWler docker images and tag them with the
@@ -163,8 +188,18 @@ To rebuild (for example after you've downloaded a new version of the CROWler
 code) the CROWler docker images from scratch, run the following command:
 
 ```bash
-./docker_rebuild build --no-cache
+./docker_rebuild.sh build --no-cache
 ```
 
 Please note: `docker_rebuild` instead of `docker-build` is used to ensure that
 the images are built from scratch.
+
+If, when you REbuild, you want to preserve your previous data (for example, the
+database), you can run the following command:
+
+```bash
+./docker_rebuild.sh build --volumes
+```
+
+This will ensure that the volumes are preserved when you rebuild the CROWler.
+The configuration will still be updated and so will plugins, rules, etc.

docker-build.sh

@@ -298,7 +298,7 @@ fi
 
 # Run Docker Compose
 # shellcheck disable=SC2086
-docker-compose ${pars}
+docker compose ${pars}
 rval=$?
 if [ $rval -ne 0 ]; then
     echo "Failed to build the Docker containers"

docker-rebuild.sh

@@ -24,7 +24,7 @@ echo "Cleaning up..."
 if [ "${preserve_volumes}" -eq 1 ]; then
     echo "Preserving volumes"
     # Stop and remove containers, networks, and volumes
-    docker-compose down --remove-orphans
+    docker compose down --remove-orphans
     # remove crowler network
     docker network rm crowler-net
     docker network rm thecrowler_crowler-net
@@ -33,7 +33,7 @@ if [ "${preserve_volumes}" -eq 1 ]; then
 else
     echo "Removing volumes"
     # Stop and remove containers, networks, and volumes
-    docker-compose down -v --remove-orphans
+    docker compose down -v --remove-orphans
     # remove crowler network
     docker network rm crowler-net
     docker network rm thecrowler_crowler-net

pkg/plugin/plugins.go

@@ -706,6 +706,90 @@ func addJSAPIClient(vm *otto.Otto) error {
 		cmn.DebugMsg(cmn.DbgLvlError, "setting post method:", err)
 	}
 
+	// Define the "get" method
+	err = apiClientObject.Set("get", func(call otto.FunctionCall) otto.Value {
+		// Get the URL (first argument)
+		url, _ := call.Argument(0).ToString()
+
+		// Get the request options object (second argument)
+		requestArg := call.Argument(1)
+		var headers map[string]string
+		var timeoutMs int64 = 30000 // Default timeout
+
+		if requestArg.IsDefined() {
+			reqMap, err := requestArg.Export()
+			if err == nil {
+				// Extract headers
+				if h, exists := reqMap.(map[string]interface{})["headers"]; exists {
+					headersInterface, _ := h.(map[string]interface{})
+					headers = make(map[string]string)
+					for k, v := range headersInterface {
+						if vStr, ok := v.(string); ok {
+							headers[k] = vStr
+						}
+					}
+				}
+				// Extract timeout
+				if t, exists := reqMap.(map[string]interface{})["timeout"]; exists {
+					if timeoutFloat, ok := t.(float64); ok { // Otto exports numbers as float64
+						timeoutMs = int64(timeoutFloat)
+					}
+				}
+			}
+		}
+
+		// Set up HTTP client with timeout
+		timeout := time.Duration(timeoutMs) * time.Millisecond
+		client := &http.Client{Timeout: timeout}
+
+		// Create the HTTP request
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			cmn.DebugMsg(cmn.DbgLvlError, "Error creating GET request:", err)
+			return otto.UndefinedValue()
+		}
+
+		// Add headers to the request
+		for key, value := range headers {
+			req.Header.Set(key, value)
+		}
+
+		// Execute the request
+		resp, err := client.Do(req)
+		if err != nil {
+			cmn.DebugMsg(cmn.DbgLvlError, "Error executing GET request:", err)
+			return otto.UndefinedValue()
+		}
+		defer resp.Body.Close() //nolint:errcheck // We can't check error here it's a defer
+
+		// Read response body
+		respBody, err := io.ReadAll(resp.Body)
+		if err != nil {
+			cmn.DebugMsg(cmn.DbgLvlError, "Error reading GET response body:", err)
+			return otto.UndefinedValue()
+		}
+
+		// Create response object for JavaScript
+		respObject, _ := vm.Object(`({})`)
+		err = respObject.Set("status", resp.StatusCode)
+		if err != nil {
+			cmn.DebugMsg(cmn.DbgLvlError, "Error setting status in response object:", err)
+		}
+		err = respObject.Set("headers", resp.Header)
+		if err != nil {
+			cmn.DebugMsg(cmn.DbgLvlError, "Error setting headers in response object:", err)
+		}
+		err = respObject.Set("body", string(respBody))
+		if err != nil {
+			cmn.DebugMsg(cmn.DbgLvlError, "Error setting body in response object:", err)
+		}
+
+		return respObject.Value()
+	})
+	if err != nil {
+		cmn.DebugMsg(cmn.DbgLvlError, "setting get method:", err)
+	}
+
 	return vm.Set("apiClient", apiClientObject)
 }