Merge pull request #160 from Luzilla/error-msg

Fix: error handling

.github/workflows/golint.yml

@@ -0,0 +1,23 @@
+name: golangci-lint
+on:
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
+          cache: false
+      - uses: golangci/golangci-lint-action@v3
+        with:
+          version: latest
+          skip-cache: true

.github/workflows/integration.yml

@@ -5,7 +5,7 @@ on:
   pull_request:
 
 jobs:
-  snapshot:
+  e2e:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -38,20 +38,31 @@ jobs:
       unbound:
         image: klutchell/unbound:latest
         ports:
-          - 5053:5053
+          - 5053:5053/tcp
           - 5053:5053/udp
     steps:
-      - name: Download artifact
-        uses: actions/download-artifact@v3
+      - uses: actions/checkout@v3
         with:
-          name: dnsbl_exporter
-      - name: Allow running exporter
-        run: chmod +x ./dnsbl-exporter
-      - name: Start dnsbl_exporter 
-        run: ./dnsbl-exporter --config.dns-resolver=unbound:5053 &
-      - name: Test "/" exists
-        run: curl -I http://127.0.0.1:9211/
-      - name: Test "/metrics" exists
-        run: curl -I http://127.0.0.1:9211/metrics
-      - name: Test "/metrics" with targets
-        run: curl -i http://127.0.0.1:9211/metrics
+          fetch-depth: 0
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
+          cache: false
+      - uses: goreleaser/goreleaser-action@v4
+        with:
+          version: latest
+          args: release --config ./.goreleaser.ci.yml --clean --snapshot
+      - run: cp targets.ini rbls.ini ./dist/dnsbl_exporter_linux_amd64_v1
+      - uses: JarvusInnovations/background-action@v1
+        with:
+          run: |
+            ls -lah && ./dnsbl-exporter --log.debug --config.dns-resolver=localhost:5053 &
+          wait-on: |
+            http-get://localhost:9211/
+            http-get://localhost:9211/metrics
+            http-get://localhost:9211/prober?target=github.com
+          tail: true # true = stderr,stdout
+          log-output-resume: stderr
+          wait-for: 1m
+          log-output: stderr,stdout
+          working-directory: ./dist/dnsbl_exporter_linux_amd64_v1

.golangci.yml

@@ -0,0 +1,2 @@
+run:
+  timeout: 2m

Makefile

@@ -4,7 +4,9 @@ build:
 
 .PHONY: run-dev
 run-dev:
-	go run dnsbl_exporter.go --log.debug
+	go run dnsbl_exporter.go \
+		--log.debug \
+		--config.dns-resolver 0.0.0.0:15353
 
 .PHONY: snapshot
 snapshot:

app/app.go

@@ -13,9 +13,12 @@ import (
 	"github.com/Luzilla/dnsbl_exporter/internal/metrics"
 	"github.com/Luzilla/dnsbl_exporter/internal/prober"
 	"github.com/Luzilla/dnsbl_exporter/internal/setup"
-	"github.com/prometheus/client_golang/prometheus"
+	"github.com/Luzilla/dnsbl_exporter/pkg/dns"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/exp/slog"
+
+	x "github.com/miekg/dns"
 )
 
 type DNSBLApp struct {
@@ -104,7 +107,6 @@ func NewApp(name string, version string) DNSBLApp {
 func (a *DNSBLApp) Bootstrap() {
 	a.App.Action = func(ctx *cli.Context) error {
 		// setup logging
-		fmt.Println("VERSION: " + appVersion)
 		handler := &slog.HandlerOptions{}
 		var writer io.Writer
 
@@ -161,7 +163,13 @@ func (a *DNSBLApp) Bootstrap() {
 
 		registry := setup.CreateRegistry()
 
-		rblCollector := setup.CreateCollector(rbls, targets, resolver, log.With("area", "metrics"))
+		dnsUtil, err := dns.New(new(x.Client), resolver, log)
+		if err != nil {
+			log.Error("failed to initialize dns client")
+			return err
+		}
+
+		rblCollector := setup.CreateCollector(rbls, targets, dnsUtil, log.With("area", "metrics"))
 		registry.MustRegister(rblCollector)
 
 		registryExporter := setup.CreateRegistry()
@@ -170,8 +178,8 @@ func (a *DNSBLApp) Bootstrap() {
 			log.Info("Exposing exporter metrics")
 
 			registryExporter.MustRegister(
-				prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
-				prometheus.NewGoCollector(),
+				collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+				collectors.NewGoCollector(),
 			)
 		}
 
@@ -183,13 +191,16 @@ func (a *DNSBLApp) Bootstrap() {
 		http.Handle(ctx.String("web.telemetry-path"), mHandler.Handler())
 
 		pHandler := prober.ProberHandler{
-			Resolver: resolver,
-			Rbls:     rbls,
-			Logger:   log.With("area", "prober"),
+			DNS:    dnsUtil,
+			Rbls:   rbls,
+			Logger: log.With("area", "prober"),
 		}
 		http.Handle("/prober", pHandler)
 
-		log.Info("Starting on: " + ctx.String("web.listen-address"))
+		log.Info("starting exporter",
+			slog.String("web.listen-address", ctx.String("web.listen-address")),
+			slog.String("resolver", resolver),
+		)
 		err = http.ListenAndServe(ctx.String("web.listen-address"), nil)
 		if err != nil {
 			return err

collector/collector.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/Luzilla/dnsbl_exporter/pkg/dns"
 	"github.com/Luzilla/dnsbl_exporter/pkg/rbl"
-	x "github.com/miekg/dns"
 	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/exp/slog"
 )
@@ -23,7 +22,7 @@ type RblCollector struct {
 	targetsMetric     *prometheus.Desc
 	durationMetric    *prometheus.Desc
 	rbls              []string
-	resolver          string
+	util              *dns.DNSUtil
 	targets           []string
 	logger            *slog.Logger
 }
@@ -33,7 +32,7 @@ func BuildFQName(metric string) string {
 }
 
 // NewRblCollector ... creates the collector
-func NewRblCollector(rbls []string, targets []string, resolver string, logger *slog.Logger) *RblCollector {
+func NewRblCollector(rbls []string, targets []string, util *dns.DNSUtil, logger *slog.Logger) *RblCollector {
 	return &RblCollector{
 		configuredMetric: prometheus.NewDesc(
 			BuildFQName("used"),
@@ -71,10 +70,10 @@ func NewRblCollector(rbls []string, targets []string, resolver string, logger *s
 			nil,
 			nil,
 		),
-		rbls:     rbls,
-		resolver: resolver,
-		targets:  targets,
-		logger:   logger,
+		rbls:    rbls,
+		util:    util,
+		targets: targets,
+		logger:  logger,
 	}
 }
 
@@ -106,36 +105,51 @@ func (c *RblCollector) Collect(ch chan<- prometheus.Metric) {
 	// this should be a map of blacklist and a counter (for listings)
 	var listed sync.Map
 
-	// iterate over hosts -> resolve to ip, check
+	resolver := rbl.NewRBLResolver(c.logger, c.util)
+
+	// iterate over hosts -> resolve to ip
+	targets := make(chan []rbl.Target)
 	for _, host := range hosts {
-		logger := c.logger.With("target", host)
+		go resolver.Do(host, targets)
+	}
+
+	// run the check
+	for _, target := range <-targets {
 
-		logger.Debug("Starting check")
+		results := make([]rbl.Result, 0)
 
-		r := rbl.New(dns.New(new(x.Client), c.resolver, logger), logger)
-		r.Update(host, c.rbls)
+		result := make(chan rbl.Result)
+		for _, blocklist := range c.rbls {
+			logger := c.logger.With("host", target.Host)
+
+			logger.Debug("starting check")
+
+			r := rbl.New(c.util, logger)
+			go r.Update(target, blocklist, result)
+			results = append(results, <-result)
+		}
 
-		for _, result := range r.Results {
+		for _, check := range results {
 			metricValue := 0
 
-			val, _ := listed.LoadOrStore(result.Rbl, 0)
-			if result.Listed {
+			val, _ := listed.LoadOrStore(check.Rbl, 0)
+			if check.Listed {
 				metricValue = 1
-				listed.Store(result.Rbl, val.(int)+1)
+				listed.Store(check.Rbl, val.(int)+1)
 			}
 
-			logger.Debug(result.Rbl+" listed?", slog.Int("v", metricValue))
+			c.logger.Debug("listed?", slog.Int("v", metricValue), slog.String("rbl", check.Rbl))
 
-			labelValues := []string{result.Rbl, result.Address, host}
+			labelValues := []string{check.Rbl, check.Target.IP.String(), check.Target.Host}
 
-			// this is an "error" from the RBL
-			if result.Error {
-				logger.Error(result.Text)
+			// this is an "error" from the RBL/transport
+			if check.Error {
+				c.logger.Error(check.ErrorType.Error(), slog.String("text", check.Text))
 				ch <- prometheus.MustNewConstMetric(
 					c.errorsMetrics,
 					prometheus.GaugeValue,
 					1,
-					[]string{result.Rbl}...,
+					[]string{check.Rbl}...,
 				)
 			}
 

config/config.go

@@ -49,7 +49,7 @@ func (c *Config) GetTargets(cfg *ini.File) []string {
 
 // LoadFile ...
 func (c *Config) LoadFile(path string) (*ini.File, error) {
-	c.Logger.Debug("Loading configuration file: " + path)
+	c.Logger.Debug("loading configuration file: " + path)
 
 	cfg, err := loadConfig(path)
 	if err != nil {

dnsbl_exporter.go

@@ -9,7 +9,7 @@ import (
 
 // The following are customized during build
 var exporterName string = "dnsbl-exporter"
-var exporterVersion string
+var exporterVersion string = "dev"
 
 func main() {
 	dnsbl := app.NewApp(exporterName, exporterVersion)
@@ -20,5 +20,4 @@ func main() {
 		fmt.Println("error: " + err.Error())
 		os.Exit(1)
 	}
-
 }

go.mod

@@ -6,6 +6,7 @@ require github.com/prometheus/client_golang v1.17.0
 
 require (
 	github.com/Luzilla/godnsbl v1.0.0
+	github.com/foxcpp/go-mockdns v1.0.0
 	github.com/miekg/dns v1.1.56
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli/v2 v2.25.7

go.sum

@@ -9,6 +9,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=
+github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
@@ -20,6 +22,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
 github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -41,18 +44,32 @@ github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
 github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
 golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=

internal/index/index.go

@@ -11,6 +11,7 @@ type IndexHandler struct {
 }
 
 func (i IndexHandler) Handler(w http.ResponseWriter, r *http.Request) {
+	//nolint:errcheck
 	w.Write([]byte(`<html>
 		<head><title>` + i.Name + `</title></head>
 		<body>

internal/prober/prober.go

@@ -4,14 +4,15 @@ import (
 	"net/http"
 
 	"github.com/Luzilla/dnsbl_exporter/internal/setup"
+	"github.com/Luzilla/dnsbl_exporter/pkg/dns"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/exp/slog"
 )
 
 type ProberHandler struct {
-	Resolver string
-	Rbls     []string
-	Logger   *slog.Logger
+	DNS    *dns.DNSUtil
+	Rbls   []string
+	Logger *slog.Logger
 }
 
 func (p ProberHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -25,7 +26,7 @@ func (p ProberHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	targets = append(targets, r.URL.Query().Get("target"))
 
 	registry := setup.CreateRegistry()
-	collector := setup.CreateCollector(p.Rbls, targets, p.Resolver, p.Logger)
+	collector := setup.CreateCollector(p.Rbls, targets, p.DNS, p.Logger)
 	registry.MustRegister(collector)
 
 	h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{