feat: add opentelemetry tracer and metrics

README.md

@@ -288,37 +288,108 @@ Controls what log levels are output. Choose from `panic`, `fatal`, `error`, `war
 
 If you wish logs to be written to a file, set `log_file` to a valid file path.
 
-### Opentracing
+### Observability
 
-Currently, only the Datadog tracer is supported.
+GoTrue has basic observability built in. It is able to export
+[OpenTelemetry](https://opentelemetry.io) metrics and traces to a collector.
 
-```properties
-GOTRUE_TRACING_ENABLED=true
-GOTRUE_TRACING_HOST=127.0.0.1
-GOTRUE_TRACING_PORT=8126
-GOTRUE_TRACING_TAGS="tag1:value1,tag2:value2"
-GOTRUE_SERVICE_NAME="gotrue"
+#### Tracing
+
+To enable tracing configure these variables:
+
+`GOTRUE_TRACING_ENABLED` - `boolean`
+
+`GOTRUE_TRACING_EXPORTER` - `string` only `opentracing` (deprecated) and
+`opentelemetry` supported
+
+Make sure you also configure the [OpenTelemetry
+Exporter](https://opentelemetry.io/docs/reference/specification/protocol/exporter/)
+configuration for your collector or service.
+
+For example, if you use
+[Honeycomb.io](https://docs.honeycomb.io/getting-data-in/opentelemetry/go-distro/#using-opentelemetry-without-the-honeycomb-distribution)
+you should set these standard OpenTelemetry OTLP variables:
+
+```
+OTEL_SERVICE_NAME=gotrue
+OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+OTEL_EXPORTER_OTLP_ENDPOINT=https://api.honeycomb.io:443
+OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=<API-KEY>,x-honeycomb-dataset=gotrue"
 ```
 
-`TRACING_ENABLED` - `bool`
+#### Metrics
 
-Whether tracing is enabled or not. Defaults to `false`.
+To enable metrics configure these variables:
 
-`TRACING_HOST` - `bool`
+`GOTRUE_METRICS_ENABLED` - `boolean`
 
-The tracing destination.
+`GOTRUE_METRICS_EXPORTER` - `string` only `opentelemetry` and `prometheus`
+supported
 
-`TRACING_PORT` - `bool`
+Make sure you also configure the [OpenTelemetry
+Exporter](https://opentelemetry.io/docs/reference/specification/protocol/exporter/)
+configuration for your collector or service.
 
-The port for the tracing host.
+If you use the `prometheus` exporter, the server host and port can be
+configured using these standard OpenTelemetry variables:
 
-`TRACING_TAGS` - `string`
+`OTEL_EXPORTER_PROMETHEUS_HOST` - IP address, default `0.0.0.0`
+
+`OTEL_EXPORTER_PROMETHEUS_PORT` - port number, default `9100`
+
+The metrics are exported on the `/` path on the server.
+
+If you use the `opentelemetry` exporter, the metrics are pushed to the
+collector.
+
+For example, if you use
+[Honeycomb.io](https://docs.honeycomb.io/getting-data-in/opentelemetry/go-distro/#using-opentelemetry-without-the-honeycomb-distribution)
+you should set these standard OpenTelemetry OTLP variables:
+
+```
+OTEL_SERVICE_NAME=gotrue
+OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+OTEL_EXPORTER_OTLP_ENDPOINT=https://api.honeycomb.io:443
+OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=<API-KEY>,x-honeycomb-dataset=gotrue"
+```
 
-A comma separated list of key:value pairs. These key value pairs will be added as tags to all opentracing spans.
+Note that Honeycomb.io requires a paid plan to ingest metrics.
+
+If you need to debug an issue with traces or metrics not being pushed, you can
+set `DEBUG=true` to get more insights from the OpenTelemetry SDK.
+
+#### Custom resource attributes
+
+When using the OpenTelemetry tracing or metrics exporter you can define custom
+resource attributes using the [standard `OTEL_RESOURCE_ATTRIBUTES` environment
+variable](https://opentelemetry.io/docs/reference/specification/resource/sdk/#specifying-resource-information-via-an-environment-variable).
+
+A default attribute `gotrue.version` is provided containing the build version.
+
+#### Tracing HTTP routes
+
+All HTTP calls to the GoTrue API are traced. Routes use the parametrized
+version of the route, and the values for the route parameters can be found as
+the `http.route.params.<route-key>` span attribute.
+
+For example, the following request:
+
+```
+GET /admin/users/4acde936-82dc-4552-b851-831fb8ce0927/
+```
+
+will be traced as:
+
+```
+http.method = GET
+http.route = /admin/users/{user_id}
+http.route.params.user_id = 4acde936-82dc-4552-b851-831fb8ce0927
+```
 
-`SERVICE_NAME` - `string`
+#### Go runtime and HTTP metrics
 
-The name to use for the service.
+All of the Go runtime metrics are exposed. Some HTTP metrics are also collected
+by default.
 
 ### JSON Web Tokens (JWT)
 

api/api.go

@@ -62,10 +62,20 @@ func NewAPIWithVersion(ctx context.Context, globalConfig *conf.GlobalConfigurati
 	logger := observability.NewStructuredLogger(logrus.StandardLogger())
 
 	r := newRouter()
-	r.UseBypass(xffmw.Handler)
 	r.Use(addRequestID(globalConfig))
+
+	if globalConfig.Tracing.Enabled {
+		switch globalConfig.Tracing.Exporter {
+		case conf.OpenTelemetryTracing:
+			r.UseBypass(observability.RequestTracing())
+
+		case conf.OpenTracing:
+			r.UseBypass(opentracer)
+		}
+	}
+
+	r.UseBypass(xffmw.Handler)
 	r.Use(recoverer)
-	r.UseBypass(tracer)
 
 	r.Get("/health", api.HealthCheck)
 

api/api_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/gofrs/uuid"
 	"github.com/netlify/gotrue/conf"
 	"github.com/netlify/gotrue/models"
 	"github.com/netlify/gotrue/storage"
@@ -20,7 +19,6 @@ const (
 
 func init() {
 	models.PasswordHashCost = bcrypt.MinCost
-
 }
 
 // setupAPIForTest creates a new API to run tests with.
@@ -30,17 +28,25 @@ func setupAPIForTest() (*API, *conf.GlobalConfiguration, error) {
 	return setupAPIForTestWithCallback(nil)
 }
 
-func setupAPIForTestWithCallback(cb func(*conf.GlobalConfiguration, *storage.Connection) (uuid.UUID, error)) (*API, *conf.GlobalConfiguration, error) {
+func setupAPIForTestWithCallback(cb func(*conf.GlobalConfiguration, *storage.Connection)) (*API, *conf.GlobalConfiguration, error) {
 	config, err := conf.LoadGlobal(apiTestConfig)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	if cb != nil {
+		cb(config, nil)
+	}
+
 	conn, err := test.SetupDBConnection(config)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	if cb != nil {
+		cb(nil, conn)
+	}
+
 	return NewAPIWithVersion(context.Background(), config, conn, apiTestVersion), config, nil
 }
 

api/opentelemetry-tracer_test.go

@@ -0,0 +1,91 @@
+package api
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/netlify/gotrue/conf"
+	"github.com/netlify/gotrue/storage"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
+)
+
+type OpenTelemetryTracerTestSuite struct {
+	suite.Suite
+	API    *API
+	Config *conf.GlobalConfiguration
+}
+
+func TestOpenTelemetryTracer(t *testing.T) {
+	api, config, err := setupAPIForTestWithCallback(func(config *conf.GlobalConfiguration, conn *storage.Connection) {
+		if config != nil {
+			config.Tracing.Enabled = true
+			config.Tracing.Exporter = conf.OpenTelemetryTracing
+		}
+	})
+
+	require.NoError(t, err)
+
+	ts := &OpenTelemetryTracerTestSuite{
+		API:    api,
+		Config: config,
+	}
+	defer api.db.Close()
+
+	suite.Run(t, ts)
+}
+
+func getAttribute(attributes []attribute.KeyValue, key attribute.Key) *attribute.Value {
+	for _, value := range attributes {
+		if value.Key == key {
+			return &value.Value
+		}
+	}
+
+	return nil
+}
+
+func (ts *OpenTelemetryTracerTestSuite) TestOpenTelemetryTracer_Spans() {
+	exporter := tracetest.NewInMemoryExporter()
+	bsp := sdktrace.NewSimpleSpanProcessor(exporter)
+	traceProvider := sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
+		sdktrace.WithSpanProcessor(bsp),
+	)
+	otel.SetTracerProvider(traceProvider)
+
+	w := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodPost, "http://localhost/something1", nil)
+	ts.API.handler.ServeHTTP(w, req)
+
+	req = httptest.NewRequest(http.MethodGet, "http://localhost/something2", nil)
+	ts.API.handler.ServeHTTP(w, req)
+
+	spanStubs := exporter.GetSpans()
+	spans := spanStubs.Snapshots()
+
+	if assert.Equal(ts.T(), 2, len(spans)) {
+		attributes1 := spans[0].Attributes()
+		method1 := getAttribute(attributes1, semconv.HTTPMethodKey)
+		assert.Equal(ts.T(), "POST", method1.AsString())
+		url1 := getAttribute(attributes1, semconv.HTTPTargetKey)
+		assert.Equal(ts.T(), "http://localhost/something1", url1.AsString())
+		statusCode1 := getAttribute(attributes1, semconv.HTTPStatusCodeKey)
+		assert.Equal(ts.T(), int64(404), statusCode1.AsInt64())
+
+		attributes2 := spans[1].Attributes()
+		method2 := getAttribute(attributes2, semconv.HTTPMethodKey)
+		assert.Equal(ts.T(), "GET", method2.AsString())
+		url2 := getAttribute(attributes2, semconv.HTTPTargetKey)
+		assert.Equal(ts.T(), "http://localhost/something2", url2.AsString())
+		statusCode2 := getAttribute(attributes2, semconv.HTTPStatusCodeKey)
+		assert.Equal(ts.T(), int64(404), statusCode2.AsInt64())
+	}
+}

api/tracer.go → api/opentracer.go

@@ -8,7 +8,7 @@ import (
 	"github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 	ddtrace_ext "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"
-	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer"
+	ddopentracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer"
 )
 
 type tracingResponseWriter struct {
@@ -25,12 +25,12 @@ func (trw *tracingResponseWriter) WriteHeader(code int) {
 	trw.ResponseWriter.WriteHeader(code)
 }
 
-func tracer(next http.Handler) http.Handler {
+func opentracer(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		clientContext, _ := opentracing.GlobalTracer().Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(r.Header))
 		span, traceCtx := opentracing.StartSpanFromContext(r.Context(), "http.handler",
 			ext.RPCServerOption(clientContext),
-			opentracer.SpanType(ddtrace_ext.AppTypeWeb),
+			ddopentracer.SpanType(ddtrace_ext.AppTypeWeb),
 		)
 		defer span.Finish()
 

api/tracer_test.go → api/opentracer_test.go

@@ -6,24 +6,31 @@ import (
 	"testing"
 
 	"github.com/netlify/gotrue/conf"
+	"github.com/netlify/gotrue/storage"
 	"github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/mocktracer"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 )
 
-type TracerTestSuite struct {
+type OpenTracerTestSuite struct {
 	suite.Suite
 	API    *API
 	Config *conf.GlobalConfiguration
 }
 
-func TestTracer(t *testing.T) {
-	api, config, err := setupAPIForTest()
+func TestOpenTracer(t *testing.T) {
+	api, config, err := setupAPIForTestWithCallback(func(config *conf.GlobalConfiguration, conn *storage.Connection) {
+		if config != nil {
+			config.Tracing.Enabled = true
+			config.Tracing.Exporter = conf.OpenTracing
+		}
+	})
+
 	require.NoError(t, err)
 
-	ts := &TracerTestSuite{
+	ts := &OpenTracerTestSuite{
 		API:    api,
 		Config: config,
 	}
@@ -32,7 +39,7 @@ func TestTracer(t *testing.T) {
 	suite.Run(t, ts)
 }
 
-func (ts *TracerTestSuite) TestTracer_Spans() {
+func (ts *OpenTracerTestSuite) TestOpenTracer_Spans() {
 	mt := mocktracer.New()
 	opentracing.SetGlobalTracer(mt)
 

cmd/migrate_cmd.go

@@ -19,7 +19,7 @@ var migrateCmd = cobra.Command{
 }
 
 func migrate(cmd *cobra.Command, args []string) {
-	globalConfig := loadGlobalConfig()
+	globalConfig := loadGlobalConfig(cmd.Context())
 
 	if globalConfig.DB.Driver == "" && globalConfig.DB.URL != "" {
 		u, err := url.Parse(globalConfig.DB.URL)